howto/converting-to-mlrun/mlrun-code.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Refactored As Operational Pipeline (with MLRun)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Install prerequisites\n",
    "%pip install mlrun lightgbm shapely"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create an MLRun project and configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2023-08-20 11:31:45,650 [info] Project loaded successfully: {'project_name': 'nyc-taxi'}\n",
      "Project name: nyc-taxi-iguazio\n"
     ]
    }
   ],
   "source": [
    "from os import path\n",
    "import mlrun\n",
    "\n",
    "project_name_base = 'nyc-taxi'\n",
    "\n",
    "project = mlrun.get_or_create_project(project_name_base, user_project=True)\n",
    "\n",
    "print(f'Project name: {project.name}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Nuclio and MLRun Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mlrun: start-code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import path\n",
    "import numpy as np \n",
    "import pandas as pd\n",
    "import datetime as dt\n",
    "from sklearn.model_selection import train_test_split\n",
    "import lightgbm as lgbm\n",
    "from mlrun.execution import MLClientCtx\n",
    "from mlrun.datastore import DataItem\n",
    "from pickle import dumps\n",
    "import shapely.wkt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_zones_dict(zones_url):\n",
    "    zones_df = pd.read_csv(zones_url)\n",
    "    \n",
    "    # Remove unecessary fields\n",
    "    zones_df.drop(['Shape_Leng', 'Shape_Area', 'zone', 'LocationID', 'borough'], axis=1, inplace=True)\n",
    "    \n",
    "    # Convert DF to dictionary\n",
    "    zones_dict = zones_df.set_index('OBJECTID').to_dict('index')\n",
    "    \n",
    "    # Add lat/long to each zone\n",
    "    for zone in zones_dict:\n",
    "        shape = shapely.wkt.loads(zones_dict[zone]['the_geom'])\n",
    "        zones_dict[zone]['long'] = shape.centroid.x\n",
    "        zones_dict[zone]['lat'] = shape.centroid.y\n",
    "    \n",
    "    return zones_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_zone_lat(zones_dict, zone_id):\n",
    "    return zones_dict[zone_id]['lat']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_zone_long(zones_dict, zone_id):\n",
    "    return zones_dict[zone_id]['long']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_df(df):\n",
    "    return df[(df.fare_amount > 0)  & (df.fare_amount <= 500) &\n",
    "             (df.PULocationID > 0) & (df.PULocationID <= 263) & \n",
    "             (df.DOLocationID > 0) & (df.DOLocationID <= 263)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To Compute Haversine distance\n",
    "def sphere_dist(pickup_lat, pickup_lon, dropoff_lat, dropoff_lon):\n",
    "    \"\"\"\n",
    "    Return distance along great radius between pickup and dropoff coordinates.\n",
    "    \"\"\"\n",
    "    #Define earth radius (km)\n",
    "    R_earth = 6371\n",
    "    #Convert degrees to radians\n",
    "    pickup_lat, pickup_lon, dropoff_lat, dropoff_lon = map(np.radians,\n",
    "                                                             [pickup_lat, pickup_lon, \n",
    "                                                              dropoff_lat, dropoff_lon])\n",
    "    #Compute distances along lat, lon dimensions\n",
    "    dlat = dropoff_lat - pickup_lat\n",
    "    dlon = dropoff_lon - pickup_lon\n",
    "    \n",
    "    #Compute haversine distance\n",
    "    a = np.sin(dlat/2.0)**2 + np.cos(pickup_lat) * np.cos(dropoff_lat) * np.sin(dlon/2.0)**2\n",
    "    return 2 * R_earth * np.arcsin(np.sqrt(a))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def radian_conv(degree):\n",
    "    \"\"\"\n",
    "    Return radian.\n",
    "    \"\"\"\n",
    "    return  np.radians(degree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_airport_dist(dataset):\n",
    "    \"\"\"\n",
    "    Return minumum distance from pickup or dropoff coordinates to each airport.\n",
    "    JFK: John F. Kennedy International Airport\n",
    "    EWR: Newark Liberty International Airport\n",
    "    LGA: LaGuardia Airport\n",
    "    SOL: Statue of Liberty \n",
    "    NYC: Newyork Central\n",
    "    \"\"\"\n",
    "    jfk_coord = (40.639722, -73.778889)\n",
    "    ewr_coord = (40.6925, -74.168611)\n",
    "    lga_coord = (40.77725, -73.872611)\n",
    "    sol_coord = (40.6892,-74.0445) # Statue of Liberty\n",
    "    nyc_coord = (40.7141667,-74.0063889) \n",
    "    \n",
    "    \n",
    "    pickup_lat = dataset['pickup_latitude']\n",
    "    dropoff_lat = dataset['dropoff_latitude']\n",
    "    pickup_lon = dataset['pickup_longitude']\n",
    "    dropoff_lon = dataset['dropoff_longitude']\n",
    "    \n",
    "    pickup_jfk = sphere_dist(pickup_lat, pickup_lon, jfk_coord[0], jfk_coord[1]) \n",
    "    dropoff_jfk = sphere_dist(jfk_coord[0], jfk_coord[1], dropoff_lat, dropoff_lon) \n",
    "    pickup_ewr = sphere_dist(pickup_lat, pickup_lon, ewr_coord[0], ewr_coord[1])\n",
    "    dropoff_ewr = sphere_dist(ewr_coord[0], ewr_coord[1], dropoff_lat, dropoff_lon) \n",
    "    pickup_lga = sphere_dist(pickup_lat, pickup_lon, lga_coord[0], lga_coord[1]) \n",
    "    dropoff_lga = sphere_dist(lga_coord[0], lga_coord[1], dropoff_lat, dropoff_lon)\n",
    "    pickup_sol = sphere_dist(pickup_lat, pickup_lon, sol_coord[0], sol_coord[1]) \n",
    "    dropoff_sol = sphere_dist(sol_coord[0], sol_coord[1], dropoff_lat, dropoff_lon)\n",
    "    pickup_nyc = sphere_dist(pickup_lat, pickup_lon, nyc_coord[0], nyc_coord[1]) \n",
    "    dropoff_nyc = sphere_dist(nyc_coord[0], nyc_coord[1], dropoff_lat, dropoff_lon)\n",
    "    \n",
    "    \n",
    "    \n",
    "    dataset['jfk_dist'] = pickup_jfk + dropoff_jfk\n",
    "    dataset['ewr_dist'] = pickup_ewr + dropoff_ewr\n",
    "    dataset['lga_dist'] = pickup_lga + dropoff_lga\n",
    "    dataset['sol_dist'] = pickup_sol + dropoff_sol\n",
    "    dataset['nyc_dist'] = pickup_nyc + dropoff_nyc\n",
    "    \n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_datetime_info(dataset):\n",
    "    #Convert to datetime format\n",
    "    dataset['pickup_datetime'] = pd.to_datetime(dataset['tpep_pickup_datetime'],format=\"%Y-%m-%d %H:%M:%S\")\n",
    "    \n",
    "    dataset['hour'] = dataset.pickup_datetime.dt.hour\n",
    "    dataset['day'] = dataset.pickup_datetime.dt.day\n",
    "    dataset['month'] = dataset.pickup_datetime.dt.month\n",
    "    dataset['weekday'] = dataset.pickup_datetime.dt.weekday\n",
    "    dataset['year'] = dataset.pickup_datetime.dt.year\n",
    "    \n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fetch_data(context : MLClientCtx, taxi_records_csv_path: DataItem, zones_csv_path: DataItem):\n",
    "    \n",
    "    context.logger.info('Reading taxi records data from {}'.format(taxi_records_csv_path))\n",
    "    taxi_records_dataset = taxi_records_csv_path.as_df()\n",
    "    \n",
    "    context.logger.info('Reading zones data from {}'.format(zones_csv_path))\n",
    "    zones_dataset = zones_csv_path.as_df()\n",
    "    \n",
    "    target_path = path.join(context.artifact_path, 'data')\n",
    "    context.logger.info('Saving datasets to {} ...'.format(target_path))\n",
    "\n",
    "    # Store the data sets in your artifacts database\n",
    "    context.log_dataset('nyc-taxi-dataset', df=taxi_records_dataset, format='csv',\n",
    "                        index=False, artifact_path=target_path)\n",
    "    context.log_dataset('zones-dataset', df=zones_dataset, format='csv',\n",
    "                        index=False, artifact_path=target_path)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_zones_dict(zones_df):\n",
    "\n",
    "    # Remove unecessary fields\n",
    "    zones_df.drop(['Shape_Leng', 'Shape_Area', 'zone', 'LocationID', 'borough'], axis=1, inplace=True)\n",
    "    \n",
    "    # Convert DF to dictionary\n",
    "    zones_dict = zones_df.set_index('OBJECTID').to_dict('index')\n",
    "    \n",
    "    # Add lat/long to each zone\n",
    "    for zone in zones_dict:\n",
    "        shape = shapely.wkt.loads(zones_dict[zone]['the_geom'])\n",
    "        zones_dict[zone]['long'] = shape.centroid.x\n",
    "        zones_dict[zone]['lat'] = shape.centroid.y\n",
    "    \n",
    "    return zones_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_zone_lat(zones_dict, zone_id):\n",
    "    return zones_dict[zone_id]['lat']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_zone_long(zones_dict, zone_id):\n",
    "    return zones_dict[zone_id]['long']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def transform_dataset(context : MLClientCtx, taxi_records_csv_path: DataItem, zones_csv_path: DataItem):\n",
    "    \n",
    "    context.logger.info('Begin datasets transform')\n",
    "    \n",
    "    context.logger.info('zones_csv_path: ' + str(zones_csv_path))\n",
    "    \n",
    "    zones_df = zones_csv_path.as_df()    \n",
    "    \n",
    "    # Get zones dictionary\n",
    "    zones_dict = get_zones_dict(zones_df)\n",
    "    \n",
    "    train_df = taxi_records_csv_path.as_df()\n",
    "    \n",
    "    # Clean DF\n",
    "    train_df = clean_df(train_df)\n",
    "    \n",
    "    # Enrich DF\n",
    "    train_df['pickup_latitude'] = train_df.apply(lambda x: get_zone_lat(zones_dict, x['PULocationID']), axis=1 )\n",
    "    train_df['pickup_longitude'] = train_df.apply(lambda x: get_zone_long(zones_dict, x['PULocationID']), axis=1 )\n",
    "    train_df['dropoff_latitude'] = train_df.apply(lambda x: get_zone_lat(zones_dict, x['DOLocationID']), axis=1 )\n",
    "    train_df['dropoff_longitude'] = train_df.apply(lambda x: get_zone_long(zones_dict, x['DOLocationID']), axis=1 )\n",
    "\n",
    "    train_df = add_datetime_info(train_df)\n",
    "    train_df = add_airport_dist(train_df)\n",
    "\n",
    "    train_df['pickup_latitude'] = radian_conv(train_df['pickup_latitude'])\n",
    "    train_df['pickup_longitude'] = radian_conv(train_df['pickup_longitude'])\n",
    "    train_df['dropoff_latitude'] = radian_conv(train_df['dropoff_latitude'])\n",
    "    train_df['dropoff_longitude'] = radian_conv(train_df['dropoff_longitude'])\n",
    "\n",
    "    train_df.drop(['VendorID', 'tpep_pickup_datetime', 'tpep_dropoff_datetime', 'congestion_surcharge', 'improvement_surcharge', 'pickup_datetime',\n",
    "                  'extra', 'mta_tax', 'tip_amount', 'tolls_amount', 'total_amount', 'RatecodeID', 'store_and_fwd_flag',\n",
    "                  'PULocationID', 'DOLocationID', 'payment_type'], \n",
    "                  axis=1, inplace=True, errors='ignore')\n",
    "    \n",
    "    # Save dataset to artifact\n",
    "    target_path = path.join(context.artifact_path, 'data')\n",
    "    context.log_dataset('nyc-taxi-dataset-transformed', df=train_df, artifact_path=target_path, format='csv')    \n",
    "    \n",
    "    context.logger.info('End dataset transform')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "        'boosting_type':'gbdt',\n",
    "        'objective': 'regression',\n",
    "        'nthread': 4,\n",
    "        'num_leaves': 31,\n",
    "        'learning_rate': 0.05,\n",
    "        'max_depth': -1,\n",
    "        'subsample': 0.8,\n",
    "        'bagging_fraction' : 1,\n",
    "        'max_bin' : 5000 ,\n",
    "        'bagging_freq': 20,\n",
    "        'colsample_bytree': 0.6,\n",
    "        'metric': 'rmse',\n",
    "        'min_split_gain': 0.5,\n",
    "        'min_child_weight': 1,\n",
    "        'min_child_samples': 10,\n",
    "        'scale_pos_weight':1,\n",
    "        'zero_as_missing': True,\n",
    "        'seed':0,\n",
    "        'num_rounds':50000,\n",
    "        'early_stopping_rounds':500,\n",
    "        'verbose_eval':500\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(context: MLClientCtx, input_ds: DataItem):\n",
    "    \n",
    "    context.logger.info('Begin training')\n",
    "    context.logger.info('LGBM version is ' + str(lgbm.__version__))\n",
    "    \n",
    "    train_df = input_ds.as_df()\n",
    "    \n",
    "    y = train_df['fare_amount']\n",
    "  \n",
    "    train_df = train_df.drop(columns=['fare_amount'])\n",
    "    train_df = train_df.drop(train_df.columns[[0]], axis=1)\n",
    "    x_train,x_test,y_train,y_test = train_test_split(train_df,y,random_state=123,test_size=0.10)\n",
    "    \n",
    "    train_set = lgbm.Dataset(x_train, y_train, categorical_feature=['year','month','day','weekday'])\n",
    "    valid_set = lgbm.Dataset(x_test, y_test, categorical_feature=['year','month','day','weekday'])\n",
    "    model = lgbm.train(params, train_set = train_set, num_boost_round=10000, valid_sets=valid_set)\n",
    "    \n",
    "    context.log_model('FareModel',\n",
    "                     body=dumps(model),\n",
    "                     artifact_path=context.artifact_subpath(\"models\"),\n",
    "                     model_file=\"FareModel.pkl\")\n",
    "    \n",
    "    context.logger.info('End training')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mlrun: end-code"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set Input Paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import mlrun"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The example uses a CSVfile from MLRun demo data repository(by default stored in Wasabi object-store service)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "taxi_records_csv_path = mlrun.get_sample_path('data/Taxi/yellow_tripdata_2019-01_subset.csv')\n",
    "zones_csv_path = mlrun.get_sample_path('data/Taxi/taxi_zones.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convert Code to a Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "taxi_func = mlrun.code_to_function(name='taxi',\n",
    "                                   kind='job',\n",
    "                                   image='mlrun/mlrun',\n",
    "                                   requirements=['lightgbm', 'shapely'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run `fetch_data` Locally\n",
    "\n",
    "We can test out code locally, by calling the function with `local` parameter set to `True`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2022-08-28 12:12:50,973 [info] starting run taxi-fetch_data uid=0b9cef3d535b446493d633d1f88bb5ee DB=http://mlrun-api:8080\n",
      "> 2022-08-28 12:12:51,145 [info] Reading taxi records data from https://s3.wasabisys.com/iguazio/data/Taxi/yellow_tripdata_2019-01_subset.csv\n",
      "> 2022-08-28 12:13:47,719 [info] Reading zones data from https://s3.wasabisys.com/iguazio/data/Taxi/taxi_zones.csv\n",
      "> 2022-08-28 12:13:49,678 [info] Saving datasets to v3io:///projects/nyc-taxi-dani/artifacts/data ...\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       ".dictlist {\n",
       "  background-color: #4EC64B;\n",
       "  text-align: center;\n",
       "  margin: 4px;\n",
       "  border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;}\n",
       ".artifact {\n",
       "  cursor: pointer;\n",
       "  background-color: #4EC64B;\n",
       "  text-align: left;\n",
       "  margin: 4px; border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;\n",
       "}\n",
       "div.block.hidden {\n",
       "  display: none;\n",
       "}\n",
       ".clickable {\n",
       "  cursor: pointer;\n",
       "}\n",
       ".ellipsis {\n",
       "  display: inline-block;\n",
       "  max-width: 60px;\n",
       "  white-space: nowrap;\n",
       "  overflow: hidden;\n",
       "  text-overflow: ellipsis;\n",
       "}\n",
       ".master-wrapper {\n",
       "  display: flex;\n",
       "  flex-flow: row nowrap;\n",
       "  justify-content: flex-start;\n",
       "  align-items: stretch;\n",
       "}\n",
       ".master-tbl {\n",
       "  flex: 3\n",
       "}\n",
       ".master-wrapper > div {\n",
       "  margin: 4px;\n",
       "  padding: 10px;\n",
       "}\n",
       "iframe.fileview {\n",
       "  border: 0 none;\n",
       "  height: 100%;\n",
       "  width: 100%;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       ".pane-header-title {\n",
       "  width: 80%;\n",
       "  font-weight: 500;\n",
       "}\n",
       ".pane-header {\n",
       "  line-height: 1;\n",
       "  background-color: #4EC64B;\n",
       "  padding: 3px;\n",
       "}\n",
       ".pane-header .close {\n",
       "  font-size: 20px;\n",
       "  font-weight: 700;\n",
       "  float: right;\n",
       "  margin-top: -5px;\n",
       "}\n",
       ".master-wrapper .right-pane {\n",
       "  border: 1px inset silver;\n",
       "  width: 40%;\n",
       "  min-height: 300px;\n",
       "  flex: 3\n",
       "  min-width: 500px;\n",
       "}\n",
       ".master-wrapper * {\n",
       "  box-sizing: border-box;\n",
       "}\n",
       "</style><script>\n",
       "function copyToClipboard(fld) {\n",
       "    if (document.queryCommandSupported && document.queryCommandSupported('copy')) {\n",
       "        var textarea = document.createElement('textarea');\n",
       "        textarea.textContent = fld.innerHTML;\n",
       "        textarea.style.position = 'fixed';\n",
       "        document.body.appendChild(textarea);\n",
       "        textarea.select();\n",
       "\n",
       "        try {\n",
       "            return document.execCommand('copy'); // Security exception may be thrown by some browsers.\n",
       "        } catch (ex) {\n",
       "\n",
       "        } finally {\n",
       "            document.body.removeChild(textarea);\n",
       "        }\n",
       "    }\n",
       "}\n",
       "function expandPanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName');\n",
       "  console.log(el.title);\n",
       "\n",
       "  document.querySelector(panelName + \"-title\").innerHTML = el.title\n",
       "  iframe = document.querySelector(panelName + \"-body\");\n",
       "\n",
       "  const tblcss = `<style> body { font-family: Arial, Helvetica, sans-serif;}\n",
       "    #csv { margin-bottom: 15px; }\n",
       "    #csv table { border-collapse: collapse;}\n",
       "    #csv table td { padding: 4px 8px; border: 1px solid silver;} </style>`;\n",
       "\n",
       "  function csvToHtmlTable(str) {\n",
       "    return '<div id=\"csv\"><table><tr><td>' +  str.replace(/[\\n\\r]+$/g, '').replace(/[\\n\\r]+/g, '</td></tr><tr><td>')\n",
       "      .replace(/,/g, '</td><td>') + '</td></tr></table></div>';\n",
       "  }\n",
       "\n",
       "  function reqListener () {\n",
       "    if (el.title.endsWith(\".csv\")) {\n",
       "      iframe.setAttribute(\"srcdoc\", tblcss + csvToHtmlTable(this.responseText));\n",
       "    } else {\n",
       "      iframe.setAttribute(\"srcdoc\", this.responseText);\n",
       "    }\n",
       "    console.log(this.responseText);\n",
       "  }\n",
       "\n",
       "  const oReq = new XMLHttpRequest();\n",
       "  oReq.addEventListener(\"load\", reqListener);\n",
       "  oReq.open(\"GET\", el.title);\n",
       "  oReq.send();\n",
       "\n",
       "\n",
       "  //iframe.src = el.title;\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.remove(\"hidden\");\n",
       "  }\n",
       "}\n",
       "function closePanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName')\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (!resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.add(\"hidden\");\n",
       "  }\n",
       "}\n",
       "\n",
       "</script>\n",
       "<div class=\"master-wrapper\">\n",
       "  <div class=\"block master-tbl\"><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>project</th>\n",
       "      <th>uid</th>\n",
       "      <th>iter</th>\n",
       "      <th>start</th>\n",
       "      <th>state</th>\n",
       "      <th>name</th>\n",
       "      <th>labels</th>\n",
       "      <th>inputs</th>\n",
       "      <th>parameters</th>\n",
       "      <th>results</th>\n",
       "      <th>artifacts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>nyc-taxi-dani</td>\n",
       "      <td><div title=\"0b9cef3d535b446493d633d1f88bb5ee\"><a href=\"https://dashboard.default-tenant.app.vmdev93.lab.iguazeng.com/mlprojects/nyc-taxi-dani/jobs/monitor/0b9cef3d535b446493d633d1f88bb5ee/overview\" target=\"_blank\" >...f88bb5ee</a></div></td>\n",
       "      <td>0</td>\n",
       "      <td>Aug 28 12:12:51</td>\n",
       "      <td>completed</td>\n",
       "      <td>taxi-fetch_data</td>\n",
       "      <td><div class=\"dictlist\">v3io_user=dani</div><div class=\"dictlist\">kind=</div><div class=\"dictlist\">owner=dani</div><div class=\"dictlist\">host=jupyter-dani-5fcc55bd99-42fs2</div></td>\n",
       "      <td><div title=\"https://s3.wasabisys.com/iguazio/data/Taxi/yellow_tripdata_2019-01_subset.csv\">taxi_records_csv_path</div><div title=\"https://s3.wasabisys.com/iguazio/data/Taxi/taxi_zones.csv\">zones_csv_path</div></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"resultcb699bb2\" title=\"files/v3io/projects/nyc-taxi-dani/artifacts/data/taxi-fetch_data/0/nyc-taxi-dataset.csv\">nyc-taxi-dataset</div><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"resultcb699bb2\" title=\"files/v3io/projects/nyc-taxi-dani/artifacts/data/taxi-fetch_data/0/zones-dataset.csv\">zones-dataset</div></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div></div>\n",
       "  <div id=\"resultcb699bb2-pane\" class=\"right-pane block hidden\">\n",
       "    <div class=\"pane-header\">\n",
       "      <span id=\"resultcb699bb2-title\" class=\"pane-header-title\">Title</span>\n",
       "      <span onclick=\"closePanel(this)\" paneName=\"resultcb699bb2\" class=\"close clickable\">&times;</span>\n",
       "    </div>\n",
       "    <iframe class=\"fileview\" id=\"resultcb699bb2-body\"></iframe>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<b> > to track results use the .show() or .logs() methods  or <a href=\"https://dashboard.default-tenant.app.vmdev93.lab.iguazeng.com/mlprojects/nyc-taxi-dani/jobs/monitor/0b9cef3d535b446493d633d1f88bb5ee/overview\" target=\"_blank\">click here</a> to open in UI</b>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2022-08-28 12:13:59,290 [info] run executed, status=completed\n"
     ]
    }
   ],
   "source": [
    "fetch_data_run = taxi_func.run(handler='fetch_data',\n",
    "                               inputs={'taxi_records_csv_path': taxi_records_csv_path,\n",
    "                                       'zones_csv_path': zones_csv_path},\n",
    "                               local=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'nyc-taxi-dataset': 'store://artifacts/nyc-taxi-dani/taxi-fetch_data_nyc-taxi-dataset:0b9cef3d535b446493d633d1f88bb5ee',\n",
       " 'zones-dataset': 'store://artifacts/nyc-taxi-dani/taxi-fetch_data_zones-dataset:0b9cef3d535b446493d633d1f88bb5ee'}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fetch_data_run.outputs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run on the Cluster"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare Cluster Function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create an MLRun function and create a custom image for it (that uses shapely)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2022-08-28 12:13:59,457 [info] Started building image: .mlrun/func-nyc-taxi-dani-taxi:latest\n",
      "\u001B[36mINFO\u001B[0m[0000] Retrieving image manifest datanode-registry.iguazio-platform.app.vmdev93.lab.iguazeng.com:80/quay.io/mlrun/mlrun:1.1.0-rc25 \n",
      "\u001B[36mINFO\u001B[0m[0000] Retrieving image datanode-registry.iguazio-platform.app.vmdev93.lab.iguazeng.com:80/quay.io/mlrun/mlrun:1.1.0-rc25 from registry datanode-registry.iguazio-platform.app.vmdev93.lab.iguazeng.com:80 \n",
      "\u001B[36mINFO\u001B[0m[0000] Built cross stage deps: map[]                \n",
      "\u001B[36mINFO\u001B[0m[0000] Retrieving image manifest datanode-registry.iguazio-platform.app.vmdev93.lab.iguazeng.com:80/quay.io/mlrun/mlrun:1.1.0-rc25 \n",
      "\u001B[36mINFO\u001B[0m[0000] Returning cached image manifest              \n",
      "\u001B[36mINFO\u001B[0m[0000] Executing 0 build triggers                   \n",
      "\u001B[36mINFO\u001B[0m[0000] Unpacking rootfs as cmd RUN python -m pip install lightgbm shapely requires it. \n",
      "\u001B[36mINFO\u001B[0m[0028] RUN python -m pip install lightgbm shapely   \n",
      "\u001B[36mINFO\u001B[0m[0028] Taking snapshot of full filesystem...        \n",
      "\u001B[36mINFO\u001B[0m[0054] cmd: /bin/sh                                 \n",
      "\u001B[36mINFO\u001B[0m[0054] args: [-c python -m pip install lightgbm shapely] \n",
      "\u001B[36mINFO\u001B[0m[0054] Running: [/bin/sh -c python -m pip install lightgbm shapely] \n",
      "Collecting lightgbm\n",
      "  Downloading lightgbm-3.3.2-py3-none-manylinux1_x86_64.whl (2.0 MB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.0/2.0 MB 9.4 MB/s eta 0:00:00\n",
      "Collecting shapely\n",
      "  Downloading Shapely-1.8.4-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (2.0 MB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.0/2.0 MB 110.7 MB/s eta 0:00:00\n",
      "Requirement already satisfied: scipy in /usr/local/lib/python3.7/site-packages (from lightgbm) (1.7.3)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.7/site-packages (from lightgbm) (1.21.6)\n",
      "Requirement already satisfied: scikit-learn!=0.22.0 in /usr/local/lib/python3.7/site-packages (from lightgbm) (1.0.2)\n",
      "Requirement already satisfied: wheel in /usr/local/lib/python3.7/site-packages (from lightgbm) (0.37.1)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/site-packages (from scikit-learn!=0.22.0->lightgbm) (3.1.0)\n",
      "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/site-packages (from scikit-learn!=0.22.0->lightgbm) (1.1.0)\n",
      "Installing collected packages: shapely, lightgbm\n",
      "Successfully installed lightgbm-3.3.2 shapely-1.8.4\n",
      "WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\n",
      "WARNING: You are using pip version 22.0.4; however, version 22.2.2 is available.\n",
      "You should consider upgrading via the '/usr/local/bin/python -m pip install --upgrade pip' command.\n",
      "\u001B[36mINFO\u001B[0m[0059] Taking snapshot of full filesystem...        \n",
      "\u001B[36mINFO\u001B[0m[0061] Pushing image to docker-registry.default-tenant.app.vmdev93.lab.iguazeng.com:80/mlrun/func-nyc-taxi-dani-taxi:latest \n",
      "\u001B[36mINFO\u001B[0m[0061] Pushed docker-registry.default-tenant.app.vmdev93.lab.iguazeng.com:80/mlrun/func-nyc-taxi-dani-taxi@sha256:7dd3cd07d9c76b8b15d3130222c31b946c0f02b0fdd4a79dab7a9a361a6646a6 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from mlrun.platforms import auto_mount\n",
    "taxi_func.apply(auto_mount())\n",
    "taxi_func.deploy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2022-08-28 12:15:08,055 [info] starting run fetch_data uid=ac3686a0e6de4ca584ce9446c3ad7c94 DB=http://mlrun-api:8080\n",
      "> 2022-08-28 12:15:08,322 [info] Job is running in the background, pod: fetch-data-vrn52\n",
      "> 2022-08-28 12:15:16,789 [info] Reading taxi records data from https://s3.wasabisys.com/iguazio/data/Taxi/yellow_tripdata_2019-01_subset.csv\n",
      "> 2022-08-28 12:15:56,364 [info] Reading zones data from https://s3.wasabisys.com/iguazio/data/Taxi/taxi_zones.csv\n",
      "> 2022-08-28 12:15:58,144 [info] Saving datasets to v3io:///projects/nyc-taxi-dani/artifacts/data ...\n",
      "> 2022-08-28 12:16:07,087 [info] run executed, status=completed\n",
      "final state: completed\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       ".dictlist {\n",
       "  background-color: #4EC64B;\n",
       "  text-align: center;\n",
       "  margin: 4px;\n",
       "  border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;}\n",
       ".artifact {\n",
       "  cursor: pointer;\n",
       "  background-color: #4EC64B;\n",
       "  text-align: left;\n",
       "  margin: 4px; border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;\n",
       "}\n",
       "div.block.hidden {\n",
       "  display: none;\n",
       "}\n",
       ".clickable {\n",
       "  cursor: pointer;\n",
       "}\n",
       ".ellipsis {\n",
       "  display: inline-block;\n",
       "  max-width: 60px;\n",
       "  white-space: nowrap;\n",
       "  overflow: hidden;\n",
       "  text-overflow: ellipsis;\n",
       "}\n",
       ".master-wrapper {\n",
       "  display: flex;\n",
       "  flex-flow: row nowrap;\n",
       "  justify-content: flex-start;\n",
       "  align-items: stretch;\n",
       "}\n",
       ".master-tbl {\n",
       "  flex: 3\n",
       "}\n",
       ".master-wrapper > div {\n",
       "  margin: 4px;\n",
       "  padding: 10px;\n",
       "}\n",
       "iframe.fileview {\n",
       "  border: 0 none;\n",
       "  height: 100%;\n",
       "  width: 100%;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       ".pane-header-title {\n",
       "  width: 80%;\n",
       "  font-weight: 500;\n",
       "}\n",
       ".pane-header {\n",
       "  line-height: 1;\n",
       "  background-color: #4EC64B;\n",
       "  padding: 3px;\n",
       "}\n",
       ".pane-header .close {\n",
       "  font-size: 20px;\n",
       "  font-weight: 700;\n",
       "  float: right;\n",
       "  margin-top: -5px;\n",
       "}\n",
       ".master-wrapper .right-pane {\n",
       "  border: 1px inset silver;\n",
       "  width: 40%;\n",
       "  min-height: 300px;\n",
       "  flex: 3\n",
       "  min-width: 500px;\n",
       "}\n",
       ".master-wrapper * {\n",
       "  box-sizing: border-box;\n",
       "}\n",
       "</style><script>\n",
       "function copyToClipboard(fld) {\n",
       "    if (document.queryCommandSupported && document.queryCommandSupported('copy')) {\n",
       "        var textarea = document.createElement('textarea');\n",
       "        textarea.textContent = fld.innerHTML;\n",
       "        textarea.style.position = 'fixed';\n",
       "        document.body.appendChild(textarea);\n",
       "        textarea.select();\n",
       "\n",
       "        try {\n",
       "            return document.execCommand('copy'); // Security exception may be thrown by some browsers.\n",
       "        } catch (ex) {\n",
       "\n",
       "        } finally {\n",
       "            document.body.removeChild(textarea);\n",
       "        }\n",
       "    }\n",
       "}\n",
       "function expandPanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName');\n",
       "  console.log(el.title);\n",
       "\n",
       "  document.querySelector(panelName + \"-title\").innerHTML = el.title\n",
       "  iframe = document.querySelector(panelName + \"-body\");\n",
       "\n",
       "  const tblcss = `<style> body { font-family: Arial, Helvetica, sans-serif;}\n",
       "    #csv { margin-bottom: 15px; }\n",
       "    #csv table { border-collapse: collapse;}\n",
       "    #csv table td { padding: 4px 8px; border: 1px solid silver;} </style>`;\n",
       "\n",
       "  function csvToHtmlTable(str) {\n",
       "    return '<div id=\"csv\"><table><tr><td>' +  str.replace(/[\\n\\r]+$/g, '').replace(/[\\n\\r]+/g, '</td></tr><tr><td>')\n",
       "      .replace(/,/g, '</td><td>') + '</td></tr></table></div>';\n",
       "  }\n",
       "\n",
       "  function reqListener () {\n",
       "    if (el.title.endsWith(\".csv\")) {\n",
       "      iframe.setAttribute(\"srcdoc\", tblcss + csvToHtmlTable(this.responseText));\n",
       "    } else {\n",
       "      iframe.setAttribute(\"srcdoc\", this.responseText);\n",
       "    }\n",
       "    console.log(this.responseText);\n",
       "  }\n",
       "\n",
       "  const oReq = new XMLHttpRequest();\n",
       "  oReq.addEventListener(\"load\", reqListener);\n",
       "  oReq.open(\"GET\", el.title);\n",
       "  oReq.send();\n",
       "\n",
       "\n",
       "  //iframe.src = el.title;\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.remove(\"hidden\");\n",
       "  }\n",
       "}\n",
       "function closePanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName')\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (!resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.add(\"hidden\");\n",
       "  }\n",
       "}\n",
       "\n",
       "</script>\n",
       "<div class=\"master-wrapper\">\n",
       "  <div class=\"block master-tbl\"><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>project</th>\n",
       "      <th>uid</th>\n",
       "      <th>iter</th>\n",
       "      <th>start</th>\n",
       "      <th>state</th>\n",
       "      <th>name</th>\n",
       "      <th>labels</th>\n",
       "      <th>inputs</th>\n",
       "      <th>parameters</th>\n",
       "      <th>results</th>\n",
       "      <th>artifacts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>nyc-taxi-dani</td>\n",
       "      <td><div title=\"ac3686a0e6de4ca584ce9446c3ad7c94\"><a href=\"https://dashboard.default-tenant.app.vmdev93.lab.iguazeng.com/mlprojects/nyc-taxi-dani/jobs/monitor/ac3686a0e6de4ca584ce9446c3ad7c94/overview\" target=\"_blank\" >...c3ad7c94</a></div></td>\n",
       "      <td>0</td>\n",
       "      <td>Aug 28 12:15:16</td>\n",
       "      <td>completed</td>\n",
       "      <td>fetch_data</td>\n",
       "      <td><div class=\"dictlist\">v3io_user=dani</div><div class=\"dictlist\">kind=job</div><div class=\"dictlist\">owner=dani</div><div class=\"dictlist\">mlrun/client_version=1.1.0-rc25</div><div class=\"dictlist\">host=fetch-data-vrn52</div></td>\n",
       "      <td><div title=\"https://s3.wasabisys.com/iguazio/data/Taxi/yellow_tripdata_2019-01_subset.csv\">taxi_records_csv_path</div><div title=\"https://s3.wasabisys.com/iguazio/data/Taxi/taxi_zones.csv\">zones_csv_path</div></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"resultabb92d03\" title=\"files/v3io/projects/nyc-taxi-dani/artifacts/data/fetch_data/0/nyc-taxi-dataset.csv\">nyc-taxi-dataset</div><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"resultabb92d03\" title=\"files/v3io/projects/nyc-taxi-dani/artifacts/data/fetch_data/0/zones-dataset.csv\">zones-dataset</div></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div></div>\n",
       "  <div id=\"resultabb92d03-pane\" class=\"right-pane block hidden\">\n",
       "    <div class=\"pane-header\">\n",
       "      <span id=\"resultabb92d03-title\" class=\"pane-header-title\">Title</span>\n",
       "      <span onclick=\"closePanel(this)\" paneName=\"resultabb92d03\" class=\"close clickable\">&times;</span>\n",
       "    </div>\n",
       "    <iframe class=\"fileview\" id=\"resultabb92d03-body\"></iframe>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<b> > to track results use the .show() or .logs() methods  or <a href=\"https://dashboard.default-tenant.app.vmdev93.lab.iguazeng.com/mlprojects/nyc-taxi-dani/jobs/monitor/ac3686a0e6de4ca584ce9446c3ad7c94/overview\" target=\"_blank\">click here</a> to open in UI</b>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2022-08-28 12:16:09,531 [info] run executed, status=completed\n"
     ]
    }
   ],
   "source": [
    "fetch_data_run = taxi_func.run(name='fetch_data',\n",
    "                               handler='fetch_data',\n",
    "                               inputs={'taxi_records_csv_path': taxi_records_csv_path,\n",
    "                                       'zones_csv_path': zones_csv_path})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'nyc-taxi-dataset': 'store://artifacts/nyc-taxi-dani/fetch_data_nyc-taxi-dataset:ac3686a0e6de4ca584ce9446c3ad7c94',\n",
       " 'zones-dataset': 'store://artifacts/nyc-taxi-dani/fetch_data_zones-dataset:ac3686a0e6de4ca584ce9446c3ad7c94'}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fetch_data_run.outputs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transform the Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2022-08-28 12:16:09,552 [info] starting run transform_dataset uid=1a39a8c7e87a4367899713e55b2fc8c6 DB=http://mlrun-api:8080\n",
      "> 2022-08-28 12:16:09,760 [info] Job is running in the background, pod: transform-dataset-2xhsd\n",
      "> 2022-08-28 12:16:18,083 [info] Begin datasets transform\n",
      "> 2022-08-28 12:16:18,084 [info] zones_csv_path: v3io:///projects/nyc-taxi-dani/artifacts/data/fetch_data/0/zones-dataset.csv\n",
      "> 2022-08-28 12:17:28,979 [info] End dataset transform\n",
      "> 2022-08-28 12:17:29,033 [info] run executed, status=completed\n",
      "final state: completed\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       ".dictlist {\n",
       "  background-color: #4EC64B;\n",
       "  text-align: center;\n",
       "  margin: 4px;\n",
       "  border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;}\n",
       ".artifact {\n",
       "  cursor: pointer;\n",
       "  background-color: #4EC64B;\n",
       "  text-align: left;\n",
       "  margin: 4px; border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;\n",
       "}\n",
       "div.block.hidden {\n",
       "  display: none;\n",
       "}\n",
       ".clickable {\n",
       "  cursor: pointer;\n",
       "}\n",
       ".ellipsis {\n",
       "  display: inline-block;\n",
       "  max-width: 60px;\n",
       "  white-space: nowrap;\n",
       "  overflow: hidden;\n",
       "  text-overflow: ellipsis;\n",
       "}\n",
       ".master-wrapper {\n",
       "  display: flex;\n",
       "  flex-flow: row nowrap;\n",
       "  justify-content: flex-start;\n",
       "  align-items: stretch;\n",
       "}\n",
       ".master-tbl {\n",
       "  flex: 3\n",
       "}\n",
       ".master-wrapper > div {\n",
       "  margin: 4px;\n",
       "  padding: 10px;\n",
       "}\n",
       "iframe.fileview {\n",
       "  border: 0 none;\n",
       "  height: 100%;\n",
       "  width: 100%;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       ".pane-header-title {\n",
       "  width: 80%;\n",
       "  font-weight: 500;\n",
       "}\n",
       ".pane-header {\n",
       "  line-height: 1;\n",
       "  background-color: #4EC64B;\n",
       "  padding: 3px;\n",
       "}\n",
       ".pane-header .close {\n",
       "  font-size: 20px;\n",
       "  font-weight: 700;\n",
       "  float: right;\n",
       "  margin-top: -5px;\n",
       "}\n",
       ".master-wrapper .right-pane {\n",
       "  border: 1px inset silver;\n",
       "  width: 40%;\n",
       "  min-height: 300px;\n",
       "  flex: 3\n",
       "  min-width: 500px;\n",
       "}\n",
       ".master-wrapper * {\n",
       "  box-sizing: border-box;\n",
       "}\n",
       "</style><script>\n",
       "function copyToClipboard(fld) {\n",
       "    if (document.queryCommandSupported && document.queryCommandSupported('copy')) {\n",
       "        var textarea = document.createElement('textarea');\n",
       "        textarea.textContent = fld.innerHTML;\n",
       "        textarea.style.position = 'fixed';\n",
       "        document.body.appendChild(textarea);\n",
       "        textarea.select();\n",
       "\n",
       "        try {\n",
       "            return document.execCommand('copy'); // Security exception may be thrown by some browsers.\n",
       "        } catch (ex) {\n",
       "\n",
       "        } finally {\n",
       "            document.body.removeChild(textarea);\n",
       "        }\n",
       "    }\n",
       "}\n",
       "function expandPanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName');\n",
       "  console.log(el.title);\n",
       "\n",
       "  document.querySelector(panelName + \"-title\").innerHTML = el.title\n",
       "  iframe = document.querySelector(panelName + \"-body\");\n",
       "\n",
       "  const tblcss = `<style> body { font-family: Arial, Helvetica, sans-serif;}\n",
       "    #csv { margin-bottom: 15px; }\n",
       "    #csv table { border-collapse: collapse;}\n",
       "    #csv table td { padding: 4px 8px; border: 1px solid silver;} </style>`;\n",
       "\n",
       "  function csvToHtmlTable(str) {\n",
       "    return '<div id=\"csv\"><table><tr><td>' +  str.replace(/[\\n\\r]+$/g, '').replace(/[\\n\\r]+/g, '</td></tr><tr><td>')\n",
       "      .replace(/,/g, '</td><td>') + '</td></tr></table></div>';\n",
       "  }\n",
       "\n",
       "  function reqListener () {\n",
       "    if (el.title.endsWith(\".csv\")) {\n",
       "      iframe.setAttribute(\"srcdoc\", tblcss + csvToHtmlTable(this.responseText));\n",
       "    } else {\n",
       "      iframe.setAttribute(\"srcdoc\", this.responseText);\n",
       "    }\n",
       "    console.log(this.responseText);\n",
       "  }\n",
       "\n",
       "  const oReq = new XMLHttpRequest();\n",
       "  oReq.addEventListener(\"load\", reqListener);\n",
       "  oReq.open(\"GET\", el.title);\n",
       "  oReq.send();\n",
       "\n",
       "\n",
       "  //iframe.src = el.title;\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.remove(\"hidden\");\n",
       "  }\n",
       "}\n",
       "function closePanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName')\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (!resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.add(\"hidden\");\n",
       "  }\n",
       "}\n",
       "\n",
       "</script>\n",
       "<div class=\"master-wrapper\">\n",
       "  <div class=\"block master-tbl\"><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>project</th>\n",
       "      <th>uid</th>\n",
       "      <th>iter</th>\n",
       "      <th>start</th>\n",
       "      <th>state</th>\n",
       "      <th>name</th>\n",
       "      <th>labels</th>\n",
       "      <th>inputs</th>\n",
       "      <th>parameters</th>\n",
       "      <th>results</th>\n",
       "      <th>artifacts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>nyc-taxi-dani</td>\n",
       "      <td><div title=\"1a39a8c7e87a4367899713e55b2fc8c6\"><a href=\"https://dashboard.default-tenant.app.vmdev93.lab.iguazeng.com/mlprojects/nyc-taxi-dani/jobs/monitor/1a39a8c7e87a4367899713e55b2fc8c6/overview\" target=\"_blank\" >...5b2fc8c6</a></div></td>\n",
       "      <td>0</td>\n",
       "      <td>Aug 28 12:16:17</td>\n",
       "      <td>completed</td>\n",
       "      <td>transform_dataset</td>\n",
       "      <td><div class=\"dictlist\">v3io_user=dani</div><div class=\"dictlist\">kind=job</div><div class=\"dictlist\">owner=dani</div><div class=\"dictlist\">mlrun/client_version=1.1.0-rc25</div><div class=\"dictlist\">host=transform-dataset-2xhsd</div></td>\n",
       "      <td><div title=\"store://artifacts/nyc-taxi-dani/fetch_data_nyc-taxi-dataset:ac3686a0e6de4ca584ce9446c3ad7c94\">taxi_records_csv_path</div><div title=\"store://artifacts/nyc-taxi-dani/fetch_data_zones-dataset:ac3686a0e6de4ca584ce9446c3ad7c94\">zones_csv_path</div></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"result95ddb7c9\" title=\"files/v3io/projects/nyc-taxi-dani/artifacts/data/transform_dataset/0/nyc-taxi-dataset-transformed.csv\">nyc-taxi-dataset-transformed</div></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div></div>\n",
       "  <div id=\"result95ddb7c9-pane\" class=\"right-pane block hidden\">\n",
       "    <div class=\"pane-header\">\n",
       "      <span id=\"result95ddb7c9-title\" class=\"pane-header-title\">Title</span>\n",
       "      <span onclick=\"closePanel(this)\" paneName=\"result95ddb7c9\" class=\"close clickable\">&times;</span>\n",
       "    </div>\n",
       "    <iframe class=\"fileview\" id=\"result95ddb7c9-body\"></iframe>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<b> > to track results use the .show() or .logs() methods  or <a href=\"https://dashboard.default-tenant.app.vmdev93.lab.iguazeng.com/mlprojects/nyc-taxi-dani/jobs/monitor/1a39a8c7e87a4367899713e55b2fc8c6/overview\" target=\"_blank\">click here</a> to open in UI</b>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2022-08-28 12:17:38,904 [info] run executed, status=completed\n"
     ]
    }
   ],
   "source": [
    "transform_dataset_run = taxi_func.run(name='transform_dataset',\n",
    "                                      handler='transform_dataset',\n",
    "                                      inputs={'taxi_records_csv_path': fetch_data_run.outputs['nyc-taxi-dataset'],\n",
    "                                              'zones_csv_path': fetch_data_run.outputs['zones-dataset']})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'nyc-taxi-dataset-transformed': 'store://artifacts/nyc-taxi-dani/transform_dataset_nyc-taxi-dataset-transformed:1a39a8c7e87a4367899713e55b2fc8c6'}"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transform_dataset_run.outputs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2022-08-28 12:17:38,940 [info] starting run train_model uid=3b0bdb3e45334eb2981784dbc2f6c584 DB=http://mlrun-api:8080\n",
      "> 2022-08-28 12:17:39,156 [info] Job is running in the background, pod: train-model-qnf4x\n",
      "> 2022-08-28 12:17:47,325 [info] Begin training\n",
      "> 2022-08-28 12:17:47,325 [info] LGBM version is 3.3.2\n",
      "[LightGBM] [Warning] Met categorical feature which contains sparse values. Consider renumbering to consecutive integers started from zero\n",
      "[LightGBM] [Warning] bagging_fraction is set=1, subsample=0.8 will be ignored. Current value: bagging_fraction=1\n",
      "[LightGBM] [Warning] bagging_fraction is set=1, subsample=0.8 will be ignored. Current value: bagging_fraction=1\n",
      "[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.010111 seconds.\n",
      "You can set `force_row_wise=true` to remove the overhead.\n",
      "And if memory is not enough, you can set `force_col_wise=true`.\n",
      "[LightGBM] [Info] Total Bins 23961\n",
      "[LightGBM] [Info] Number of data points in the train set: 879294, number of used features: 16\n",
      "[LightGBM] [Warning] bagging_fraction is set=1, subsample=0.8 will be ignored. Current value: bagging_fraction=1\n",
      "[LightGBM] [Info] Start training from score 12.418691\n",
      "Training until validation scores don't improve for 500 rounds\n",
      "[500]\tvalid_0's rmse: 3.01302\n",
      "[1000]\tvalid_0's rmse: 2.98594\n",
      "[1500]\tvalid_0's rmse: 2.98293\n",
      "[2000]\tvalid_0's rmse: 2.98666\n",
      "Early stopping, best iteration is:\n",
      "[1551]\tvalid_0's rmse: 2.98227\n",
      "> 2022-08-28 12:19:33,166 [info] End training\n",
      "> 2022-08-28 12:19:33,252 [info] run executed, status=completed\n",
      "Found `num_rounds` in params. Will use it instead of argument\n",
      "'early_stopping_rounds' argument is deprecated and will be removed in a future release of LightGBM. Pass 'early_stopping()' callback via 'callbacks' argument instead.\n",
      "Using categorical_feature in Dataset.\n",
      "'verbose_eval' argument is deprecated and will be removed in a future release of LightGBM. Pass 'log_evaluation()' callback via 'callbacks' argument instead.\n",
      "'silent' argument is deprecated and will be removed in a future release of LightGBM. Pass 'verbose' parameter via 'params' instead.\n",
      "Overriding the parameters from Reference Dataset.\n",
      "categorical_column in param dict is overridden.\n",
      "final state: completed\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       ".dictlist {\n",
       "  background-color: #4EC64B;\n",
       "  text-align: center;\n",
       "  margin: 4px;\n",
       "  border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;}\n",
       ".artifact {\n",
       "  cursor: pointer;\n",
       "  background-color: #4EC64B;\n",
       "  text-align: left;\n",
       "  margin: 4px; border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;\n",
       "}\n",
       "div.block.hidden {\n",
       "  display: none;\n",
       "}\n",
       ".clickable {\n",
       "  cursor: pointer;\n",
       "}\n",
       ".ellipsis {\n",
       "  display: inline-block;\n",
       "  max-width: 60px;\n",
       "  white-space: nowrap;\n",
       "  overflow: hidden;\n",
       "  text-overflow: ellipsis;\n",
       "}\n",
       ".master-wrapper {\n",
       "  display: flex;\n",
       "  flex-flow: row nowrap;\n",
       "  justify-content: flex-start;\n",
       "  align-items: stretch;\n",
       "}\n",
       ".master-tbl {\n",
       "  flex: 3\n",
       "}\n",
       ".master-wrapper > div {\n",
       "  margin: 4px;\n",
       "  padding: 10px;\n",
       "}\n",
       "iframe.fileview {\n",
       "  border: 0 none;\n",
       "  height: 100%;\n",
       "  width: 100%;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       ".pane-header-title {\n",
       "  width: 80%;\n",
       "  font-weight: 500;\n",
       "}\n",
       ".pane-header {\n",
       "  line-height: 1;\n",
       "  background-color: #4EC64B;\n",
       "  padding: 3px;\n",
       "}\n",
       ".pane-header .close {\n",
       "  font-size: 20px;\n",
       "  font-weight: 700;\n",
       "  float: right;\n",
       "  margin-top: -5px;\n",
       "}\n",
       ".master-wrapper .right-pane {\n",
       "  border: 1px inset silver;\n",
       "  width: 40%;\n",
       "  min-height: 300px;\n",
       "  flex: 3\n",
       "  min-width: 500px;\n",
       "}\n",
       ".master-wrapper * {\n",
       "  box-sizing: border-box;\n",
       "}\n",
       "</style><script>\n",
       "function copyToClipboard(fld) {\n",
       "    if (document.queryCommandSupported && document.queryCommandSupported('copy')) {\n",
       "        var textarea = document.createElement('textarea');\n",
       "        textarea.textContent = fld.innerHTML;\n",
       "        textarea.style.position = 'fixed';\n",
       "        document.body.appendChild(textarea);\n",
       "        textarea.select();\n",
       "\n",
       "        try {\n",
       "            return document.execCommand('copy'); // Security exception may be thrown by some browsers.\n",
       "        } catch (ex) {\n",
       "\n",
       "        } finally {\n",
       "            document.body.removeChild(textarea);\n",
       "        }\n",
       "    }\n",
       "}\n",
       "function expandPanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName');\n",
       "  console.log(el.title);\n",
       "\n",
       "  document.querySelector(panelName + \"-title\").innerHTML = el.title\n",
       "  iframe = document.querySelector(panelName + \"-body\");\n",
       "\n",
       "  const tblcss = `<style> body { font-family: Arial, Helvetica, sans-serif;}\n",
       "    #csv { margin-bottom: 15px; }\n",
       "    #csv table { border-collapse: collapse;}\n",
       "    #csv table td { padding: 4px 8px; border: 1px solid silver;} </style>`;\n",
       "\n",
       "  function csvToHtmlTable(str) {\n",
       "    return '<div id=\"csv\"><table><tr><td>' +  str.replace(/[\\n\\r]+$/g, '').replace(/[\\n\\r]+/g, '</td></tr><tr><td>')\n",
       "      .replace(/,/g, '</td><td>') + '</td></tr></table></div>';\n",
       "  }\n",
       "\n",
       "  function reqListener () {\n",
       "    if (el.title.endsWith(\".csv\")) {\n",
       "      iframe.setAttribute(\"srcdoc\", tblcss + csvToHtmlTable(this.responseText));\n",
       "    } else {\n",
       "      iframe.setAttribute(\"srcdoc\", this.responseText);\n",
       "    }\n",
       "    console.log(this.responseText);\n",
       "  }\n",
       "\n",
       "  const oReq = new XMLHttpRequest();\n",
       "  oReq.addEventListener(\"load\", reqListener);\n",
       "  oReq.open(\"GET\", el.title);\n",
       "  oReq.send();\n",
       "\n",
       "\n",
       "  //iframe.src = el.title;\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.remove(\"hidden\");\n",
       "  }\n",
       "}\n",
       "function closePanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName')\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (!resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.add(\"hidden\");\n",
       "  }\n",
       "}\n",
       "\n",
       "</script>\n",
       "<div class=\"master-wrapper\">\n",
       "  <div class=\"block master-tbl\"><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>project</th>\n",
       "      <th>uid</th>\n",
       "      <th>iter</th>\n",
       "      <th>start</th>\n",
       "      <th>state</th>\n",
       "      <th>name</th>\n",
       "      <th>labels</th>\n",
       "      <th>inputs</th>\n",
       "      <th>parameters</th>\n",
       "      <th>results</th>\n",
       "      <th>artifacts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>nyc-taxi-dani</td>\n",
       "      <td><div title=\"3b0bdb3e45334eb2981784dbc2f6c584\"><a href=\"https://dashboard.default-tenant.app.vmdev93.lab.iguazeng.com/mlprojects/nyc-taxi-dani/jobs/monitor/3b0bdb3e45334eb2981784dbc2f6c584/overview\" target=\"_blank\" >...c2f6c584</a></div></td>\n",
       "      <td>0</td>\n",
       "      <td>Aug 28 12:17:46</td>\n",
       "      <td>completed</td>\n",
       "      <td>train_model</td>\n",
       "      <td><div class=\"dictlist\">v3io_user=dani</div><div class=\"dictlist\">kind=job</div><div class=\"dictlist\">owner=dani</div><div class=\"dictlist\">mlrun/client_version=1.1.0-rc25</div><div class=\"dictlist\">host=train-model-qnf4x</div></td>\n",
       "      <td><div title=\"store://artifacts/nyc-taxi-dani/transform_dataset_nyc-taxi-dataset-transformed:1a39a8c7e87a4367899713e55b2fc8c6\">input_ds</div></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td><div title=\"v3io:///projects/nyc-taxi-dani/artifacts/models/train_model/0/FareModel/\">FareModel</div></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div></div>\n",
       "  <div id=\"result21114a20-pane\" class=\"right-pane block hidden\">\n",
       "    <div class=\"pane-header\">\n",
       "      <span id=\"result21114a20-title\" class=\"pane-header-title\">Title</span>\n",
       "      <span onclick=\"closePanel(this)\" paneName=\"result21114a20\" class=\"close clickable\">&times;</span>\n",
       "    </div>\n",
       "    <iframe class=\"fileview\" id=\"result21114a20-body\"></iframe>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<b> > to track results use the .show() or .logs() methods  or <a href=\"https://dashboard.default-tenant.app.vmdev93.lab.iguazeng.com/mlprojects/nyc-taxi-dani/jobs/monitor/3b0bdb3e45334eb2981784dbc2f6c584/overview\" target=\"_blank\">click here</a> to open in UI</b>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2022-08-28 12:19:34,765 [info] run executed, status=completed\n"
     ]
    }
   ],
   "source": [
    "train_model_run = taxi_func.run(name='train_model',\n",
    "                                handler='train_model',\n",
    "                                inputs={'input_ds': transform_dataset_run.outputs['nyc-taxi-dataset-transformed']})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'FareModel': 'store://artifacts/nyc-taxi-dani/train_model_FareModel:3b0bdb3e45334eb2981784dbc2f6c584'}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_model_run.outputs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Serving"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The model serving class is in model-serving.ipynb."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2022-08-28 12:19:48,103 [info] Starting remote function deploy\n",
      "2022-08-28 12:19:48  (info) Deploying function\n",
      "2022-08-28 12:19:48  (info) Building\n",
      "2022-08-28 12:19:48  (info) Staging files and preparing base images\n",
      "2022-08-28 12:19:48  (info) Building processor image\n",
      "2022-08-28 12:20:44  (info) Build complete\n",
      "2022-08-28 12:20:58  (info) Function deploy complete\n",
      "> 2022-08-28 12:20:59,829 [info] successfully deployed function: {'internal_invocation_urls': ['nuclio-nyc-taxi-dani-model-serving.default-tenant.svc.cluster.local:8080'], 'external_invocation_urls': ['nyc-taxi-dani-model-serving-nyc-taxi-dani.default-tenant.app.vmdev93.lab.iguazeng.com/']}\n"
     ]
    }
   ],
   "source": [
    "serving = mlrun.code_to_function(filename=path.abspath('model-serving.ipynb'),kind = 'serving',image='mlrun/mlrun').apply(auto_mount())\n",
    "\n",
    "serving.spec.default_class = 'LGBMModel'\n",
    "serving.spec.build.commands = ['pip install lightgbm']\n",
    "serving.add_model('taxi-serving', train_model_run.outputs['FareModel'])\n",
    "serving_address = serving.deploy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2022-08-28 12:20:59,906 [info] invoking function: {'method': 'POST', 'path': 'http://nuclio-nyc-taxi-dani-model-serving.default-tenant.svc.cluster.local:8080/v2/models/taxi-serving/predict'}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'id': '175d1e04-9b5b-488d-9fc4-7deef4ae671b',\n",
       " 'model_name': 'taxi-serving',\n",
       " 'outputs': [9.52302976897415]}"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_data = '''{\"inputs\":[[1,0.80,0.711950,-1.291073,0.712059,1.290988,13,1,1,1,2019,47.274013,40.386065,16.975747,26.587155,18.925788]]}'''\n",
    "serving.invoke('/v2/models/taxi-serving/predict', my_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Kubeflow Pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set project functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<mlrun.runtimes.serving.ServingRuntime at 0x7fe93f6ad9d0>"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "project.set_function(f'db://{project.name}/taxi')\n",
    "project.set_function(f'db://{project.name}/model-serving')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create the Workflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing /User/test/demos/howto/converting-to-mlrun/conf/workflow.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile {'workflow.py'}\n",
    "from kfp import dsl\n",
    "from mlrun.platforms import auto_mount\n",
    "import os\n",
    "import sys\n",
    "import mlrun\n",
    "\n",
    "funcs = {}\n",
    "\n",
    "# init functions is used to configure function resources and local settings\n",
    "def init_functions(functions: dict, project=None, secrets=None):\n",
    "    for f in functions.values():\n",
    "        f.apply(auto_mount())\n",
    "\n",
    "@dsl.pipeline(\n",
    "    name=\"NYC Taxi Demo\",\n",
    "    description=\"Convert ML script to MLRun\"\n",
    ")\n",
    "\n",
    "def kfpipeline():\n",
    "\n",
    "    taxi_records_csv_path = mlrun.get_sample_path('data/Taxi/yellow_tripdata_2019-01_subset.csv')\n",
    "    zones_csv_path = mlrun.get_sample_path('data/Taxi/taxi_zones.csv')\n",
    "    \n",
    "    # build our ingestion function (container image)\n",
    "    builder = funcs['taxi'].deploy_step()\n",
    "    \n",
    "    # run the ingestion function with the new image and params\n",
    "    ingest = funcs['taxi'].as_step(\n",
    "        name=\"fetch_data\",\n",
    "        handler='fetch_data',\n",
    "        image=builder.outputs['image'],\n",
    "        inputs={'taxi_records_csv_path': taxi_records_csv_path,\n",
    "                'zones_csv_path': zones_csv_path},\n",
    "        outputs=['nyc-taxi-dataset', 'zones-dataset'])\n",
    "\n",
    "    # Join and transform the data sets \n",
    "    transform = funcs[\"taxi\"].as_step(\n",
    "        name=\"transform_dataset\",\n",
    "        handler='transform_dataset',\n",
    "        inputs={\"taxi_records_csv_path\": ingest.outputs['nyc-taxi-dataset'],\n",
    "                \"zones_csv_path\" : ingest.outputs['zones-dataset']},\n",
    "        outputs=['nyc-taxi-dataset-transformed'])\n",
    "\n",
    "    # Train the model\n",
    "    train = funcs[\"taxi\"].as_step(\n",
    "        name=\"train\",\n",
    "        handler=\"train_model\",\n",
    "        inputs={\"input_ds\" : transform.outputs['nyc-taxi-dataset-transformed']},\n",
    "        outputs=['FareModel'])\n",
    "    \n",
    "    # Deploy the model\n",
    "    deploy = funcs[\"model-serving\"].deploy_step(models={\"taxi-serving_v1\": train.outputs['FareModel']}, tag='v2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "project.set_workflow('main', 'workflow.py', embed=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<mlrun.projects.project.MlrunProject at 0x7fe93d6df490>"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "project.save()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run the Workflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>Pipeline running (id=c08abbf2-8d1f-4b5c-9df1-2a5706bb52eb), <a href=\"https://dashboard.default-tenant.app.vmdev93.lab.iguazeng.com/mlprojects/nyc-taxi-dani/jobs/monitor-workflows/workflow/c08abbf2-8d1f-4b5c-9df1-2a5706bb52eb\" target=\"_blank\"><b>click here</b></a> to view the details in MLRun UI</div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/svg+xml": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n -->\n<!-- Title: kfp Pages: 1 -->\n<svg width=\"198pt\" height=\"332pt\"\n viewBox=\"0.00 0.00 197.78 332.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 328)\">\n<title>kfp</title>\n<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-328 193.7832,-328 193.7832,4 -4,4\"/>\n<!-- nyc&#45;taxi&#45;demo&#45;xllnp&#45;3449952920 -->\n<g id=\"node1\" class=\"node\">\n<title>nyc&#45;taxi&#45;demo&#45;xllnp&#45;3449952920</title>\n<ellipse fill=\"#00ff00\" stroke=\"#000000\" cx=\"94.8916\" cy=\"-234\" rx=\"57.6901\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"94.8916\" y=\"-230.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">fetch&#45;data</text>\n</g>\n<!-- nyc&#45;taxi&#45;demo&#45;xllnp&#45;452990864 -->\n<g id=\"node2\" class=\"node\">\n<title>nyc&#45;taxi&#45;demo&#45;xllnp&#45;452990864</title>\n<ellipse fill=\"#00ff00\" stroke=\"#000000\" cx=\"94.8916\" cy=\"-162\" rx=\"94.7833\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"94.8916\" y=\"-158.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">transform&#45;dataset</text>\n</g>\n<!-- nyc&#45;taxi&#45;demo&#45;xllnp&#45;3449952920&#45;&gt;nyc&#45;taxi&#45;demo&#45;xllnp&#45;452990864 -->\n<g id=\"edge1\" class=\"edge\">\n<title>nyc&#45;taxi&#45;demo&#45;xllnp&#45;3449952920&#45;&gt;nyc&#45;taxi&#45;demo&#45;xllnp&#45;452990864</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M94.8916,-215.8314C94.8916,-208.131 94.8916,-198.9743 94.8916,-190.4166\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"98.3917,-190.4132 94.8916,-180.4133 91.3917,-190.4133 98.3917,-190.4132\"/>\n</g>\n<!-- nyc&#45;taxi&#45;demo&#45;xllnp&#45;4232508115 -->\n<g id=\"node4\" class=\"node\">\n<title>nyc&#45;taxi&#45;demo&#45;xllnp&#45;4232508115</title>\n<ellipse fill=\"#00ff00\" stroke=\"#000000\" cx=\"94.8916\" cy=\"-90\" rx=\"33.2948\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"94.8916\" y=\"-86.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">train</text>\n</g>\n<!-- nyc&#45;taxi&#45;demo&#45;xllnp&#45;452990864&#45;&gt;nyc&#45;taxi&#45;demo&#45;xllnp&#45;4232508115 -->\n<g id=\"edge3\" class=\"edge\">\n<title>nyc&#45;taxi&#45;demo&#45;xllnp&#45;452990864&#45;&gt;nyc&#45;taxi&#45;demo&#45;xllnp&#45;4232508115</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M94.8916,-143.8314C94.8916,-136.131 94.8916,-126.9743 94.8916,-118.4166\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"98.3917,-118.4132 94.8916,-108.4133 91.3917,-118.4133 98.3917,-118.4132\"/>\n</g>\n<!-- nyc&#45;taxi&#45;demo&#45;xllnp&#45;3605105989 -->\n<g id=\"node3\" class=\"node\">\n<title>nyc&#45;taxi&#45;demo&#45;xllnp&#45;3605105989</title>\n<polygon fill=\"#00ff00\" stroke=\"#000000\" points=\"179.8916,-36 13.8916,-36 9.8916,-32 9.8916,0 175.8916,0 179.8916,-4 179.8916,-36\"/>\n<polyline fill=\"none\" stroke=\"#000000\" points=\"175.8916,-32 9.8916,-32 \"/>\n<polyline fill=\"none\" stroke=\"#000000\" points=\"175.8916,-32 175.8916,0 \"/>\n<polyline fill=\"none\" stroke=\"#000000\" points=\"175.8916,-32 179.8916,-36 \"/>\n<text text-anchor=\"middle\" x=\"94.8916\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">deploy&#45;model&#45;serving</text>\n</g>\n<!-- nyc&#45;taxi&#45;demo&#45;xllnp&#45;4232508115&#45;&gt;nyc&#45;taxi&#45;demo&#45;xllnp&#45;3605105989 -->\n<g id=\"edge2\" class=\"edge\">\n<title>nyc&#45;taxi&#45;demo&#45;xllnp&#45;4232508115&#45;&gt;nyc&#45;taxi&#45;demo&#45;xllnp&#45;3605105989</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M94.8916,-71.8314C94.8916,-64.131 94.8916,-54.9743 94.8916,-46.4166\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"98.3917,-46.4132 94.8916,-36.4133 91.3917,-46.4133 98.3917,-46.4132\"/>\n</g>\n<!-- nyc&#45;taxi&#45;demo&#45;xllnp&#45;752872435 -->\n<g id=\"node5\" class=\"node\">\n<title>nyc&#45;taxi&#45;demo&#45;xllnp&#45;752872435</title>\n<polygon fill=\"#00ff00\" stroke=\"#000000\" points=\"142.3916,-324 47.3916,-324 47.3916,-288 142.3916,-288 142.3916,-324\"/>\n<text text-anchor=\"middle\" x=\"94.8916\" y=\"-302.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">deploy&#45;taxi</text>\n</g>\n<!-- nyc&#45;taxi&#45;demo&#45;xllnp&#45;752872435&#45;&gt;nyc&#45;taxi&#45;demo&#45;xllnp&#45;3449952920 -->\n<g id=\"edge4\" class=\"edge\">\n<title>nyc&#45;taxi&#45;demo&#45;xllnp&#45;752872435&#45;&gt;nyc&#45;taxi&#45;demo&#45;xllnp&#45;3449952920</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M94.8916,-287.8314C94.8916,-280.131 94.8916,-270.9743 94.8916,-262.4166\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"98.3917,-262.4132 94.8916,-252.4133 91.3917,-262.4133 98.3917,-262.4132\"/>\n</g>\n</g>\n</svg>\n",
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7fe93f40f3d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<h2>Run Results</h2>Workflow c08abbf2-8d1f-4b5c-9df1-2a5706bb52eb finished, state=Succeeded<br>click the hyper links below to see detailed results<br><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>uid</th>\n",
       "      <th>start</th>\n",
       "      <th>state</th>\n",
       "      <th>name</th>\n",
       "      <th>parameters</th>\n",
       "      <th>results</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td><div title=\"740a1b560f6744b59a6338ecf69dd707\"><a href=\"https://dashboard.default-tenant.app.vmdev93.lab.iguazeng.com/mlprojects/nyc-taxi-dani/jobs/monitor/740a1b560f6744b59a6338ecf69dd707/overview\" target=\"_blank\" >...f69dd707</a></div></td>\n",
       "      <td>Aug 28 12:26:47</td>\n",
       "      <td>completed</td>\n",
       "      <td>train</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><div title=\"d5ae493f1b6e4c2897a5fbcc0563ca9b\"><a href=\"https://dashboard.default-tenant.app.vmdev93.lab.iguazeng.com/mlprojects/nyc-taxi-dani/jobs/monitor/d5ae493f1b6e4c2897a5fbcc0563ca9b/overview\" target=\"_blank\" >...0563ca9b</a></div></td>\n",
       "      <td>Aug 28 12:25:01</td>\n",
       "      <td>completed</td>\n",
       "      <td>transform_dataset</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><div title=\"d45e824cce454aa8ae62be9205f20edb\"><a href=\"https://dashboard.default-tenant.app.vmdev93.lab.iguazeng.com/mlprojects/nyc-taxi-dani/jobs/monitor/d45e824cce454aa8ae62be9205f20edb/overview\" target=\"_blank\" >...05f20edb</a></div></td>\n",
       "      <td>Aug 28 12:22:43</td>\n",
       "      <td>completed</td>\n",
       "      <td>fetch_data</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "artifact_path = path.abspath('./pipe/{{workflow.uid}}')\n",
    "run_id = project.run(\n",
    "    'main',    \n",
    "    artifact_path=artifact_path, \n",
    "    dirty=True,\n",
    "    watch=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}