Suman.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "298e978a-16bc-4769-9ea6-301229a1902a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import intake\n",
    "from dask.diagnostics import ProgressBar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "eb894fec-dc18-43b7-b2c9-bcf6e70177bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "from cmip6_preprocessing.utils import google_cmip_col\n",
    "\n",
    "# Initialize the Pangeo CMIP6 cloud collection\n",
    "col = google_cmip_col() \n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "227a9a73-f821-415f-b975-ab45ea509846",
   "metadata": {},
   "source": [
    "## Filtering the data using specifications "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bfe7b597-748c-4c40-bed7-3a23c5236930",
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_list(experiment_id1,experiment_id2,variable_id1,variable_id2):\n",
    "    list1 = col.df[col.df.experiment_id == experiment_id1].source_id.unique()\n",
    "    list2 = col.df[col.df.experiment_id == experiment_id2].source_id.unique()\n",
    "\n",
    "    elist = np.intersect1d(list1,list2)\n",
    "\n",
    "    # all source_id with variable_id == 'no3os' or 'tos'\n",
    "    vlist1 = col.df[col.df.variable_id == variable_id1].source_id.unique()\n",
    "    vlist2 = col.df[col.df.variable_id == variable_id2].source_id.unique()\n",
    "\n",
    "    vlist = np.intersect1d(vlist1,vlist2)\n",
    "\n",
    "    model = np.intersect1d(vlist,elist)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "163960f5-8ab3-400e-a2d4-ad34d5376668",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['ACCESS-ESM1-5', 'CESM2', 'CESM2-WACCM', 'CNRM-ESM2-1', 'CanESM5',\n",
       "       'CanESM5-CanOE', 'GFDL-ESM4', 'GISS-E2-1-G', 'IPSL-CM6A-LR',\n",
       "       'MIROC-ES2L', 'MPI-ESM1-2-HR', 'MPI-ESM1-2-LR', 'NorESM2-LM',\n",
       "       'NorESM2-MM', 'UKESM1-0-LL'], dtype=object)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_list('historical','ssp585','tos','no3os')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b6ce7eaf-81a3-432b-9b1f-c9e25fab4a23",
   "metadata": {},
   "source": [
    "***This is how we can know that how many different models are there with that perticular specifications***"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "694d0a98-5963-475d-b4b8-2fb89d08da6e",
   "metadata": {},
   "source": [
    "But this has one issue that few models make their way in the list which have only one `variable_d`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6627b908-b010-4c67-ae8b-0e8457c4664c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>experiment_id</th>\n",
       "      <th>variable_id</th>\n",
       "      <th>table_id</th>\n",
       "      <th>member_id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>source_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ACCESS-ESM1-5</th>\n",
       "      <td>8</td>\n",
       "      <td>59</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CESM2</th>\n",
       "      <td>2</td>\n",
       "      <td>109</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CESM2-WACCM</th>\n",
       "      <td>5</td>\n",
       "      <td>108</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CanESM5</th>\n",
       "      <td>16</td>\n",
       "      <td>66</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CanESM5-CanOE</th>\n",
       "      <td>7</td>\n",
       "      <td>68</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GFDL-ESM4</th>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GISS-E2-1-G</th>\n",
       "      <td>6</td>\n",
       "      <td>29</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IPSL-CM6A-LR</th>\n",
       "      <td>11</td>\n",
       "      <td>109</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MPI-ESM1-2-HR</th>\n",
       "      <td>6</td>\n",
       "      <td>91</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MPI-ESM1-2-LR</th>\n",
       "      <td>9</td>\n",
       "      <td>93</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NorESM2-LM</th>\n",
       "      <td>3</td>\n",
       "      <td>52</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NorESM2-MM</th>\n",
       "      <td>2</td>\n",
       "      <td>45</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               experiment_id  variable_id  table_id  member_id\n",
       "source_id                                                     \n",
       "ACCESS-ESM1-5              8           59         1          2\n",
       "CESM2                      2          109         1          2\n",
       "CESM2-WACCM                5          108         1          2\n",
       "CanESM5                   16           66         1          4\n",
       "CanESM5-CanOE              7           68         1          2\n",
       "GFDL-ESM4                  3           10         1          2\n",
       "GISS-E2-1-G                6           29         1          2\n",
       "IPSL-CM6A-LR              11          109         1          2\n",
       "MPI-ESM1-2-HR              6           91         1          2\n",
       "MPI-ESM1-2-LR              9           93         1          2\n",
       "NorESM2-LM                 3           52         1          2\n",
       "NorESM2-MM                 2           45         1          2"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "models = model_list('historical','ssp585','tos','no3os')\n",
    "\n",
    "cat = col.search(\n",
    "    source_id=models,\n",
    "    grid_label='gn',\n",
    "    table_id='Omon',\n",
    "    member_id = ['r2i1p1f1', 'r3i1p1f1', 'r2i1p2f1', 'r3i1p2f1'] \n",
    ")\n",
    "cat.df.groupby(\"source_id\")[\n",
    "    [\"experiment_id\", \"variable_id\", \"table_id\", \"member_id\"]\n",
    "].nunique()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a4b0dfb0-25ed-4b76-a98c-f44663c1a297",
   "metadata": {},
   "source": [
    "We can see that `GFDL-ESM4` , `GISS-E2-1-G` and `NorESM2-MM` have only one variable_id and this can create problem in our analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0635607b-7ebb-4684-97ca-204b8a1fc2fe",
   "metadata": {},
   "source": [
    "***One another way is to use `search` method and force it to apply all query***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "acd03c13-be92-4dae-8bec-645f3b56307d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/srv/conda/lib/python3.9/site-packages/intake_esm/search.py:104: UserWarning: Query returned zero results.\n",
      "  warn(message)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>experiment_id</th>\n",
       "      <th>variable_id</th>\n",
       "      <th>table_id</th>\n",
       "      <th>member_id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>source_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [experiment_id, variable_id, table_id, member_id]\n",
       "Index: []"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#these are the models that julius has taken in his notebook\n",
    "\n",
    "models = ['CanESM5-CanOE', 'IPSL-CM6A-LR']\n",
    "\n",
    "# Define our query\n",
    "query = dict(\n",
    "    variable_id=['tos','no3os'],\n",
    "    source_id=models,\n",
    "    experiment_id=['historical', 'ssp585'],\n",
    "    grid_label='gn',\n",
    "    table_id='Omon',\n",
    "    member_id = ['r2i1p1f1', 'r3i1p1f1', 'r2i1p2f1', 'r3i1p2f1'] \n",
    ")\n",
    "\n",
    "cat = col.search(require_all_on=[\"source_id\"], **query)\n",
    "cat.df.groupby(\"source_id\")[\n",
    "    [\"experiment_id\", \"variable_id\", \"table_id\", \"member_id\"]\n",
    "].nunique()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a085481a-5584-46e1-bd3e-8eee95df0ab4",
   "metadata": {},
   "source": [
    "***We can see that it is producing zero result***"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ccdee19-63d0-41d8-98ec-2fe0e2cfaaac",
   "metadata": {},
   "source": [
    "***Where as if I comment this member_id it will give me total models. That means No model is satisfying exactly these queries***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4a5df1d1-a804-488c-937c-b5035be703df",
   "metadata": {},
   "outputs": [],
   "source": [
    "query = dict(\n",
    "    variable_id=[\"tos\", \"no3os\"],\n",
    "    experiment_id=[\"historical\", \"ssp585\"],\n",
    "    table_id=[\"Omon\"],\n",
    "    source_id=['CanESM5-CanOE', 'IPSL-CM6A-LR'],\n",
    "#     member_id = ['r2i1p1f1', 'r3i1p1f1', 'r2i1p2f1', 'r3i1p2f1'],\n",
    "    grid_label='gn'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3b9c18dc-1b86-4a79-867a-7dd99214549e",
   "metadata": {},
   "outputs": [],
   "source": [
    "col_subset = col.search(require_all_on=[\"source_id\"], **query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "41902892-e111-4c12-a9a1-0fa2ee124ffc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>experiment_id</th>\n",
       "      <th>variable_id</th>\n",
       "      <th>table_id</th>\n",
       "      <th>member_id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>source_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CanESM5-CanOE</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IPSL-CM6A-LR</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               experiment_id  variable_id  table_id  member_id\n",
       "source_id                                                     \n",
       "CanESM5-CanOE              2            2         1          3\n",
       "IPSL-CM6A-LR               2            2         1         32"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "col_subset.df.groupby(\"source_id\")[[\"experiment_id\", \"variable_id\", \"table_id\", \"member_id\"]].nunique()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84ec8042-409f-4153-990b-dcdf072976b9",
   "metadata": {},
   "source": [
    "## Now going with one method described above and try to analyse it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1b1de78e-7d56-48a1-9073-a84f5acd346a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "--> The keys in the returned dictionary of datasets are constructed as follows:\n",
      "\t'activity_id.institution_id.source_id.experiment_id.member_id.table_id.variable_id.grid_label.zstore.dcpp_init_year.version'\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='1315' class='' max='1315' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [1315/1315 01:10<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "models = ['ACCESS-ESM1-5','CESM2','CESM2-WACCM']\n",
    "\n",
    "cat = col.search(\n",
    "    source_id=models,\n",
    "    grid_label='gn',\n",
    "    table_id='Omon',\n",
    "    member_id = ['r2i1p1f1', 'r3i1p1f1', 'r2i1p2f1', 'r3i1p2f1'] \n",
    ")\n",
    "cat.df.groupby(\"source_id\")[\n",
    "    [\"experiment_id\", \"variable_id\", \"table_id\", \"member_id\"]\n",
    "].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e8a7c74b-fc33-4278-8f51-119ccbde3bb7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "--> The keys in the returned dictionary of datasets are constructed as follows:\n",
      "\t'activity_id.institution_id.source_id.experiment_id.member_id.table_id.variable_id.grid_label.zstore.dcpp_init_year.version'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/srv/conda/lib/python3.9/site-packages/cmip6_preprocessing/preprocessing.py:233: UserWarning: ScenarioMIP.CSIRO.ACCESS-ESM1-5.ssp585.r3i1p1f1.Omon.gn.v20191203.sos: Unit correction failed with: Cannot parse units:\n",
      "    invalid units for variable 'sos': 0.001 (attribute) (reason: Unit expression cannot have a scaling factor.)\n",
      "  warnings.warn(\n",
      "/srv/conda/lib/python3.9/site-packages/cmip6_preprocessing/preprocessing.py:233: UserWarning: CMIP.NCAR.CESM2-WACCM.historical.r2i1p1f1.Omon.gn.none.sos: Unit correction failed with: Cannot parse units:\n",
      "    invalid units for variable 'sos': 0.001 (attribute) (reason: Unit expression cannot have a scaling factor.)\n",
      "  warnings.warn(\n",
      "/srv/conda/lib/python3.9/site-packages/cmip6_preprocessing/preprocessing.py:233: UserWarning: CMIP.CSIRO.ACCESS-ESM1-5.historical.r3i1p1f1.Omon.gn.v20191203.sos: Unit correction failed with: Cannot parse units:\n",
      "    invalid units for variable 'sos': 0.001 (attribute) (reason: Unit expression cannot have a scaling factor.)\n",
      "  warnings.warn(\n",
      "/srv/conda/lib/python3.9/site-packages/cmip6_preprocessing/preprocessing.py:233: UserWarning: CMIP.NCAR.CESM2.historical.r3i1p1f1.Omon.gn.none.sos: Unit correction failed with: Cannot parse units:\n",
      "    invalid units for variable 'sos': 0.001 (attribute) (reason: Unit expression cannot have a scaling factor.)\n",
      "  warnings.warn(\n",
      "/srv/conda/lib/python3.9/site-packages/cmip6_preprocessing/preprocessing.py:233: UserWarning: CMIP.CSIRO.ACCESS-ESM1-5.abrupt-4xCO2.r2i1p1f1.Omon.gn.v20200901.sos: Unit correction failed with: Cannot parse units:\n",
      "    invalid units for variable 'sos': 0.001 (attribute) (reason: Unit expression cannot have a scaling factor.)\n",
      "  warnings.warn(\n",
      "/srv/conda/lib/python3.9/site-packages/cmip6_preprocessing/preprocessing.py:233: UserWarning: ScenarioMIP.NCAR.CESM2-WACCM.ssp585.r2i1p1f1.Omon.gn.none.sos: Unit correction failed with: Cannot parse units:\n",
      "    invalid units for variable 'sos': 0.001 (attribute) (reason: Unit expression cannot have a scaling factor.)\n",
      "  warnings.warn(\n",
      "/srv/conda/lib/python3.9/site-packages/cmip6_preprocessing/preprocessing.py:233: UserWarning: CMIP.NCAR.CESM2.historical.r2i1p1f1.Omon.gn.none.sos: Unit correction failed with: Cannot parse units:\n",
      "    invalid units for variable 'sos': 0.001 (attribute) (reason: Unit expression cannot have a scaling factor.)\n",
      "  warnings.warn(\n",
      "/srv/conda/lib/python3.9/site-packages/cmip6_preprocessing/preprocessing.py:233: UserWarning: CMIP.CSIRO.ACCESS-ESM1-5.esm-hist.r2i1p1f1.Omon.gn.v20191203.sos: Unit correction failed with: Cannot parse units:\n",
      "    invalid units for variable 'sos': 0.001 (attribute) (reason: Unit expression cannot have a scaling factor.)\n",
      "  warnings.warn(\n",
      "/srv/conda/lib/python3.9/site-packages/cmip6_preprocessing/preprocessing.py:233: UserWarning: ScenarioMIP.NCAR.CESM2-WACCM.ssp245.r2i1p1f1.Omon.gn.none.sos: Unit correction failed with: Cannot parse units:\n",
      "    invalid units for variable 'sos': 0.001 (attribute) (reason: Unit expression cannot have a scaling factor.)\n",
      "  warnings.warn(\n",
      "/srv/conda/lib/python3.9/site-packages/cmip6_preprocessing/preprocessing.py:233: UserWarning: ScenarioMIP.CSIRO.ACCESS-ESM1-5.ssp126.r3i1p1f1.Omon.gn.v20191201.sos: Unit correction failed with: Cannot parse units:\n",
      "    invalid units for variable 'sos': 0.001 (attribute) (reason: Unit expression cannot have a scaling factor.)\n",
      "  warnings.warn(\n",
      "/srv/conda/lib/python3.9/site-packages/cmip6_preprocessing/preprocessing.py:233: UserWarning: ScenarioMIP AerChemMIP.CSIRO.ACCESS-ESM1-5.ssp370.r2i1p1f1.Omon.gn.v20191129.sos: Unit correction failed with: Cannot parse units:\n",
      "    invalid units for variable 'sos': 0.001 (attribute) (reason: Unit expression cannot have a scaling factor.)\n",
      "  warnings.warn(\n",
      "/srv/conda/lib/python3.9/site-packages/cmip6_preprocessing/preprocessing.py:233: UserWarning: ScenarioMIP.NCAR.CESM2-WACCM.ssp370.r3i1p1f1.Omon.gn.none.sos: Unit correction failed with: Cannot parse units:\n",
      "    invalid units for variable 'sos': 0.001 (attribute) (reason: Unit expression cannot have a scaling factor.)\n",
      "  warnings.warn(\n",
      "/srv/conda/lib/python3.9/site-packages/cmip6_preprocessing/preprocessing.py:233: UserWarning: ScenarioMIP.CSIRO.ACCESS-ESM1-5.ssp585.r2i1p1f1.Omon.gn.v20191129.sos: Unit correction failed with: Cannot parse units:\n",
      "    invalid units for variable 'sos': 0.001 (attribute) (reason: Unit expression cannot have a scaling factor.)\n",
      "  warnings.warn(\n",
      "/srv/conda/lib/python3.9/site-packages/cmip6_preprocessing/preprocessing.py:233: UserWarning: ScenarioMIP.CSIRO.ACCESS-ESM1-5.ssp126.r2i1p1f1.Omon.gn.v20191129.sos: Unit correction failed with: Cannot parse units:\n",
      "    invalid units for variable 'sos': 0.001 (attribute) (reason: Unit expression cannot have a scaling factor.)\n",
      "  warnings.warn(\n",
      "/srv/conda/lib/python3.9/site-packages/cmip6_preprocessing/preprocessing.py:233: UserWarning: ScenarioMIP.CSIRO.ACCESS-ESM1-5.ssp245.r2i1p1f1.Omon.gn.v20191129.sos: Unit correction failed with: Cannot parse units:\n",
      "    invalid units for variable 'sos': 0.001 (attribute) (reason: Unit expression cannot have a scaling factor.)\n",
      "  warnings.warn(\n",
      "/srv/conda/lib/python3.9/site-packages/cmip6_preprocessing/preprocessing.py:233: UserWarning: ScenarioMIP.CSIRO.ACCESS-ESM1-5.ssp245.r3i1p1f1.Omon.gn.v20191203.sos: Unit correction failed with: Cannot parse units:\n",
      "    invalid units for variable 'sos': 0.001 (attribute) (reason: Unit expression cannot have a scaling factor.)\n",
      "  warnings.warn(\n",
      "/srv/conda/lib/python3.9/site-packages/cmip6_preprocessing/preprocessing.py:233: UserWarning: ScenarioMIP.NCAR.CESM2-WACCM.ssp585.r3i1p1f1.Omon.gn.none.sos: Unit correction failed with: Cannot parse units:\n",
      "    invalid units for variable 'sos': 0.001 (attribute) (reason: Unit expression cannot have a scaling factor.)\n",
      "  warnings.warn(\n",
      "/srv/conda/lib/python3.9/site-packages/cmip6_preprocessing/preprocessing.py:233: UserWarning: ScenarioMIP.NCAR.CESM2-WACCM.ssp245.r3i1p1f1.Omon.gn.none.sos: Unit correction failed with: Cannot parse units:\n",
      "    invalid units for variable 'sos': 0.001 (attribute) (reason: Unit expression cannot have a scaling factor.)\n",
      "  warnings.warn(\n",
      "/srv/conda/lib/python3.9/site-packages/cmip6_preprocessing/preprocessing.py:233: UserWarning: ScenarioMIP.NCAR.CESM2-WACCM.ssp370.r2i1p1f1.Omon.gn.none.sos: Unit correction failed with: Cannot parse units:\n",
      "    invalid units for variable 'sos': 0.001 (attribute) (reason: Unit expression cannot have a scaling factor.)\n",
      "  warnings.warn(\n",
      "/srv/conda/lib/python3.9/site-packages/cmip6_preprocessing/preprocessing.py:233: UserWarning: ScenarioMIP AerChemMIP.CSIRO.ACCESS-ESM1-5.ssp370.r3i1p1f1.Omon.gn.v20191203.sos: Unit correction failed with: Cannot parse units:\n",
      "    invalid units for variable 'sos': 0.001 (attribute) (reason: Unit expression cannot have a scaling factor.)\n",
      "  warnings.warn(\n",
      "/srv/conda/lib/python3.9/site-packages/cmip6_preprocessing/preprocessing.py:233: UserWarning: CMIP.CSIRO.ACCESS-ESM1-5.esm-hist.r3i1p1f1.Omon.gn.v20191115.sos: Unit correction failed with: Cannot parse units:\n",
      "    invalid units for variable 'sos': 0.001 (attribute) (reason: Unit expression cannot have a scaling factor.)\n",
      "  warnings.warn(\n",
      "/srv/conda/lib/python3.9/site-packages/cmip6_preprocessing/preprocessing.py:233: UserWarning: ScenarioMIP.NCAR.CESM2-WACCM.ssp534-over.r2i1p1f1.Omon.gn.none.sos: Unit correction failed with: Cannot parse units:\n",
      "    invalid units for variable 'sos': 0.001 (attribute) (reason: Unit expression cannot have a scaling factor.)\n",
      "  warnings.warn(\n",
      "/srv/conda/lib/python3.9/site-packages/cmip6_preprocessing/preprocessing.py:233: UserWarning: CMIP.NCAR.CESM2-WACCM.historical.r3i1p1f1.Omon.gn.none.sos: Unit correction failed with: Cannot parse units:\n",
      "    invalid units for variable 'sos': 0.001 (attribute) (reason: Unit expression cannot have a scaling factor.)\n",
      "  warnings.warn(\n",
      "/srv/conda/lib/python3.9/site-packages/cmip6_preprocessing/preprocessing.py:233: UserWarning: CMIP.CSIRO.ACCESS-ESM1-5.historical.r2i1p1f1.Omon.gn.v20191128.sos: Unit correction failed with: Cannot parse units:\n",
      "    invalid units for variable 'sos': 0.001 (attribute) (reason: Unit expression cannot have a scaling factor.)\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "Failed to apply pre-processing function: combined_preprocessing",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m/srv/conda/lib/python3.9/site-packages/xarray/core/dataset.py\u001b[0m in \u001b[0;36m_construct_dataarray\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m   1397\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1398\u001b[0;31m             \u001b[0mvariable\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_variables\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1399\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'x'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m/srv/conda/lib/python3.9/site-packages/intake_esm/merge_util.py\u001b[0m in \u001b[0;36m_open_asset\u001b[0;34m(path, data_format, zarr_kwargs, cdf_kwargs, preprocess, varname, requested_variables)\u001b[0m\n\u001b[1;32m    314\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 315\u001b[0;31m             \u001b[0mds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpreprocess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    316\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/lib/python3.9/site-packages/cmip6_preprocessing/preprocessing.py\u001b[0m in \u001b[0;36mcombined_preprocessing\u001b[0;34m(ds)\u001b[0m\n\u001b[1;32m    457\u001b[0m     \u001b[0;31m# broadcast lon/lat\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 458\u001b[0;31m     \u001b[0mds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbroadcast_lonlat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    459\u001b[0m     \u001b[0;31m# shift all lons to consistent 0-360\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/lib/python3.9/site-packages/cmip6_preprocessing/preprocessing.py\u001b[0m in \u001b[0;36mbroadcast_lonlat\u001b[0;34m(ds, verbose)\u001b[0m\n\u001b[1;32m    128\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;34m\"lon\"\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvariables\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 129\u001b[0;31m         \u001b[0mds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcoords\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"lon\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"x\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    130\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;34m\"lat\"\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvariables\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/lib/python3.9/site-packages/xarray/core/dataset.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   1501\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mhashable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1502\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_construct_dataarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1503\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/lib/python3.9/site-packages/xarray/core/dataset.py\u001b[0m in \u001b[0;36m_construct_dataarray\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m   1399\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1400\u001b[0;31m             _, name, variable = _get_virtual_variable(\n\u001b[0m\u001b[1;32m   1401\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_variables\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_level_coords\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdims\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/lib/python3.9/site-packages/xarray/core/dataset.py\u001b[0m in \u001b[0;36m_get_virtual_variable\u001b[0;34m(variables, key, level_vars, dim_sizes)\u001b[0m\n\u001b[1;32m    172\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 173\u001b[0;31m         \u001b[0mref_var\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvariables\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mref_name\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    174\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'x'",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_649/2019317723.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mcmip6_preprocessing\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpreprocessing\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mcombined_preprocessing\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;31m# load all datasets into a python dictionary\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m ddict = cat.to_dataset_dict(\n\u001b[0m\u001b[1;32m      4\u001b[0m     \u001b[0mzarr_kwargs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m'consolidated'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'use_cftime'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;31m# recommended for faster reading and better time handling\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mstorage_options\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m'token'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m'anon'\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;31m# needed to access the public CMIP6 data on google\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/lib/python3.9/site-packages/intake_esm/core.py\u001b[0m in \u001b[0;36mto_dataset_dict\u001b[0;34m(self, zarr_kwargs, cdf_kwargs, preprocess, storage_options, progressbar, aggregate)\u001b[0m\n\u001b[1;32m    928\u001b[0m             ]\n\u001b[1;32m    929\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtask\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconcurrent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfutures\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_completed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfuture_tasks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 930\u001b[0;31m                 \u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtask\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    931\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_datasets\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mds\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    932\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprogressbar\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/lib/python3.9/concurrent/futures/_base.py\u001b[0m in \u001b[0;36mresult\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    436\u001b[0m                     \u001b[0;32mraise\u001b[0m \u001b[0mCancelledError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    437\u001b[0m                 \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_state\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mFINISHED\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 438\u001b[0;31m                     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__get_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    439\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    440\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_condition\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/lib/python3.9/concurrent/futures/_base.py\u001b[0m in \u001b[0;36m__get_result\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    388\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exception\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    389\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 390\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    391\u001b[0m             \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    392\u001b[0m                 \u001b[0;31m# Break a reference cycle with the exception in self._exception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/lib/python3.9/concurrent/futures/thread.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     50\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 52\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     53\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mBaseException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfuture\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/lib/python3.9/site-packages/intake_esm/core.py\u001b[0m in \u001b[0;36m_load_source\u001b[0;34m(key, source)\u001b[0m\n\u001b[1;32m    914\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    915\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0m_load_source\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msource\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 916\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msource\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_dask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    917\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    918\u001b[0m         \u001b[0msources\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0msource\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0msource_kwargs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msource\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/lib/python3.9/site-packages/intake_esm/source.py\u001b[0m in \u001b[0;36mto_dask\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     87\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mto_dask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     88\u001b[0m         \u001b[0;34m\"\"\"Return xarray object (which will have chunks)\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 89\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_load_metadata\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     90\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ds\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/lib/python3.9/site-packages/intake/source/base.py\u001b[0m in \u001b[0;36m_load_metadata\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    234\u001b[0m         \u001b[0;34m\"\"\"load metadata only if needed\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    235\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_schema\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 236\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_schema\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_schema\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    237\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_schema\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    238\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_schema\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/lib/python3.9/site-packages/intake_esm/source.py\u001b[0m in \u001b[0;36m_get_schema\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     55\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     56\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ds\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 57\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_open_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     58\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m             metadata = {\n",
      "\u001b[0;32m/srv/conda/lib/python3.9/site-packages/intake_esm/source.py\u001b[0m in \u001b[0;36m_open_dataset\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     73\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_open_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     74\u001b[0m         \u001b[0mmapper\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_path_to_mapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath_column\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstorage_options\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata_format\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 75\u001b[0;31m         ds = _open_asset(\n\u001b[0m\u001b[1;32m     76\u001b[0m             \u001b[0mmapper\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     77\u001b[0m             \u001b[0mdata_format\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata_format\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/lib/python3.9/site-packages/intake_esm/merge_util.py\u001b[0m in \u001b[0;36m_open_asset\u001b[0;34m(path, data_format, zarr_kwargs, cdf_kwargs, preprocess, varname, requested_variables)\u001b[0m\n\u001b[1;32m    315\u001b[0m             \u001b[0mds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpreprocess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    316\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 317\u001b[0;31m             raise RuntimeError(\n\u001b[0m\u001b[1;32m    318\u001b[0m                 \u001b[0;34mf'Failed to apply pre-processing function: {preprocess.__name__}'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    319\u001b[0m             ) from exc\n",
      "\u001b[0;31mRuntimeError\u001b[0m: Failed to apply pre-processing function: combined_preprocessing"
     ]
    }
   ],
   "source": [
    "from cmip6_preprocessing.preprocessing import combined_preprocessing\n",
    "# load all datasets into a python dictionary\n",
    "ddict = cat.to_dataset_dict(\n",
    "    zarr_kwargs={'consolidated':True, 'use_cftime':True}, # recommended for faster reading and better time handling\n",
    "    storage_options={'token': 'anon'}, # needed to access the public CMIP6 data on google\n",
    "    aggregate=False,\n",
    "    preprocess=combined_preprocessing, # this applies the preprocessing to all datasets\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0393db6c-e030-4711-ac93-608c8b4d1788",
   "metadata": {},
   "outputs": [],
   "source": [
    "# quick example of how to loop over several models\n",
    "from cmip6_preprocessing.utils import cmip6_dataset_id\n",
    "fig, axarr = plt.subplots(ncols=3, nrows=4, figsize=[30,20])\n",
    "for ax, (name, ds) in zip(axarr.flat, ddict.items()):\n",
    "    # select the first time step\n",
    "    ds = ds.isel(time=100)\n",
    "    # select the datavariable\n",
    "    da = ds[ds.variable_id]\n",
    "    # plot\n",
    "    \n",
    "    da.plot(ax=ax)\n",
    "    ax.set_title('.'.join(cmip6_dataset_id(ds).split('.')[2:6]))\n",
    "    \n",
    "fig.subplots_adjust(hspace=0.5, wspace=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d518020d-9208-49d4-9b4f-2d809640794c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from cmip6_preprocessing.postprocessing import merge_variables, concat_members, concat_experiments\n",
    "ddict_combined_a = merge_variables(ddict)\n",
    "def maybe_remove_attrs(ds, attr):\n",
    "    if attr in ds.attrs.keys():\n",
    "        del ds.attrs[attr]\n",
    "    return ds\n",
    "# remove the `variable_id` attr manually\n",
    "ddict_combined_aa = {k:maybe_remove_attrs(ds, 'variable_id') for k,ds in ddict_combined_a.items()}\n",
    "ddict_combined_b = concat_experiments(ddict_combined_aa)\n",
    "# same thing for experiment_id\n",
    "ddict_combined_bb = {k:maybe_remove_attrs(ds, 'experiment_id') for k,ds in ddict_combined_b.items()}\n",
    "ddict_combined = concat_members(ddict_combined_bb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c0ae1ed-5692-4c13-a1bd-57d104136318",
   "metadata": {},
   "outputs": [],
   "source": [
    "ddict_combined.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93193462-5918-4e44-8ed5-1db5fcd17404",
   "metadata": {},
   "outputs": [],
   "source": [
    "ddict_combined['MPI-ESM1-2-LR.gn.Omon']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e924601-3a5a-4cb3-a0e0-18e92619650f",
   "metadata": {},
   "outputs": [],
   "source": [
    "ddict_combined['ACCESS-ESM1-5.gn.Omon']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "59f3cf2d-75ac-4fe5-af13-0de88c6d88c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axarr = plt.subplots(ncols=2, nrows=1, figsize=[15,5])\n",
    "for ax, (name, ds) in zip(axarr.flat, ddict_combined.items()):\n",
    "    # show just the first time step\n",
    "    ds = ds.isel(time=0)\n",
    "    \n",
    "    # mask out the SST where the SST is above 25 deg\n",
    "    da = ds.tos.where(ds.tos>=25)\n",
    "    \n",
    "    # average all members\n",
    "    if 'member_id' in da.dims:\n",
    "        da = da.mean('member_id')\n",
    "    da.plot(ax=ax, robust=True)\n",
    "    ax.set_title(name)\n",
    "fig.subplots_adjust(hspace=0.5, wspace=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a4d0110-0908-4e26-8afe-38b19d6e9030",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}