ingest.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ingest Pancan + GTEX\n",
    "\n",
    "Download [Pancan/TCGA](https://www.cell.com/pb-assets/consortium/pancanceratlas/pancani3/index.html) and [GTEX](https://gtexportal.org)[Kallisto](https://pachterlab.github.io/kallisto/) clinical labels and RNA-Seq transcript expression computed by the [Toil recompute](https://xenabrowser.net/datapages/?hub=https://toil.xenahubs.net:443) from [Xena](https://xenabrowser.net), convert transcripts to [HUGO](https://www.genenames.org/) gene names and expression, wrangle, prune and store in a single h5 file for machine learning."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# Switch to a scratch data directory so all paths are local\n",
    "os.makedirs(os.path.expanduser(\"~/data/pancan-gtex\"), exist_ok=True)\n",
    "os.chdir(os.path.expanduser(\"~/data/pancan-gtex\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ingest Samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download raw files from xena\n",
    "!wget -q -N https://toil.xenahubs.net/download/tcga_Kallisto_tpm.gz\n",
    "!wget -q -N https://toil.xenahubs.net/download/gtex_Kallisto_tpm.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 91 µs, sys: 0 ns, total: 91 µs\n",
      "Wall time: 99.4 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Convert to float32, Transpose to ML style rows = samples and hdf for significantly faster reading\n",
    "if not os.path.exists(os.path.expanduser(\"tcga_Kallisto_tpm.T.fp32.h5\")):\n",
    "    pd.read_table(os.path.expanduser(\"tcga_Kallisto_tpm.gz\"), index_col=0, engine='c') \\\n",
    "        .astype(np.float32).T \\\n",
    "        .to_hdf(os.path.expanduser(\"tcga_Kallisto_tpm.T.fp32.h5\"), \"expression\", mode=\"w\", format=\"fixed\")\n",
    "if not os.path.exists(os.path.expanduser(\"gtex_Kallisto_tpm.T.fp32.h5\")):\n",
    "    pd.read_table(os.path.expanduser(\"gtex_Kallisto_tpm.gz\"), index_col=0, engine='c') \\\n",
    "        .astype(np.float32).T \\\n",
    "        .to_hdf(os.path.expanduser(\"gtex_Kallisto_tpm.T.fp32.h5\"), \"expression\", mode=\"w\", format=\"fixed\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.68 s, sys: 26.2 s, total: 27.9 s\n",
      "Wall time: 45.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "tcga_samples = pd.read_hdf(os.path.expanduser(\"tcga_Kallisto_tpm.T.fp32.h5\"))\n",
    "gtex_samples = pd.read_hdf(os.path.expanduser(\"gtex_Kallisto_tpm.T.fp32.h5\"))\n",
    "\n",
    "# Make sure they have the exact same set of transcript names\n",
    "assert tcga_samples.columns.equals(gtex_samples.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ingested 18525 samples with 197044 features\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>sample</th>\n",
       "      <th>ENST00000548312.5</th>\n",
       "      <th>ENST00000527779.1</th>\n",
       "      <th>ENST00000454820.5</th>\n",
       "      <th>ENST00000535093.1</th>\n",
       "      <th>ENST00000346219.7</th>\n",
       "      <th>ENST00000570899.1</th>\n",
       "      <th>ENST00000557761.1</th>\n",
       "      <th>ENST00000625998.2</th>\n",
       "      <th>ENST00000583693.5</th>\n",
       "      <th>ENST00000383738.6</th>\n",
       "      <th>...</th>\n",
       "      <th>ENST00000380620.8</th>\n",
       "      <th>ENST00000548698.5</th>\n",
       "      <th>ENST00000542429.2</th>\n",
       "      <th>ENST00000602837.1</th>\n",
       "      <th>ENST00000422233.5</th>\n",
       "      <th>ENST00000377138.1</th>\n",
       "      <th>ENST00000463473.2</th>\n",
       "      <th>ENST00000380293.3</th>\n",
       "      <th>ENST00000288710.6</th>\n",
       "      <th>ENST00000250055.2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>TCGA-E9-A1N3-01</th>\n",
       "      <td>0.227062</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.765630</td>\n",
       "      <td>0.000208</td>\n",
       "      <td>0.090963</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.403153</td>\n",
       "      <td>0.273409</td>\n",
       "      <td>0.014030</td>\n",
       "      <td>...</td>\n",
       "      <td>0.043985</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.386267</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.024769</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.347058</td>\n",
       "      <td>0.384339</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TCGA-EL-A3ZP-01</th>\n",
       "      <td>0.408150</td>\n",
       "      <td>0.886119</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.237821</td>\n",
       "      <td>0.486171</td>\n",
       "      <td>0.243753</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.931843</td>\n",
       "      <td>0.876822</td>\n",
       "      <td>0.007793</td>\n",
       "      <td>...</td>\n",
       "      <td>0.147270</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.111492</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.115896</td>\n",
       "      <td>0.095810</td>\n",
       "      <td>0.032398</td>\n",
       "      <td>0.371729</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TCGA-E2-A152-01</th>\n",
       "      <td>0.101443</td>\n",
       "      <td>0.305870</td>\n",
       "      <td>0.128104</td>\n",
       "      <td>0.180419</td>\n",
       "      <td>0.017127</td>\n",
       "      <td>0.146706</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.182633</td>\n",
       "      <td>0.042321</td>\n",
       "      <td>...</td>\n",
       "      <td>7.393230</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.446730</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.042321</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TCGA-66-2734-01</th>\n",
       "      <td>0.215029</td>\n",
       "      <td>1.765263</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.165132</td>\n",
       "      <td>0.036271</td>\n",
       "      <td>0.102213</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.175764</td>\n",
       "      <td>0.466617</td>\n",
       "      <td>0.017338</td>\n",
       "      <td>...</td>\n",
       "      <td>0.102135</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.541126</td>\n",
       "      <td>0.556174</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.074598</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.054823</td>\n",
       "      <td>21.661081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TCGA-BQ-5885-01</th>\n",
       "      <td>0.072516</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.128417</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.065330</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.001991</td>\n",
       "      <td>0.407951</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.484620</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.216381</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.694537</td>\n",
       "      <td>0.136738</td>\n",
       "      <td>0.310586</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 197044 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "sample           ENST00000548312.5  ENST00000527779.1  ENST00000454820.5  \\\n",
       "TCGA-E9-A1N3-01           0.227062           0.000000           0.000000   \n",
       "TCGA-EL-A3ZP-01           0.408150           0.886119           0.000000   \n",
       "TCGA-E2-A152-01           0.101443           0.305870           0.128104   \n",
       "TCGA-66-2734-01           0.215029           1.765263           0.000000   \n",
       "TCGA-BQ-5885-01           0.072516           0.000000           0.000000   \n",
       "\n",
       "sample           ENST00000535093.1  ENST00000346219.7  ENST00000570899.1  \\\n",
       "TCGA-E9-A1N3-01           1.765630           0.000208           0.090963   \n",
       "TCGA-EL-A3ZP-01           0.237821           0.486171           0.243753   \n",
       "TCGA-E2-A152-01           0.180419           0.017127           0.146706   \n",
       "TCGA-66-2734-01           0.165132           0.036271           0.102213   \n",
       "TCGA-BQ-5885-01           0.128417           0.000000           0.065330   \n",
       "\n",
       "sample           ENST00000557761.1  ENST00000625998.2  ENST00000583693.5  \\\n",
       "TCGA-E9-A1N3-01                0.0           1.403153           0.273409   \n",
       "TCGA-EL-A3ZP-01                0.0           2.931843           0.876822   \n",
       "TCGA-E2-A152-01                0.0           0.000000           0.182633   \n",
       "TCGA-66-2734-01                0.0           0.175764           0.466617   \n",
       "TCGA-BQ-5885-01                0.0           3.001991           0.407951   \n",
       "\n",
       "sample           ENST00000383738.6        ...          ENST00000380620.8  \\\n",
       "TCGA-E9-A1N3-01           0.014030        ...                   0.043985   \n",
       "TCGA-EL-A3ZP-01           0.007793        ...                   0.147270   \n",
       "TCGA-E2-A152-01           0.042321        ...                   7.393230   \n",
       "TCGA-66-2734-01           0.017338        ...                   0.102135   \n",
       "TCGA-BQ-5885-01           0.000000        ...                   0.484620   \n",
       "\n",
       "sample           ENST00000548698.5  ENST00000542429.2  ENST00000602837.1  \\\n",
       "TCGA-E9-A1N3-01                0.0           0.000000           0.000000   \n",
       "TCGA-EL-A3ZP-01                0.0           0.000000           0.111492   \n",
       "TCGA-E2-A152-01                0.0           0.000000           0.446730   \n",
       "TCGA-66-2734-01                0.0           0.541126           0.556174   \n",
       "TCGA-BQ-5885-01                0.0           0.000000           0.216381   \n",
       "\n",
       "sample           ENST00000422233.5  ENST00000377138.1  ENST00000463473.2  \\\n",
       "TCGA-E9-A1N3-01           0.386267                0.0           0.024769   \n",
       "TCGA-EL-A3ZP-01           0.000000                0.0           0.115896   \n",
       "TCGA-E2-A152-01           0.000000                0.0           0.000000   \n",
       "TCGA-66-2734-01           0.000000                0.0           0.074598   \n",
       "TCGA-BQ-5885-01           0.000000                0.0           0.000000   \n",
       "\n",
       "sample           ENST00000380293.3  ENST00000288710.6  ENST00000250055.2  \n",
       "TCGA-E9-A1N3-01           0.000000           0.347058           0.384339  \n",
       "TCGA-EL-A3ZP-01           0.095810           0.032398           0.371729  \n",
       "TCGA-E2-A152-01           0.000000           0.042321           0.000000  \n",
       "TCGA-66-2734-01           0.000000           0.054823          21.661081  \n",
       "TCGA-BQ-5885-01           0.694537           0.136738           0.310586  \n",
       "\n",
       "[5 rows x 197044 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Combine into a single dataset and convert from log2(TPM+0.001) back into TPM\n",
    "all_samples = pd.concat([tcga_samples, gtex_samples], axis=\"index\").apply(np.exp2).subtract(0.001).clip_lower(0)\n",
    "print(\"Ingested {} samples with {} features\".format(all_samples.shape[0], all_samples.shape[1]))\n",
    "all_samples.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TCGA-E9-A1N3-01                 1.000000e+06\n",
       "TCGA-CR-5249-01                 9.999975e+05\n",
       "TCGA-27-1835-01                 1.000000e+06\n",
       "GTEX-117YX-1426-SM-5H12H        9.999982e+05\n",
       "GTEX-12ZZX-0011-R3b-SM-5EGLH    9.999986e+05\n",
       "dtype: float32"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check that TPM (transcripts per million) per sample sum to 1M\n",
    "all_samples.iloc[::all_samples.shape[0]//5].sum(axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transcript to Hugo\n",
    "\n",
    "Kallisto outputs expression by Ensemble transcript. Download and create a mapping table to convert transcript to HUGO (ie gene) names for use in pruning and inference annotation as that is what most pathway databases use as well as the primary way the clinical world talks about drug targeting etc..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "!wget -q -N http://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_23/gencode.v23.chr_patch_hapl_scaff.annotation.gtf.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 216139 transcript to hugo mappings\n",
      "0 transcripts have no hugo mapping\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gene</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transcript</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ENST00000545363.6</th>\n",
       "      <td>RP11-705C15.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENST00000480456.5</th>\n",
       "      <td>JAM2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENST00000633293.1</th>\n",
       "      <td>CPEB1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENST00000564050.6</th>\n",
       "      <td>PRMT7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENST00000622921.1</th>\n",
       "      <td>LINC01128</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            gene\n",
       "transcript                      \n",
       "ENST00000545363.6  RP11-705C15.2\n",
       "ENST00000480456.5           JAM2\n",
       "ENST00000633293.1          CPEB1\n",
       "ENST00000564050.6          PRMT7\n",
       "ENST00000622921.1      LINC01128"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "import gzip\n",
    "matches = re.findall(r'transcript_id\\W+?\"(.*?)\".*?gene_name\\W+?\"(.*?)\"', \n",
    "                     gzip.open(\"gencode.v23.chr_patch_hapl_scaff.annotation.gtf.gz\", \"rt\").read(), \n",
    "                     flags=re.MULTILINE)\n",
    "ensemble_to_hugo = pd.DataFrame(\n",
    "    list(set(matches)), columns=[\"transcript\", \"gene\"]).set_index(\"transcript\")\n",
    "ensemble_to_hugo.to_csv(\"ensemble-to-hugo.tsv\", sep=\"\\t\")\n",
    "\n",
    "# Load ensemble to hugo mapping table\n",
    "ensemble_to_hugo = pd.read_table(\"ensemble-to-hugo.tsv\", index_col=0)\n",
    "print(\"Found {} transcript to hugo mappings\".format(ensemble_to_hugo.shape[0]))\n",
    "ensemble_to_hugo.head()\n",
    "\n",
    "missing = list(filter(lambda x: x not in ensemble_to_hugo.index, all_samples.columns.values))\n",
    "print(\"{} transcripts have no hugo mapping\".format(len(missing)))\n",
    "\n",
    "ensemble_to_hugo.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "After collapsing genes we have 44792 features\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TCGA-E9-A1N3-01                 1000000.125\n",
       "TCGA-CR-5249-01                  999997.625\n",
       "TCGA-27-1835-01                 1000000.125\n",
       "GTEX-117YX-1426-SM-5H12H         999998.125\n",
       "GTEX-12ZZX-0011-R3b-SM-5EGLH     999998.625\n",
       "dtype: float32"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a new dataframe replacing the index with hugo names\n",
    "hugo_samples = all_samples.copy()\n",
    "hugo_samples.columns = ensemble_to_hugo.reindex(all_samples.columns).gene.values\n",
    "\n",
    "# Multiple Ensemble genes map to the same Hugo name so sum them to generate TPM for a single HUGO gene\n",
    "hugo_samples = hugo_samples.groupby(axis=\"columns\", level=0).aggregate(np.sum)\n",
    "print(\"After collapsing genes we have {} features\".format(hugo_samples.shape[1]))\n",
    "\n",
    "# Check that we still sum to 1M\n",
    "hugo_samples.iloc[::hugo_samples.shape[0] // 5].sum(axis=\"columns\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prune Features\n",
    "Reduce the number of features/genes from ~20k down to a smaller number so the features vs. samples ratio is less out of whack but also so that a colab notebook can download the training in reasonable time when computering SHAP values against a tissue cohort (see infer.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pruning from 44792 down to 5540 features/genes\n"
     ]
    }
   ],
   "source": [
    "# Prune X to only KEGG pathway genes (http://software.broadinstitute.org/gsea/msigdb/)\n",
    "with open(\"c2.cp.kegg.v6.2.symbols.gmt\") as f:\n",
    "    kegg_genes = list(set().union(*[line.strip().split(\"\\t\")[2:] for line in f.readlines()]))\n",
    "\n",
    "# Prune X to only Cosmic Cancer Genes (https://cancer.sanger.ac.uk/census)\n",
    "cosmic_genes = pd.read_table(\"cosmic-26-11-2018.tsv\")[\"Gene Symbol\"].values\n",
    "\n",
    "subset_of_genes = list(set(kegg_genes).union(set(cosmic_genes)))\n",
    "\n",
    "pruned_samples = hugo_samples.drop(labels=(set(hugo_samples.columns) - set(subset_of_genes)), axis=1)\n",
    "\n",
    "print(\"Pruning from {} down to {} features/genes\".format(hugo_samples.shape[1], pruned_samples.shape[1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ingest Labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "!wget -q -N https://pancanatlas.xenahubs.net/download/Survival_SupplementalTable_S1_20171025_xena_sp.gz\n",
    "!wget -q -N https://toil.xenahubs.net/download/TcgaTargetGTEX_phenotype.txt.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "survival_labels = pd.read_table(\n",
    "    \"Survival_SupplementalTable_S1_20171025_xena_sp.gz\", compression=\"gzip\", \n",
    "    header=0, sep=\"\\t\", encoding=\"ISO-8859-1\", index_col=0, dtype=\"str\").sort_index(axis=\"index\")\n",
    "\n",
    "tcga_gtex_labels = pd.read_table(\n",
    "    \"TcgaTargetGTEX_phenotype.txt.gz\", compression=\"gzip\", \n",
    "    header=0, sep=\"\\t\", encoding=\"ISO-8859-1\", index_col=0, dtype=\"str\").sort_index(axis=\"index\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ingested 39 labels for 21226 samples\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>detailed_category</th>\n",
       "      <th>primary disease or tissue</th>\n",
       "      <th>_primary_site</th>\n",
       "      <th>_sample_type</th>\n",
       "      <th>_gender</th>\n",
       "      <th>_study</th>\n",
       "      <th>_PATIENT</th>\n",
       "      <th>cancer type abbreviation</th>\n",
       "      <th>age_at_initial_pathologic_diagnosis</th>\n",
       "      <th>gender</th>\n",
       "      <th>...</th>\n",
       "      <th>residual_tumor</th>\n",
       "      <th>OS</th>\n",
       "      <th>OS.time</th>\n",
       "      <th>DSS</th>\n",
       "      <th>DSS.time</th>\n",
       "      <th>DFI</th>\n",
       "      <th>DFI.time</th>\n",
       "      <th>PFI</th>\n",
       "      <th>PFI.time</th>\n",
       "      <th>Redaction</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sample</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>GTEX-1117F-0226-SM-5GZZ7</th>\n",
       "      <td>Adipose - Subcutaneous</td>\n",
       "      <td>Adipose - Subcutaneous</td>\n",
       "      <td>Adipose Tissue</td>\n",
       "      <td>Normal Tissue</td>\n",
       "      <td>Female</td>\n",
       "      <td>GTEX</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>...</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GTEX-QDVN-2126-SM-33HBS</th>\n",
       "      <td>Adipose - Subcutaneous</td>\n",
       "      <td>Adipose - Subcutaneous</td>\n",
       "      <td>Adipose Tissue</td>\n",
       "      <td>Normal Tissue</td>\n",
       "      <td>Male</td>\n",
       "      <td>GTEX</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>...</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TARGET-50-PAJNAA-01</th>\n",
       "      <td>Wilms Tumor</td>\n",
       "      <td>Wilms Tumor</td>\n",
       "      <td>Kidney</td>\n",
       "      <td>Primary Solid Tumor</td>\n",
       "      <td>nan</td>\n",
       "      <td>TARGET</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>...</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TCGA-AW-A1PO-01</th>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>TCGA-AW-A1PO</td>\n",
       "      <td>UCEC</td>\n",
       "      <td>66</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>...</td>\n",
       "      <td>nan</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TCGA-EI-6508-01</th>\n",
       "      <td>Rectum Adenocarcinoma</td>\n",
       "      <td>Rectum Adenocarcinoma</td>\n",
       "      <td>Rectum</td>\n",
       "      <td>Primary Tumor</td>\n",
       "      <td>Female</td>\n",
       "      <td>TCGA</td>\n",
       "      <td>TCGA-EI-6508</td>\n",
       "      <td>READ</td>\n",
       "      <td>48</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>...</td>\n",
       "      <td>nan</td>\n",
       "      <td>0</td>\n",
       "      <td>636</td>\n",
       "      <td>0</td>\n",
       "      <td>636</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>0</td>\n",
       "      <td>636</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TCGA-ZX-AA5X-01</th>\n",
       "      <td>Cervical &amp; Endocervical Cancer</td>\n",
       "      <td>Cervical &amp; Endocervical Cancer</td>\n",
       "      <td>Cervix</td>\n",
       "      <td>Primary Tumor</td>\n",
       "      <td>Female</td>\n",
       "      <td>TCGA</td>\n",
       "      <td>TCGA-ZX-AA5X</td>\n",
       "      <td>CESC</td>\n",
       "      <td>64</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>...</td>\n",
       "      <td>nan</td>\n",
       "      <td>0</td>\n",
       "      <td>119</td>\n",
       "      <td>0</td>\n",
       "      <td>119</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>0</td>\n",
       "      <td>119</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6 rows × 39 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       detailed_category  \\\n",
       "sample                                                     \n",
       "GTEX-1117F-0226-SM-5GZZ7          Adipose - Subcutaneous   \n",
       "GTEX-QDVN-2126-SM-33HBS           Adipose - Subcutaneous   \n",
       "TARGET-50-PAJNAA-01                          Wilms Tumor   \n",
       "TCGA-AW-A1PO-01                                      nan   \n",
       "TCGA-EI-6508-01                    Rectum Adenocarcinoma   \n",
       "TCGA-ZX-AA5X-01           Cervical & Endocervical Cancer   \n",
       "\n",
       "                               primary disease or tissue   _primary_site  \\\n",
       "sample                                                                     \n",
       "GTEX-1117F-0226-SM-5GZZ7          Adipose - Subcutaneous  Adipose Tissue   \n",
       "GTEX-QDVN-2126-SM-33HBS           Adipose - Subcutaneous  Adipose Tissue   \n",
       "TARGET-50-PAJNAA-01                          Wilms Tumor          Kidney   \n",
       "TCGA-AW-A1PO-01                                      nan             nan   \n",
       "TCGA-EI-6508-01                    Rectum Adenocarcinoma          Rectum   \n",
       "TCGA-ZX-AA5X-01           Cervical & Endocervical Cancer          Cervix   \n",
       "\n",
       "                                 _sample_type _gender  _study      _PATIENT  \\\n",
       "sample                                                                        \n",
       "GTEX-1117F-0226-SM-5GZZ7        Normal Tissue  Female    GTEX           nan   \n",
       "GTEX-QDVN-2126-SM-33HBS         Normal Tissue    Male    GTEX           nan   \n",
       "TARGET-50-PAJNAA-01       Primary Solid Tumor     nan  TARGET           nan   \n",
       "TCGA-AW-A1PO-01                           nan     nan     nan  TCGA-AW-A1PO   \n",
       "TCGA-EI-6508-01                 Primary Tumor  Female    TCGA  TCGA-EI-6508   \n",
       "TCGA-ZX-AA5X-01                 Primary Tumor  Female    TCGA  TCGA-ZX-AA5X   \n",
       "\n",
       "                         cancer type abbreviation  \\\n",
       "sample                                              \n",
       "GTEX-1117F-0226-SM-5GZZ7                      nan   \n",
       "GTEX-QDVN-2126-SM-33HBS                       nan   \n",
       "TARGET-50-PAJNAA-01                           nan   \n",
       "TCGA-AW-A1PO-01                              UCEC   \n",
       "TCGA-EI-6508-01                              READ   \n",
       "TCGA-ZX-AA5X-01                              CESC   \n",
       "\n",
       "                         age_at_initial_pathologic_diagnosis  gender  \\\n",
       "sample                                                                 \n",
       "GTEX-1117F-0226-SM-5GZZ7                                 nan     nan   \n",
       "GTEX-QDVN-2126-SM-33HBS                                  nan     nan   \n",
       "TARGET-50-PAJNAA-01                                      nan     nan   \n",
       "TCGA-AW-A1PO-01                                           66  FEMALE   \n",
       "TCGA-EI-6508-01                                           48  FEMALE   \n",
       "TCGA-ZX-AA5X-01                                           64  FEMALE   \n",
       "\n",
       "                            ...    residual_tumor   OS OS.time  DSS DSS.time  \\\n",
       "sample                      ...                                                \n",
       "GTEX-1117F-0226-SM-5GZZ7    ...               nan  nan     nan  nan      nan   \n",
       "GTEX-QDVN-2126-SM-33HBS     ...               nan  nan     nan  nan      nan   \n",
       "TARGET-50-PAJNAA-01         ...               nan  nan     nan  nan      nan   \n",
       "TCGA-AW-A1PO-01             ...               nan    0      17    0       17   \n",
       "TCGA-EI-6508-01             ...               nan    0     636    0      636   \n",
       "TCGA-ZX-AA5X-01             ...               nan    0     119    0      119   \n",
       "\n",
       "                          DFI DFI.time  PFI PFI.time Redaction  \n",
       "sample                                                          \n",
       "GTEX-1117F-0226-SM-5GZZ7  nan      nan  nan      nan       nan  \n",
       "GTEX-QDVN-2126-SM-33HBS   nan      nan  nan      nan       nan  \n",
       "TARGET-50-PAJNAA-01       nan      nan  nan      nan       nan  \n",
       "TCGA-AW-A1PO-01           nan      nan    0       17       nan  \n",
       "TCGA-EI-6508-01           nan      nan    0      636       nan  \n",
       "TCGA-ZX-AA5X-01           nan      nan    0      119       nan  \n",
       "\n",
       "[6 rows x 39 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_labels = pd.merge(tcga_gtex_labels, survival_labels, left_index=True, right_index=True, how=\"outer\").astype('str')\n",
    "print(\"Ingested {} labels for {} samples\".format(all_labels.shape[1], all_labels.shape[0]))\n",
    "all_labels.iloc[::all_labels.shape[0]//5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Wrangle and Prune\n",
    "\n",
    "Drop samples with missing values for fields we want to train on, or that have too few of the class and transform field values for training etc..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting with 18397 labeled sample pairs\n",
      "18397 with _primary_site\n",
      "18397 with _gender\n",
      "17640 too few examples by _primary_site\n",
      "17277 not Cell Line\n",
      "17277 with _sample_type\n",
      "17277 labels after pruning\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>detailed_category</th>\n",
       "      <th>primary disease or tissue</th>\n",
       "      <th>_primary_site</th>\n",
       "      <th>_sample_type</th>\n",
       "      <th>_gender</th>\n",
       "      <th>_study</th>\n",
       "      <th>_PATIENT</th>\n",
       "      <th>cancer type abbreviation</th>\n",
       "      <th>age_at_initial_pathologic_diagnosis</th>\n",
       "      <th>gender</th>\n",
       "      <th>...</th>\n",
       "      <th>OS</th>\n",
       "      <th>OS.time</th>\n",
       "      <th>DSS</th>\n",
       "      <th>DSS.time</th>\n",
       "      <th>DFI</th>\n",
       "      <th>DFI.time</th>\n",
       "      <th>PFI</th>\n",
       "      <th>PFI.time</th>\n",
       "      <th>Redaction</th>\n",
       "      <th>tumor_normal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>GTEX-1117F-0226-SM-5GZZ7</th>\n",
       "      <td>Adipose - Subcutaneous</td>\n",
       "      <td>Adipose - Subcutaneous</td>\n",
       "      <td>Adipose Tissue</td>\n",
       "      <td>Normal Tissue</td>\n",
       "      <td>Female</td>\n",
       "      <td>GTEX</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>...</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GTEX-OHPK-0226-SM-3MJH6</th>\n",
       "      <td>Adipose - Subcutaneous</td>\n",
       "      <td>Adipose - Subcutaneous</td>\n",
       "      <td>Adipose Tissue</td>\n",
       "      <td>Normal Tissue</td>\n",
       "      <td>Female</td>\n",
       "      <td>GTEX</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>...</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GTEX-ZXG5-0326-SM-5GICH</th>\n",
       "      <td>Muscle - Skeletal</td>\n",
       "      <td>Muscle - Skeletal</td>\n",
       "      <td>Muscle</td>\n",
       "      <td>Normal Tissue</td>\n",
       "      <td>Male</td>\n",
       "      <td>GTEX</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>...</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TCGA-BA-5556-01</th>\n",
       "      <td>Head &amp; Neck Squamous Cell Carcinoma</td>\n",
       "      <td>Head &amp; Neck Squamous Cell Carcinoma</td>\n",
       "      <td>Head and Neck region</td>\n",
       "      <td>Primary Tumor</td>\n",
       "      <td>Female</td>\n",
       "      <td>TCGA</td>\n",
       "      <td>TCGA-BA-5556</td>\n",
       "      <td>HNSC</td>\n",
       "      <td>58</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>725</td>\n",
       "      <td>0</td>\n",
       "      <td>725</td>\n",
       "      <td>0</td>\n",
       "      <td>725</td>\n",
       "      <td>0</td>\n",
       "      <td>725</td>\n",
       "      <td>nan</td>\n",
       "      <td>Tumor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TCGA-EM-A2OW-01</th>\n",
       "      <td>Thyroid Carcinoma</td>\n",
       "      <td>Thyroid Carcinoma</td>\n",
       "      <td>Thyroid Gland</td>\n",
       "      <td>Primary Tumor</td>\n",
       "      <td>Female</td>\n",
       "      <td>TCGA</td>\n",
       "      <td>TCGA-EM-A2OW</td>\n",
       "      <td>THCA</td>\n",
       "      <td>37</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1227</td>\n",
       "      <td>0</td>\n",
       "      <td>1227</td>\n",
       "      <td>0</td>\n",
       "      <td>1227</td>\n",
       "      <td>0</td>\n",
       "      <td>1227</td>\n",
       "      <td>nan</td>\n",
       "      <td>Tumor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TCGA-ZT-A8OM-01</th>\n",
       "      <td>Thymoma</td>\n",
       "      <td>Thymoma</td>\n",
       "      <td>Thymus</td>\n",
       "      <td>Primary Tumor</td>\n",
       "      <td>Female</td>\n",
       "      <td>TCGA</td>\n",
       "      <td>TCGA-ZT-A8OM</td>\n",
       "      <td>THYM</td>\n",
       "      <td>73</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1398</td>\n",
       "      <td>0</td>\n",
       "      <td>1398</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>0</td>\n",
       "      <td>1398</td>\n",
       "      <td>nan</td>\n",
       "      <td>Tumor</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6 rows × 40 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            detailed_category  \\\n",
       "GTEX-1117F-0226-SM-5GZZ7               Adipose - Subcutaneous   \n",
       "GTEX-OHPK-0226-SM-3MJH6                Adipose - Subcutaneous   \n",
       "GTEX-ZXG5-0326-SM-5GICH                     Muscle - Skeletal   \n",
       "TCGA-BA-5556-01           Head & Neck Squamous Cell Carcinoma   \n",
       "TCGA-EM-A2OW-01                             Thyroid Carcinoma   \n",
       "TCGA-ZT-A8OM-01                                       Thymoma   \n",
       "\n",
       "                                    primary disease or tissue  \\\n",
       "GTEX-1117F-0226-SM-5GZZ7               Adipose - Subcutaneous   \n",
       "GTEX-OHPK-0226-SM-3MJH6                Adipose - Subcutaneous   \n",
       "GTEX-ZXG5-0326-SM-5GICH                     Muscle - Skeletal   \n",
       "TCGA-BA-5556-01           Head & Neck Squamous Cell Carcinoma   \n",
       "TCGA-EM-A2OW-01                             Thyroid Carcinoma   \n",
       "TCGA-ZT-A8OM-01                                       Thymoma   \n",
       "\n",
       "                                 _primary_site   _sample_type _gender _study  \\\n",
       "GTEX-1117F-0226-SM-5GZZ7        Adipose Tissue  Normal Tissue  Female   GTEX   \n",
       "GTEX-OHPK-0226-SM-3MJH6         Adipose Tissue  Normal Tissue  Female   GTEX   \n",
       "GTEX-ZXG5-0326-SM-5GICH                 Muscle  Normal Tissue    Male   GTEX   \n",
       "TCGA-BA-5556-01           Head and Neck region  Primary Tumor  Female   TCGA   \n",
       "TCGA-EM-A2OW-01                  Thyroid Gland  Primary Tumor  Female   TCGA   \n",
       "TCGA-ZT-A8OM-01                         Thymus  Primary Tumor  Female   TCGA   \n",
       "\n",
       "                              _PATIENT cancer type abbreviation  \\\n",
       "GTEX-1117F-0226-SM-5GZZ7           nan                      nan   \n",
       "GTEX-OHPK-0226-SM-3MJH6            nan                      nan   \n",
       "GTEX-ZXG5-0326-SM-5GICH            nan                      nan   \n",
       "TCGA-BA-5556-01           TCGA-BA-5556                     HNSC   \n",
       "TCGA-EM-A2OW-01           TCGA-EM-A2OW                     THCA   \n",
       "TCGA-ZT-A8OM-01           TCGA-ZT-A8OM                     THYM   \n",
       "\n",
       "                         age_at_initial_pathologic_diagnosis  gender  \\\n",
       "GTEX-1117F-0226-SM-5GZZ7                                 nan     nan   \n",
       "GTEX-OHPK-0226-SM-3MJH6                                  nan     nan   \n",
       "GTEX-ZXG5-0326-SM-5GICH                                  nan     nan   \n",
       "TCGA-BA-5556-01                                           58  FEMALE   \n",
       "TCGA-EM-A2OW-01                                           37  FEMALE   \n",
       "TCGA-ZT-A8OM-01                                           73  FEMALE   \n",
       "\n",
       "                             ...        OS OS.time  DSS DSS.time  DFI  \\\n",
       "GTEX-1117F-0226-SM-5GZZ7     ...       nan     nan  nan      nan  nan   \n",
       "GTEX-OHPK-0226-SM-3MJH6      ...       nan     nan  nan      nan  nan   \n",
       "GTEX-ZXG5-0326-SM-5GICH      ...       nan     nan  nan      nan  nan   \n",
       "TCGA-BA-5556-01              ...         0     725    0      725    0   \n",
       "TCGA-EM-A2OW-01              ...         0    1227    0     1227    0   \n",
       "TCGA-ZT-A8OM-01              ...         0    1398    0     1398  nan   \n",
       "\n",
       "                         DFI.time  PFI PFI.time Redaction tumor_normal  \n",
       "GTEX-1117F-0226-SM-5GZZ7      nan  nan      nan       nan       Normal  \n",
       "GTEX-OHPK-0226-SM-3MJH6       nan  nan      nan       nan       Normal  \n",
       "GTEX-ZXG5-0326-SM-5GICH       nan  nan      nan       nan       Normal  \n",
       "TCGA-BA-5556-01               725    0      725       nan        Tumor  \n",
       "TCGA-EM-A2OW-01              1227    0     1227       nan        Tumor  \n",
       "TCGA-ZT-A8OM-01               nan    0     1398       nan        Tumor  \n",
       "\n",
       "[6 rows x 40 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Include only labels for samples that we have\n",
    "pruned_labels = all_labels.loc[all_labels.index.intersection(pruned_samples.index)]\n",
    "print(\"Starting with {} labeled sample pairs\".format(pruned_labels.shape[0]))\n",
    "\n",
    "# Drop that are missing labels we plan to classify\n",
    "pruned_labels = pruned_labels.dropna(subset=[\"_primary_site\"])\n",
    "print(pruned_labels.shape[0], \"with _primary_site\")\n",
    "pruned_labels = pruned_labels.dropna(subset=[\"_gender\"])\n",
    "print(pruned_labels.shape[0], \"with _gender\")\n",
    "\n",
    "# Drop primary site with less then 100 examples\n",
    "counts = pruned_labels._primary_site.value_counts()\n",
    "pruned_labels = pruned_labels[pruned_labels._primary_site.isin(counts[counts > 100].index)]\n",
    "print(pruned_labels.shape[0], \"too few examples by _primary_site\")\n",
    "\n",
    "# Some of the cell line are normal and in any case not clear they are reliable signal\n",
    "pruned_labels = pruned_labels[pruned_labels._sample_type != \"Cell Line\"]\n",
    "print(pruned_labels.shape[0], \"not Cell Line\")\n",
    "\n",
    "# Generate a Tumor/Normal label\n",
    "pruned_labels = pruned_labels.dropna(subset=[\"_sample_type\"])\n",
    "print(pruned_labels.shape[0], \"with _sample_type\")\n",
    "pruned_labels[\"tumor_normal\"] = pruned_labels.apply(\n",
    "    lambda row: \"Normal\" if row[\"_sample_type\"] in [\"Normal Tissue\", \"Solid Tissue Normal\"]\n",
    "    else \"Tumor\", axis=1)\n",
    "\n",
    "print(\"{} labels after pruning\".format(pruned_labels.shape[0]))\n",
    "pruned_labels.iloc[::pruned_labels.shape[0]//5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Export\n",
    "\n",
    "Export the full dataset as an h5 file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exporting 17277 samples\n",
      "CPU times: user 6.1 s, sys: 2.52 s, total: 8.62 s\n",
      "Wall time: 8.72 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Include only ids that we have both sample and label for\n",
    "# REMIND: Should we store as a sparse matric?\n",
    "# https://www.hdfgroup.org/2018/06/hdf5-or-how-i-learned-to-love-data-compression-and-partial-i-o\n",
    "# https://stackoverflow.com/questions/43390038/storing-scipy-sparse-matrix-as-hdf5\n",
    "sample_ids = pruned_samples.index.intersection(pruned_labels.index)\n",
    "print(\"Exporting {} samples\".format(len(sample_ids)))\n",
    "\n",
    "# NOTE: Setting complevel to 9 reduces the size of the resulting h5 file from 3G down to 2.1G\n",
    "# but increases the read time from 2.79s to 20.8s and the write time from 19.9s to 25m\n",
    "pruned_samples.loc[sample_ids].sort_index(axis=\"index\").sort_index(axis=\"columns\").to_hdf(\n",
    "    os.path.expanduser(\"pancan-gtex-pruned.h5\"), key=\"samples\", mode=\"w\", format=\"fixed\", complevel=0)\n",
    "pruned_labels.loc[sample_ids].sort_index(axis=\"index\").sort_index(axis=\"columns\").to_hdf(\n",
    "    os.path.expanduser(\"pancan-gtex-pruned.h5\"), key=\"labels\", mode=\"a\", format=\"fixed\", complevel=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-01-25 04:23:03    5544666 ensemble-to-hugo.tsv\r\n",
      "2019-01-25 04:22:59  388947218 pancan-gtex-pruned.h5\r\n"
     ]
    }
   ],
   "source": [
    "# # Empty the path\n",
    "# !aws --profile {os.getenv(\"AWS_PROFILE\")} --endpoint {os.getenv(\"AWS_S3_ENDPOINT\")} \\\n",
    "#     s3 rm --recursive s3://stuartlab/pancan-gtex/\n",
    "    \n",
    "!aws --profile {os.getenv(\"AWS_PROFILE\")} --endpoint {os.getenv(\"AWS_S3_ENDPOINT\")} \\\n",
    "    s3 ls s3://stuartlab/pancan-gtex/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "upload: ./pancan-gtex-pruned.h5 to s3://stuartlab/pancan-gtex/pancan-gtex-pruned.h5\n",
      "upload: ./ensemble-to-hugo.tsv to s3://stuartlab/pancan-gtex/ensemble-to-hugo.tsv\n"
     ]
    }
   ],
   "source": [
    "# Copy to S3/PRP for training as well as public_html for faster access in colab infer\n",
    "!aws --profile {os.getenv(\"AWS_PROFILE\")} --endpoint {os.getenv(\"AWS_S3_ENDPOINT\")} \\\n",
    "    s3 cp pancan-gtex-pruned.h5 s3://stuartlab/pancan-gtex/ --acl public-read\n",
    "\n",
    "!aws --profile {os.getenv(\"AWS_PROFILE\")} --endpoint {os.getenv(\"AWS_S3_ENDPOINT\")} \\\n",
    "    s3 cp ensemble-to-hugo.tsv s3://stuartlab/pancan-gtex/ --acl public-read"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}