gene_multiple_hit_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "\n",
    "sys.path.append('/Users/friedman/Desktop/mnt/ifs/work/taylorlab/friedman/myUtils')\n",
    "import analysis_utils \n",
    "import mutationSigUtils \n",
    "import maf_analysis_utils\n",
    "import mutation_modeling_util\n",
    "\n",
    "pathPrefix = '/Users/friedman/Desktop/mnt'\n",
    "\n",
    "from collections import Counter\n",
    "import clonality_analysis_util\n",
    "import numpy as np\n",
    "import mutation_permutation_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#reload as needed\n",
    "reload(clonality_analysis_util)\n",
    "reload(maf_analysis_utils)\n",
    "reload(analysis_utils)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#TODO:\n",
    "#analyze clonality etc in the cases and see how many cases have some subclonal mutations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py:2714: DtypeWarning: Columns (17,18,19,45,46,48,50,51,52,53,54,55,56,57,60,61,62,63,64,65,74,75,76,77,78,80,81,85,86,89,90,91,92,93,98,101,102,103,104,105,106,107,108,110,111,112,113,114,115,116,117,121) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "endometrialDf = pd.read_table(pathPrefix + '/ifs/work/taylorlab/friedman/myAdjustedDataFiles/subsettedMafs/Endometrial_HypermutantCaseMuts_MAF_ANNO.maf')\n",
    "colorectalDf = pd.read_table(pathPrefix + '/ifs/work/taylorlab/friedman/myAdjustedDataFiles/subsettedMafs/Colorectal_HypermutantCaseMuts_MAF_ANNO.maf')\n",
    "gliomaDf = pd.read_table(pathPrefix + '/ifs/work/taylorlab/friedman/myAdjustedDataFiles/subsettedMafs/Glioma_HypermutantCaseMuts_MAF_ANNO.maf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py:2714: DtypeWarning: Columns (13,36,39,85,136) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "allImpactMuts = pd.read_table(pathPrefix + '/ifs/work/taylorlab/friedman/hypermutationAnalysisProj/projectDataAndConfigFiles/all_impact_mutations_annotated_cohort.maf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "endometrialDf['cancer_type'] = 'Endometrial Cancer'\n",
    "colorectalDf['cancer_type'] = 'Colorectal Cancer'\n",
    "gliomaDf['cancer_type'] = 'Glioma'\n",
    "#for now lets just analyze endometrial and colorectal\n",
    "#combinedDf = pd.concat([endometrialDf, colorectalDf])\n",
    "combinedDf = pd.concat([endometrialDf, colorectalDf, gliomaDf])\n",
    "\n",
    "clonalThresh = .8 #the threshold to use for ccf mcopies for clonal vs subclonal\n",
    "combinedDf['clonal'] = combinedDf['ccf_expected_upper_em'].apply(lambda x: 1 if x > clonalThresh else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#JUST A LITTLE QC AREA\n",
    "clonalThresh = .8 #the threshold to use for ccf mcopies for clonal vs subclonal\n",
    "for case in set(combinedDf['Tumor_Sample_Barcode']):\n",
    "    caseMaf = combinedDf[combinedDf['Tumor_Sample_Barcode'] == case]\n",
    "    if caseMaf[caseMaf['ccf_expected_upper_em'].notnull()].shape[0] > 0:\n",
    "        caseMafClonal = caseMaf[caseMaf['ccf_expected_upper_em'] > clonalThresh]\n",
    "        caseMafSubclonal = caseMaf[caseMaf['ccf_expected_upper_em'] <= clonalThresh]\n",
    "        print 'clonal:', caseMafClonal.shape[0], 'subclonal:', caseMafSubclonal.shape[0], case\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#COME UP WITH rankings based on normal fracs\n",
    "reload(maf_analysis_utils)\n",
    "\n",
    "endometrialNormalIds = analysis_utils.get_ids_by_hypermutant_status(hypermutantIdDir=pathPrefix +'/ifs/work/taylorlab/friedman/hypermutationAnalysisProj/projectDataAndConfigFiles/hypermutationStatusIds', cancerType='Endometrial Cancer', hypermutantStatus = 'Normal')\n",
    "colorectalNormalIds = analysis_utils.get_ids_by_hypermutant_status(hypermutantIdDir=pathPrefix +'/ifs/work/taylorlab/friedman/hypermutationAnalysisProj/projectDataAndConfigFiles/hypermutationStatusIds', cancerType='Colorectal Cancer', hypermutantStatus = 'Normal')\n",
    "gliomaNormalIds = analysis_utils.get_ids_by_hypermutant_status(hypermutantIdDir=pathPrefix +'/ifs/work/taylorlab/friedman/hypermutationAnalysisProj/projectDataAndConfigFiles/hypermutationStatusIds', cancerType='Glioma', hypermutantStatus = 'Normal')\n",
    "\n",
    "endoNormalDf = allImpactMuts[allImpactMuts['Tumor_Sample_Barcode'].isin(endometrialNormalIds)]\n",
    "colorectalNormalDf = allImpactMuts[allImpactMuts['Tumor_Sample_Barcode'].isin(colorectalNormalIds)]\n",
    "gliomaNormalDf = allImpactMuts[allImpactMuts['Tumor_Sample_Barcode'].isin(gliomaNormalIds)]\n",
    "\n",
    "\n",
    "endoNormalRanking = maf_analysis_utils.enumerate_gene_mut_ranking_for_cohort(endoNormalDf)\n",
    "colorectalNormalRanking = maf_analysis_utils.enumerate_gene_mut_ranking_for_cohort(colorectalNormalDf)\n",
    "gliomaNormalRanking = maf_analysis_utils.enumerate_gene_mut_ranking_for_cohort(gliomaNormalDf)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "countsDict = {'Glioma': gliomaNormalRanking, 'Endometrial Cancer': endoNormalRanking, 'Colorectal Cancer': colorectalNormalRanking}\n",
    "combinedDf['ranking'] = combinedDf.apply(lambda row: \n",
    "                                                                               countsDict[row['cancer_type']][row['Hugo_Symbol']] \n",
    "                                                                               if row['Hugo_Symbol'] in countsDict[row['cancer_type']]\n",
    "                                                                               else None, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "listOfDicts = []\n",
    "\n",
    "#CHANGE THESE AND SAVE THE FILE DIFFERENTLY\n",
    "analysisDf = combinedDf[combinedDf['oncogenic'].notnull()]\n",
    "#analysisDf = combinedDf[combinedDf['is-a-hotspot'] == 'Y']\n",
    "\n",
    "totalClonal = analysisDf[analysisDf['clonal'] == 1].shape[0]\n",
    "totalSubclonal = analysisDf[analysisDf['clonal'] == 0].shape[0]\n",
    "for gene in set(analysisDf['Hugo_Symbol']):\n",
    "    rank = analysisDf[analysisDf['Hugo_Symbol'] == gene]['ranking'].iloc[0]\n",
    "    genesMoreCommon = analysisDf[analysisDf['ranking'] < rank]\n",
    "    \n",
    "    genesMoreCommonClonalCount = genesMoreCommon[genesMoreCommon['clonal'] == 1].shape[0]\n",
    "    genesMoreCommonSubclonalCount = genesMoreCommon[genesMoreCommon['clonal'] == 0].shape[0]\n",
    "    \n",
    "    listOfDicts.append({\n",
    "        'Hugo_Symbol': gene, 'class': 'Clonal', 'rank': rank, 'frac': (1.0*genesMoreCommonClonalCount)/totalClonal\n",
    "    })\n",
    "\n",
    "    \n",
    "    listOfDicts.append({\n",
    "        'Hugo_Symbol': gene, 'class': 'Subclonal', 'rank': rank, 'frac': (1.0*genesMoreCommonSubclonalCount)/totalSubclonal\n",
    "    })\n",
    "    \n",
    "\n",
    "df = pd.DataFrame(listOfDicts)\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('/Users/friedman/Desktop/WORK/dataForLocalPlotting/clonalVsSubclonalOncMutUtilization_oncogenic.tsv', index=False, sep='\\t')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('/Users/friedman/Desktop/WORK/dataForLocalPlotting/clonalVsSubclonalOncMutUtilization_hotspot.tsv', index=False, sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Permutation test** <br/> <br/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_observed_multiplet_counts(observedDf, permutedDf):\n",
    "    dObserved = {}\n",
    "    observedDf['geneCase'] = observedDf.apply(lambda row: row['Tumor_Sample_Barcode'] + '_' + row['Hugo_Symbol'], axis=1)\n",
    "    \n",
    "    mutationsValDict = dict(observedDf['geneCase'].value_counts())\n",
    "    df = pd.DataFrame([])\n",
    "    df['Tumor_Sample_Barcode'] = mutationsValDict.keys()\n",
    "    df['Count'] = mutationsValDict.values()\n",
    "\n",
    "    cols = permutedDf.columns.values\n",
    "    for col in cols:\n",
    "        dObserved[col] = df[df['Count'] == col].shape[0]\n",
    "    return dObserved    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "#calculates fractions of tests that work etc\n",
    "def calculate_permutation_test_by_different_cohorts(maf, geneGroups, n=100):\n",
    "    \n",
    "    returnDict = {}\n",
    "    impact468Genes = set(['ABL1', 'ACVR1', 'AGO2', 'AKT1', 'AKT2', 'AKT3', 'ALK', 'ALOX12B', 'ANKRD11', 'APC', 'AR', 'ARAF', 'ARID1A', 'ARID1B', 'ARID2', 'ARID5B', 'ASXL1', 'ASXL2', 'ATM', 'ATR', 'ATRX', 'AURKA', 'AURKB', 'AXIN1', 'AXIN2', 'AXL', 'B2M', 'BABAM1', 'BAP1', 'BARD1', 'BBC3', 'BCL10', 'BCL2', 'BCL2L1', 'BCL2L11', 'BCL6', 'BCOR', 'BIRC3', 'BLM', 'BMPR1A', 'BRAF', 'BRCA1', 'BRCA2', 'BRD4', 'BRIP1', 'BTK', 'CALR', 'CARD11', 'CARM1', 'CASP8', 'CBFB', 'CBL', 'CCND1', 'CCND2', 'CCND3', 'CCNE1', 'CD274', 'CD276', 'CD79A', 'CD79B', 'CDC42', 'CDC73', 'CDH1', 'CDK12', 'CDK4', 'CDK6', 'CDK8', 'CDKN1A', 'CDKN1B', 'CDKN2A', 'CDKN2B', 'CDKN2C', 'CEBPA', 'CENPA', 'CHEK1', 'CHEK2', 'CIC', 'CREBBP', 'CRKL', 'CRLF2', 'CSDE1', 'CSF1R', 'CSF3R', 'CTCF', 'CTLA4', 'CTNNB1', 'CUL3', 'CXCR4', 'CYLD', 'CYSLTR2', 'DAXX', 'DCUN1D1', 'DDR2', 'DICER1', 'DIS3', 'DNAJB1', 'DNMT1', 'DNMT3A', 'DNMT3B', 'DOT1L', 'DROSHA', 'DUSP4', 'E2F3', 'EED', 'EGFL7', 'EGFR', 'EIF1AX', 'EIF4A2', 'EIF4E', 'ELF3', 'EP300', 'EPAS1', 'EPCAM', 'EPHA3', 'EPHA5', 'EPHA7', 'EPHB1', 'ERBB2', 'ERBB3', 'ERBB4', 'ERCC2', 'ERCC3', 'ERCC4', 'ERCC5', 'ERF', 'ERG', 'ERRFI1', 'ESR1', 'ETV1', 'ETV6', 'EZH1', 'EZH2', 'FAM123B', 'FAM175A', 'FAM46C', 'FAM58A', 'FANCA', 'FANCC', 'FAT1', 'FBXW7', 'FGF19', 'FGF3', 'FGF4', 'FGFR1', 'FGFR2', 'FGFR3', 'FGFR4', 'FH', 'FLCN', 'FLT1', 'FLT3', 'FLT4', 'FOXA1', 'FOXL2', 'FOXO1', 'FOXP1', 'FUBP1', 'FYN', 'GATA1', 'GATA2', 'GATA3', 'GLI1', 'GNA11', 'GNAQ', 'GNAS', 'GPS2', 'GREM1', 'GRIN2A', 'GSK3B', 'H3F3A', 'H3F3B', 'H3F3C', 'HGF', 'HIST1H1C', 'HIST1H2BD', 'HIST1H3A', 'HIST1H3B', 'HIST1H3C', 'HIST1H3D', 'HIST1H3E', 'HIST1H3F', 'HIST1H3G', 'HIST1H3H', 'HIST1H3I', 'HIST1H3J', 'HIST2H3C', 'HIST2H3D', 'HIST3H3', 'HLA-A', 'HLA-B', 'HNF1A', 'HOXB13', 'HRAS', 'ICOSLG', 'ID3', 'IDH1', 'IDH2', 'IFNGR1', 'IGF1', 'IGF1R', 'IGF2', 'IKBKE', 'IKZF1', 'IL10', 'IL7R', 'INHA', 'INHBA', 'INPP4A', 'INPP4B', 'INPPL1', 'INSR', 'IRF4', 'IRS1', 'IRS2', 'JAK1', 'JAK2', 'JAK3', 'JUN', 'KDM5A', 'KDM5C', 'KDM6A', 'KDR', 'KEAP1', 'KIT', 'KLF4', 'KMT2B', 'KMT5A', 'KNSTRN', 'KRAS', 'LATS1', 'LATS2', 'LMO1', 'LYN', 'MALT1', 'MAP2K1', 'MAP2K2', 'MAP2K4', 'MAP3K1', 'MAP3K13', 'MAP3K14', 'MAPK1', 'MAPK3', 'MAPKAP1', 'MAX', 'MCL1', 'MDC1', 'MDM2', 'MDM4', 'MED12', 'MEF2B', 'MEN1', 'MET', 'MGA', 'MITF', 'MLH1', 'MLL', 'MLL2', 'MLL3', 'MPL', 'MRE11A', 'MSH2', 'MSH3', 'MSH6', 'MSI1', 'MSI2', 'MST1', 'MST1R', 'MTOR', 'MUTYH', 'MYC', 'MYCL1', 'MYCN', 'MYD88', 'MYOD1', 'NBN', 'NCOA3', 'NCOR1', 'NEGR1', 'NF1', 'NF2', 'NFE2L2', 'NFKBIA', 'NKX2-1', 'NKX3-1', 'NOTCH1', 'NOTCH2', 'NOTCH3', 'NOTCH4', 'NPM1', 'NRAS', 'NSD1', 'NTHL1', 'NTRK1', 'NTRK2', 'NTRK3', 'NUF2', 'NUP93', 'PAK1', 'PAK7', 'PALB2', 'PARK2', 'PARP1', 'PAX5', 'PBRM1', 'PDCD1', 'PDCD1LG2', 'PDGFRA', 'PDGFRB', 'PDPK1', 'PGR', 'PHOX2B', 'PIK3C2G', 'PIK3C3', 'PIK3CA', 'PIK3CB', 'PIK3CD', 'PIK3CG', 'PIK3R1', 'PIK3R2', 'PIK3R3', 'PIM1', 'PLCG2', 'PLK2', 'PMAIP1', 'PMS1', 'PMS2', 'PNRC1', 'POLD1', 'POLE', 'PPARG', 'PPM1D', 'PPP2R1A', 'PPP4R2', 'PPP6C', 'PRDM1', 'PRDM14', 'PREX2', 'PRKAR1A', 'PRKCI', 'PRKD1', 'PTCH1', 'PTEN', 'PTP4A1', 'PTPN11', 'PTPRD', 'PTPRS', 'PTPRT', 'RAB35', 'RAC1', 'RAC2', 'RAD21', 'RAD50', 'RAD51', 'RAD51C', 'RAD51L1', 'RAD51L3', 'RAD52', 'RAD54L', 'RAF1', 'RARA', 'RASA1', 'RB1', 'RBM10', 'RECQL', 'RECQL4', 'REL', 'RET', 'RFWD2', 'RHEB', 'RHOA', 'RICTOR', 'RIT1', 'RNF43', 'ROS1', 'RPS6KA4', 'RPS6KB2', 'RPTOR', 'RRAGC', 'RRAS', 'RRAS2', 'RTEL1', 'RUNX1', 'RXRA', 'RYBP', 'SDHA', 'SDHAF2', 'SDHB', 'SDHC', 'SDHD', 'SESN1', 'SESN2', 'SESN3', 'SETD2', 'SF3B1', 'SH2B3', 'SH2D1A', 'SHOC2', 'SHQ1', 'SLX4', 'SMAD2', 'SMAD3', 'SMAD4', 'SMARCA4', 'SMARCB1', 'SMARCD1', 'SMO', 'SMYD3', 'SOCS1', 'SOS1', 'SOX17', 'SOX2', 'SOX9', 'SPEN', 'SPOP', 'SPRED1', 'SRC', 'SRSF2', 'STAG2', 'STAT3', 'STAT5A', 'STAT5B', 'STK11', 'STK19', 'STK40', 'SUFU', 'SUZ12', 'SYK', 'TAP1', 'TAP2', 'TBX3', 'TCEB1', 'TCF3', 'TCF7L2', 'TEK', 'TERT', 'TET1', 'TET2', 'TGFBR1', 'TGFBR2', 'TMEM127', 'TMPRSS2', 'TNFAIP3', 'TNFRSF14', 'TOP1', 'TP53', 'TP53BP1', 'TP63', 'TRAF2', 'TRAF7', 'TSC1', 'TSC2', 'TSHR', 'U2AF1', 'UPF1', 'VEGFA', 'VHL', 'VTCN1', 'WHSC1', 'WHSC1L1', 'WT1', 'WWTR1', 'XIAP', 'XPO1', 'XRCC2', 'YAP1', 'YES1', 'ZFHX3', 'ZRSR2'])\n",
    "    \n",
    "    for label, genes in geneGroups.items():\n",
    "        geneMaf = maf[maf['Hugo_Symbol'].isin(genes)]\n",
    "        mutsToShuffle = maf[maf['Hugo_Symbol'].isin(genes)]\n",
    "        summaryDf = mutation_permutation_utils.do_n_iterations_of_shuffling(mutsToShuffle, 100, impact468Genes)\n",
    "        dObserved = calculate_observed_multiplet_counts(geneMaf, summaryDf)\n",
    "        differencesDict = mutation_permutation_utils.calculate_n_tests_that_differed(summaryDf, dObserved)\n",
    "        returnDict[label] = differencesDict\n",
    "    return returnDict\n",
    "\n",
    "#this function is used to map genes to their categories of recurrence in the cancer type \n",
    "def enumerate_genes_by_ranking(genesToConsider, rankingDict, \n",
    "                               breaks=[.03, .5, 1]): #breaks are a list of pcts between 0 and 1\n",
    "    dictOfSets = dict([(str(i), set([])) for i in breaks])\n",
    "    for gene in genesToConsider:\n",
    "        for b in breaks:\n",
    "            if gene not in rankingDict:\n",
    "                break\n",
    "            if rankingDict[gene] < b:\n",
    "                dictOfSets[str(b)] = dictOfSets[str(b)] | set([gene])\n",
    "                break\n",
    "    return dictOfSets\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_results_as_df_for_plotting(resultsD, n):\n",
    "    listOfDicts = []\n",
    "    for cohort, values in resultsD.items():\n",
    "        twoMutPValue = (1.0*values[2])/n\n",
    "        listOfDicts.append({\n",
    "            'cohort': cohort,\n",
    "            'p': twoMutPValue\n",
    "        })\n",
    "    \n",
    "    return pd.DataFrame(listOfDicts)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'mutation_permutation_utils' from '/Users/friedman/Desktop/mnt/ifs/work/taylorlab/friedman/myUtils/mutation_permutation_utils.py'>"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reload(mutation_permutation_utils)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "endometrialNormalIds = analysis_utils.get_ids_by_hypermutant_status(cancerType='Endometrial Cancer', hypermutantStatus = 'Normal', hypermutantIdDir=pathPrefix + '/ifs/work/taylorlab/friedman/hypermutationAnalysisProj/projectDataAndConfigFiles/hypermutationStatusIds')\n",
    "colorectalNormalIds = analysis_utils.get_ids_by_hypermutant_status(cancerType='Colorectal Cancer', hypermutantStatus = 'Normal', hypermutantIdDir=pathPrefix + '/ifs/work/taylorlab/friedman/hypermutationAnalysisProj/projectDataAndConfigFiles/hypermutationStatusIds')\n",
    "gliomaNormalIds = analysis_utils.get_ids_by_hypermutant_status(cancerType='Glioma', hypermutantStatus = 'Normal', hypermutantIdDir=pathPrefix + '/ifs/work/taylorlab/friedman/hypermutationAnalysisProj/projectDataAndConfigFiles/hypermutationStatusIds')\n",
    "\n",
    "endoNormalDf = allImpactMuts[allImpactMuts['Tumor_Sample_Barcode'].isin(endometrialNormalIds)]\n",
    "colorectalNormalDf = allImpactMuts[allImpactMuts['Tumor_Sample_Barcode'].isin(colorectalNormalIds)]\n",
    "gliomaNormalDf = allImpactMuts[allImpactMuts['Tumor_Sample_Barcode'].isin(gliomaNormalIds)]\n",
    "\n",
    "endoNormalRanking = maf_analysis_utils.enumerate_gene_mut_ranking_for_cohort(endoNormalDf)\n",
    "colorectalNormalRanking = maf_analysis_utils.enumerate_gene_mut_ranking_for_cohort(colorectalNormalDf)\n",
    "gliomaNormalRanking = maf_analysis_utils.enumerate_gene_mut_ranking_for_cohort(gliomaNormalDf)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "nTrials = 1000\n",
    "\n",
    "#We define some gene categories (t suppresor, oncogenes, all oncogenes besides pik3ca and erbb)\n",
    "impact468Genes = set(['ABL1', 'ACVR1', 'AGO2', 'AKT1', 'AKT2', 'AKT3', 'ALK', 'ALOX12B', 'ANKRD11', 'APC', 'AR', 'ARAF', 'ARID1A', 'ARID1B', 'ARID2', 'ARID5B', 'ASXL1', 'ASXL2', 'ATM', 'ATR', 'ATRX', 'AURKA', 'AURKB', 'AXIN1', 'AXIN2', 'AXL', 'B2M', 'BABAM1', 'BAP1', 'BARD1', 'BBC3', 'BCL10', 'BCL2', 'BCL2L1', 'BCL2L11', 'BCL6', 'BCOR', 'BIRC3', 'BLM', 'BMPR1A', 'BRAF', 'BRCA1', 'BRCA2', 'BRD4', 'BRIP1', 'BTK', 'CALR', 'CARD11', 'CARM1', 'CASP8', 'CBFB', 'CBL', 'CCND1', 'CCND2', 'CCND3', 'CCNE1', 'CD274', 'CD276', 'CD79A', 'CD79B', 'CDC42', 'CDC73', 'CDH1', 'CDK12', 'CDK4', 'CDK6', 'CDK8', 'CDKN1A', 'CDKN1B', 'CDKN2A', 'CDKN2B', 'CDKN2C', 'CEBPA', 'CENPA', 'CHEK1', 'CHEK2', 'CIC', 'CREBBP', 'CRKL', 'CRLF2', 'CSDE1', 'CSF1R', 'CSF3R', 'CTCF', 'CTLA4', 'CTNNB1', 'CUL3', 'CXCR4', 'CYLD', 'CYSLTR2', 'DAXX', 'DCUN1D1', 'DDR2', 'DICER1', 'DIS3', 'DNAJB1', 'DNMT1', 'DNMT3A', 'DNMT3B', 'DOT1L', 'DROSHA', 'DUSP4', 'E2F3', 'EED', 'EGFL7', 'EGFR', 'EIF1AX', 'EIF4A2', 'EIF4E', 'ELF3', 'EP300', 'EPAS1', 'EPCAM', 'EPHA3', 'EPHA5', 'EPHA7', 'EPHB1', 'ERBB2', 'ERBB3', 'ERBB4', 'ERCC2', 'ERCC3', 'ERCC4', 'ERCC5', 'ERF', 'ERG', 'ERRFI1', 'ESR1', 'ETV1', 'ETV6', 'EZH1', 'EZH2', 'FAM123B', 'FAM175A', 'FAM46C', 'FAM58A', 'FANCA', 'FANCC', 'FAT1', 'FBXW7', 'FGF19', 'FGF3', 'FGF4', 'FGFR1', 'FGFR2', 'FGFR3', 'FGFR4', 'FH', 'FLCN', 'FLT1', 'FLT3', 'FLT4', 'FOXA1', 'FOXL2', 'FOXO1', 'FOXP1', 'FUBP1', 'FYN', 'GATA1', 'GATA2', 'GATA3', 'GLI1', 'GNA11', 'GNAQ', 'GNAS', 'GPS2', 'GREM1', 'GRIN2A', 'GSK3B', 'H3F3A', 'H3F3B', 'H3F3C', 'HGF', 'HIST1H1C', 'HIST1H2BD', 'HIST1H3A', 'HIST1H3B', 'HIST1H3C', 'HIST1H3D', 'HIST1H3E', 'HIST1H3F', 'HIST1H3G', 'HIST1H3H', 'HIST1H3I', 'HIST1H3J', 'HIST2H3C', 'HIST2H3D', 'HIST3H3', 'HLA-A', 'HLA-B', 'HNF1A', 'HOXB13', 'HRAS', 'ICOSLG', 'ID3', 'IDH1', 'IDH2', 'IFNGR1', 'IGF1', 'IGF1R', 'IGF2', 'IKBKE', 'IKZF1', 'IL10', 'IL7R', 'INHA', 'INHBA', 'INPP4A', 'INPP4B', 'INPPL1', 'INSR', 'IRF4', 'IRS1', 'IRS2', 'JAK1', 'JAK2', 'JAK3', 'JUN', 'KDM5A', 'KDM5C', 'KDM6A', 'KDR', 'KEAP1', 'KIT', 'KLF4', 'KMT2B', 'KMT5A', 'KNSTRN', 'KRAS', 'LATS1', 'LATS2', 'LMO1', 'LYN', 'MALT1', 'MAP2K1', 'MAP2K2', 'MAP2K4', 'MAP3K1', 'MAP3K13', 'MAP3K14', 'MAPK1', 'MAPK3', 'MAPKAP1', 'MAX', 'MCL1', 'MDC1', 'MDM2', 'MDM4', 'MED12', 'MEF2B', 'MEN1', 'MET', 'MGA', 'MITF', 'MLH1', 'MLL', 'MLL2', 'MLL3', 'MPL', 'MRE11A', 'MSH2', 'MSH3', 'MSH6', 'MSI1', 'MSI2', 'MST1', 'MST1R', 'MTOR', 'MUTYH', 'MYC', 'MYCL1', 'MYCN', 'MYD88', 'MYOD1', 'NBN', 'NCOA3', 'NCOR1', 'NEGR1', 'NF1', 'NF2', 'NFE2L2', 'NFKBIA', 'NKX2-1', 'NKX3-1', 'NOTCH1', 'NOTCH2', 'NOTCH3', 'NOTCH4', 'NPM1', 'NRAS', 'NSD1', 'NTHL1', 'NTRK1', 'NTRK2', 'NTRK3', 'NUF2', 'NUP93', 'PAK1', 'PAK7', 'PALB2', 'PARK2', 'PARP1', 'PAX5', 'PBRM1', 'PDCD1', 'PDCD1LG2', 'PDGFRA', 'PDGFRB', 'PDPK1', 'PGR', 'PHOX2B', 'PIK3C2G', 'PIK3C3', 'PIK3CA', 'PIK3CB', 'PIK3CD', 'PIK3CG', 'PIK3R1', 'PIK3R2', 'PIK3R3', 'PIM1', 'PLCG2', 'PLK2', 'PMAIP1', 'PMS1', 'PMS2', 'PNRC1', 'POLD1', 'POLE', 'PPARG', 'PPM1D', 'PPP2R1A', 'PPP4R2', 'PPP6C', 'PRDM1', 'PRDM14', 'PREX2', 'PRKAR1A', 'PRKCI', 'PRKD1', 'PTCH1', 'PTEN', 'PTP4A1', 'PTPN11', 'PTPRD', 'PTPRS', 'PTPRT', 'RAB35', 'RAC1', 'RAC2', 'RAD21', 'RAD50', 'RAD51', 'RAD51C', 'RAD51L1', 'RAD51L3', 'RAD52', 'RAD54L', 'RAF1', 'RARA', 'RASA1', 'RB1', 'RBM10', 'RECQL', 'RECQL4', 'REL', 'RET', 'RFWD2', 'RHEB', 'RHOA', 'RICTOR', 'RIT1', 'RNF43', 'ROS1', 'RPS6KA4', 'RPS6KB2', 'RPTOR', 'RRAGC', 'RRAS', 'RRAS2', 'RTEL1', 'RUNX1', 'RXRA', 'RYBP', 'SDHA', 'SDHAF2', 'SDHB', 'SDHC', 'SDHD', 'SESN1', 'SESN2', 'SESN3', 'SETD2', 'SF3B1', 'SH2B3', 'SH2D1A', 'SHOC2', 'SHQ1', 'SLX4', 'SMAD2', 'SMAD3', 'SMAD4', 'SMARCA4', 'SMARCB1', 'SMARCD1', 'SMO', 'SMYD3', 'SOCS1', 'SOS1', 'SOX17', 'SOX2', 'SOX9', 'SPEN', 'SPOP', 'SPRED1', 'SRC', 'SRSF2', 'STAG2', 'STAT3', 'STAT5A', 'STAT5B', 'STK11', 'STK19', 'STK40', 'SUFU', 'SUZ12', 'SYK', 'TAP1', 'TAP2', 'TBX3', 'TCEB1', 'TCF3', 'TCF7L2', 'TEK', 'TERT', 'TET1', 'TET2', 'TGFBR1', 'TGFBR2', 'TMEM127', 'TMPRSS2', 'TNFAIP3', 'TNFRSF14', 'TOP1', 'TP53', 'TP53BP1', 'TP63', 'TRAF2', 'TRAF7', 'TSC1', 'TSC2', 'TSHR', 'U2AF1', 'UPF1', 'VEGFA', 'VHL', 'VTCN1', 'WHSC1', 'WHSC1L1', 'WT1', 'WWTR1', 'XIAP', 'XPO1', 'XRCC2', 'YAP1', 'YES1', 'ZFHX3', 'ZRSR2'])\n",
    "tumorSuppressors = set(['ERRFI1', 'ASXL2', 'PMAIP1', 'ACTG1', 'SUFU', 'FBXO11', 'MEN1', 'FAM58A', 'B2M', 'RB1', 'DUSP22', 'SESN1', 'GPS2', 'RAD51D', 'SMG1', 'CDC73', 'MAP3K1', 'SMARCB1', 'INPP4B', 'PARK2', 'SMAD4', 'CBFB', 'CDH1', 'PPP6C', 'SETDB1', 'SETDB2', 'NF2', 'CDKN2B', 'CDKN2C', 'CDKN2A', 'DDX3X', 'PIK3R1', 'BARD1', 'PDS5B', 'KLF4', 'SPRED1', 'VHL', 'SMAD2', 'PMS1', 'PMS2', 'SETD2', 'GATA3', 'TBL1XR1', 'MUTYH', 'SOCS1', 'FAM175A', 'ROBO1', 'ARID1B', 'ARID1A', 'TCF7L2', 'STK11', 'FOXA1', 'PTEN', 'FAT1', 'FAS', 'CYLD', 'MAX', 'SH2D1A', 'APC', 'NTHL1', 'CTCF', 'KDM5C', 'KMT2C', 'ZFHX3', 'FOXP1', 'PIGA', 'CDKN1B', 'CDKN1A', 'FUBP1', 'MSH2', 'ID3', 'TNFRSF14', 'TRAF3', 'EP400', 'BRIP1', 'ARID4A', 'ARID4B', 'XRCC2', 'DAXX', 'SDHAF2', 'ASXL1', 'AMER1', 'RASA1', 'EGR1', 'MST1', 'SOX17', 'RUNX1', 'PIK3R3', 'NCOR1', 'NF1', 'JAK1', 'PTPRD', 'CHEK2', 'CHEK1', 'SMC1A', 'TMEM127', 'STAG1', 'RAD51', 'TCF3', 'STAG2', 'ARID2', 'RAD50', 'RNF43', 'PARP1', 'BLM', 'CUX1', 'RECQL', 'RAD21', 'PTPN2', 'PTPN1', 'SLX4', 'INHA', 'PAX5', 'IRF1', 'TP53', 'HLA-A', 'IRF8', 'CBL', 'TOP1', 'SHQ1', 'PRDM1', 'NSD1', 'ATXN2', 'CREBBP', 'HDAC4', 'SESN2', 'PPP2R1A', 'EPHA7', 'ATM', 'EPHA3', 'POT1', 'SMAD3', 'MOB3B', 'TBX3', 'POLE', 'ATR', 'FANCD2', 'FH', 'BCORL1', 'SOX9', 'IKZF3', 'TSC1', 'TP63', 'MRE11A', 'SDHC', 'BTG1', 'POLD1', 'CIITA', 'SMC3', 'SAMHD1', 'RTEL1', 'ECT2L', 'PIK3R2', 'CRBN', 'FANCC', 'NBN', 'FANCA', 'HLA-B', 'RECQL4', 'DUSP4', 'ERCC2', 'FBXW7', 'TGFBR2', 'TGFBR1', 'MSH3', 'RBM15', 'TET1', 'TET3', 'SESN3', 'MGA', 'LTB', 'FOXL2', 'SH2B3', 'BCOR', 'HIST1H1D', 'ATRX', 'EP300', 'RAD51C', 'RAD51B', 'HIST1H1B', 'TNFAIP3', 'DICER1', 'ARID5B', 'LATS2', 'FOXO1', 'KEAP1', 'EZH2', 'SP140', 'NKX3-1', 'PBRM1', 'PALB2', 'CIC', 'BRCA1', 'DTX1', 'FLCN', 'SPEN', 'CD58', 'ERCC3', 'ERCC4', 'MSH6', 'BCL11B', 'BMPR1A', 'ERF', 'BRCA2', 'NOTCH2', 'EED', 'MITF', 'ELF3', 'SMARCA4', 'BBC3', 'ANKRD11', 'CEBPA', 'BCL2L11', 'AXIN2', 'AXIN1', 'CDK12', 'ESCO2', 'MLH1', 'SDHB', 'MED12', 'HNF1A', 'RYBP', 'ATP6V1B2', 'DNMT3B', 'KMT2B', 'KMT2A', 'DNMT3A', 'NFKBIA', 'TRAF5', 'KMT2D', 'SPOP', 'RBM10', 'P2RY8', 'TP53BP1', 'TSC2', 'KDM6A', 'EPCAM', 'PHOX2B', 'NPM1', 'BCL10', 'LATS1', 'HOXB13', 'ARID3A', 'PTPRT', 'PTPRS', 'INPPL1', 'NOTCH4', 'TET2', 'NOTCH1', 'CASP8', 'NOTCH3', 'GRIN2A', 'MAP2K4', 'WT1', 'BACH2', 'SDHA', 'BAP1', 'PTCH1', 'SDHD'])\n",
    "oncogenes = impact468Genes - tumorSuppressors\n",
    "oncogenesSansKnownTwoMutOncogenes = oncogenes - set(['PIK3CA', 'ERBB2']) #we calculate it using all oncogenes and oncogenes minus pik3ca\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#does the steps to get 2 mut enrichment information for a cancer type\n",
    "#TODO Later im lazy\n",
    "#def get_enrichment_info_for_cancer_type(cancerType):\n",
    "#    cancerTypeFilteredDf  = allImpactMuts[allImpactMuts['Tumor_Sample_Barcode'].isin(set(endometrialDf['Tumor_Sample_Barcode']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#do this for endometrials\n",
    "\n",
    "endometrialFilteredDf = allImpactMuts[allImpactMuts['Tumor_Sample_Barcode'].isin(set(endometrialDf['Tumor_Sample_Barcode']))]\n",
    "endometrialOncMutations = endometrialFilteredDf[endometrialFilteredDf['oncogenic'].notnull()]\n",
    "endometrialOncMutations['shuffleColumn'] = endometrialOncMutations['Hugo_Symbol']\n",
    "\n",
    "#endometrialTumorSuppresorPercentiles = enumerate_genes_by_ranking(tumorSuppresors, endoNormalRanking, breaks=[.025, .25, .5, .75, 1])\n",
    "endometrialTumorSuppresorPercentiles = enumerate_genes_by_ranking(tumorSuppresors, endoNormalRanking,\n",
    "                                                                  breaks=[.1, .2, .3, .4,.5,.6,.7,.8,.9,1])\n",
    "\n",
    "\n",
    "fullDict = endometrialTumorSuppresorPercentiles\n",
    "fullDict['_oncogenes'] = oncogenes\n",
    "fullDict['_oncogenesSansPik3caErbb2'] = oncogenesSansKnownTwoMutOncogenes\n",
    "\n",
    "results = calculate_permutation_test_by_different_cohorts(endometrialOncMutations, fullDict, n=nTrials)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#COLORECTAL\n",
    "\n",
    "colorectalFilteredDf = allImpactMuts[allImpactMuts['Tumor_Sample_Barcode'].isin(set(colorectalDf['Tumor_Sample_Barcode']))]\n",
    "colorectalOncMutations = colorectalFilteredDf[colorectalFilteredDf['oncogenic'].notnull()]\n",
    "colorectalOncMutations['shuffleColumn'] = colorectalOncMutations['Hugo_Symbol']\n",
    "\n",
    "#endometrialTumorSuppresorPercentiles = enumerate_genes_by_ranking(tumorSuppresors, endoNormalRanking, breaks=[.025, .25, .5, .75, 1])\n",
    "colorectalTumorSuppresorPercentiles = enumerate_genes_by_ranking(tumorSuppresors, colorectalNormalRanking,\n",
    "                                                                  breaks=[.1, .2, .3, .4,.5,.6,.7,.8,.9,1])\n",
    "\n",
    "\n",
    "fullDict = colorectalTumorSuppresorPercentiles\n",
    "fullDict['_oncogenes'] = oncogenes\n",
    "fullDict['_oncogenesSansPik3caErbb2'] = oncogenesSansKnownTwoMutOncogenes\n",
    "\n",
    "resultsColo = calculate_permutation_test_by_different_cohorts(colorectalOncMutations, fullDict, n=nTrials)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#GLIOMA\n",
    "\n",
    "gliomaFilteredDf = allImpactMuts[allImpactMuts['Tumor_Sample_Barcode'].isin(set(gliomaDf['Tumor_Sample_Barcode']))]\n",
    "gliomaOncMutations = gliomaFilteredDf[gliomaFilteredDf['oncogenic'].notnull()]\n",
    "gliomaOncMutations['shuffleColumn'] = gliomaOncMutations['Hugo_Symbol']\n",
    "\n",
    "#endometrialTumorSuppresorPercentiles = enumerate_genes_by_ranking(tumorSuppresors, endoNormalRanking, breaks=[.025, .25, .5, .75, 1])\n",
    "gliomaTumorSuppresorPercentiles = enumerate_genes_by_ranking(tumorSuppresors, gliomaNormalRanking,\n",
    "                                                                  breaks=[.1, .2, .3, .4,.5,.6,.7,.8,.9,1])\n",
    "\n",
    "fullDict = gliomaTumorSuppresorPercentiles\n",
    "fullDict['_oncogenes'] = oncogenes\n",
    "fullDict['_oncogenesSansPik3caErbb2'] = oncogenesSansKnownTwoMutOncogenes\n",
    "\n",
    "resultsGliomas = calculate_permutation_test_by_different_cohorts(gliomaOncMutations, fullDict, n=nTrials)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = format_results_as_df_for_plotting(resultsGliomas, nTrials)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('/Users/friedman/Desktop/WORK/dataForLocalPlotting/permutationTestDoubleMutGlioma.tsv', index=False, sep='\\t')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                       cohort      p\n",
      "0                         0.9  0.005\n",
      "1                         0.8  0.049\n",
      "2                         0.1  0.023\n",
      "3                         0.3  0.002\n",
      "4                         0.2  0.005\n",
      "5                         0.5  0.001\n",
      "6                         0.4  0.079\n",
      "7                         0.7  0.000\n",
      "8                         0.6  0.014\n",
      "9                           1  0.011\n",
      "10  _oncogenesSansPik3caErbb2  0.090\n",
      "11                 _oncogenes  0.089\n"
     ]
    }
   ],
   "source": [
    "dfColo = format_results_as_df_for_plotting(resultsColo, nTrials)\n",
    "print df\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'0.025': {1: 91, 2: 0, 3: 100, 4: 100, 5: 98, 6: 59, 7: 18},\n",
       " '0.25': {1: 100, 2: 0, 3: 56, 4: 34, 5: 53, 6: 12, 7: 2},\n",
       " '0.5': {1: 99, 2: 0, 3: 17, 4: 27, 5: 5},\n",
       " '0.75': {1: 51, 2: 63, 3: 2, 4: 1},\n",
       " '1': {1: 91, 2: 23, 3: 5, 4: 14, 5: 1},\n",
       " 'oncogenes': {1: 45,\n",
       "  2: 3,\n",
       "  3: 54,\n",
       "  4: 99,\n",
       "  5: 73,\n",
       "  6: 24,\n",
       "  7: 12,\n",
       "  8: 3,\n",
       "  9: 1,\n",
       "  10: 1},\n",
       " 'oncogenesSansPik3ca': {1: 67, 2: 25, 3: 29, 4: 11, 5: 1}}"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "impact468Genes = set(['ABL1', 'ACVR1', 'AGO2', 'AKT1', 'AKT2', 'AKT3', 'ALK', 'ALOX12B', 'ANKRD11', 'APC', 'AR', 'ARAF', 'ARID1A', 'ARID1B', 'ARID2', 'ARID5B', 'ASXL1', 'ASXL2', 'ATM', 'ATR', 'ATRX', 'AURKA', 'AURKB', 'AXIN1', 'AXIN2', 'AXL', 'B2M', 'BABAM1', 'BAP1', 'BARD1', 'BBC3', 'BCL10', 'BCL2', 'BCL2L1', 'BCL2L11', 'BCL6', 'BCOR', 'BIRC3', 'BLM', 'BMPR1A', 'BRAF', 'BRCA1', 'BRCA2', 'BRD4', 'BRIP1', 'BTK', 'CALR', 'CARD11', 'CARM1', 'CASP8', 'CBFB', 'CBL', 'CCND1', 'CCND2', 'CCND3', 'CCNE1', 'CD274', 'CD276', 'CD79A', 'CD79B', 'CDC42', 'CDC73', 'CDH1', 'CDK12', 'CDK4', 'CDK6', 'CDK8', 'CDKN1A', 'CDKN1B', 'CDKN2A', 'CDKN2B', 'CDKN2C', 'CEBPA', 'CENPA', 'CHEK1', 'CHEK2', 'CIC', 'CREBBP', 'CRKL', 'CRLF2', 'CSDE1', 'CSF1R', 'CSF3R', 'CTCF', 'CTLA4', 'CTNNB1', 'CUL3', 'CXCR4', 'CYLD', 'CYSLTR2', 'DAXX', 'DCUN1D1', 'DDR2', 'DICER1', 'DIS3', 'DNAJB1', 'DNMT1', 'DNMT3A', 'DNMT3B', 'DOT1L', 'DROSHA', 'DUSP4', 'E2F3', 'EED', 'EGFL7', 'EGFR', 'EIF1AX', 'EIF4A2', 'EIF4E', 'ELF3', 'EP300', 'EPAS1', 'EPCAM', 'EPHA3', 'EPHA5', 'EPHA7', 'EPHB1', 'ERBB2', 'ERBB3', 'ERBB4', 'ERCC2', 'ERCC3', 'ERCC4', 'ERCC5', 'ERF', 'ERG', 'ERRFI1', 'ESR1', 'ETV1', 'ETV6', 'EZH1', 'EZH2', 'FAM123B', 'FAM175A', 'FAM46C', 'FAM58A', 'FANCA', 'FANCC', 'FAT1', 'FBXW7', 'FGF19', 'FGF3', 'FGF4', 'FGFR1', 'FGFR2', 'FGFR3', 'FGFR4', 'FH', 'FLCN', 'FLT1', 'FLT3', 'FLT4', 'FOXA1', 'FOXL2', 'FOXO1', 'FOXP1', 'FUBP1', 'FYN', 'GATA1', 'GATA2', 'GATA3', 'GLI1', 'GNA11', 'GNAQ', 'GNAS', 'GPS2', 'GREM1', 'GRIN2A', 'GSK3B', 'H3F3A', 'H3F3B', 'H3F3C', 'HGF', 'HIST1H1C', 'HIST1H2BD', 'HIST1H3A', 'HIST1H3B', 'HIST1H3C', 'HIST1H3D', 'HIST1H3E', 'HIST1H3F', 'HIST1H3G', 'HIST1H3H', 'HIST1H3I', 'HIST1H3J', 'HIST2H3C', 'HIST2H3D', 'HIST3H3', 'HLA-A', 'HLA-B', 'HNF1A', 'HOXB13', 'HRAS', 'ICOSLG', 'ID3', 'IDH1', 'IDH2', 'IFNGR1', 'IGF1', 'IGF1R', 'IGF2', 'IKBKE', 'IKZF1', 'IL10', 'IL7R', 'INHA', 'INHBA', 'INPP4A', 'INPP4B', 'INPPL1', 'INSR', 'IRF4', 'IRS1', 'IRS2', 'JAK1', 'JAK2', 'JAK3', 'JUN', 'KDM5A', 'KDM5C', 'KDM6A', 'KDR', 'KEAP1', 'KIT', 'KLF4', 'KMT2B', 'KMT5A', 'KNSTRN', 'KRAS', 'LATS1', 'LATS2', 'LMO1', 'LYN', 'MALT1', 'MAP2K1', 'MAP2K2', 'MAP2K4', 'MAP3K1', 'MAP3K13', 'MAP3K14', 'MAPK1', 'MAPK3', 'MAPKAP1', 'MAX', 'MCL1', 'MDC1', 'MDM2', 'MDM4', 'MED12', 'MEF2B', 'MEN1', 'MET', 'MGA', 'MITF', 'MLH1', 'MLL', 'MLL2', 'MLL3', 'MPL', 'MRE11A', 'MSH2', 'MSH3', 'MSH6', 'MSI1', 'MSI2', 'MST1', 'MST1R', 'MTOR', 'MUTYH', 'MYC', 'MYCL1', 'MYCN', 'MYD88', 'MYOD1', 'NBN', 'NCOA3', 'NCOR1', 'NEGR1', 'NF1', 'NF2', 'NFE2L2', 'NFKBIA', 'NKX2-1', 'NKX3-1', 'NOTCH1', 'NOTCH2', 'NOTCH3', 'NOTCH4', 'NPM1', 'NRAS', 'NSD1', 'NTHL1', 'NTRK1', 'NTRK2', 'NTRK3', 'NUF2', 'NUP93', 'PAK1', 'PAK7', 'PALB2', 'PARK2', 'PARP1', 'PAX5', 'PBRM1', 'PDCD1', 'PDCD1LG2', 'PDGFRA', 'PDGFRB', 'PDPK1', 'PGR', 'PHOX2B', 'PIK3C2G', 'PIK3C3', 'PIK3CA', 'PIK3CB', 'PIK3CD', 'PIK3CG', 'PIK3R1', 'PIK3R2', 'PIK3R3', 'PIM1', 'PLCG2', 'PLK2', 'PMAIP1', 'PMS1', 'PMS2', 'PNRC1', 'POLD1', 'POLE', 'PPARG', 'PPM1D', 'PPP2R1A', 'PPP4R2', 'PPP6C', 'PRDM1', 'PRDM14', 'PREX2', 'PRKAR1A', 'PRKCI', 'PRKD1', 'PTCH1', 'PTEN', 'PTP4A1', 'PTPN11', 'PTPRD', 'PTPRS', 'PTPRT', 'RAB35', 'RAC1', 'RAC2', 'RAD21', 'RAD50', 'RAD51', 'RAD51C', 'RAD51L1', 'RAD51L3', 'RAD52', 'RAD54L', 'RAF1', 'RARA', 'RASA1', 'RB1', 'RBM10', 'RECQL', 'RECQL4', 'REL', 'RET', 'RFWD2', 'RHEB', 'RHOA', 'RICTOR', 'RIT1', 'RNF43', 'ROS1', 'RPS6KA4', 'RPS6KB2', 'RPTOR', 'RRAGC', 'RRAS', 'RRAS2', 'RTEL1', 'RUNX1', 'RXRA', 'RYBP', 'SDHA', 'SDHAF2', 'SDHB', 'SDHC', 'SDHD', 'SESN1', 'SESN2', 'SESN3', 'SETD2', 'SF3B1', 'SH2B3', 'SH2D1A', 'SHOC2', 'SHQ1', 'SLX4', 'SMAD2', 'SMAD3', 'SMAD4', 'SMARCA4', 'SMARCB1', 'SMARCD1', 'SMO', 'SMYD3', 'SOCS1', 'SOS1', 'SOX17', 'SOX2', 'SOX9', 'SPEN', 'SPOP', 'SPRED1', 'SRC', 'SRSF2', 'STAG2', 'STAT3', 'STAT5A', 'STAT5B', 'STK11', 'STK19', 'STK40', 'SUFU', 'SUZ12', 'SYK', 'TAP1', 'TAP2', 'TBX3', 'TCEB1', 'TCF3', 'TCF7L2', 'TEK', 'TERT', 'TET1', 'TET2', 'TGFBR1', 'TGFBR2', 'TMEM127', 'TMPRSS2', 'TNFAIP3', 'TNFRSF14', 'TOP1', 'TP53', 'TP53BP1', 'TP63', 'TRAF2', 'TRAF7', 'TSC1', 'TSC2', 'TSHR', 'U2AF1', 'UPF1', 'VEGFA', 'VHL', 'VTCN1', 'WHSC1', 'WHSC1L1', 'WT1', 'WWTR1', 'XIAP', 'XPO1', 'XRCC2', 'YAP1', 'YES1', 'ZFHX3', 'ZRSR2'])\n",
    "tumorSuppressors = set(['ERRFI1', 'ASXL2', 'PMAIP1', 'ACTG1', 'SUFU', 'FBXO11', 'MEN1', 'FAM58A', 'B2M', 'RB1', 'DUSP22', 'SESN1', 'GPS2', 'RAD51D', 'SMG1', 'CDC73', 'MAP3K1', 'SMARCB1', 'INPP4B', 'PARK2', 'SMAD4', 'CBFB', 'CDH1', 'PPP6C', 'SETDB1', 'SETDB2', 'NF2', 'CDKN2B', 'CDKN2C', 'CDKN2A', 'DDX3X', 'PIK3R1', 'BARD1', 'PDS5B', 'KLF4', 'SPRED1', 'VHL', 'SMAD2', 'PMS1', 'PMS2', 'SETD2', 'GATA3', 'TBL1XR1', 'MUTYH', 'SOCS1', 'FAM175A', 'ROBO1', 'ARID1B', 'ARID1A', 'TCF7L2', 'STK11', 'FOXA1', 'PTEN', 'FAT1', 'FAS', 'CYLD', 'MAX', 'SH2D1A', 'APC', 'NTHL1', 'CTCF', 'KDM5C', 'KMT2C', 'ZFHX3', 'FOXP1', 'PIGA', 'CDKN1B', 'CDKN1A', 'FUBP1', 'MSH2', 'ID3', 'TNFRSF14', 'TRAF3', 'EP400', 'BRIP1', 'ARID4A', 'ARID4B', 'XRCC2', 'DAXX', 'SDHAF2', 'ASXL1', 'AMER1', 'RASA1', 'EGR1', 'MST1', 'SOX17', 'RUNX1', 'PIK3R3', 'NCOR1', 'NF1', 'JAK1', 'PTPRD', 'CHEK2', 'CHEK1', 'SMC1A', 'TMEM127', 'STAG1', 'RAD51', 'TCF3', 'STAG2', 'ARID2', 'RAD50', 'RNF43', 'PARP1', 'BLM', 'CUX1', 'RECQL', 'RAD21', 'PTPN2', 'PTPN1', 'SLX4', 'INHA', 'PAX5', 'IRF1', 'TP53', 'HLA-A', 'IRF8', 'CBL', 'TOP1', 'SHQ1', 'PRDM1', 'NSD1', 'ATXN2', 'CREBBP', 'HDAC4', 'SESN2', 'PPP2R1A', 'EPHA7', 'ATM', 'EPHA3', 'POT1', 'SMAD3', 'MOB3B', 'TBX3', 'POLE', 'ATR', 'FANCD2', 'FH', 'BCORL1', 'SOX9', 'IKZF3', 'TSC1', 'TP63', 'MRE11A', 'SDHC', 'BTG1', 'POLD1', 'CIITA', 'SMC3', 'SAMHD1', 'RTEL1', 'ECT2L', 'PIK3R2', 'CRBN', 'FANCC', 'NBN', 'FANCA', 'HLA-B', 'RECQL4', 'DUSP4', 'ERCC2', 'FBXW7', 'TGFBR2', 'TGFBR1', 'MSH3', 'RBM15', 'TET1', 'TET3', 'SESN3', 'MGA', 'LTB', 'FOXL2', 'SH2B3', 'BCOR', 'HIST1H1D', 'ATRX', 'EP300', 'RAD51C', 'RAD51B', 'HIST1H1B', 'TNFAIP3', 'DICER1', 'ARID5B', 'LATS2', 'FOXO1', 'KEAP1', 'EZH2', 'SP140', 'NKX3-1', 'PBRM1', 'PALB2', 'CIC', 'BRCA1', 'DTX1', 'FLCN', 'SPEN', 'CD58', 'ERCC3', 'ERCC4', 'MSH6', 'BCL11B', 'BMPR1A', 'ERF', 'BRCA2', 'NOTCH2', 'EED', 'MITF', 'ELF3', 'SMARCA4', 'BBC3', 'ANKRD11', 'CEBPA', 'BCL2L11', 'AXIN2', 'AXIN1', 'CDK12', 'ESCO2', 'MLH1', 'SDHB', 'MED12', 'HNF1A', 'RYBP', 'ATP6V1B2', 'DNMT3B', 'KMT2B', 'KMT2A', 'DNMT3A', 'NFKBIA', 'TRAF5', 'KMT2D', 'SPOP', 'RBM10', 'P2RY8', 'TP53BP1', 'TSC2', 'KDM6A', 'EPCAM', 'PHOX2B', 'NPM1', 'BCL10', 'LATS1', 'HOXB13', 'ARID3A', 'PTPRT', 'PTPRS', 'INPPL1', 'NOTCH4', 'TET2', 'NOTCH1', 'CASP8', 'NOTCH3', 'GRIN2A', 'MAP2K4', 'WT1', 'BACH2', 'SDHA', 'BAP1', 'PTCH1', 'SDHD'])\n",
    "\n",
    "endometrialFilteredDf = allImpactMuts[allImpactMuts['Tumor_Sample_Barcode'].isin(set(endometrialDf['Tumor_Sample_Barcode']))]\n",
    "\n",
    "endometrialOncMutations = endometrialFilteredDf[endometrialFilteredDf['oncogenic'].notnull()]\n",
    "endometrialOncMutations['shuffleColumn'] = endometrialOncMutations['Hugo_Symbol']\n",
    "\n",
    "\"\"\"endometrialOncTumorSuppressors = endometrialOncMutations[endometrialOncMutations['Hugo_Symbol'].isin(tumorSuppressors - \n",
    "                                                                                                     set(['ARID1A','BCOR','CTCF',\n",
    "  'FBXW7','MED12','PIK3R1',\n",
    "  'PPP2R1A','PTEN','TP53','ZFHX3']))]\n",
    "  \"\"\"\n",
    "endometrialOncTumorSuppressors = endometrialOncMutations[endometrialOncMutations['Hugo_Symbol'].isin(endometrialUncommonTS)]\n",
    "\n",
    "\n",
    "\n",
    "endometrialOncOncogenes = endometrialOncMutations[~endometrialOncMutations['Hugo_Symbol'].isin(tumorSuppressors | set(['PIK3CA']))]\n",
    "\n",
    "summaryDfHyperAll = mutation_permutation_utils.do_n_iterations_of_shuffling(endometrialOncMutations, 100, impact468Genes)\n",
    "summaryDfHyperTumorSuppressor = mutation_permutation_utils.do_n_iterations_of_shuffling(endometrialOncTumorSuppressors, 100, impact468Genes)\n",
    "summaryDfHyperOncogene = mutation_permutation_utils.do_n_iterations_of_shuffling(endometrialOncOncogenes, 100, impact468Genes)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda2/lib/python2.7/site-packages/ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    }
   ],
   "source": [
    "dObserved = calculate_observed_multiplet_counts(endometrialOncMutations, summaryDfHyperAll)\n",
    "dObservedTS = calculate_observed_multiplet_counts(endometrialOncTumorSuppressors, summaryDfHyperTumorSuppressor)\n",
    "dObservedO = calculate_observed_multiplet_counts(endometrialOncOncogenes, summaryDfHyperOncogene)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{1: 100, 2: 0, 3: 96, 4: 100, 5: 100, 6: 99, 7: 79, 8: 44, 9: 18, 10: 9, 11: 4, 12: 1, 13: 2}\n",
      "{1: 91, 2: 33, 3: 1, 4: 10}\n",
      "{1: 73, 2: 28, 3: 17, 4: 11}\n"
     ]
    }
   ],
   "source": [
    "print mutation_permutation_utils.calculate_n_tests_that_differed(summaryDfHyperAll, dObserved)\n",
    "print mutation_permutation_utils.calculate_n_tests_that_differed(summaryDfHyperTumorSuppressor, dObservedTS)\n",
    "print mutation_permutation_utils.calculate_n_tests_that_differed(summaryDfHyperOncogene, dObservedO)\n",
    "\n",
    "#TODO Graph and explainfi\n",
    "\n",
    "#print summaryDfHyper.columns.values\n",
    "#print '________'\n",
    "#print summaryDfHyper[summaryDfHyper[2] > dObserved[2]]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "endometrialTumorSuppresorPercentiles = enumerate_genes_by_ranking(tumorSuppresors, endoNormalRanking)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}