count_mutations_in_related_and_unrelated_genes_hypermutators.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import argparse\n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import scipy.stats as stats\n",
    "\n",
    "from collections import Counter\n",
    "\n",
    "pathPrefix = '/Users/friedman/Desktop/mnt'\n",
    "\n",
    "sys.path.append(pathPrefix + '/ifs/work/taylorlab/friedman/myUtils')\n",
    "import analysis_utils \n",
    "import mutationSigUtils \n",
    "import maf_analysis_utils\n",
    "import mutation_modeling_util"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_eos_muts_obs_expected(maf, eosGenes):\n",
    "    \n",
    "    #impact341Genes = set(['ABL1', 'AKT1', 'AKT2', 'AKT3', 'ALK', 'ALOX12B', 'APC', 'AR', 'ARAF', 'ARID1A', 'ARID1B', 'ARID2', 'ARID5B', 'ASXL1', 'ASXL2', 'ATM', 'ATR', 'ATRX', 'AURKA', 'AURKB', 'AXIN1', 'AXIN2', 'AXL', 'B2M', 'BAP1', 'BARD1', 'BBC3', 'BCL2', 'BCL2L1', 'BCL2L11', 'BCL6', 'BCOR', 'BLM', 'BMPR1A', 'BRAF', 'BRCA1', 'BRCA2', 'BRD4', 'BRIP1', 'BTK', 'CARD11', 'CASP8', 'CBFB', 'CBL', 'CCND1', 'CCND2', 'CCND3', 'CCNE1', 'CD274', 'CD276', 'CD79B', 'CDC73', 'CDH1', 'CDK12', 'CDK4', 'CDK6', 'CDK8', 'CDKN1A', 'CDKN1B', 'CDKN2A', 'CDKN2B', 'CDKN2C', 'CHEK1', 'CHEK2', 'CIC', 'CREBBP', 'CRKL', 'CRLF2', 'CSF1R', 'CTCF', 'CTLA4', 'CTNNB1', 'CUL3', 'DAXX', 'DCUN1D1', 'DDR2', 'DICER1', 'DIS3', 'DNMT1', 'DNMT3A', 'DNMT3B', 'DOT1L', 'E2F3', 'EED', 'EGFL7', 'EGFR', 'EIF1AX', 'EP300', 'EPCAM', 'EPHA3', 'EPHA5', 'EPHB1', 'ERBB2', 'ERBB3', 'ERBB4', 'ERCC2', 'ERCC3', 'ERCC4', 'ERCC5', 'ERG', 'ESR1', 'ETV1', 'ETV6', 'EZH2', 'FAM123B', 'FAM175A', 'FAM46C', 'FANCA', 'FANCC', 'FAT1', 'FBXW7', 'FGF19', 'FGF3', 'FGF4', 'FGFR1', 'FGFR2', 'FGFR3', 'FGFR4', 'FH', 'FLCN', 'FLT1', 'FLT3', 'FLT4', 'FOXA1', 'FOXL2', 'FOXP1', 'FUBP1', 'GATA1', 'GATA2', 'GATA3', 'GNA11', 'GNAQ', 'GNAS', 'GREM1', 'GRIN2A', 'GSK3B', 'H3F3C', 'HGF', 'HIST1H1C', 'HIST1H2BD', 'HIST1H3B', 'HNF1A', 'HRAS', 'ICOSLG', 'IDH1', 'IDH2', 'IFNGR1', 'IGF1', 'IGF1R', 'IGF2', 'IKBKE', 'IKZF1', 'IL10', 'IL7R', 'INPP4A', 'INPP4B', 'INSR', 'IRF4', 'IRS1', 'IRS2', 'JAK1', 'JAK2', 'JAK3', 'JUN', 'KDM5A', 'KDM5C', 'KDM6A', 'KDR', 'KEAP1', 'KIT', 'KLF4', 'KRAS', 'LATS1', 'LATS2', 'LMO1', 'MAP2K1', 'MAP2K2', 'MAP2K4', 'MAP3K1', 'MAP3K13', 'MAPK1', 'MAX', 'MCL1', 'MDC1', 'MDM2', 'MDM4', 'MED12', 'MEF2B', 'MEN1', 'MET', 'MITF', 'MLH1', 'MLL', 'MLL2', 'MLL3', 'MPL', 'MRE11A', 'MSH2', 'MSH6', 'MTOR', 'MUTYH', 'MYC', 'MYCL1', 'MYCN', 'MYD88', 'MYOD1', 'NBN', 'NCOR1', 'NF1', 'NF2', 'NFE2L2', 'NKX2-1', 'NKX3-1', 'NOTCH1', 'NOTCH2', 'NOTCH3', 'NOTCH4', 'NPM1', 'NRAS', 'NSD1', 'NTRK1', 'NTRK2', 'NTRK3', 'PAK1', 'PAK7', 'PALB2', 'PARK2', 'PARP1', 'PAX5', 'PBRM1', 'PDCD1', 'PDGFRA', 'PDGFRB', 'PDPK1', 'PHOX2B', 'PIK3C2G', 'PIK3C3', 'PIK3CA', 'PIK3CB', 'PIK3CD', 'PIK3CG', 'PIK3R1', 'PIK3R2', 'PIK3R3', 'PIM1', 'PLK2', 'PMAIP1', 'PMS1', 'PMS2', 'PNRC1', 'POLE', 'PPP2R1A', 'PRDM1', 'PRKAR1A', 'PTCH1', 'PTEN', 'PTPN11', 'PTPRD', 'PTPRS', 'PTPRT', 'RAC1', 'RAD50', 'RAD51', 'RAD51B', 'RAD51C', 'RAD51D', 'RAD52', 'RAD54L', 'RAF1', 'RARA', 'RASA1', 'RB1', 'RBM10', 'RECQL4', 'REL', 'RET', 'RFWD2', 'RHOA', 'RICTOR', 'RIT1', 'RNF43', 'ROS1', 'RPS6KA4', 'RPS6KB2', 'RPTOR', 'RUNX1', 'RYBP', 'SDHA', 'SDHAF2', 'SDHB', 'SDHC', 'SDHD', 'SETD2', 'SF3B1', 'SH2D1A', 'SHQ1', 'SMAD2', 'SMAD3', 'SMAD4', 'SMARCA4', 'SMARCB1', 'SMARCD1', 'SMO', 'SOCS1', 'SOX17', 'SOX2', 'SOX9', 'SPEN', 'SPOP', 'SRC', 'STAG2', 'STK11', 'STK40', 'SUFU', 'SUZ12', 'SYK', 'TBX3', 'TERT', 'TET1', 'TET2', 'TGFBR1', 'TGFBR2', 'TMEM127', 'TMPRSS2', 'TNFAIP3', 'TNFRSF14', 'TOP1', 'TP53', 'TP63', 'TRAF7', 'TSC1', 'TSC2', 'TSHR', 'U2AF1', 'VHL', 'VTCN1', 'WT1', 'XIAP', 'XPO1', 'YAP1', 'YES1'])\n",
    "    listOfDicts = []\n",
    "    for case in set(maf['Tumor_Sample_Barcode']):\n",
    "        #caseMaf341Snps = maf[(maf['Tumor_Sample_Barcode'] == case) \n",
    "        #    & (maf['Hugo_Symbol'].isin(impact341Genes)) & (maf['Variant_Type'] == 'SNP')]\n",
    "        caseMaf = maf[maf['Tumor_Sample_Barcode'] == case]\n",
    "        nmut = caseMaf.shape[0]\n",
    "        nmutMb = maf_analysis_utils.get_nmut_mb_from_impact_id(case, nmut)\n",
    "        #nmut341 = caseMaf341Snps.shape[0]\n",
    "        oncogenicCaseMaf = caseMaf[caseMaf['oncogenic'].notnull()]\n",
    "        \n",
    "        nObsOncMuts = len(set(oncogenicCaseMaf['Hugo_Symbol']))\n",
    "        \n",
    "        oncoCaseMafEos = oncogenicCaseMaf[oncogenicCaseMaf['Hugo_Symbol'].isin(eosGenes)]\n",
    "        oncoCaseMafNotEos = oncogenicCaseMaf[~oncogenicCaseMaf['Hugo_Symbol'].isin(eosGenes)]\n",
    "        \n",
    "        nOncoEosMuts, nOncoEosGenes = oncoCaseMafEos.shape[0], len(set(oncoCaseMafEos['Hugo_Symbol']))\n",
    "        nOncoNotEosMuts, nOncoNotEosGenes = oncoCaseMafNotEos.shape[0], len(set(oncoCaseMafNotEos['Hugo_Symbol']))\n",
    "        \n",
    "        listOfDicts.append({'nmut_Mb': nmutMb, 'nOncoEosMuts': nOncoEosMuts, 'nOncoEosGenes': nOncoEosGenes,\n",
    "                           'nOncoNotEosMuts': nOncoNotEosMuts, 'nOncoNotEosGenes': nOncoNotEosGenes})\n",
    "    \n",
    "    df = pd.DataFrame(listOfDicts)\n",
    "    return df\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_oncogenic_mut_susceptibility_of_genes_by_signature(oncogenicSDict, suffix='_hotspot_rate'):\n",
    "    listOfDicts = []\n",
    "    sigNames = ['Signature.' + str(i) for i in range(1,31)]\n",
    "    for i in range(1,31):\n",
    "        curSig = 'Signature.' + str(i)\n",
    "        d = {}\n",
    "        for s in sigNames:\n",
    "            d[s] = 0\n",
    "        d[curSig] = 1\n",
    "        #PRETEND we got a case with 100% signature i on the decomposition\n",
    "        quadNucFractions = mutation_modeling_util.get_quadnuc_fracs_given_decomposition(d, spectraPath = pathPrefix + '/ifs/work/taylorlab/friedman/noahFirstProject/signature_sig_copy/mutation-signatures/Stratton_signatures30.txt')\n",
    "        #v = mutation_modeling_util.get_expected_oncogenic_val_given_quadnuc_fractions(quadNucFractions, oncogenicSDict, 'IMPACT_468')\n",
    "        #ALERT NOAH I CHANGED THIS HERE\n",
    "        v = mutation_modeling_util.get_expected_oncogenic_val_given_quadnuc_fractions_v2(quadNucFractions, oncogenicSDict, suffix)\n",
    "\n",
    "        listOfDicts.append({'Signature_Name': curSig, 'ExpectedFracOfMutsOncogenic': v})\n",
    "    return pd.DataFrame(listOfDicts)\n",
    "\n",
    "def quantify_quadnuc_oncogenic_susceptibility_per_mutation(simMafData, relatedGenes=None):\n",
    "    allBases = ['A', 'C', 'G', 'T']\n",
    "    changes = ['CA', 'CG', 'CT', 'TA', 'TC', 'TG'] #format: 'CA' means a change from C>A\n",
    "    allQuadNucs = [firstBase + change + lastBase for firstBase in allBases for change in changes for lastBase in allBases] #enumerate all 96 quadnucs for signatures\n",
    "   \n",
    "    d = {}\n",
    "    \n",
    "    for quadNuc in allQuadNucs:\n",
    "        nPossibleMuts = sum(simMafData[quadNuc]) - sum(simMafData[quadNuc + '_silent'])\n",
    "        nPossibleHotspotMuts = sum(simMafData[quadNuc + '_oncogenic'])\n",
    "        d[quadNuc + '_oncogenic_rate'] = (1.0*nPossibleHotspotMuts)/(1.0*nPossibleMuts)\n",
    "        \n",
    "    return d\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "eosDict = {'Endometrial Cancer': ['CASP8', 'MSH6', 'JAK1', 'INPPL1', 'MSH2', 'MAP3K1', 'ARID5B', 'RNF43', 'APC', 'RB1',\n",
    "                      'CCND1', 'NFE2L2', 'RASA1', 'ARID1A', 'PTEN', 'PIK3R1', 'ATM', 'EPHA3', 'B2M', 'TP53',\n",
    "                      'BCOR', 'CTNNB1', 'KRAS', 'FGFR2', 'PPP2R1A', 'POLE', 'NF1', 'FBXW7', 'XPO1', 'ERBB2', 'ERBB3',\n",
    "                      'PTPRD', 'CTCF', 'PIK3CA'],\n",
    "\n",
    "            'Colorectal Cancer': ['CASP8', 'MSH2', 'HLA-A', 'HLA-B', 'PIK3R1', 'B2M', 'KRAS', 'PTEN', 'APC',\n",
    "                     'PIK3CA', 'FBXW7', 'RNF43', 'AMER1', 'TCF7L2', 'SMAD4', 'NRAS', 'BRAF', 'SOX9',\n",
    "                      'TP53', 'CDKN2A', 'SMAD2', 'ARID1A', 'POLE', 'MSH3', 'MLH1', 'MSH6'],\n",
    "\n",
    "            'Glioma': ['IDH1', 'TP53', 'TERT', 'MSH6', 'CIC', 'PIK3CA', 'MSH6',\n",
    "                 'MSH2', 'NF1', 'TSC2', 'PTEN', 'ATRX', 'EGFR', 'PIK3R1', 'RB1']\n",
    "          }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "allImpactMuts = analysis_utils.load_in_df_with_progress(filePath = pathPrefix + '/ifs/work/taylorlab/friedman/hypermutationAnalysisProj/projectDataAndConfigFiles/all_impact_mutations_annotated_cohort.maf', nLinesFile = 275000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "normalEndometrial = analysis_utils.get_ids_by_hypermutant_status(hypermutantIdDir=pathPrefix +'/ifs/work/taylorlab/friedman/hypermutationAnalysisProj/projectDataAndConfigFiles/hypermutationStatusIds', cancerType='Endometrial Cancer', hypermutantStatus = 'Normal')\n",
    "normalColorectal = analysis_utils.get_ids_by_hypermutant_status(hypermutantIdDir=pathPrefix +'/ifs/work/taylorlab/friedman/hypermutationAnalysisProj/projectDataAndConfigFiles/hypermutationStatusIds', cancerType='Colorectal Cancer', hypermutantStatus = 'Normal')\n",
    "normalGlioma = analysis_utils.get_ids_by_hypermutant_status(hypermutantIdDir=pathPrefix +'/ifs/work/taylorlab/friedman/hypermutationAnalysisProj/projectDataAndConfigFiles/hypermutationStatusIds', cancerType='Glioma', hypermutantStatus = 'Normal')\n",
    "\n",
    "hyperEndometrial = analysis_utils.get_ids_by_hypermutant_status(hypermutantIdDir=pathPrefix +'/ifs/work/taylorlab/friedman/hypermutationAnalysisProj/projectDataAndConfigFiles/hypermutationStatusIds', cancerType='Endometrial Cancer', hypermutantStatus = 'Hypermutated')\n",
    "hyperColorectal = analysis_utils.get_ids_by_hypermutant_status(hypermutantIdDir=pathPrefix +'/ifs/work/taylorlab/friedman/hypermutationAnalysisProj/projectDataAndConfigFiles/hypermutationStatusIds', cancerType='Colorectal Cancer', hypermutantStatus = 'Hypermutated')\n",
    "hyperGlioma = analysis_utils.get_ids_by_hypermutant_status(hypermutantIdDir=pathPrefix +'/ifs/work/taylorlab/friedman/hypermutationAnalysisProj/projectDataAndConfigFiles/hypermutationStatusIds', cancerType='Glioma', hypermutantStatus = 'Hypermutated')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "impactSigs = pd.read_table(pathPrefix + '/ifs/work/taylorlab/friedman/hypermutationAnalysisProj/projectDataAndConfigFiles/signatures_from_unfiltered_maf.txt')\n",
    "impactSigs = mutationSigUtils.merge_signature_columns(impactSigs)\n",
    "impactSigs['dominantSignature'] = impactSigs.apply(lambda row: \n",
    "        mutationSigUtils.get_dominant_signature(row.to_dict(), cols=None, prefix='mean', notEnoughMuts= True), axis=1)\n",
    "\n",
    "casesWithMSISignature = set(impactSigs[(impactSigs['dominantSignature'] == 'mean_MMR')\n",
    "                                      |((impactSigs['Nmut_Mb'] >= 30) & (impactSigs['dominantSignature'].isin(set(['mean_1']))))\n",
    "                                        ]['Tumor_Sample_Barcode'])\n",
    "\n",
    "casesWithPOLESignature = set(impactSigs[(impactSigs['dominantSignature'] == 'mean_10') & (impactSigs['Nmut_Mb'] > 30)]['Tumor_Sample_Barcode'])\n",
    "casesWithTMZSignature = set(impactSigs[(impactSigs['dominantSignature'] == 'mean_11') & (impactSigs['Nmut_Mb'] > 30)]['Tumor_Sample_Barcode'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "endometrialHyperMuts = allImpactMuts[allImpactMuts['Tumor_Sample_Barcode'].isin(hyperEndometrial)]\n",
    "colorectalHyperMuts = allImpactMuts[allImpactMuts['Tumor_Sample_Barcode'].isin(hyperColorectal)]\n",
    "gliomaHyperMuts = allImpactMuts[allImpactMuts['Tumor_Sample_Barcode'].isin(hyperGlioma)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "simOncogenicitySummary = pd.read_table(pathPrefix + '/ifs/work/taylorlab/friedman/hypermutationAnalysisProj/projectDataAndConfigFiles/simulatedMutationSummary.tsv')\n",
    "oncogenicSusceptibilityDict = mutation_modeling_util.calculate_quadnuc_based_oncogenic_susceptibility_dict(simOncogenicitySummary)\n",
    "impact341Genes = set(['ABL1', 'AKT1', 'AKT2', 'AKT3', 'ALK', 'ALOX12B', 'APC', 'AR', 'ARAF', 'ARID1A', 'ARID1B', 'ARID2', 'ARID5B', 'ASXL1', 'ASXL2', 'ATM', 'ATR', 'ATRX', 'AURKA', 'AURKB', 'AXIN1', 'AXIN2', 'AXL', 'B2M', 'BAP1', 'BARD1', 'BBC3', 'BCL2', 'BCL2L1', 'BCL2L11', 'BCL6', 'BCOR', 'BLM', 'BMPR1A', 'BRAF', 'BRCA1', 'BRCA2', 'BRD4', 'BRIP1', 'BTK', 'CARD11', 'CASP8', 'CBFB', 'CBL', 'CCND1', 'CCND2', 'CCND3', 'CCNE1', 'CD274', 'CD276', 'CD79B', 'CDC73', 'CDH1', 'CDK12', 'CDK4', 'CDK6', 'CDK8', 'CDKN1A', 'CDKN1B', 'CDKN2A', 'CDKN2B', 'CDKN2C', 'CHEK1', 'CHEK2', 'CIC', 'CREBBP', 'CRKL', 'CRLF2', 'CSF1R', 'CTCF', 'CTLA4', 'CTNNB1', 'CUL3', 'DAXX', 'DCUN1D1', 'DDR2', 'DICER1', 'DIS3', 'DNMT1', 'DNMT3A', 'DNMT3B', 'DOT1L', 'E2F3', 'EED', 'EGFL7', 'EGFR', 'EIF1AX', 'EP300', 'EPCAM', 'EPHA3', 'EPHA5', 'EPHB1', 'ERBB2', 'ERBB3', 'ERBB4', 'ERCC2', 'ERCC3', 'ERCC4', 'ERCC5', 'ERG', 'ESR1', 'ETV1', 'ETV6', 'EZH2', 'FAM123B', 'FAM175A', 'FAM46C', 'FANCA', 'FANCC', 'FAT1', 'FBXW7', 'FGF19', 'FGF3', 'FGF4', 'FGFR1', 'FGFR2', 'FGFR3', 'FGFR4', 'FH', 'FLCN', 'FLT1', 'FLT3', 'FLT4', 'FOXA1', 'FOXL2', 'FOXP1', 'FUBP1', 'GATA1', 'GATA2', 'GATA3', 'GNA11', 'GNAQ', 'GNAS', 'GREM1', 'GRIN2A', 'GSK3B', 'H3F3C', 'HGF', 'HIST1H1C', 'HIST1H2BD', 'HIST1H3B', 'HNF1A', 'HRAS', 'ICOSLG', 'IDH1', 'IDH2', 'IFNGR1', 'IGF1', 'IGF1R', 'IGF2', 'IKBKE', 'IKZF1', 'IL10', 'IL7R', 'INPP4A', 'INPP4B', 'INSR', 'IRF4', 'IRS1', 'IRS2', 'JAK1', 'JAK2', 'JAK3', 'JUN', 'KDM5A', 'KDM5C', 'KDM6A', 'KDR', 'KEAP1', 'KIT', 'KLF4', 'KRAS', 'LATS1', 'LATS2', 'LMO1', 'MAP2K1', 'MAP2K2', 'MAP2K4', 'MAP3K1', 'MAP3K13', 'MAPK1', 'MAX', 'MCL1', 'MDC1', 'MDM2', 'MDM4', 'MED12', 'MEF2B', 'MEN1', 'MET', 'MITF', 'MLH1', 'MLL', 'MLL2', 'MLL3', 'MPL', 'MRE11A', 'MSH2', 'MSH6', 'MTOR', 'MUTYH', 'MYC', 'MYCL1', 'MYCN', 'MYD88', 'MYOD1', 'NBN', 'NCOR1', 'NF1', 'NF2', 'NFE2L2', 'NKX2-1', 'NKX3-1', 'NOTCH1', 'NOTCH2', 'NOTCH3', 'NOTCH4', 'NPM1', 'NRAS', 'NSD1', 'NTRK1', 'NTRK2', 'NTRK3', 'PAK1', 'PAK7', 'PALB2', 'PARK2', 'PARP1', 'PAX5', 'PBRM1', 'PDCD1', 'PDGFRA', 'PDGFRB', 'PDPK1', 'PHOX2B', 'PIK3C2G', 'PIK3C3', 'PIK3CA', 'PIK3CB', 'PIK3CD', 'PIK3CG', 'PIK3R1', 'PIK3R2', 'PIK3R3', 'PIM1', 'PLK2', 'PMAIP1', 'PMS1', 'PMS2', 'PNRC1', 'POLE', 'PPP2R1A', 'PRDM1', 'PRKAR1A', 'PTCH1', 'PTEN', 'PTPN11', 'PTPRD', 'PTPRS', 'PTPRT', 'RAC1', 'RAD50', 'RAD51', 'RAD51B', 'RAD51C', 'RAD51D', 'RAD52', 'RAD54L', 'RAF1', 'RARA', 'RASA1', 'RB1', 'RBM10', 'RECQL4', 'REL', 'RET', 'RFWD2', 'RHOA', 'RICTOR', 'RIT1', 'RNF43', 'ROS1', 'RPS6KA4', 'RPS6KB2', 'RPTOR', 'RUNX1', 'RYBP', 'SDHA', 'SDHAF2', 'SDHB', 'SDHC', 'SDHD', 'SETD2', 'SF3B1', 'SH2D1A', 'SHQ1', 'SMAD2', 'SMAD3', 'SMAD4', 'SMARCA4', 'SMARCB1', 'SMARCD1', 'SMO', 'SOCS1', 'SOX17', 'SOX2', 'SOX9', 'SPEN', 'SPOP', 'SRC', 'STAG2', 'STK11', 'STK40', 'SUFU', 'SUZ12', 'SYK', 'TBX3', 'TERT', 'TET1', 'TET2', 'TGFBR1', 'TGFBR2', 'TMEM127', 'TMPRSS2', 'TNFAIP3', 'TNFRSF14', 'TOP1', 'TP53', 'TP63', 'TRAF7', 'TSC1', 'TSC2', 'TSHR', 'U2AF1', 'VHL', 'VTCN1', 'WT1', 'XIAP', 'XPO1', 'YAP1', 'YES1'])\n",
    "simulatedDataSummary341 = simOncogenicitySummary[simOncogenicitySummary['Hugo_Symbol'].isin(impact341Genes)]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "simOncogenicitySummary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "\n",
    "quadNucOncogenicSusceptibilityDict = quantify_quadnuc_oncogenic_susceptibility_per_mutation(simulatedDataSummary341)\n",
    "signatureOncogenicSusceptibilityDf = calculate_oncogenic_mut_susceptibility_of_genes_by_signature(quadNucOncogenicSusceptibilityDict, suffix='_oncogenic_rate')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "endometrialHyperPole = endometrialHyperMuts[endometrialHyperMuts['Tumor_Sample_Barcode'].isin(casesWithPOLESignature)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "endoSummary = count_eos_muts_obs_expected(endometrialHyperMuts, eosGenesEndometrial)\n",
    "coloSummary = count_eos_muts_obs_expected(colorectalHyperMuts, eosGenesColorectal)\n",
    "gliomaSummary = count_eos_muts_obs_expected(gliomaHyperMuts, eosGenesGlioma)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "endoSummary['Quantile_rank']=pd.qcut(endoSummary['nmut_Mb'],10,labels=False)\n",
    "coloSummary['Quantile_rank']=pd.qcut(coloSummary['nmut_Mb'],10,labels=False)\n",
    "gliomaSummary['Quantile_rank']=pd.qcut(gliomaSummary['nmut_Mb'],10,labels=False)\n",
    "\n",
    "endoSummary['cancerType'] = 'Endometrial Cancer'\n",
    "coloSummary['cancerType'] = 'Colorectal Cancer'\n",
    "gliomaSummary['cancerType'] = 'Glioma'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "combinedDf = pd.concat([endoSummary, coloSummary, gliomaSummary])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "combinedDf['decile'] = combinedDf['Quantile_rank'].apply(lambda x:\n",
    "                                                         str(10*x) + '-' + str(10*(x+1)) + 'th')\n",
    "                                                           \n",
    "                                                           #if x == 0 else '25-50th' if x == 1\n",
    "                                            #else '50-75th' if x == 2 else '75th-100th')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "combinedDf.to_csv('/Users/friedman/Desktop/WORK/dataForLocalPlotting/eosMutsByMutBurden.tsv', index=False, sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Count observed vs expected by EOS/not eos mutations**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "fullCaseGeneExpectedMutRate = pd.read_table(pathPrefix + '/ifs/work/taylorlab/friedman/hypermutationAnalysisProj/mutSimulation/expectedMutationTables/allHypermutatorsExpectedGeneMutInfo.tsv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "allImpactMuts = analysis_utils.load_in_df_with_progress(filePath = pathPrefix + '/ifs/work/taylorlab/friedman/hypermutationAnalysisProj/projectDataAndConfigFiles/all_impact_mutations_annotated_cohort.maf', nLinesFile = 275000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "im341Genes = set(['ABL1', 'AKT1', 'AKT2', 'AKT3', 'ALK', 'ALOX12B', 'APC', 'AR', 'ARAF', 'ARID1A', 'ARID1B', 'ARID2', 'ARID5B', 'ASXL1', 'ASXL2', 'ATM', 'ATR', 'ATRX', 'AURKA', 'AURKB', 'AXIN1', 'AXIN2', 'AXL', 'B2M', 'BAP1', 'BARD1', 'BBC3', 'BCL2', 'BCL2L1', 'BCL2L11', 'BCL6', 'BCOR', 'BLM', 'BMPR1A', 'BRAF', 'BRCA1', 'BRCA2', 'BRD4', 'BRIP1', 'BTK', 'CARD11', 'CASP8', 'CBFB', 'CBL', 'CCND1', 'CCND2', 'CCND3', 'CCNE1', 'CD274', 'CD276', 'CD79B', 'CDC73', 'CDH1', 'CDK12', 'CDK4', 'CDK6', 'CDK8', 'CDKN1A', 'CDKN1B', 'CDKN2A', 'CDKN2B', 'CDKN2C', 'CHEK1', 'CHEK2', 'CIC', 'CREBBP', 'CRKL', 'CRLF2', 'CSF1R', 'CTCF', 'CTLA4', 'CTNNB1', 'CUL3', 'DAXX', 'DCUN1D1', 'DDR2', 'DICER1', 'DIS3', 'DNMT1', 'DNMT3A', 'DNMT3B', 'DOT1L', 'E2F3', 'EED', 'EGFL7', 'EGFR', 'EIF1AX', 'EP300', 'EPCAM', 'EPHA3', 'EPHA5', 'EPHB1', 'ERBB2', 'ERBB3', 'ERBB4', 'ERCC2', 'ERCC3', 'ERCC4', 'ERCC5', 'ERG', 'ESR1', 'ETV1', 'ETV6', 'EZH2', 'FAM123B', 'FAM175A', 'FAM46C', 'FANCA', 'FANCC', 'FAT1', 'FBXW7', 'FGF19', 'FGF3', 'FGF4', 'FGFR1', 'FGFR2', 'FGFR3', 'FGFR4', 'FH', 'FLCN', 'FLT1', 'FLT3', 'FLT4', 'FOXA1', 'FOXL2', 'FOXP1', 'FUBP1', 'GATA1', 'GATA2', 'GATA3', 'GNA11', 'GNAQ', 'GNAS', 'GREM1', 'GRIN2A', 'GSK3B', 'H3F3C', 'HGF', 'HIST1H1C', 'HIST1H2BD', 'HIST1H3B', 'HNF1A', 'HRAS', 'ICOSLG', 'IDH1', 'IDH2', 'IFNGR1', 'IGF1', 'IGF1R', 'IGF2', 'IKBKE', 'IKZF1', 'IL10', 'IL7R', 'INPP4A', 'INPP4B', 'INSR', 'IRF4', 'IRS1', 'IRS2', 'JAK1', 'JAK2', 'JAK3', 'JUN', 'KDM5A', 'KDM5C', 'KDM6A', 'KDR', 'KEAP1', 'KIT', 'KLF4', 'KRAS', 'LATS1', 'LATS2', 'LMO1', 'MAP2K1', 'MAP2K2', 'MAP2K4', 'MAP3K1', 'MAP3K13', 'MAPK1', 'MAX', 'MCL1', 'MDC1', 'MDM2', 'MDM4', 'MED12', 'MEF2B', 'MEN1', 'MET', 'MITF', 'MLH1', 'MLL', 'MLL2', 'MLL3', 'MPL', 'MRE11A', 'MSH2', 'MSH6', 'MTOR', 'MUTYH', 'MYC', 'MYCL1', 'MYCN', 'MYD88', 'MYOD1', 'NBN', 'NCOR1', 'NF1', 'NF2', 'NFE2L2', 'NKX2-1', 'NKX3-1', 'NOTCH1', 'NOTCH2', 'NOTCH3', 'NOTCH4', 'NPM1', 'NRAS', 'NSD1', 'NTRK1', 'NTRK2', 'NTRK3', 'PAK1', 'PAK7', 'PALB2', 'PARK2', 'PARP1', 'PAX5', 'PBRM1', 'PDCD1', 'PDGFRA', 'PDGFRB', 'PDPK1', 'PHOX2B', 'PIK3C2G', 'PIK3C3', 'PIK3CA', 'PIK3CB', 'PIK3CD', 'PIK3CG', 'PIK3R1', 'PIK3R2', 'PIK3R3', 'PIM1', 'PLK2', 'PMAIP1', 'PMS1', 'PMS2', 'PNRC1', 'POLE', 'PPP2R1A', 'PRDM1', 'PRKAR1A', 'PTCH1', 'PTEN', 'PTPN11', 'PTPRD', 'PTPRS', 'PTPRT', 'RAC1', 'RAD50', 'RAD51', 'RAD51B', 'RAD51C', 'RAD51D', 'RAD52', 'RAD54L', 'RAF1', 'RARA', 'RASA1', 'RB1', 'RBM10', 'RECQL4', 'REL', 'RET', 'RFWD2', 'RHOA', 'RICTOR', 'RIT1', 'RNF43', 'ROS1', 'RPS6KA4', 'RPS6KB2', 'RPTOR', 'RUNX1', 'RYBP', 'SDHA', 'SDHAF2', 'SDHB', 'SDHC', 'SDHD', 'SETD2', 'SF3B1', 'SH2D1A', 'SHQ1', 'SMAD2', 'SMAD3', 'SMAD4', 'SMARCA4', 'SMARCB1', 'SMARCD1', 'SMO', 'SOCS1', 'SOX17', 'SOX2', 'SOX9', 'SPEN', 'SPOP', 'SRC', 'STAG2', 'STK11', 'STK40', 'SUFU', 'SUZ12', 'SYK', 'TBX3', 'TERT', 'TET1', 'TET2', 'TGFBR1', 'TGFBR2', 'TMEM127', 'TMPRSS2', 'TNFAIP3', 'TNFRSF14', 'TOP1', 'TP53', 'TP63', 'TRAF7', 'TSC1', 'TSC2', 'TSHR', 'U2AF1', 'VHL', 'VTCN1', 'WT1', 'XIAP', 'XPO1', 'YAP1', 'YES1'])\n",
    "allIm341Muts = allImpactMuts[allImpactMuts['Hugo_Symbol'].isin(im341Genes)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "hyperEndometrial = analysis_utils.get_ids_by_hypermutant_status(hypermutantIdDir=pathPrefix + '/ifs/work/taylorlab/friedman/hypermutationAnalysisProj/projectDataAndConfigFiles/hypermutationStatusIds', cancerType='Endometrial Cancer', hypermutantStatus = 'Hypermutated')\n",
    "hyperColorectal = analysis_utils.get_ids_by_hypermutant_status(hypermutantIdDir=pathPrefix +'/ifs/work/taylorlab/friedman/hypermutationAnalysisProj/projectDataAndConfigFiles/hypermutationStatusIds', cancerType='Colorectal Cancer', hypermutantStatus = 'Hypermutated')\n",
    "hyperGlioma = analysis_utils.get_ids_by_hypermutant_status(hypermutantIdDir=pathPrefix +'/ifs/work/taylorlab/friedman/hypermutationAnalysisProj/projectDataAndConfigFiles/hypermutationStatusIds', cancerType='Glioma', hypermutantStatus = 'Hypermutated')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#map cases to cancer type\n",
    "cancerTypeDict ={}\n",
    "for case in hyperEndometrial:\n",
    "    cancerTypeDict[case] = 'Endometrial Cancer'\n",
    "for case in hyperColorectal:\n",
    "    cancerTypeDict[case] = 'Colorectal Cancer'\n",
    "for case in hyperGlioma:\n",
    "    cancerTypeDict[case] = 'Glioma'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "50\n",
      "100\n",
      "150\n",
      "200\n",
      "250\n",
      "300\n",
      "350\n",
      "400\n",
      "450\n",
      "500\n",
      "550\n"
     ]
    }
   ],
   "source": [
    "mode = 'hotspot'\n",
    "listOfDicts = []\n",
    "cntr = 0\n",
    "for case, cancerType in cancerTypeDict.items():\n",
    "    \n",
    "    if cntr%50 == 0: print cntr \n",
    "    cntr += 1\n",
    "    #HERE we only look at snps\n",
    "    caseMaf = allIm341Muts[(allIm341Muts['Tumor_Sample_Barcode'] == case) & (allIm341Muts['Variant_Type'] == 'SNP')]\n",
    "    nmutIm341 = caseMaf.shape[0]\n",
    "    \n",
    "    eosGenes = eosDict[cancerType]\n",
    "    notEosGenes = im341Genes - set(eosGenes)\n",
    "    eosChance = sum(fullCaseGeneExpectedMutRate[(\n",
    "        fullCaseGeneExpectedMutRate['case'] == case) & (fullCaseGeneExpectedMutRate['gene'].isin(eosGenes))][mode+'Chance'])\n",
    "    notEosChance = sum(fullCaseGeneExpectedMutRate[(\n",
    "        fullCaseGeneExpectedMutRate['case'] == case) & (~fullCaseGeneExpectedMutRate['gene'].isin(eosGenes))][mode+'Chance'])\n",
    "\n",
    "    caseMafSelected = None\n",
    "    if mode == 'hotspot':\n",
    "        caseMafSelected = caseMaf[caseMaf['is-a-hotspot'] == 'Y']\n",
    "    if mode == 'oncogenic':\n",
    "        caseMafSelected = caseMaf[caseMaf['oncogenic'].notnull()]\n",
    "    if mode == 'truncating':\n",
    "        caseMafSelected = caseMaf[caseMaf['Consequence'] == 'stop_gained']\n",
    "    nMutFocusEos = caseMafSelected[caseMafSelected['Hugo_Symbol'].isin(eosGenes)].shape[0]\n",
    "    nMutFocusNotEos = caseMafSelected[~caseMafSelected['Hugo_Symbol'].isin(eosGenes)].shape[0]\n",
    "    listOfDicts.append({'Tumor_Sample_Barcode': case, 'class': 'EOS', 'cancerType':cancerType,\n",
    "                        'expected': nmutIm341*eosChance, 'observed': nMutFocusEos})\n",
    "    \n",
    "    listOfDicts.append({'Tumor_Sample_Barcode': case, 'class': 'not-EOS', 'cancerType':cancerType,\n",
    "                        'expected': nmutIm341*notEosChance, 'observed': nMutFocusNotEos})\n",
    "                        \n",
    "df = pd.DataFrame(listOfDicts)\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['sigClass'] = df['Tumor_Sample_Barcode'].apply(lambda x: 'POLE' if x in casesWithPOLESignature\n",
    "                                                 else 'MMR' if x in casesWithMSISignature\n",
    "                                                 else 'TMZ' if x in casesWithTMZSignature\n",
    "                                                 else None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('/Users/friedman/Desktop/WORK/dataForLocalPlotting/obsExpectedByGeneClass_hotspot.tsv',index=False, sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Count observed and expected in Oncogenes**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "fullCaseGeneExpectedMutRate = pd.read_table(pathPrefix + '/ifs/work/taylorlab/friedman/hypermutationAnalysisProj/mutSimulation/expectedMutationTables/allHypermutatorsExpectedGeneMutInfo.tsv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "allImpactMuts = analysis_utils.load_in_df_with_progress(filePath = pathPrefix + '/ifs/work/taylorlab/friedman/hypermutationAnalysisProj/projectDataAndConfigFiles/all_impact_mutations_annotated_cohort.maf', nLinesFile = 275000)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "impactSigs = pd.read_table(pathPrefix + '/ifs/work/taylorlab/friedman/hypermutationAnalysisProj/projectDataAndConfigFiles/signatures_from_unfiltered_maf.txt')\n",
    "impactSigs = mutationSigUtils.merge_signature_columns(impactSigs)\n",
    "impactSigs['dominantSignature'] = impactSigs.apply(lambda row: \n",
    "        mutationSigUtils.get_dominant_signature(row.to_dict(), cols=None, prefix='mean', notEnoughMuts= True), axis=1)\n",
    "\n",
    "casesWithPOLESignature = set(impactSigs[(impactSigs['dominantSignature'] == 'mean_10') & (impactSigs['Nmut_Mb'] > 30)]['Tumor_Sample_Barcode'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "poleCaseExpectation = fullCaseGeneExpectedMutRate[fullCaseGeneExpectedMutRate['case'].isin(casesWithPOLESignature)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "impact341genes = set(['ABL1', 'AKT1', 'AKT2', 'AKT3', 'ALK', 'ALOX12B', 'APC', 'AR', 'ARAF', 'ARID1A', 'ARID1B', 'ARID2', 'ARID5B', 'ASXL1', 'ASXL2', 'ATM', 'ATR', 'ATRX', 'AURKA', 'AURKB', 'AXIN1', 'AXIN2', 'AXL', 'B2M', 'BAP1', 'BARD1', 'BBC3', 'BCL2', 'BCL2L1', 'BCL2L11', 'BCL6', 'BCOR', 'BLM', 'BMPR1A', 'BRAF', 'BRCA1', 'BRCA2', 'BRD4', 'BRIP1', 'BTK', 'CARD11', 'CASP8', 'CBFB', 'CBL', 'CCND1', 'CCND2', 'CCND3', 'CCNE1', 'CD274', 'CD276', 'CD79B', 'CDC73', 'CDH1', 'CDK12', 'CDK4', 'CDK6', 'CDK8', 'CDKN1A', 'CDKN1B', 'CDKN2A', 'CDKN2B', 'CDKN2C', 'CHEK1', 'CHEK2', 'CIC', 'CREBBP', 'CRKL', 'CRLF2', 'CSF1R', 'CTCF', 'CTLA4', 'CTNNB1', 'CUL3', 'DAXX', 'DCUN1D1', 'DDR2', 'DICER1', 'DIS3', 'DNMT1', 'DNMT3A', 'DNMT3B', 'DOT1L', 'E2F3', 'EED', 'EGFL7', 'EGFR', 'EIF1AX', 'EP300', 'EPCAM', 'EPHA3', 'EPHA5', 'EPHB1', 'ERBB2', 'ERBB3', 'ERBB4', 'ERCC2', 'ERCC3', 'ERCC4', 'ERCC5', 'ERG', 'ESR1', 'ETV1', 'ETV6', 'EZH2', 'FAM123B', 'FAM175A', 'FAM46C', 'FANCA', 'FANCC', 'FAT1', 'FBXW7', 'FGF19', 'FGF3', 'FGF4', 'FGFR1', 'FGFR2', 'FGFR3', 'FGFR4', 'FH', 'FLCN', 'FLT1', 'FLT3', 'FLT4', 'FOXA1', 'FOXL2', 'FOXP1', 'FUBP1', 'GATA1', 'GATA2', 'GATA3', 'GNA11', 'GNAQ', 'GNAS', 'GREM1', 'GRIN2A', 'GSK3B', 'H3F3C', 'HGF', 'HIST1H1C', 'HIST1H2BD', 'HIST1H3B', 'HNF1A', 'HRAS', 'ICOSLG', 'IDH1', 'IDH2', 'IFNGR1', 'IGF1', 'IGF1R', 'IGF2', 'IKBKE', 'IKZF1', 'IL10', 'IL7R', 'INPP4A', 'INPP4B', 'INSR', 'IRF4', 'IRS1', 'IRS2', 'JAK1', 'JAK2', 'JAK3', 'JUN', 'KDM5A', 'KDM5C', 'KDM6A', 'KDR', 'KEAP1', 'KIT', 'KLF4', 'KRAS', 'LATS1', 'LATS2', 'LMO1', 'MAP2K1', 'MAP2K2', 'MAP2K4', 'MAP3K1', 'MAP3K13', 'MAPK1', 'MAX', 'MCL1', 'MDC1', 'MDM2', 'MDM4', 'MED12', 'MEF2B', 'MEN1', 'MET', 'MITF', 'MLH1', 'MLL', 'MLL2', 'MLL3', 'MPL', 'MRE11A', 'MSH2', 'MSH6', 'MTOR', 'MUTYH', 'MYC', 'MYCL1', 'MYCN', 'MYD88', 'MYOD1', 'NBN', 'NCOR1', 'NF1', 'NF2', 'NFE2L2', 'NKX2-1', 'NKX3-1', 'NOTCH1', 'NOTCH2', 'NOTCH3', 'NOTCH4', 'NPM1', 'NRAS', 'NSD1', 'NTRK1', 'NTRK2', 'NTRK3', 'PAK1', 'PAK7', 'PALB2', 'PARK2', 'PARP1', 'PAX5', 'PBRM1', 'PDCD1', 'PDGFRA', 'PDGFRB', 'PDPK1', 'PHOX2B', 'PIK3C2G', 'PIK3C3', 'PIK3CA', 'PIK3CB', 'PIK3CD', 'PIK3CG', 'PIK3R1', 'PIK3R2', 'PIK3R3', 'PIM1', 'PLK2', 'PMAIP1', 'PMS1', 'PMS2', 'PNRC1', 'POLE', 'PPP2R1A', 'PRDM1', 'PRKAR1A', 'PTCH1', 'PTEN', 'PTPN11', 'PTPRD', 'PTPRS', 'PTPRT', 'RAC1', 'RAD50', 'RAD51', 'RAD51B', 'RAD51C', 'RAD51D', 'RAD52', 'RAD54L', 'RAF1', 'RARA', 'RASA1', 'RB1', 'RBM10', 'RECQL4', 'REL', 'RET', 'RFWD2', 'RHOA', 'RICTOR', 'RIT1', 'RNF43', 'ROS1', 'RPS6KA4', 'RPS6KB2', 'RPTOR', 'RUNX1', 'RYBP', 'SDHA', 'SDHAF2', 'SDHB', 'SDHC', 'SDHD', 'SETD2', 'SF3B1', 'SH2D1A', 'SHQ1', 'SMAD2', 'SMAD3', 'SMAD4', 'SMARCA4', 'SMARCB1', 'SMARCD1', 'SMO', 'SOCS1', 'SOX17', 'SOX2', 'SOX9', 'SPEN', 'SPOP', 'SRC', 'STAG2', 'STK11', 'STK40', 'SUFU', 'SUZ12', 'SYK', 'TBX3', 'TERT', 'TET1', 'TET2', 'TGFBR1', 'TGFBR2', 'TMEM127', 'TMPRSS2', 'TNFAIP3', 'TNFRSF14', 'TOP1', 'TP53', 'TP63', 'TRAF7', 'TSC1', 'TSC2', 'TSHR', 'U2AF1', 'VHL', 'VTCN1', 'WT1', 'XIAP', 'XPO1', 'YAP1', 'YES1'])\n",
    "tumorSupressors = set(['ERRFI1', 'ASXL2', 'PMAIP1', 'ACTG1', 'SUFU', 'FBXO11', 'MEN1', 'FAM58A', 'B2M', 'RB1', 'DUSP22', 'SESN1', 'GPS2', 'RAD51D', 'SMG1', 'CDC73', 'MAP3K1', 'SMARCB1', 'INPP4B', 'PARK2', 'SMAD4', 'CBFB', 'CDH1', 'PPP6C', 'SETDB1', 'SETDB2', 'NF2', 'CDKN2B', 'CDKN2C', 'CDKN2A', 'DDX3X', 'PIK3R1', 'BARD1', 'PDS5B', 'KLF4', 'SPRED1', 'VHL', 'SMAD2', 'PMS1', 'PMS2', 'SETD2', 'GATA3', 'TBL1XR1', 'MUTYH', 'SOCS1', 'FAM175A', 'ROBO1', 'ARID1B', 'ARID1A', 'TCF7L2', 'STK11', 'FOXA1', 'PTEN', 'FAT1', 'FAS', 'CYLD', 'MAX', 'SH2D1A', 'APC', 'NTHL1', 'CTCF', 'KDM5C', 'KMT2C', 'ZFHX3', 'FOXP1', 'PIGA', 'CDKN1B', 'CDKN1A', 'FUBP1', 'MSH2', 'ID3', 'TNFRSF14', 'TRAF3', 'EP400', 'BRIP1', 'ARID4A', 'ARID4B', 'XRCC2', 'DAXX', 'SDHAF2', 'ASXL1', 'AMER1', 'RASA1', 'EGR1', 'MST1', 'SOX17', 'RUNX1', 'PIK3R3', 'NCOR1', 'NF1', 'JAK1', 'PTPRD', 'CHEK2', 'CHEK1', 'SMC1A', 'TMEM127', 'STAG1', 'RAD51', 'TCF3', 'STAG2', 'ARID2', 'RAD50', 'RNF43', 'PARP1', 'BLM', 'CUX1', 'RECQL', 'RAD21', 'PTPN2', 'PTPN1', 'SLX4', 'INHA', 'PAX5', 'IRF1', 'TP53', 'HLA-A', 'IRF8', 'CBL', 'TOP1', 'SHQ1', 'PRDM1', 'NSD1', 'ATXN2', 'CREBBP', 'HDAC4', 'SESN2', 'PPP2R1A', 'EPHA7', 'ATM', 'EPHA3', 'POT1', 'SMAD3', 'MOB3B', 'TBX3', 'POLE', 'ATR', 'FANCD2', 'FH', 'BCORL1', 'SOX9', 'IKZF3', 'TSC1', 'TP63', 'MRE11A', 'SDHC', 'BTG1', 'POLD1', 'CIITA', 'SMC3', 'SAMHD1', 'RTEL1', 'ECT2L', 'PIK3R2', 'CRBN', 'FANCC', 'NBN', 'FANCA', 'HLA-B', 'RECQL4', 'DUSP4', 'ERCC2', 'FBXW7', 'TGFBR2', 'TGFBR1', 'MSH3', 'RBM15', 'TET1', 'TET3', 'SESN3', 'MGA', 'LTB', 'FOXL2', 'SH2B3', 'BCOR', 'HIST1H1D', 'ATRX', 'EP300', 'RAD51C', 'RAD51B', 'HIST1H1B', 'TNFAIP3', 'DICER1', 'ARID5B', 'LATS2', 'FOXO1', 'KEAP1', 'EZH2', 'SP140', 'NKX3-1', 'PBRM1', 'PALB2', 'CIC', 'BRCA1', 'DTX1', 'FLCN', 'SPEN', 'CD58', 'ERCC3', 'ERCC4', 'MSH6', 'BCL11B', 'BMPR1A', 'ERF', 'BRCA2', 'NOTCH2', 'EED', 'MITF', 'ELF3', 'SMARCA4', 'BBC3', 'ANKRD11', 'CEBPA', 'BCL2L11', 'AXIN2', 'AXIN1', 'CDK12', 'ESCO2', 'MLH1', 'SDHB', 'MED12', 'HNF1A', 'RYBP', 'ATP6V1B2', 'DNMT3B', 'KMT2B', 'KMT2A', 'DNMT3A', 'NFKBIA', 'TRAF5', 'KMT2D', 'SPOP', 'RBM10', 'P2RY8', 'TP53BP1', 'TSC2', 'KDM6A', 'EPCAM', 'PHOX2B', 'NPM1', 'BCL10', 'LATS1', 'HOXB13', 'ARID3A', 'PTPRT', 'PTPRS', 'INPPL1', 'NOTCH4', 'TET2', 'NOTCH1', 'CASP8', 'NOTCH3', 'GRIN2A', 'MAP2K4', 'WT1', 'BACH2', 'SDHA', 'BAP1', 'PTCH1', 'SDHD'])\n",
    "poleCaseExpectationOncogenes = poleCaseExpectation[~poleCaseExpectation['gene'].isin(tumorSupressors)]\n",
    "poleCaseExpectationOncogenes341 = poleCaseExpectationOncogenes[poleCaseExpectationOncogenes['gene'].isin(impact341genes)]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "poleMuts = allImpactMuts[allImpactMuts['Tumor_Sample_Barcode'].isin(casesWithPOLESignature)]\n",
    "pole341Muts = poleMuts[poleMuts['Hugo_Symbol'].isin(impact341genes)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "nmutsDict = dict(poleMuts['Tumor_Sample_Barcode'].value_counts())\n",
    "d = dict([(gene, 0) for gene in set(poleCaseExpectationOncogenes['gene'])])\n",
    "for case in set(poleCaseExpectationOncogenes['case']):\n",
    "    caseExpectation = poleCaseExpectationOncogenes[poleCaseExpectationOncogenes['case'] == case]\n",
    "    geneExpDict = dict(zip(caseExpectation['gene'], caseExpectation['truncatingChance']))\n",
    "    for gene in set(caseExpectation['gene']):\n",
    "        d[gene] += geneExpDict[gene]*nmutsDict[case]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "essentialGenesInImpact = [\"AURKA\",\"AURKB\",\"BRD4\",\"CCND1\",\"CDC42\",\"CENPA\",\"CHEK1\",\n",
    "                          \"NUF2\",\"NUP93\",\"PIK3C3\",\"POLD1\",\"POLE\",\"PPP2R1A\",\"RAD21\",\n",
    "                        \"RAD51\",\"SF3B1\",\"U2AF1\", \"UPF1\", \"VHL\", \"XPO1\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "truncatingPoleOnc = pole341Muts[pole341Muts['Consequence'] == 'stop_gained']\n",
    "geneCounts = truncatingPoleOnc['Hugo_Symbol'].value_counts()\n",
    "\n",
    "listOfDicts = []\n",
    "for gene, exp in d.items():\n",
    "    countObs = 0\n",
    "    if gene in geneCounts:\n",
    "        countObs = geneCounts[gene]\n",
    "        \n",
    "    listOfDicts.append({'gene': gene, 'obs': countObs, 'exp': exp})\n",
    "    \n",
    "df = pd.DataFrame(listOfDicts)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[~df['gene'].isin(['MLL2', 'MLL3'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['isImpactEssentialGene'] = df['gene'].apply(lambda x: True if x in essentialGenesInImpact else False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "poleHotspotDict = dict(pole341Muts[pole341Muts['is-a-hotspot'] == 'Y']['Hugo_Symbol'].value_counts())\n",
    "df['nHotspots'] = df['gene'].apply(lambda x: poleHotspotDict[x] if x in poleHotspotDict else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('/Users/friedman/Desktop/WORK/dataForLocalPlotting/oncogeneMutsObsExpPole.tsv', index=False, sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "#MSI obs/expected analyses\n",
    "\n",
    "casesWithMSISignature = set(impactSigs[(impactSigs['dominantSignature'] == 'mean_MMR')\n",
    "                                      |((impactSigs['Nmut_Mb'] >= 30) & (impactSigs['dominantSignature'].isin(set(['mean_1']))))\n",
    "                                        ]['Tumor_Sample_Barcode'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "msiCaseMuts = allImpactMuts[allImpactMuts['Tumor_Sample_Barcode'].isin(casesWithMSISignature)]\n",
    "msiHotspotDict = dict(msiCaseMuts[msiCaseMuts['is-a-hotspot'] == 'Y']['Hugo_Symbol'].value_counts())\n",
    "msiOncogene341Muts = msiCaseMuts[(msiCaseMuts['Hugo_Symbol'].isin(impact341genes)) & ~(msiCaseMuts['Hugo_Symbol'].isin(tumorSupressors))]\n",
    "msiOncogeneIndels = msiOncogene341Muts[(msiOncogene341Muts['Variant_Type'].isin(['INS', 'DEL'])) & (msiOncogene341Muts['Consequence'] == 'frameshift_variant')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "#msiOncogene341Muts['Variant_Type']\n",
    "msiOncogeneIndels = msiOncogeneIndels[~msiOncogeneIndels['Hugo_Symbol'].isin(['MLL2', 'MLL3'])]\n",
    "countDict = dict(msiOncogeneIndels['Hugo_Symbol'].value_counts())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "oncogenes341 = set(impact341genes) - set(tumorSupressors)\n",
    "listOfDicts = []\n",
    "for gene in oncogenes341:\n",
    "    nIndels = 0\n",
    "    if gene in countDict:\n",
    "        nIndels = countDict[gene]\n",
    "    nHotspots = 0\n",
    "    if gene in msiHotspotDict:\n",
    "        nHotspots = msiHotspotDict[gene]\n",
    "    listOfDicts.append({'gene': gene, 'nIndels': nIndels, 'nHotspots': nHotspots})\n",
    "\n",
    "df = pd.DataFrame(listOfDicts)\n",
    "df['isImpactEssentialGene'] = df['gene'].apply(lambda x: True if x in essentialGenesInImpact else False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('/Users/friedman/Desktop/WORK/dataForLocalPlotting/oncogeneMutsObsExpMSI.tsv', index=False, sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "set(['ACTG'])\n"
     ]
    }
   ],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}