encdd_v0.3.9.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "cf8a31c2-d560-4a61-832e-94643bf3799d",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08ea20fa-68b0-4fd0-8c81-8a9db45bbbbc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# encdd_v0.3.9.ipynb, corresponding to unenc 0.4\n",
    "# From encdd_v0.2.ipynb\n",
    "\n",
    "# template for encdd_v0.4 where there will be four loss term\n",
    "    # can do y-mapping with two cancer types\n",
    "    # or fix distanc functions to take multiple model and cancer types\n",
    "    # want mult model and cancer type versions of both the cosine similarity and the MMD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d3b96e0-4f50-465f-978f-0884e8df02ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2023-11-09 restart\n",
    "# Submit abstract per current ops - \n",
    "    # add note on digital twin - \n",
    "# clear outputs and re-check code - \n",
    "# logs/v0.3.9_prot_20231102-080655\n",
    "    # last out put dir, find latest unenc figs\n",
    "# PUT NON-CLEARED OUTPUT VERSION TO UNCONTROLLED PARENT DIR\n",
    "\n",
    "# ended with loss plots on 11/02"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "079b7168-6b52-4b8e-8106-988ed0e6f5ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "%whos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "627d1ca0-f8a3-4299-b1bd-632764c4225f",
   "metadata": {},
   "outputs": [],
   "source": [
    "v = 'v0.3.9' # results_encdd dir created"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85207222-f8c7-4efa-bb79-2b20aef3a42c",
   "metadata": {},
   "outputs": [],
   "source": [
    "v"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b27bd417-cb79-4e0e-b773-0f53df5b9593",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## log_dir devel record"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c88ebbc6-c470-4389-baa4-b61837b766b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import datetime\n",
    "log_dir = Path(\"logs\", datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c78949de-497b-4ed7-8f71-529eefd273ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b56097a3-5d9f-455d-817e-8f90e838aa75",
   "metadata": {},
   "outputs": [],
   "source": [
    "log_dir = Path(\"logs\", v + \"_prot_\" + datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\"))\n",
    "log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe862301-9537-466d-8cf9-01b43b15ba69",
   "metadata": {},
   "outputs": [],
   "source": [
    "log_dir = Path(\"logs\", v + log_dta + datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\"))\n",
    "log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06ba65d4-d323-4c65-bd06-20a489ed6425",
   "metadata": {},
   "outputs": [],
   "source": [
    "dta_ttl = 'proteomics'\n",
    "dta_typ = 'prot'\n",
    "log_dta = '_prot_'\n",
    "dta_typ_unenc = pd.read_csv('data/cl_cp_prot_850.tsv',\n",
    "                   sep = '\\t', index_col = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a90b43b-ac2f-4e0c-8d32-a8516ce475e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ^sent inline to script\n",
    "    # three parts - imports, set path, set plot label / file naming vars"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4adc1520-e199-4e72-b139-38536cf2b946",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Build VAE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f772bbb-6147-40b0-862f-f14e1346bf9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "\n",
    "import datetime # Nested in Path func\n",
    "from pathlib import Path # log dir\n",
    "\n",
    "# from argparse import ArgumentParser\n",
    "from scipy.spatial import distance  # for cosine similarity calc\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "model_type_encoder = LabelEncoder()\n",
    "cancer_type_encoder = LabelEncoder()\n",
    "\n",
    "def calculate_cosine_similarity(df: pd.DataFrame, model_type_column='model_type') -> (dict, dict):\n",
    "    # Separate the DataFrame into two based on the model_type\n",
    "    df_cell_line = df[df[model_type_column] == 'cell line'].drop(columns=[model_type_column])\n",
    "    df_tumor = df[df[model_type_column] == 'Tumor'].drop(columns=[model_type_column])\n",
    "\n",
    "    # Initialize dictionaries to store the results\n",
    "    cosine_similarities_cell_line = {}\n",
    "    cosine_similarities_tumor = {}\n",
    "\n",
    "    # Identify the columns that are common and valid for mean calculation\n",
    "    valid_columns = df_cell_line.select_dtypes(include=[np.number]).columns\n",
    "\n",
    "    # Calculate the mean vector for each class for the relevant columns\n",
    "    mean_vector_cell_line = df_cell_line[valid_columns].mean(axis=0).values\n",
    "    mean_vector_tumor = df_tumor[valid_columns].mean(axis=0).values\n",
    "\n",
    "    # Loop through each sample in cell_line and calculate cosine similarity to both mean vectors\n",
    "    for index, row in df_cell_line.iterrows():\n",
    "        sample_vector = row[valid_columns].values\n",
    "        sim_to_cell_line = distance.cosine(sample_vector, mean_vector_cell_line)\n",
    "        sim_to_tumor = distance.cosine(sample_vector, mean_vector_tumor)\n",
    "        cosine_similarities_cell_line[index] = (sim_to_cell_line, sim_to_tumor)\n",
    "\n",
    "    # Loop through each sample in tumor and calculate cosine similarity to both mean vectors\n",
    "    for index, row in df_tumor.iterrows():\n",
    "        sample_vector = row[valid_columns].values\n",
    "        sim_to_cell_line = distance.cosine(sample_vector, mean_vector_cell_line)\n",
    "        sim_to_tumor = distance.cosine(sample_vector, mean_vector_tumor)\n",
    "        cosine_similarities_tumor[index] = (sim_to_cell_line, sim_to_tumor)\n",
    "\n",
    "    # Combine the 2 dist dicts into one dict\n",
    "    cosine_similarities_tumor.update(cosine_similarities_cell_line)\n",
    "    cosine_similarities = cosine_similarities_tumor\n",
    "\n",
    "    # Prefilled 0s lists, number of dictionary keys\n",
    "    intra_cluster_tensor = list(range(df.shape[0]))\n",
    "    inter_cluster_tensor = list(range(df.shape[0]))\n",
    "\n",
    "    assert len(intra_cluster_tensor) == len(df), \"Length of list is not as expected\"\n",
    "    assert len(inter_cluster_tensor) == len(df), \"Length of list is not as expected\"\n",
    "\n",
    "    # Populate the zero lists with dist scores\n",
    "    for key in cosine_similarities.keys():\n",
    "        intra_cluster_tensor[key] = cosine_similarities[key][0]\n",
    "        inter_cluster_tensor[key] = cosine_similarities[key][1]\n",
    "\n",
    "    # Convert lists to tensors\n",
    "    intra_cluster_tensor = tf.convert_to_tensor(intra_cluster_tensor, dtype=np.float32)\n",
    "    inter_cluster_tensor = tf.convert_to_tensor(inter_cluster_tensor, dtype=np.float32)\n",
    "\n",
    "    return intra_cluster_tensor, inter_cluster_tensor\n",
    "\n",
    "\n",
    "# Define the Sampling Layer\n",
    "class Sampling(layers.Layer):\n",
    "    \"\"\"Uses (z_mean, z_log_var) to sample z, the vector encoding a digit.\"\"\"\n",
    "    # Stock TensorFlow\n",
    "    def call(self, inputs):\n",
    "        z_mean, z_log_var = inputs\n",
    "        batch = tf.shape(z_mean)[0]\n",
    "        dim = tf.shape(z_mean)[1]\n",
    "        epsilon = tf.random.normal(shape=(batch, dim))\n",
    "        return z_mean + tf.exp(0.5 * z_log_var) * epsilon\n",
    "\n",
    "\n",
    "# Define the VAE class\n",
    "class VAE(keras.Model):\n",
    "    def __init__(self, encoder, decoder, columns, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.encoder: keras.Model = encoder\n",
    "        self.decoder: keras.Model = decoder\n",
    "        self.total_loss_tracker = keras.metrics.Mean(name=\"total_loss\")\n",
    "        self.reconstruction_loss_tracker = keras.metrics.Mean(name=\"reconstruction_loss\")\n",
    "        self.kl_loss_tracker = keras.metrics.Mean(name=\"kl_loss\")\n",
    "        self.distance_loss_tracker = keras.metrics.Mean(name=\"distance_loss\")\n",
    "        self.columns = columns\n",
    "\n",
    "    @property\n",
    "    def metrics(self):\n",
    "        return [\n",
    "            self.total_loss_tracker,\n",
    "            self.reconstruction_loss_tracker,\n",
    "            self.kl_loss_tracker,\n",
    "            self.distance_loss_tracker,\n",
    "        ] \n",
    "\n",
    "    def train_step(self, data):\n",
    "        with tf.GradientTape() as tape:\n",
    "            converted_data: pd.DataFrame = pd.DataFrame(data.numpy(), columns=self.columns)\n",
    "            model_type = converted_data[\"model_type\"]\n",
    "            model_type = model_type.astype(int)\n",
    "            model_type = model_type_encoder.inverse_transform(model_type)\n",
    "            data = converted_data.drop(columns=[\"model_type\"])\n",
    "            assert \"model_type\" not in data.columns, \"model_type should not be in data\"\n",
    "\n",
    "            cancer_type = converted_data[\"cancer_type\"]\n",
    "            cancer_type = cancer_type.astype(int)\n",
    "            cancer_type = cancer_type_encoder.inverse_transform(cancer_type)\n",
    "            data = data.drop(columns=[\"cancer_type\"])\n",
    "            assert \"cancer_type\" not in data.columns, \"cancer_type should not be in data\"\n",
    "\n",
    "            data = tf.convert_to_tensor(data)\n",
    "\n",
    "            z_mean, z_log_var, z = self.encoder(data)\n",
    "\n",
    "            labeled_embeddings: pd.DataFrame = pd.DataFrame(z.numpy())\n",
    "            labeled_embeddings[\"model_type\"] = model_type\n",
    "\n",
    "            intra_cluster_distance, inter_cluster_distance = calculate_cosine_similarity(\n",
    "                df=labeled_embeddings,\n",
    "                model_type_column='model_type')\n",
    "\n",
    "            reconstruction = self.decoder(z)\n",
    "            reconstruction_loss = .1 * data.shape[1] * keras.losses.binary_crossentropy(data, reconstruction)\n",
    "            kl_loss = - 0.5 * tf.reduce_sum(1 + z_log_var - tf.square(z_mean) - tf.exp(z_log_var), axis=-1)\n",
    "            distance_loss = - 100 * inter_cluster_distance\n",
    "            total_loss = reconstruction_loss + kl_loss + distance_loss\n",
    "\n",
    "        grads = tape.gradient(total_loss, self.trainable_weights)\n",
    "        self.optimizer.apply_gradients(zip(grads, self.trainable_weights))\n",
    "        self.total_loss_tracker.update_state(total_loss)\n",
    "        self.reconstruction_loss_tracker.update_state(reconstruction_loss)\n",
    "        self.kl_loss_tracker.update_state(kl_loss)\n",
    "        self.distance_loss_tracker.update_state(distance_loss)\n",
    "        return {\n",
    "            \"loss\": self.total_loss_tracker.result(),\n",
    "            \"reconstruction_loss\": self.reconstruction_loss_tracker.result(),\n",
    "            \"kl_loss\": self.kl_loss_tracker.result(),\n",
    "            \"distance_loss\": self.distance_loss_tracker.result(),\n",
    "        }\n",
    "\n",
    "\n",
    "# Build Encoder\n",
    "def build_encoder(feature_dim, latent_dim) -> keras.Model:\n",
    "    encoder_inputs = keras.Input(shape=(feature_dim,), name=\"input_1\")\n",
    "    x = keras.layers.Dense(latent_dim, kernel_initializer='glorot_uniform', name=\"encoder_dense_1\")(encoder_inputs)\n",
    "    x = keras.layers.BatchNormalization(name=\"batchnorm\")(x)\n",
    "    z_mean = layers.Dense(latent_dim, name=\"z_mean\")(x)\n",
    "    z_log_var = layers.Dense(latent_dim, name=\"z_log_var\")(x)\n",
    "    z = Sampling()([z_mean, z_log_var])\n",
    "    encoder = keras.Model(encoder_inputs, [z_mean, z_log_var, z], name=\"encoder\")\n",
    "    return encoder\n",
    "\n",
    "\n",
    "# Build Decoder\n",
    "def build_decoder(feature_dim, latent_dim) -> keras.Model:\n",
    "    latent_inputs = keras.Input(shape=(latent_dim,))\n",
    "    x = keras.layers.Dense(feature_dim, kernel_initializer='glorot_uniform', activation='sigmoid')(latent_inputs)\n",
    "    decoder_outputs = x\n",
    "    decoder = keras.Model(latent_inputs, decoder_outputs, name=\"decoder\")\n",
    "    return decoder\n",
    "\n",
    "latent_dim = 50\n",
    "learning_rate = 0.001\n",
    "\n",
    "epochs = 30\n",
    "epochs = 5\n",
    "epochs = 10\n",
    "\n",
    "batch_size = 128"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ae16ea38-d949-4395-98fa-b3a9331e1673",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Run VAE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da9150c7-1c3d-4362-a0c9-4413abe758ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "dta_ttl = 'proteomics'\n",
    "dta_typ = 'prot'\n",
    "log_dta = '_prot_'\n",
    "dta_typ_unenc = pd.read_csv('data/cl_cp_prot_850.tsv',\n",
    "                   sep = '\\t', index_col = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e8460b0-fed6-4db3-bfbf-89f806510c7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "log_dir = Path(\"logs\", v + log_dta + datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\"))\n",
    "\n",
    "# if __name__ == '__main__':\n",
    "# parser = ArgumentParser()\n",
    "# parser.add_argument(\"-f\", \"--file\", action=\"store\", type=str, required=True)\n",
    "\n",
    "# args = parser.parse_args()\n",
    "\n",
    "# file: str = args.file\n",
    "file = 'data/cl_cp_prot_850.tsv'\n",
    "\n",
    "# d_typ_obj = pd.read_csv('data/cl_cp_cnvr_996.tsv', sep='\\t', index_col=0)\n",
    "df = pd.read_csv(file, sep='\\t', index_col=0)\n",
    "\n",
    "# v0 - scale data for baseline encoding\n",
    "selected_df = df.iloc[:, 2:]\n",
    "\n",
    "# Count features, to encoder build\n",
    "feature_count = selected_df.shape[1]\n",
    "\n",
    "scaler = MinMaxScaler()\n",
    "selected_df = pd.DataFrame(\n",
    "    scaler.fit_transform(selected_df),\n",
    "    columns=selected_df.columns,\n",
    "    index=selected_df.index)\n",
    "\n",
    "selected_df[\"model_type\"] = df[\"model_type\"]\n",
    "selected_df[\"cancer_type\"] = df[\"cancer_type\"]\n",
    "\n",
    "selected_df[\"model_type\"] = model_type_encoder.fit_transform(selected_df[\"model_type\"])\n",
    "# convert model type to int\n",
    "selected_df[\"model_type\"] = selected_df[\"model_type\"].astype(int)\n",
    "assert selected_df[\"model_type\"].nunique() == 2, \"There should be two classes\"\n",
    "\n",
    "selected_df[\"cancer_type\"] = cancer_type_encoder.fit_transform(selected_df[\"cancer_type\"])\n",
    "# convert model type to int\n",
    "selected_df[\"cancer_type\"] = selected_df[\"cancer_type\"].astype(int)\n",
    "\n",
    "print(selected_df.shape)\n",
    "\n",
    "# Build VAE\n",
    "encoder = build_encoder(feature_count, latent_dim)  # feat count set above, lat dim is a var\n",
    "decoder = build_decoder(feature_count, latent_dim)\n",
    "vae = VAE(encoder, decoder, columns=selected_df.columns)\n",
    "vae.compile(optimizer=keras.optimizers.Adam(learning_rate=learning_rate), run_eagerly=True)\n",
    "\n",
    "tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)\n",
    "cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=Path(log_dir, \"weights\"),\n",
    "                                                 save_weights_only=True,\n",
    "                                                 verbose=1)\n",
    "\n",
    "history = vae.fit(selected_df, epochs=epochs, batch_size=batch_size, shuffle=True,\n",
    "                  callbacks=[tensorboard_callback, cp_callback])\n",
    "\n",
    "# save history\n",
    "history_df = pd.DataFrame(history.history)\n",
    "# history_df.to_csv(Path(log_dir, \"history.tsv\", sep='\\t'), index=False) # sep in bracket, comma delim?\n",
    "history_df.to_csv(Path(log_dir, \"history.tsv\"), sep='\\t', index=False)\n",
    "# save model\n",
    "vae.encoder.save(Path(log_dir, \"encoder\"))\n",
    "vae.decoder.save(Path(log_dir, \"decoder\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff9dea3c-1593-4327-b274-89d20bf65d0a",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Loss curve plot, #0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10c47656-fe5e-47ab-9bdc-83384dddad3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# add cosine similarity label to distance loss curve sub component"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b88e6229-8962-435b-a4fb-5455a730cd68",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import glob\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b8403ee-d544-44c3-a5e6-b940a43352dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "81b7694d-034f-4b98-803b-115ecc2d24ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8226a86-f58d-4c9d-b2f7-9688440b1bab",
   "metadata": {},
   "outputs": [],
   "source": [
    "ls logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d079e92-f8ab-4edb-9368-7f6679351040",
   "metadata": {},
   "outputs": [],
   "source": [
    "ls logs/v0.3.9_prot_20231102-080655/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ecb59569-4a42-41d9-8f13-552140ac7e16",
   "metadata": {},
   "outputs": [],
   "source": [
    "prot_hist = pd.read_csv('logs/v0.3.9_prot_20231102-080655/history.tsv', sep = '\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ff81652-c170-4c71-868d-4d84b8c0519a",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_title = 'proteomics'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "014d6e63-a9f7-4f7d-b040-1817fd3da44c",
   "metadata": {},
   "outputs": [],
   "source": [
    "dta_ttl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e27f25f4-1646-4e30-982a-7e9a2ac7687b",
   "metadata": {},
   "outputs": [],
   "source": [
    "prot_hist # Observations\n",
    "            # 2023-11-01, same as previous test run on 5 epochs:\n",
    "            # recon flatlines, KL goes down, distance becomes increasingly negative\n",
    "            # Thoughts\n",
    "                # Add constant like +100 to the distance loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "119726b4-9749-4263-81dc-deb08535f5de",
   "metadata": {},
   "outputs": [],
   "source": [
    "Path(log_dir, \"test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f6aef2c-2500-454b-8c0e-ee406346e054",
   "metadata": {},
   "outputs": [],
   "source": [
    "type(Path(log_dir, \"test\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "065828d6-6652-4176-93b5-c1484fa854d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e46026f-dbe7-4156-9372-fd9ee2c82655",
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_dir_name = log_dir.parts[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6aa6dae4-75f8-4d5b-98ed-1ed131d3a07c",
   "metadata": {},
   "outputs": [],
   "source": [
    "log_dir.parts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62a1173d-ae44-4fe3-a229-36da5b9a2129",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(log_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0133e673-31b2-415b-a6e7-e9807a993c38",
   "metadata": {},
   "outputs": [],
   "source": [
    "log_dir.parts[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "107c4730-5279-4cd3-b02d-a39f3dcc1576",
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_plot_dir_matching_the_VAE_output_dir = log_dir.parts[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "745face4-50ba-4c14-9ebc-bcfa2f2366c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "history_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b2b41d3-17e7-4e40-b76e-163eca98814b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create main plot for overall loss\n",
    "df = history_df\n",
    "xlab = 'Epoch'\n",
    "axis_font_size = 18\n",
    "\n",
    "fig, main_ax = plt.subplots(figsize=(10, 8))\n",
    "main_ax.plot(df['loss'], label='Total Loss', color='blue')\n",
    "main_ax.set_title('Overall Loss and Individual Loss Components, ' + dta_ttl, fontsize = 20)\n",
    "main_ax.set_xlabel(xlab, fontsize = axis_font_size)\n",
    "main_ax.set_ylabel('Total Loss', fontsize = axis_font_size)\n",
    "main_ax.legend()\n",
    "\n",
    "width, height = .25, .25\n",
    "a, b = .3, .65\n",
    "inset_title_font_size = 14\n",
    "\n",
    "# Reconstruction loss, upper left\n",
    "ax1 = main_ax.inset_axes([a, b, width, height])  # x, y, width, height\n",
    "ax1.plot(df['reconstruction_loss'], label='Recon Loss', color='green')\n",
    "ax1.set_title('Reconstruction Loss', fontsize = inset_title_font_size)\n",
    "ax1.set_xlabel(xlab)\n",
    "ax1.set_ylabel('Recon Loss')\n",
    "\n",
    "# KL loss, upper right\n",
    "ax2 = main_ax.inset_axes([b, b, width, height])\n",
    "ax2.plot(df['kl_loss'], label='KL Loss', color='red')\n",
    "ax2.set_title('KL Loss', fontsize = inset_title_font_size)\n",
    "ax2.set_xlabel(xlab)\n",
    "ax2.set_ylabel('KL Loss')\n",
    "\n",
    "# Distance loss, lower left - move to lower right\n",
    "ax3 = main_ax.inset_axes([b, a-.02, width, height])\n",
    "ax3.plot(df['distance_loss'], label='Distance Loss', color='orange')\n",
    "ax3.set_title('Distance Loss', fontsize = inset_title_font_size)\n",
    "ax3.set_xlabel(xlab)\n",
    "ax3.set_ylabel('Distance Loss')\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig('loss_plots/'+loss_plot_dir_matching_the_VAE_output_dir+'cmposit_loss_plot.png')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0d9e890b-d746-4852-a4a7-47faf47169d3",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Quant setups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d083d556-fe39-4be7-9dde-e02e5e04e520",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Non-redundant, all 5 plots in sequential order\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "import umap\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import classification_report, f1_score\n",
    "\n",
    "from scipy.spatial.distance import euclidean\n",
    "from matplotlib.colors import ListedColormap\n",
    "from joypy import joyplot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68596fea-490b-445a-a900-00b07c7b48fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "grey_colors = pd.read_csv('plot_color_files/grey_scale.tsv', sep = '\\t', index_col = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "46394f7b-fd74-437b-8c00-328d8931223a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# TCGA colors on plots, build versions for alt y-mappings\n",
    "cancer_type_abbreviation_mapping = {\n",
    "    'Clear cell renal cell carcinoma': 'KIRCKICH', # combined-in chromophobe\n",
    "    'Colon adenocarcinoma': 'COADREAD', # combined-in rectal adeno\n",
    "    'Pancreatic ductal adenocarcinoma': 'PAAD',\n",
    "    'Breast carcinoma': 'BRCA',\n",
    "    'Glioblastoma': 'LGGGBM', # combined-in low grade glioma\n",
    "    'Lung squamous cell carcinoma': 'LUSC',\n",
    "    'Lung adenocarcinoma': 'LUAD',\n",
    "    'Endometrial carcinoma': 'UCEC',\n",
    "    'Head and neck squamous cell carcinoma': 'HNSC',\n",
    "    'Ovarian carcinoma': 'OV'\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f27a008a-af2e-4e8f-8e37-c68dd5eedfee",
   "metadata": {},
   "source": [
    "Design goal  \n",
    "A plot design change should update both the pre and post plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3156eab-dbc8-42fe-8111-cb61c8557016",
   "metadata": {},
   "outputs": [],
   "source": [
    "# From first complete LaTeX version of unencoded plots\n",
    "    # unenc_v0.4.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "962d6a63-1f6c-4237-ba07-cf9879430d18",
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = tf.keras.models.load_model(\"logs/v0.3.9_prot_20231102-080655/encoder\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2474cfc-74ca-44d5-9b28-301594650609",
   "metadata": {},
   "source": [
    "## Plot labeling, naming, disk reads"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "885694a9-e78a-4d23-9f62-23f01f0855df",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "### lat dim / epochs / notes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e3bfcc8-443a-4169-8c21-3e493df3c5ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "# For intermediate disk load only, ADD EPOCH count to log dir file naming\n",
    "latent_dim = 50 # Already in RAM if VAE run in same session\n",
    "epochs = 10      # ' ' ' ' '"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36c3dbdc-8b66-4e0c-b79e-307b241cca29",
   "metadata": {},
   "outputs": [],
   "source": [
    "# LATENT FILE LOC:\n",
    "    # ____________ nope\n",
    "    # Looks like a trained ecoder is saved and loads the file (above) - done\n",
    "    # what is the encdd plot target dir? - results_encdd - above unenc - subopt\n",
    "    # how does the version and data type file naming line up with \n",
    "    # the unenc plots, - v0.4, just sitting in the results_unenc dir - done\n",
    "        # kind of a misnomer as far as resluts go, its just plots, not VAE results\n",
    "        # Manual archive process - done\n",
    "    # save model - in logs/version_._/data_type/ - done\n",
    "    # loss plots??\n",
    "\n",
    "    # Starting to come back now, the v0.4 refered to the 4th loss term\n",
    "    # This will not work for the current 7-cancer y label setup\n",
    "    # because the model-type loss function is hard-coded for binary cell line vs. human\n",
    "    # So, there is a mis mathch in finishing this run\n",
    "    # Just get the LaTeX plot formatting form unenc v0.4 into this\n",
    "    # Then go to y-mapping (file aready started?)\n",
    "    # and pick up the pieces from there"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9cccb809-2798-4b2c-bc36-647cbb7029d7",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "### Toggle zero  \n",
    "Toggle after plot two, back after plot four"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67b8eccf-0329-4156-bfdb-3211e45079f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Flow starts with UMAP then this goes to plot #2 greyscale model logreg bar\n",
    "    # THIS DOESN'T PERTAIN TO UMAP, SHOULD GO LATER - copy down to point of use - \n",
    "mode_ttl = 'model type'\n",
    "mode = 'model_type'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84241954-4253-418e-96a8-541248902ce0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save this as notes version, manually copy to uncontrolled parent dir, clear this version"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "41a156ea-b92f-48e3-b9d4-5242017f37bf",
   "metadata": {},
   "source": [
    "### Toggle one  \n",
    "Leave for all five plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7a6c73e-af6e-4e7b-a3a5-ca1a390d4d1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "mdls_ttl = 'Cell line + CPTAC'\n",
    "mdls = 'cptac_+_cell_line'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e123186c-7dc0-47a6-8f5e-c862a54e1b18",
   "metadata": {},
   "outputs": [],
   "source": [
    "mdls_ttl = 'Cell line + CPTAC + HCMI'\n",
    "mdls = 'cptac_+_cell_line_+_hcmi'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "07234555-2ccc-4fd2-8406-c4d705fa1ae6",
   "metadata": {},
   "source": [
    "### Toggle set two  \n",
    "Leave for all five plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ccffbca5-51df-457e-baa6-a43dcddde2ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "dta_ttl = 'copy number'\n",
    "dta_typ = 'cnvr'\n",
    "dta_typ_unenc = pd.read_csv('data/cl_cp_cnvr_996.tsv',\n",
    "                       sep = '\\t', index_col = 0)\n",
    "dta_typ_obj = pd.DataFrame(encoder.predict(dta_typ_unenc.iloc[:, 2:])[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a9aa8aa-ce35-4170-a35e-917d9080667c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Obs - encoder shape error if run on wrong data type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58b4c975-b49a-4d3c-9ef7-f0b613b6d472",
   "metadata": {},
   "outputs": [],
   "source": [
    "dta_ttl = 'gene expression'\n",
    "dta_typ = 'gexp'\n",
    "dta_typ_unenc = pd.read_csv('data/cl_cp_gexp_998.tsv',\n",
    "                       sep = '\\t', index_col = 0)\n",
    "dta_typ_obj = pd.DataFrame(encoder.predict(dta_typ_unenc.iloc[:, 2:])[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0b660e7-8be2-4c6e-b1f7-56817f656f14",
   "metadata": {},
   "outputs": [],
   "source": [
    "dta_ttl = 'proteomics'\n",
    "dta_typ = 'prot'\n",
    "log_dta = '_prot_'\n",
    "dta_typ_unenc = pd.read_csv('data/cl_cp_prot_850.tsv',\n",
    "                   sep = '\\t', index_col = 0)\n",
    "dta_typ_obj = pd.DataFrame(encoder.predict(dta_typ_unenc.iloc[:, 2:])[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c36ffa7b-4be3-4060-98e1-ae1123e01c67",
   "metadata": {},
   "source": [
    "### Continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9ee08fb-bad7-49ee-b9ce-48fe107596f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(dta_ttl)\n",
    "dta_typ_obj.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e4f3315-5965-45fd-836b-6698f2c8f71e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Labels onto latent object\n",
    "feature_columns = dta_typ_obj.columns\n",
    "dta_typ_obj['cancer_type'] =  list(dta_typ_unenc['cancer_type'])\n",
    "dta_typ_obj['model_type'] =  list(dta_typ_unenc['model_type'])\n",
    "new_cols = ['cancer_type', 'model_type'] + list(feature_columns)\n",
    "dta_typ_obj = dta_typ_obj[new_cols]\n",
    "dta_typ_obj.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7c8395a-08bc-4121-bbac-537a2921fdf7",
   "metadata": {},
   "outputs": [],
   "source": [
    "dta_typ_obj.columns # Why does this run with numeric column names the logreg function now?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8ba782a-35ad-41f7-9d13-1141f91495d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write into results dir, manually at completion of run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ea8bbe2-f39a-4e59-aa07-1e4751d31895",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Encodded vars, unenc_v0.4\n",
    "rslts_dir = 'results_encdd'\n",
    "encdg_stts_ttl = ', encoded'\n",
    "encdg_stts_nam = 'encoded'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81ba3493-09ba-4441-8cc9-3cab66e0fecf",
   "metadata": {},
   "source": [
    "## UMAP #1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa8ef009-ea8a-42f3-babb-a3f274f4f376",
   "metadata": {},
   "outputs": [],
   "source": [
    "reducer = umap.UMAP(n_components=2)\n",
    "scaled_data = StandardScaler().fit_transform(dta_typ_obj.iloc[:, 2:])\n",
    "embedding = reducer.fit_transform(scaled_data)\n",
    "emb_df = pd.DataFrame(embedding, index = dta_typ_obj.index)\n",
    "emb_lbld = pd.concat([emb_df, dta_typ_obj[['cancer_type', 'model_type']]], axis = 1)\n",
    "emb_lbld.columns = ['UMAP_1', 'UMAP_2', 'cancer_type', 'model_type']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a722ba5-a7bf-457b-a6c0-f45bd5f914be",
   "metadata": {},
   "outputs": [],
   "source": [
    "mdls_ttl, dta_ttl, dta_typ, mdls, latent_dim, encdg_stts_ttl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79dfcf94-3f20-4d38-88b1-7d177af03e77",
   "metadata": {},
   "outputs": [],
   "source": [
    "def umap_plot_to_disk(emb_lbld, mdls_ttl, dta_ttl, dta_typ, mdls):\n",
    "    tcga_colors = pd.read_csv('plot_color_files/tcga_colors.tsv', sep = '\\t', index_col = 0)\n",
    "    unique_cancer_types = emb_lbld['cancer_type'].unique()\n",
    "    custom_palette = {cancer_type: tcga_colors.loc[cancer_type_abbreviation_mapping.get(\n",
    "        cancer_type, 'Unknown'), 'cohort_color'] for cancer_type in unique_cancer_types}\n",
    "    emb_lbld.columns = ['UMAP_1', 'UMAP_2', 'Cancer type', 'Model type'] # Re-run reducer\n",
    "    plt.figure(figsize=(5, 5))                                           # for iterations\n",
    "    marker_dict = {'Tumor': '^', 'cell line': 'o'}\n",
    "    sns.scatterplot(data=emb_lbld, x='UMAP_1', y='UMAP_2',\n",
    "                    hue='Cancer type', style='Model type', markers=marker_dict,\n",
    "                    palette=custom_palette, legend='full',\n",
    "                    s = 200)\n",
    "    plt.xlabel('UMAP_2', fontsize=16)\n",
    "    plt.ylabel('UMAP_2', fontsize=16)\n",
    "    plt.legend(title='Cancer Type', loc='upper left', bbox_to_anchor=(1, 1))\n",
    "    plt.suptitle(mdls_ttl +', '+ dta_ttl, y = 1.002, fontsize = 20)\n",
    "    plt.title('n = '+str(len(emb_lbld))+encdg_stts_ttl, fontsize = 18)\n",
    "    legend = plt.legend(title='Sample attributes', title_fontsize='14', loc='upper left',\n",
    "                        bbox_to_anchor=(1, 1), fontsize=12)\n",
    "    headers_to_bold = ['Cancer type', 'Model type']\n",
    "    for text in legend.texts:\n",
    "        if text.get_text() in headers_to_bold:\n",
    "            text.set_weight('bold')\n",
    "    plt.rcParams['text.usetex'] = True\n",
    "    plt.savefig(rslts_dir+'/umap_'+dta_typ+'_'+mdls+'_'+encdg_stts_nam+'_'+v+'.png',\n",
    "                bbox_inches = 'tight', dpi = 300)\n",
    "    return 'UMAP written to disk'\n",
    "umap_plot_to_disk(emb_lbld, mdls_ttl, dta_ttl, dta_typ, mdls)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d6f35e8f-8347-4e05-812b-5118831dad37",
   "metadata": {},
   "source": [
    "## LogReg function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba72a4e7-981f-4fea-ae86-19129547349e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model is model type or cancer type\n",
    "def log_reg(dta_typ_obj, mode):\n",
    "    col_X_strt = 2 # <-- Skip label columns\n",
    "    f1_stor_frm = pd.DataFrame()\n",
    "\n",
    "    # Logistic regression repeat loop\n",
    "    for i in list(range(0, 15)): # <-- Error control\n",
    "\n",
    "        # Train test split\n",
    "        trn = dta_typ_obj.sample(round(len(dta_typ_obj) * .8))\n",
    "        tst = dta_typ_obj.loc[~dta_typ_obj.index.isin(trn.index)]\n",
    "    \n",
    "        X_trn = trn.iloc[:, col_X_strt:]\n",
    "        X_tst = tst.iloc[:, col_X_strt:]\n",
    "\n",
    "        # Prediction targets, y is either model system or cancer type\n",
    "        y_trn = trn[mode]\n",
    "        y_tst = tst[mode]\n",
    "        \n",
    "        clf = LogisticRegression().fit(X_trn, y_trn)\n",
    "        y_pred = clf.predict(X_tst)\n",
    "        \n",
    "        f1_by_class = f1_score(y_tst, y_pred, average=None)\n",
    "        f1_df = pd.DataFrame({'Label': list(y_tst.unique()),\n",
    "                              'F1_Score': f1_by_class})\n",
    "        f1_stor_frm = pd.concat([f1_stor_frm, f1_df], axis = 0)\n",
    "    return f1_stor_frm"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff6e6a9e-ab23-4d84-ab1d-b4364ce4cae7",
   "metadata": {},
   "source": [
    "## Logreg model-type plot - greyscale #2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd568fd3-665c-477c-b985-052c0cb1e9ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "mode_ttl = 'model type'\n",
    "mode = 'model_type'\n",
    "f1_stor_frm = log_reg(dta_typ_obj, mode)\n",
    "sample_counts = dict(dta_typ_obj.model_type.value_counts())\n",
    "def logreg_model_plot(f1_stor_frm, mdls, dta_typ, latent_dim, epochs, mode):\n",
    "    plt.figure(figsize=(8, 4.5))\n",
    "    sns.set_style(\"whitegrid\")\n",
    "    sns.set(font_scale=1.5)\n",
    "    \n",
    "    sns.barplot(x='Label', y='F1_Score', data=f1_stor_frm, palette=['#666666', '#999999'],\n",
    "               errorbar=None)\n",
    "    sns.swarmplot(x='Label', y='F1_Score', data=f1_stor_frm, color='#333333', size=14)\n",
    "    \n",
    "    plt.suptitle('Logistic regression, '+mode_ttl+', '+dta_ttl,\n",
    "                 fontsize=24, y = 1.03)\n",
    "    plt.title(mdls_ttl+encdg_stts_ttl, fontsize=20)\n",
    "    plt.xlabel('Model Type', fontsize=20)\n",
    "    plt.ylabel('F1 Score', fontsize=20)\n",
    "\n",
    "    # Sample counts is global var\n",
    "    new_labels = [f\"{label}, n = {sample_counts[label]}\" for label in sample_counts.keys()]\n",
    "    plt.xticks(ticks=range(len(new_labels)), labels=new_labels, fontsize=20)\n",
    "    \n",
    "    plt.grid(color='grey', linestyle='-', linewidth=0.25, alpha=0.5)\n",
    "    plt.rcParams['text.usetex'] = True\n",
    "    plt.savefig(rslts_dir+'/log_reg_'+mode+'_'+dta_typ+'_'+encdg_stts_nam+'_'+v+'.png',\n",
    "                bbox_inches='tight')\n",
    "logreg_model_plot(f1_stor_frm, mdls, dta_typ, latent_dim, epochs, mode)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c6cb7a4-631c-41c1-b4fc-94ec85f279e1",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "#### LogReg string column fix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e765191a-0b48-4ee1-8e60-a5b65d7db68b",
   "metadata": {},
   "outputs": [],
   "source": [
    "dta_typ_obj.columns = dta_typ_obj.columns.astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6305adf0-f89a-4127-8219-1bf5540ac485",
   "metadata": {},
   "outputs": [],
   "source": [
    "dta_typ_obj.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26a76859-1e64-41e9-a10c-746ad86a4ff9",
   "metadata": {},
   "outputs": [],
   "source": [
    "dta_typ_obj.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8abe740-c7fb-43f2-b4e0-0cf65892b059",
   "metadata": {},
   "outputs": [],
   "source": [
    "f1_stor_frm = log_reg(dta_typ_obj, mode) # Single reuse function, no errors expected (?)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c9adc195-4962-4edb-b018-c3f3e8028346",
   "metadata": {},
   "source": [
    "## LogReg cancer-type plot - TCGA colors #3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "696a1786-ece0-4064-ae0f-2baf028a3831",
   "metadata": {},
   "outputs": [],
   "source": [
    "mode_ttl = 'cancer type'\n",
    "mode = 'cancer_type'\n",
    "f1_stor_frm = log_reg(dta_typ_obj, mode)\n",
    "tcga_colors = pd.read_csv('plot_color_files/tcga_colors.tsv',\n",
    "                          sep = '\\t')\n",
    "tcga_color_mapping = dict(zip(tcga_colors['tcga_cohorts'], tcga_colors['cohort_color']))\n",
    "unique_labels = f1_stor_frm['Label'].unique()\n",
    "palette_dict = {}\n",
    "for label in unique_labels:\n",
    "    tcga_abbreviation = cancer_type_abbreviation_mapping.get(label)\n",
    "    color = tcga_color_mapping.get(tcga_abbreviation)\n",
    "    if color:\n",
    "        palette_dict[label] = color\n",
    "def lgrg_plt_fnc(f1_stor_frm, mdls, data_name, latent_dim, epochs, mode):\n",
    "    plt.figure(figsize=(8, 4))\n",
    "    sns.set_style(\"whitegrid\")\n",
    "\n",
    "    ax = sns.barplot(\n",
    "        x='Label', y='F1_Score', data=f1_stor_frm,\n",
    "        palette=palette_dict,\n",
    "        errorbar=None)\n",
    "    sns.swarmplot(x='Label', y='F1_Score', data=f1_stor_frm,\n",
    "                  color='#333333', size=7)\n",
    "    \n",
    "    plt.suptitle('Logistic regression, '+mode_ttl+', '+dta_ttl,\n",
    "             fontsize=24, y = 1.04)\n",
    "    plt.title(mdls_ttl+encdg_stts_ttl, fontsize=20)\n",
    "    plt.xlabel('Cancer type', fontsize=20)\n",
    "    plt.ylabel('F1 Score', fontsize=20)\n",
    "    \n",
    "    plt.xticks(rotation=45, ha = 'right', fontsize = 16)\n",
    "    plt.grid(color='grey', linestyle='-', linewidth=0.25, alpha=0.5)\n",
    "    plt.rcParams['text.usetex'] = True\n",
    "    plt.savefig(rslts_dir+'/log_reg_'+mode+'_'+dta_typ+'_'+v+'.png',\n",
    "                bbox_inches='tight')\n",
    "lgrg_plt_fnc(f1_stor_frm, mdls, dta_typ, latent_dim, epochs, mode)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac8f826a-277f-4292-becd-62ccf160b95d",
   "metadata": {},
   "source": [
    "## Euclidean, #4 & 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "492266ea-34dc-45cf-a783-e878dcef07d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Euclicean distance, model type\n",
    "def mdl_typ_dist(sample, features, df):\n",
    "    other_types = df[df['model_type'] != sample['model_type']]\n",
    "    mean_features_other_types = other_types[features].mean()\n",
    "    distance = euclidean(sample[features], mean_features_other_types)\n",
    "    return distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a8f1d21-f487-4275-aa1f-957886b9d682",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Euclidean distance, cancer type\n",
    "def cncr_typ_dist(sample, features, df):\n",
    "    other_types = df[df['cancer_type'] != sample['cancer_type']]\n",
    "    mean_features_other_types = other_types[features].mean()\n",
    "    distance = euclidean(sample[features], mean_features_other_types)\n",
    "    return distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3636e1c7-d857-435a-aed1-0d593a5a48e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# dta_typ_obj = pd.read_csv('data/'+file,\n",
    "#                    sep = '\\t', index_col = 0)\n",
    "feature_columns = dta_typ_obj.columns[2:]\n",
    "dta_typ_obj['mdl_typ_dstncs'] = dta_typ_obj.apply(\n",
    "    lambda row: mdl_typ_dist(row, feature_columns, dta_typ_obj), axis=1)\n",
    "dta_typ_obj['cncr_typ_dstncs'] = dta_typ_obj.apply(\n",
    "    lambda row: cncr_typ_dist(row, feature_columns, dta_typ_obj), axis=1)\n",
    "new_cols = ['cancer_type', 'model_type', 'cncr_typ_dstncs', 'mdl_typ_dstncs'] + list(feature_columns)\n",
    "dta_typ_obj = dta_typ_obj[new_cols]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "314c0745-a57f-45cb-bb38-a99ed06f8ca1",
   "metadata": {},
   "source": [
    "## Eucldn Colrs, #4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15c52a23-2a96-4b4f-b6b1-71ad0dcd7d1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "mode_ttl = 'cancer type'\n",
    "mode = 'cancer_type'\n",
    "dstnc_typ = 'cncr_typ_dstncs'\n",
    "average_distances = dta_typ_obj.groupby(\n",
    "    mode)[dstnc_typ].mean().sort_values(ascending=False)\n",
    "sorted_df = dta_typ_obj.loc[dta_typ_obj[mode].isin(average_distances.index)]\n",
    "sorted_df[mode] = pd.Categorical(\n",
    "    sorted_df[mode], categories=average_distances.index, ordered=True)\n",
    "sorted_df = sorted_df.sort_values(mode)\n",
    "tcga_colors = pd.read_csv('plot_color_files/tcga_colors.tsv', sep = '\\t', index_col = 0)\n",
    "custom_color_list = [tcga_colors.loc[cancer_type_abbreviation_mapping[cancer_type],'cohort_color'] for cancer_type in average_distances.index]\n",
    "custom_colormap = ListedColormap(custom_color_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2e76ec8-e1c6-4e44-bd74-18a50c378649",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_values = np.linspace(0.75, 0.068, 7)\n",
    "plt.figure()\n",
    "joyplot(data=sorted_df[[mode, dstnc_typ]], by=mode,\n",
    "    figsize=(10, 6.5), colormap=custom_colormap,\n",
    "    fade=True)\n",
    "\n",
    "for y_value, cancer_type in zip(y_values, sorted_df[mode].unique()):\n",
    "    count = dict(sorted_df[mode].value_counts())[cancer_type]\n",
    "    x_position = sorted_df[dstnc_typ].max()\n",
    "    plt.annotate(f\"n={count}\", xy=(x_position, y_value), verticalalignment='center')\n",
    "\n",
    "plt.suptitle('Euclidean Distances, '+mode_ttl+', '+dta_ttl,\n",
    "             fontsize=30, y = 1.01)\n",
    "plt.title(mdls_ttl+ ', n = '+str(dta_typ_obj.shape[0])+encdg_stts_ttl,\n",
    "          y = .92, x = .22, fontsize = 26)\n",
    "\n",
    "plt.rcParams['text.usetex'] = True\n",
    "\n",
    "plt.annotate(\n",
    "    r'Variance of means: $\\mathbf{' + f'{average_distances.var():.3f}' + '}$',\n",
    "    xy=(0.01, 0.87), xycoords='axes fraction',\n",
    "    ha='right', va='top')\n",
    "\n",
    "plt.savefig(rslts_dir+'/euc-rdgln_'+mode+'_'+mdls+'_'+dta_typ+'_'+v+'.png',\n",
    "            bbox_inches = 'tight', dpi = 300)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c327270-f435-4b82-841b-969397c18e8c",
   "metadata": {},
   "source": [
    "### Build grey Euc ridge obj"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e864565c-0cee-4278-b871-f3c9b866456c",
   "metadata": {},
   "outputs": [],
   "source": [
    "mode_ttl = 'model type'\n",
    "mode = 'model_type'\n",
    "dstnc_typ = 'mdl_typ_dstncs'\n",
    "\n",
    "abbreviation_mapping = {\n",
    "    'cell line': 'cell line',\n",
    "    'Tumor': 'Tumor',\n",
    "    'HCMI': 'HCMI', # devel\n",
    "}\n",
    "\n",
    "average_distances = dta_typ_obj.groupby(\n",
    "    mode)[dstnc_typ].mean().sort_values(ascending=False)\n",
    "sorted_df = dta_typ_obj.loc[dta_typ_obj[mode].isin(average_distances.index)]\n",
    "sorted_df[mode] = pd.Categorical(\n",
    "    sorted_df[mode], categories=average_distances.index, ordered=True)\n",
    "sorted_df = sorted_df.sort_values(mode)\n",
    "custom_color_list = [grey_colors.loc[\n",
    "                     abbreviation_mapping[\n",
    "                     model_type],'quant_mode_color'] for model_type in average_distances.index]\n",
    "custom_colormap = ListedColormap(custom_color_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "50e998d4-93f3-43fc-b717-6697232c22e0",
   "metadata": {},
   "source": [
    "### Euc ridge grey plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9be6bae3-77a0-4d7c-9998-040c013345b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_values = np.linspace(0.52, 0.15, 2)\n",
    "plt.figure()\n",
    "joyplot(data=sorted_df[[mode, dstnc_typ]], by=mode,\n",
    "    figsize=(10, 6.5), colormap=custom_colormap,\n",
    "    fade=True)\n",
    "\n",
    "for y_value, cancer_type in zip(y_values, sorted_df[mode].unique()):\n",
    "    count = dict(sorted_df[mode].value_counts())[cancer_type]\n",
    "    x_position = sorted_df[dstnc_typ].max()\n",
    "    plt.annotate(f\"n={count}\", xy=(x_position, y_value), verticalalignment='center', fontsize = 24)\n",
    "\n",
    "plt.suptitle('Euclidean Distances, '+mode_ttl+', '+dta_ttl,\n",
    "             fontsize=30, y = 1.06)\n",
    "plt.title(mdls_ttl+ ', n = '+str(dta_typ_obj.shape[0])+encdg_stts_ttl,\n",
    "          y = .97, x = .4, fontsize = 26)\n",
    "\n",
    "plt.rcParams['text.usetex'] = True\n",
    "\n",
    "plt.annotate(\n",
    "    r'Variance of means: $\\mathbf{' + f'{average_distances.var():.3f}' + '}$',\n",
    "    xy=(0.2, 0.87), xycoords='axes fraction',\n",
    "    ha='right', va='top')\n",
    "\n",
    "plt.savefig(rslts_dir+'/euc-rdgln_'+mode+'_'+mdls+'_'+dta_typ+'_'+v+'.png',\n",
    "            bbox_inches = 'tight', dpi = 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4e313b1-95a8-451b-a9b5-ec03ca338fb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "mode_ttl = 'model type'\n",
    "mode = 'model_type'\n",
    "dist_typ = 'mdl_typ_dstncs'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d6fb36c-f3fa-466e-b721-dda2eb812fd7",
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted_df, custom_colormap = avrg_dstnc(mode, dstnc_typ)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "242c2ff0-6f1b-4370-bfd2-1425c34708a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "abbreviation_mapping = {\n",
    "    'cell line': 'cell line',\n",
    "    'Tumor': 'Tumor',\n",
    "    'HCMI': 'HCMI', # test\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13836b34-3c0c-4623-b0d5-496123ddee95",
   "metadata": {},
   "outputs": [],
   "source": [
    "average_distances = dta_typ_obj.groupby(\n",
    "    mode)[dstnc_typ].mean().sort_values(ascending=False)\n",
    "sorted_df = dta_typ_obj.loc[dta_typ_obj[mode].isin(average_distances.index)]\n",
    "sorted_df[mode] = pd.Categorical(\n",
    "    sorted_df[mode], categories=average_distances.index, ordered=True)\n",
    "sorted_df = sorted_df.sort_values(mode)\n",
    "tcga_colors = pd.read_csv('plot_color_files/tcga_colors.tsv', sep = '\\t', index_col = 0) #\n",
    "custom_color_list = [grey_colors.loc[\n",
    "                     abbreviation_mapping[\n",
    "                     model_type],'quant_mode_color'] for model_type in average_distances.index]\n",
    "custom_colormap = ListedColormap(custom_color_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "53481f46-3058-4c74-b10e-404fd555a720",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_values = np.linspace(0.52, 0.15, 2)\n",
    "plt.figure()\n",
    "joyplot(data=sorted_df[[mode, dstnc_typ]], by=mode,\n",
    "    figsize=(10, 6.5), colormap=custom_colormap,\n",
    "    fade=True)\n",
    "\n",
    "for y_value, cancer_type in zip(y_values, sorted_df[mode].unique()):\n",
    "    count = dict(sorted_df[mode].value_counts())[cancer_type]\n",
    "    x_position = sorted_df[dstnc_typ].max()\n",
    "    plt.annotate(f\"n={count}\", xy=(x_position, y_value), verticalalignment='center', fontsize = 24)\n",
    "\n",
    "plt.suptitle('Euclidean Distances, '+mode_ttl+', '+dta_ttl,\n",
    "             fontsize=30, y = 1.06)\n",
    "plt.title(mdls_ttl+ ', n = '+str(dta_typ_obj.shape[0]),\n",
    "          y = .97, x = .4, fontsize = 26)\n",
    "\n",
    "plt.rcParams['text.usetex'] = True\n",
    "\n",
    "plt.annotate(\n",
    "    r'Variance of means: $\\mathbf{' + f'{average_distances.var():.3f}' + '}$',\n",
    "    xy=(0.2, 0.87), xycoords='axes fraction',\n",
    "    ha='right', va='top')\n",
    "\n",
    "plt.savefig('results_encdd/rdgln_'+mode+'_'+mdls+'_'+dta_typ+'_v0.2.png',\n",
    "            bbox_inches = 'tight', dpi = 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10d21cd8-40fa-41ff-82af-d2fff5ed1463",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}