load_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data from Vizier\n",
    "\n",
    "#### Index<a name=\"index\"></a>\n",
    "1. [Import packages](#imports)\n",
    "2. [Load raw data](#loadData)\n",
    "3. [Create examples folder](#createFolder)\n",
    "4. [Create and save data](#createData)\n",
    "    1. [True clusters](#createTrue)\n",
    "    1. [False clusters](#createFalse)\n",
    "\n",
    "## 1. Import packages<a name=\"imports\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "\n",
    "from astroquery.vizier import Vizier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Aestetic settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "sns.set(font_scale=1.3, style=\"ticks\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Load raw data<a name=\"loadData\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Vizier.ROW_LIMIT = 3000000\n",
    "catalogs = Vizier.get_catalogs('J/A+A/618/A93/members')\n",
    "print(catalogs)  # astroquery.utils.commons.TableList"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since `catalogs` only has 1 table, we are only interested in `catalogs[0]`. We will transform it to a `pandas` Data Frame for ease of handling."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "catalog = catalogs[0].to_pandas()\n",
    "catalog = catalog.applymap(lambda x: x.decode() if isinstance(x, bytes) else x)\n",
    "catalog.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Create examples folder<a name=\"createFolder\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists('examples'):\n",
    "    os.makedirs('examples')\n",
    "if not os.path.exists('neg_examples'):\n",
    "    os.makedirs('neg_examples')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Create and save data<a name=\"createData\"></a>\n",
    "\n",
    "First choose if you want to see in the notebook the scatter plots and histograms created."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_scatter = True\n",
    "plot_hist = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.1. True clusters<a name=\"createTrue\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clusters = np.unique(catalog['Cluster'])\n",
    "number_clusters = len(clusters)\n",
    "\n",
    "true_clusters_2d_hist = []\n",
    "for cluster in clusters:\n",
    "    is_cluster = catalog['Cluster'] == cluster\n",
    "    data_cluster = catalog[is_cluster]\n",
    "    \n",
    "    # Remove stars for which Gaia didn't measure magnitude or colour\n",
    "    gmag = data_cluster['Gmag']\n",
    "    bp_rp = data_cluster['BP-RP']\n",
    "    is_nan = (np.isnan(bp_rp)) | (np.isnan(gmag))\n",
    "    data_cluster = data_cluster[~is_nan]\n",
    "    \n",
    "    gmag = data_cluster['Gmag']\n",
    "    bp_rp = data_cluster['BP-RP']\n",
    "    \n",
    "    if plot_scatter:\n",
    "        plt.scatter(bp_rp, gmag, marker='.')\n",
    "        plt.xlabel(r'$B_p - R_p$ (mag)')\n",
    "        plt.ylabel(r'$G$ (mag)')\n",
    "        plt.title(f'Cluster {cluster}')\n",
    "        plt.ylim([18,8])\n",
    "        plt.show()\n",
    "\n",
    "    hist, xedges, yedges = np.histogram2d(bp_rp, gmag, bins=20, \n",
    "                                          density=True)\n",
    "\n",
    "    plt.imshow(hist, interpolation='nearest')\n",
    "    plt.title(f'Cluster {cluster}')\n",
    "    plt.savefig(f'examples/{cluster}.jpg', bbox_inches='tight')\n",
    "    if plot_hist:\n",
    "        plt.show()\n",
    "    plt.close()\n",
    "\n",
    "    hist_linear = hist.reshape((400))\n",
    "    true_clusters_2d_hist.append(hist_linear)\n",
    "\n",
    "true_clusters_2d_hist_file = f'true_clusters_2d_hist_{number_clusters}.npy'\n",
    "#np.save(true_clusters_2d_hist_file, true_clusters_2d_hist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(true_clusters_2d_hist_file, true_clusters_2d_hist)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Confirm the file was well saved by loading it. If the output of the cell is False, the saving went wrong."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "true_clusters_2d_hist_saved = np.load(true_clusters_2d_hist_file, allow_pickle=True)\n",
    "np.allclose(true_clusters_2d_hist_saved, true_clusters_2d_hist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Number of positive samples: ', len(true_clusters_2d_hist))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.2. False clusters<a name=\"createFalse\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "false_clusters_2d_hist = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range (0, number_clusters):\n",
    "    number_stars = int(random.uniform (30, 500)) \n",
    "    gmag = np.random.uniform (8, 18, size=number_stars)\n",
    "    bp_rp = np.random.uniform (0, 5, size=number_stars)\n",
    "    \n",
    "    hist, xedges, yedges = np.histogram2d(bp_rp, gmag, bins=20, \n",
    "                                          density=True)\n",
    "\n",
    "    if i <= 10:\n",
    "        plt.imshow(hist, interpolation='nearest')\n",
    "        plt.title(f'False Cluster {i}')\n",
    "        plt.savefig(f'neg_examples/false_cluster_{i}.jpg', \n",
    "                    bbox_inches='tight')\n",
    "    \n",
    "    if plot_hist:\n",
    "        plt.show()\n",
    "    plt.close()\n",
    "    \n",
    "    hist_linear = hist.reshape((400))\n",
    "    false_clusters_2d_hist.append(hist_linear)\n",
    "    \n",
    "false_clusters_2d_hist_file = f'false_clusters_2d_hist_{number_clusters}.npy'\n",
    "#np.save(false_clusters_2d_hist_file, false_clusters_2d_hist)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Confirm the file was well saved by loading it. If the output of the cell is False, the saving went wrong."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "false_clusters_2d_hist_saved = np.load(false_clusters_2d_hist_file, allow_pickle=True)\n",
    "np.allclose(false_clusters_2d_hist_saved, false_clusters_2d_hist)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Go to [Index](#index)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}