create_ddsm_image_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "from mammo_utils import convert_images_to_array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# read in the label data\n",
    "labels_tr = pd.read_pickle(os.path.join(\"data\",\"train_labels.pkl\"))\n",
    "# reset the index, drop duplicate rows, and then reset the index\n",
    "labels_tr = labels_tr.reset_index().drop_duplicates(subset='IMAGE_NAME', keep='first', inplace=False)\n",
    "labels_tr = labels_tr.set_index(\"IMAGE_NAME\")\n",
    "\n",
    "labels_te = pd.read_pickle(os.path.join(\"data\",\"test_labels.pkl\"))\n",
    "# reset the index, drop duplicate rows, and then reset the index\n",
    "labels_te = labels_te.reset_index().drop_duplicates(subset='IMAGE_NAME', keep='first', inplace=False)\n",
    "labels_te = labels_te.set_index(\"IMAGE_NAME\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert the images to an array\n",
    "mass_train_images, mass_train_labels = convert_images_to_array(\"data/new/Mass Train/AllJPEGS299\", label_data=labels_tr)\n",
    "calc_train_images, calc_train_labels = convert_images_to_array(\"data/new/Calc Train/AllJPEGS299\", label_data=labels_tr)\n",
    "\n",
    "train_images = np.concatenate([mass_train_images, calc_train_images], axis=0)\n",
    "train_labels = np.concatenate([mass_train_labels, calc_train_labels], axis=0)\n",
    "\n",
    "# discard the last two channels since the images are grayscale and all channels are identical\n",
    "train_images = train_images[:,:,:,0].reshape(-1,299,299,1)\n",
    "\n",
    "# save the files\n",
    "np.save(os.path.join(\"data\", \"train_images299.npy\"), train_images)\n",
    "np.save(os.path.join(\"data\", \"train_labels.npy\"), train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Labels: 2458\n",
      "Train Data: (2458, 299, 299, 1)\n"
     ]
    }
   ],
   "source": [
    "print(\"Train Labels:\",len(train_labels))\n",
    "print(\"Train Data:\", train_images.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert the images to arrays\n",
    "mass_test_images, mass_test_labels = convert_images_to_array(\"data/new/Mass Test/AllJPEGS299\", label_data=labels_te)\n",
    "calc_test_images, calc_test_labels = convert_images_to_array(\"data/new/Calc Test/AllJPEGS299\", label_data=labels_te)\n",
    "\n",
    "test_images = np.concatenate([mass_test_images, calc_test_images], axis=0)\n",
    "test_labels = np.concatenate([mass_test_labels, calc_test_labels], axis=0)\n",
    "\n",
    "# discard the last two channels since the images are grayscale and all channels are identical\n",
    "test_images = test_images[:,:,:,0].reshape(-1,299,299,1)\n",
    "\n",
    "\n",
    "# save the file\n",
    "np.save(os.path.join(\"data\", \"test_images299.npy\"), test_images)\n",
    "np.save(os.path.join(\"data\", \"test_labels.npy\"), test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Labels: 643\n",
      "Test Data: (643, 299, 299, 1)\n"
     ]
    }
   ],
   "source": [
    "print(\"Test Labels:\",len(test_labels))\n",
    "print(\"Test Data:\", test_images.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_labels = np.load(\"data/train_labels.npy\")\n",
    "test_labels = np.load(\"data/test_labels.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_labels[train_labels == 'BENIGN_WITHOUT_CALLBACK_mass'] = \"BENIGN_mass\"\n",
    "train_labels[train_labels == 'BENIGN_WITHOUT_CALLBACK_calcification'] = \"BENIGN_calcification\"\n",
    "\n",
    "test_labels[test_labels == 'BENIGN_WITHOUT_CALLBACK_mass'] = \"BENIGN_mass\"\n",
    "test_labels[test_labels == 'BENIGN_WITHOUT_CALLBACK_calcification'] = \"BENIGN_calcification\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.save(\"data/train_labels.npy\", train_labels)\n",
    "np.save(\"data/test_labels.npy\", test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BENIGN_mass                0.357724\n",
       "BENIGN_calcification       0.255285\n",
       "MALIGNANT_mass             0.235772\n",
       "MALIGNANT_calcification    0.151220\n",
       "dtype: float64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.value_counts(test_labels, normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}