build_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b30148a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import json\n",
    "import re\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "import copy\n",
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "70959bce",
   "metadata": {},
   "outputs": [],
   "source": [
    "def dataloader_pipeline(file_list: list, desired_filename) -> list:\n",
    "    \"\"\"\n",
    "    Pipeline for loading in data\n",
    "    \n",
    "    @param file_list: A list of files to load in\n",
    "    @returns data: A list of playlists from these files\n",
    "    \"\"\"\n",
    "    # Resulting data (hopefully to be stored in list)\n",
    "    data = []\n",
    "    \n",
    "    # Traversing through available datafiles\n",
    "    print(\"Starting Dataloading...\")\n",
    "    for file in tqdm(file_list):\n",
    "        \n",
    "        # Ensuring filename valid\n",
    "        if desired_filename.match(file):\n",
    "            \n",
    "            # Opening and preprocessing\n",
    "            with open(\"./data/\" + file, 'r') as file_reader:\n",
    "                data += json.load(file_reader)[\"playlists\"]\n",
    "    \n",
    "    print(\"Finished Dataloading...\")\n",
    "   \n",
    "    return data\n",
    "\n",
    "def build_relevant_ds(songs: list):\n",
    "    \"\"\"\n",
    "    Preprocesses data, simultaneously building relevant data structures\n",
    "    \n",
    "    @param data - a data list of playlist dictionaries to preprocess\n",
    "    @returns a list of tracks per user, users per track, watered down data list\n",
    "    \"\"\"\n",
    "    \n",
    "    def process_uri(uri:str):\n",
    "        \"\"\"URI Processing method\"\"\"\n",
    "        return uri.split(\":\")[2]\n",
    "        \n",
    "    print(\"Preprocessing started...\")\n",
    "    tracks_per_user, users_per_track, users_per_artist, artists_per_user = defaultdict(list), defaultdict(list), defaultdict(list), defaultdict(list)\n",
    "    \n",
    "    # Traversing through data and preprocessing\n",
    "    for song in songs:\n",
    "\n",
    "        # Obtaining user\n",
    "        user = song['user']\n",
    "\n",
    "        # obtaining necessary data\n",
    "        track, artist, album = song['track_name'], song['artist_name'], song['album_name']\n",
    "\n",
    "        # Appending data to data structures\n",
    "        tracks_per_user[user].append(track)\n",
    "        users_per_track[track].append(user)\n",
    "        users_per_artist[artist].append(user)\n",
    "        artists_per_user[user].append(artist)\n",
    "\n",
    "    return tracks_per_user, users_per_track, users_per_artist, artists_per_user\n",
    "\n",
    "def enrich_song(song: dict, playlist: dict):\n",
    "    \"\"\"Playlist enrichment procedure\"\"\"\n",
    "    song['playlist_name'] = playlist['name']\n",
    "    song['playlist_mod_at'] = playlist['modified_at']\n",
    "    song['playlist_num_followers'] = playlist['num_followers']\n",
    "    song['user'] = playlist['pid']\n",
    "    song['listened'] = True\n",
    "    return song"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bffcdb9d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Dataloading...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 50/50 [00:06<00:00,  8.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Dataloading...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Listing directory\n",
    "files = os.listdir(\"./data\")\n",
    "\n",
    "# Regular expression for desired filenames\n",
    "desired_filename = re.compile(\"mpd.*\")\n",
    "\n",
    "# Loading data\n",
    "data = dataloader_pipeline(files, desired_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0a56188f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building Song Dataset...\n"
     ]
    }
   ],
   "source": [
    "# Constructing a song centric dataset\n",
    "print(\"Building Song Dataset...\")\n",
    "new_data = {\"data\": [enrich_song(song, playlist) for playlist in data for song in playlist['tracks']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "efe12cf1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3196773"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(new_data['data'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "05626641",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing data to file...\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[0;32mIn [7]\u001b[0m, in \u001b[0;36m<cell line: 3>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mWriting data to file...\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfull_song_data.json\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mw\u001b[39m\u001b[38;5;124m'\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m file_writer:\n\u001b[0;32m----> 4\u001b[0m     \u001b[43mjson\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdump\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnew_data\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfile_writer\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/json/__init__.py:179\u001b[0m, in \u001b[0;36mdump\u001b[0;34m(obj, fp, skipkeys, ensure_ascii, check_circular, allow_nan, cls, indent, separators, default, sort_keys, **kw)\u001b[0m\n\u001b[1;32m    173\u001b[0m     iterable \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mcls\u001b[39m(skipkeys\u001b[38;5;241m=\u001b[39mskipkeys, ensure_ascii\u001b[38;5;241m=\u001b[39mensure_ascii,\n\u001b[1;32m    174\u001b[0m         check_circular\u001b[38;5;241m=\u001b[39mcheck_circular, allow_nan\u001b[38;5;241m=\u001b[39mallow_nan, indent\u001b[38;5;241m=\u001b[39mindent,\n\u001b[1;32m    175\u001b[0m         separators\u001b[38;5;241m=\u001b[39mseparators,\n\u001b[1;32m    176\u001b[0m         default\u001b[38;5;241m=\u001b[39mdefault, sort_keys\u001b[38;5;241m=\u001b[39msort_keys, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkw)\u001b[38;5;241m.\u001b[39miterencode(obj)\n\u001b[1;32m    177\u001b[0m \u001b[38;5;66;03m# could accelerate with writelines in some versions of Python, at\u001b[39;00m\n\u001b[1;32m    178\u001b[0m \u001b[38;5;66;03m# a debuggability cost\u001b[39;00m\n\u001b[0;32m--> 179\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk \u001b[38;5;129;01min\u001b[39;00m iterable:\n\u001b[1;32m    180\u001b[0m     fp\u001b[38;5;241m.\u001b[39mwrite(chunk)\n",
      "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/json/encoder.py:431\u001b[0m, in \u001b[0;36m_make_iterencode.<locals>._iterencode\u001b[0;34m(o, _current_indent_level)\u001b[0m\n\u001b[1;32m    429\u001b[0m     \u001b[38;5;28;01myield from\u001b[39;00m _iterencode_list(o, _current_indent_level)\n\u001b[1;32m    430\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(o, \u001b[38;5;28mdict\u001b[39m):\n\u001b[0;32m--> 431\u001b[0m     \u001b[38;5;28;01myield from\u001b[39;00m _iterencode_dict(o, _current_indent_level)\n\u001b[1;32m    432\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    433\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m markers \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/json/encoder.py:405\u001b[0m, in \u001b[0;36m_make_iterencode.<locals>._iterencode_dict\u001b[0;34m(dct, _current_indent_level)\u001b[0m\n\u001b[1;32m    403\u001b[0m         \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    404\u001b[0m             chunks \u001b[38;5;241m=\u001b[39m _iterencode(value, _current_indent_level)\n\u001b[0;32m--> 405\u001b[0m         \u001b[38;5;28;01myield from\u001b[39;00m chunks\n\u001b[1;32m    406\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m newline_indent \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    407\u001b[0m     _current_indent_level \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n",
      "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/json/encoder.py:325\u001b[0m, in \u001b[0;36m_make_iterencode.<locals>._iterencode_list\u001b[0;34m(lst, _current_indent_level)\u001b[0m\n\u001b[1;32m    323\u001b[0m         \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    324\u001b[0m             chunks \u001b[38;5;241m=\u001b[39m _iterencode(value, _current_indent_level)\n\u001b[0;32m--> 325\u001b[0m         \u001b[38;5;28;01myield from\u001b[39;00m chunks\n\u001b[1;32m    326\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m newline_indent \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    327\u001b[0m     _current_indent_level \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n",
      "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/json/encoder.py:382\u001b[0m, in \u001b[0;36m_make_iterencode.<locals>._iterencode_dict\u001b[0;34m(dct, _current_indent_level)\u001b[0m\n\u001b[1;32m    380\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    381\u001b[0m     \u001b[38;5;28;01myield\u001b[39;00m item_separator\n\u001b[0;32m--> 382\u001b[0m \u001b[38;5;28;01myield\u001b[39;00m \u001b[43m_encoder\u001b[49m\u001b[43m(\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    383\u001b[0m \u001b[38;5;28;01myield\u001b[39;00m _key_separator\n\u001b[1;32m    384\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(value, \u001b[38;5;28mstr\u001b[39m):\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# Writing song centric dataset to file\n",
    "print(\"Writing data to file...\")\n",
    "with open('full_song_data.json', 'w') as file_writer:\n",
    "    json.dump(new_data, file_writer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "57b62640",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Splitting into train and test sets...\n"
     ]
    }
   ],
   "source": [
    "# Splitting up to train and test, writing train and test to their own files\n",
    "print(\"Splitting into train and test sets...\")\n",
    "my_data = new_data['data']\n",
    "np.random.shuffle(my_data)\n",
    "\n",
    "# Splitting train and test by threshold\n",
    "split_threshold = int(len(my_data) * (0.75))\n",
    "train = my_data[:800_000]\n",
    "\n",
    "# Attaining positive instances\n",
    "test_positives = my_data[800_000:1_000_000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d4c49518",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing training data to file...\n"
     ]
    }
   ],
   "source": [
    "# Writing song centric dataset to file\n",
    "print(\"Writing training data to file...\")\n",
    "with open('data_train.json', 'w') as file_writer:\n",
    "    json.dump(train, file_writer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "041ca65b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Set Size: 200000\n"
     ]
    }
   ],
   "source": [
    "print(f\"Test Set Size: {len(test_positives)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0a0405c1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing test data to file...\n"
     ]
    }
   ],
   "source": [
    "# Writing song centric dataset to file\n",
    "print(\"Writing test data to file...\")\n",
    "with open('data_test.json', 'w') as file_writer:\n",
    "    json.dump(test_positives, file_writer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4582c235",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "48ac449d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}