projector.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tensorboardX import SummaryWriter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('tmp/vectors/bert_all.csv.gz', compression='gzip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['vectors'] = df['vectors'].map(eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['opencorpora', 'criminal_code', 'criminal_court_orders',\n",
       "       'civil_code', 'civil_court_orders'], dtype=object)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['source'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = []\n",
    "for source in df['source'].unique():\n",
    "    tmp = df[df['source'] == source]\n",
    "    if source in ['opencorpora', 'criminal_court_orders', 'civil_court_orders']:\n",
    "        tmp = tmp.sample(500)\n",
    "    arr.append(tmp)\n",
    "tmp = pd.concat(arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "codes = tmp[tmp['source'].isin(['criminal_code', 'civil_code'])]\n",
    "civil_code = tmp[tmp['source'].isin(['civil_code'])]\n",
    "criminal_code = tmp[tmp['source'].isin(['criminal_code'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "with SummaryWriter() as writer:\n",
    "    for d, name in zip([tmp, codes, civil_code, criminal_code], [\"all\", \"all_codes\", \"civil_code\", \"criminal_code\"]):\n",
    "        vectors = np.array(d['vectors'].tolist())\n",
    "        meta_header = ['name', 'source']\n",
    "        metadata = d[meta_header].applymap(lambda x: '' if pd.isnull(x) else str(x)).values.tolist()\n",
    "        writer.add_embedding(mat=vectors, metadata=metadata, tag=name, metadata_header=meta_header)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "TensorBoard 1.12.0 at http://tensor-flow-lh.test.ru.kworld.kpmg.com:20017 (Press CTRL+C to quit)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorboard import default\n",
    "from tensorboard import program\n",
    "\n",
    "tb = program.TensorBoard()\n",
    "tb.configure(argv=[None, '--logdir', 'runs', '--port', '20017'])\n",
    "tb.main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6 (testing)",
   "language": "python",
   "name": "testing"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}