bert-embeddings.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!test -f newsticker2019.db || wget https://datanizing.com/bert/heise/newsticker2019.db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sqlite3\n",
    "sql = sqlite3.connect(\"newsticker2019.db\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install pytorch_pretrained_bert\n",
    "import torch\n",
    "from pytorch_pretrained_bert import BertTokenizer, BertModel, BertForMaskedLM\n",
    "\n",
    "# OPTIONAL: if you want to have more information on what's happening, activate the logger as follows\n",
    "import logging\n",
    "#logging.basicConfig(level=logging.INFO)\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "# uncomment for GPU\n",
    "#n_gpu = torch.cuda.device_count()\n",
    "#torch.cuda.get_device_name(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = pd.read_sql(\"SELECT text, time FROM news\", sql, parse_dates=[\"time\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load pre-trained model tokenizer (vocabulary)\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-uncased')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tokenization, model evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load pre-trained model (weights)\n",
    "model = BertModel.from_pretrained('bert-base-multilingual-uncased')\n",
    "\n",
    "# Put the model in \"evaluation\" mode, meaning feed-forward operation.\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "tokenized_texts = []\n",
    "indexed_tokens = []\n",
    "encoded_layers = []\n",
    "for text in tqdm(texts[\"text\"]):\n",
    "    marked_text = \"[CLS] \" + str(text) + \" [SEP]\"\n",
    "    tokenized = tokenizer.tokenize(marked_text)\n",
    "    tokenized_texts.append(tokenized)\n",
    "    indexed = tokenizer.convert_tokens_to_ids(tokenized)\n",
    "    indexed_tokens.append(indexed)\n",
    "    # segments => all in one sentence\n",
    "    segments_ids = [1] * len(tokenized)\n",
    "    # Convert inputs to PyTorch tensors\n",
    "    tokens_tensor = torch.tensor([indexed])\n",
    "    segments_tensors = torch.tensor([segments_ids])\n",
    "    # Predict hidden states features for each layer\n",
    "    with torch.no_grad():\n",
    "        el, _ = model(tokens_tensor, segments_tensors)\n",
    "    encoded_layers.append(el)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BERT embedding für jede Headline berechnen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "headline_embeddings = []\n",
    "for e in encoded_layers:\n",
    "    token_vecs = e[11][0]\n",
    "    embedding = torch.mean(token_vecs, dim=0)\n",
    "    headline_embeddings.append(embedding.numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "# Cosinus-Ähnlichkeit aller Dokumente miteinander berechnen\n",
    "sim = cosine_similarity(headline_embeddings, headline_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ergebnis ist eine Matrix\n",
    "sim.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Größter Wert wird für identische Meldungen gefunden\n",
    "sim.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for threshhold in [0.7, 0.8, 0.9, 0.95]:\n",
    "    s = sim.copy()\n",
    "    # alle über dem Threshhold ausmaskieren\n",
    "    s[s > threshhold] = 0\n",
    "    (max_a, max_b) = np.unravel_index(np.argmax(s), s.shape)\n",
    "    print(\"%f '%s' '%s'\" % (threshhold, texts.iloc[max_a][\"text\"], texts.iloc[max_b][\"text\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Alternative Aufgabe: Ähnlichste Nachrichten in unterschiedlichen Monaten finden"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}