semantic_search_mvp.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ryderwishart/biblical-machine-learning/blob/main/semantic_search_mvp.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install gensim"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KKKv_sTLfl34",
        "outputId": "bf3dcc9d-1280-41fe-f087-43ad2340fbf2"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: gensim in /usr/local/lib/python3.9/dist-packages (4.3.1)\n",
            "Requirement already satisfied: scipy>=1.7.0 in /usr/local/lib/python3.9/dist-packages (from gensim) (1.10.1)\n",
            "Requirement already satisfied: numpy>=1.18.5 in /usr/local/lib/python3.9/dist-packages (from gensim) (1.22.4)\n",
            "Requirement already satisfied: smart-open>=1.8.1 in /usr/local/lib/python3.9/dist-packages (from gensim) (6.3.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "0xWPCjQBfU9E"
      },
      "outputs": [],
      "source": [
        "from gensim.models.doc2vec import Doc2Vec, TaggedDocument\n",
        "from gensim.corpora.dictionary import Dictionary\n",
        "import os\n",
        "import re"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Download the KJV text file\n",
        "filename = 'pg10.txt'\n",
        "if filename not in [path for path in os.listdir()]:\n",
        "    !wget -q 'https://www.gutenberg.org/cache/epub/10/pg10.txt'\n",
        "os.listdir()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X09xIFirffK8",
        "outputId": "f9ffe0d8-6753-4b6e-bc08-7ebc0058e130"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['.config', 'pg10.txt', 'sample_data']"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# stop_words = ['the', 'and', 'of', 'to', 'And', 'that', 'in', 'shall', 'he', 'unto', 'I', 'his', 'a', 'for', 'they', 'be', 'is', 'him', 'not', 'them', 'with', 'it', 'all', 'thou', 'was', 'thy', 'which', 'my', 'me', 'said', 'their', 'have', 'thee', 'will', 'ye', 'from', 'as', 'are', 'were', 'out', 'upon', 'you', 'by', 'when', 'this', 'but']"
      ],
      "metadata": {
        "id": "sCVKnLFnpsgb"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Load the KJV text file and extract the sentences\n",
        "sentences = []\n",
        "with open(filename, 'r') as f:\n",
        "    # open the whole text so the verses can be properly split\n",
        "    text = f.read()\n",
        "    paragraphs = text.split('\\n\\n') # blank line for paragraph\n",
        "    for paragraph in paragraphs:\n",
        "        # Extract paragraphs that begin with a verse reference\n",
        "        if re.match(r'\\d+:\\d+', paragraph):\n",
        "            # Clean the sentence by removing the verse reference number and punctuation\n",
        "            sentence = paragraph.replace('\\n', ' ').strip().split(' ')[1:]\n",
        "            sentence = [word.strip('.,!?;:-') for word in sentence]\n",
        "            sentences.append(sentence)\n",
        "\n",
        "print('Sentences in corpus: ', len(sentences))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CRHwPOatfgh-",
        "outputId": "29dd350d-d09a-403a-a5a9-42aced19ef10"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sentences in corpus:  24337\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Count the frequency of each word in the corpus\n",
        "word_counts = {}\n",
        "for sentence in sentences:\n",
        "    for word in sentence:\n",
        "        if word not in word_counts:\n",
        "            word_counts[word] = 1\n",
        "        else:\n",
        "            word_counts[word] += 1\n",
        "\n",
        "stop_words = [word for word, count in sorted(word_counts.items(), key=lambda item: item[1], reverse=True)[:50]]\n",
        "exceptions = ['LORD', 'Israel', 'man', 'God']\n",
        "stop_words = [word for word in stop_words if word not in exceptions]\n",
        "print(stop_words)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JedlEit_nxcv",
        "outputId": "f6549673-7f32-41df-82a6-eb0634df748f"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['the', 'and', 'of', 'to', 'And', 'that', 'in', 'shall', 'he', 'unto', 'I', 'his', 'a', 'for', 'they', 'be', 'is', 'him', 'not', 'them', 'with', 'it', 'all', 'thou', 'was', 'thy', 'which', 'my', 'me', 'said', 'their', 'have', 'thee', 'will', 'ye', 'from', 'as', 'are', 'were', 'out', 'upon', 'you', 'by', 'when', 'this', 'but']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Build the vocabulary of words that appear in the corpus\n",
        "dictionary = Dictionary(sentences)"
      ],
      "metadata": {
        "id": "X6RolMX5gwMn"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Train the Doc2Vec model on the sentences\n",
        "documents = [TaggedDocument(\n",
        "    words=[word for word in sentence if word not in stop_words], \n",
        "    tags=[str(i)]) for i, sentence in enumerate(sentences)\n",
        "]\n",
        "model = Doc2Vec(\n",
        "    documents=documents, \n",
        "    vector_size=100, \n",
        "    window=5, \n",
        "    min_count=5, \n",
        "    workers=4, \n",
        "    epochs=2\n",
        ")\n",
        "model.build_vocab(documents)"
      ],
      "metadata": {
        "id": "UjpKHr4vfh2S",
        "outputId": "6bdc4212-c43b-4542-eb3a-721b67eb2b08",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:gensim.models.keyedvectors:sorting after vectors have been allocated is expensive & error-prone\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.train(documents, total_examples=model.corpus_count, epochs=model.epochs)"
      ],
      "metadata": {
        "id": "q77rgvxniURS",
        "outputId": "b4a2208e-b2c0-400f-cace-31afb1b4312d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:gensim.models.word2vec:Effective 'alpha' higher than previous training cycles\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Get the five most similar sentences to an input sentence\n",
        "input_sentence = \"Love your neighbor\"\n",
        "vector = model.infer_vector(input_sentence.split())\n",
        "similar_sentences = model.docvecs.most_similar(positive=[vector], topn=10)"
      ],
      "metadata": {
        "id": "u6Rm9XdbfjGz",
        "outputId": "7eb1a6d4-8b1f-44be-ffce-fbc83337a646",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-12-b98e1dab7848>:4: DeprecationWarning: Call to deprecated `docvecs` (The `docvecs` property has been renamed `dv`.).\n",
            "  similar_sentences = model.docvecs.most_similar(positive=[vector], topn=10)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Print the similar sentences\n",
        "print('*NOTE: this model is trained on a tiny, toy corpus (the KJV). The results cannot be good, and this should serve as a proof of concept only.\\n')\n",
        "for index, similarity in similar_sentences:\n",
        "    print(\"{:.2f}\".format(similarity), ' '.join(sentences[int(index)]))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4EXSgDNGfkOk",
        "outputId": "da0d8328-7639-4db3-cee8-f3bd69317ac7"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "*NOTE: this model is trained on a tiny, toy corpus (the KJV). The results cannot be good, and this should serve as a proof of concept only.\n",
            "\n",
            "0.75 And this is the number of them thirty chargers of gold a thousand chargers of silver nine and twenty knives 1:10 Thirty basons of gold silver basons of a second sort four hundred and ten and other vessels a thousand\n",
            "0.75 Then I heard one saint speaking and another saint said unto that certain saint which spake How long shall be the vision concerning the daily sacrifice and the transgression of desolation to give both the sanctuary and the host to be trodden under foot  8:14 And he said unto me Unto two thousand and three hundred days then shall the sanctuary be cleansed\n",
            "0.74 And the oracle in the forepart was twenty cubits in length and twenty cubits in breadth and twenty cubits in the height thereof and he overlaid it with pure gold and so covered the altar which was of cedar\n",
            "0.74 And he made two cherubims of gold beaten out of one piece made he them on the two ends of the mercy seat 37:8 One cherub on the end on this side and another cherub on the other end on that side out of the mercy seat made he the cherubims on the two ends thereof\n",
            "0.74 And he built twenty cubits on the sides of the house both the floor and the walls with boards of cedar he even built them for it within even for the oracle even for the most holy place\n",
            "0.74 And the men of Israel beside Benjamin were numbered four hundred thousand men that drew sword all these were men of war\n",
            "0.73 And three hundred shields made he of beaten gold three hundred shekels of gold went to one shield And the king put them in the house of the forest of Lebanon\n",
            "0.73 And he made ten bases of brass four cubits was the length of one base and four cubits the breadth thereof and three cubits the height of it\n",
            "0.73 Zedekiah was twenty and one years old when he began to reign and he reigned eleven years in Jerusalem And his mother’s name was Hamutal the daughter of Jeremiah of Libnah\n",
            "0.73 And he made the altar of burnt offering of shittim wood five cubits was the length thereof and five cubits the breadth thereof it was foursquare and three cubits the height thereof\n"
          ]
        }
      ]
    }
  ]
}