DataExtraction.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "DataExtraction.ipynb",
      "provenance": [],
      "mount_file_id": "1paWoyFMG0aK7PCnLpLLxwUxuP7-BxJeB",
      "authorship_tag": "ABX9TyPcAeNAuW53TkcCiAC2/28M",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/PadmajaVB/AI_Composer/blob/master/DataExtraction.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "59_TuW42KylM"
      },
      "source": [
        "### Script to get the data for everyday Sexism project"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h4D1R9JiNWiK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 820
        },
        "outputId": "11261e74-0941-4e88-c7aa-ff86047d6fb6"
      },
      "source": [
        "!pip install ipdb"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting ipdb\n",
            "  Downloading https://files.pythonhosted.org/packages/5f/43/eb2be141dac56e502b6e35c1e4a9b1bbb2d4dcbec773c0f6563e79758909/ipdb-0.13.8.tar.gz\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from ipdb) (56.1.0)\n",
            "Collecting ipython>=7.17.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/8d/84/de1b1488556909a759e4fed39e21a41d4bf63d06f88def27bf9cc2b75434/ipython-7.24.0-py3-none-any.whl (785kB)\n",
            "\u001b[K     |████████████████████████████████| 788kB 9.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: toml>=0.10.2 in /usr/local/lib/python3.7/dist-packages (from ipdb) (0.10.2)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.7/dist-packages (from ipdb) (4.4.2)\n",
            "Requirement already satisfied: pexpect>4.3; sys_platform != \"win32\" in /usr/local/lib/python3.7/dist-packages (from ipython>=7.17.0->ipdb) (4.8.0)\n",
            "Requirement already satisfied: matplotlib-inline in /usr/local/lib/python3.7/dist-packages (from ipython>=7.17.0->ipdb) (0.1.2)\n",
            "Requirement already satisfied: jedi>=0.16 in /usr/local/lib/python3.7/dist-packages (from ipython>=7.17.0->ipdb) (0.18.0)\n",
            "Requirement already satisfied: backcall in /usr/local/lib/python3.7/dist-packages (from ipython>=7.17.0->ipdb) (0.2.0)\n",
            "Requirement already satisfied: traitlets>=4.2 in /usr/local/lib/python3.7/dist-packages (from ipython>=7.17.0->ipdb) (5.0.5)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.7/dist-packages (from ipython>=7.17.0->ipdb) (2.6.1)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.7/dist-packages (from ipython>=7.17.0->ipdb) (0.7.5)\n",
            "Collecting prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/eb/e6/4b4ca4fa94462d4560ba2f4e62e62108ab07be2e16a92e594e43b12d3300/prompt_toolkit-3.0.18-py3-none-any.whl (367kB)\n",
            "\u001b[K     |████████████████████████████████| 368kB 37.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.7/dist-packages (from pexpect>4.3; sys_platform != \"win32\"->ipython>=7.17.0->ipdb) (0.7.0)\n",
            "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /usr/local/lib/python3.7/dist-packages (from jedi>=0.16->ipython>=7.17.0->ipdb) (0.8.2)\n",
            "Requirement already satisfied: ipython-genutils in /usr/local/lib/python3.7/dist-packages (from traitlets>=4.2->ipython>=7.17.0->ipdb) (0.2.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython>=7.17.0->ipdb) (0.2.5)\n",
            "Building wheels for collected packages: ipdb\n",
            "  Building wheel for ipdb (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for ipdb: filename=ipdb-0.13.8-cp37-none-any.whl size=11599 sha256=7deeecdad7ec374911d0d8dfbabb814be13bf4264263f64591c7eb6c69b6b5e3\n",
            "  Stored in directory: /root/.cache/pip/wheels/0d/d6/5a/2fdf30b75ca5099e18f66a0a4d439ba031e1aa239e12b39c24\n",
            "Successfully built ipdb\n",
            "\u001b[31mERROR: jupyter-console 5.2.0 has requirement prompt-toolkit<2.0.0,>=1.0.0, but you'll have prompt-toolkit 3.0.18 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: google-colab 1.0.0 has requirement ipython~=5.5.0, but you'll have ipython 7.24.0 which is incompatible.\u001b[0m\n",
            "Installing collected packages: prompt-toolkit, ipython, ipdb\n",
            "  Found existing installation: prompt-toolkit 1.0.18\n",
            "    Uninstalling prompt-toolkit-1.0.18:\n",
            "      Successfully uninstalled prompt-toolkit-1.0.18\n",
            "  Found existing installation: ipython 5.5.0\n",
            "    Uninstalling ipython-5.5.0:\n",
            "      Successfully uninstalled ipython-5.5.0\n",
            "Successfully installed ipdb-0.13.8 ipython-7.24.0 prompt-toolkit-3.0.18\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "IPython",
                  "prompt_toolkit"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_2D3qViJNq0u",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "653c5e04-4dcb-45e7-c68a-6492c4fae9fc"
      },
      "source": [
        "!apt-get install libmagic-dev\n",
        "!pip install python-magic"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "The following package was automatically installed and is no longer required:\n",
            "  libnvidia-common-460\n",
            "Use 'apt autoremove' to remove it.\n",
            "The following additional packages will be installed:\n",
            "  libmagic-mgc libmagic1\n",
            "Suggested packages:\n",
            "  file\n",
            "The following NEW packages will be installed:\n",
            "  libmagic-dev libmagic-mgc libmagic1\n",
            "0 upgraded, 3 newly installed, 0 to remove and 34 not upgraded.\n",
            "Need to get 332 kB of archives.\n",
            "After this operation, 5,552 kB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libmagic-mgc amd64 1:5.32-2ubuntu0.4 [184 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libmagic1 amd64 1:5.32-2ubuntu0.4 [68.6 kB]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libmagic-dev amd64 1:5.32-2ubuntu0.4 [79.7 kB]\n",
            "Fetched 332 kB in 1s (289 kB/s)\n",
            "Selecting previously unselected package libmagic-mgc.\n",
            "(Reading database ... 160706 files and directories currently installed.)\n",
            "Preparing to unpack .../libmagic-mgc_1%3a5.32-2ubuntu0.4_amd64.deb ...\n",
            "Unpacking libmagic-mgc (1:5.32-2ubuntu0.4) ...\n",
            "Selecting previously unselected package libmagic1:amd64.\n",
            "Preparing to unpack .../libmagic1_1%3a5.32-2ubuntu0.4_amd64.deb ...\n",
            "Unpacking libmagic1:amd64 (1:5.32-2ubuntu0.4) ...\n",
            "Selecting previously unselected package libmagic-dev:amd64.\n",
            "Preparing to unpack .../libmagic-dev_1%3a5.32-2ubuntu0.4_amd64.deb ...\n",
            "Unpacking libmagic-dev:amd64 (1:5.32-2ubuntu0.4) ...\n",
            "Setting up libmagic-mgc (1:5.32-2ubuntu0.4) ...\n",
            "Setting up libmagic1:amd64 (1:5.32-2ubuntu0.4) ...\n",
            "Setting up libmagic-dev:amd64 (1:5.32-2ubuntu0.4) ...\n",
            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
            "Processing triggers for libc-bin (2.27-3ubuntu1.2) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/python3.7/dist-packages/ideep4py/lib/libmkldnn.so.0 is not a symbolic link\n",
            "\n",
            "Collecting python-magic\n",
            "  Downloading https://files.pythonhosted.org/packages/ba/7c/1d1d4bdda29bfec662b9b50951dee2dddf7747d3cbf7777f3d1c63372bd0/python_magic-0.4.22-py2.py3-none-any.whl\n",
            "Installing collected packages: python-magic\n",
            "Successfully installed python-magic-0.4.22\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1tzpYpiKv4sB",
        "outputId": "35ba1cbc-edf4-476b-cb95-bf9ad1d09be4"
      },
      "source": [
        "!pip install ftfy"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting ftfy\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/af/da/d215a091986e5f01b80f5145cff6f22e2dc57c6b048aab2e882a07018473/ftfy-6.0.3.tar.gz (64kB)\n",
            "\r\u001b[K     |█████                           | 10kB 21.9MB/s eta 0:00:01\r\u001b[K     |██████████▏                     | 20kB 27.9MB/s eta 0:00:01\r\u001b[K     |███████████████▎                | 30kB 21.2MB/s eta 0:00:01\r\u001b[K     |████████████████████▍           | 40kB 18.7MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▌      | 51kB 8.3MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▋ | 61kB 8.1MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 71kB 5.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from ftfy) (0.2.5)\n",
            "Building wheels for collected packages: ftfy\n",
            "  Building wheel for ftfy (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for ftfy: filename=ftfy-6.0.3-cp37-none-any.whl size=41916 sha256=5b7f41ad8ddcfa3f3617c8301dea126ca0895145cfc2fef78f570f446c3bdf19\n",
            "  Stored in directory: /root/.cache/pip/wheels/99/2c/e6/109c8a28fef7a443f67ba58df21fe1d0067ac3322e75e6b0b7\n",
            "Successfully built ftfy\n",
            "Installing collected packages: ftfy\n",
            "Successfully installed ftfy-6.0.3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u6J0tNny0j6a"
      },
      "source": [
        "import re\n",
        "import urllib.request as urllib2\n",
        "import os\n",
        "import ipdb\n",
        "from bs4 import BeautifulSoup\n",
        "import time\n",
        "import sys\n",
        "import codecs\n",
        "import pandas as pd\n",
        "import csv\n",
        "import json\n",
        "from os import listdir\n",
        "from os.path import isfile, join\n",
        "import io\n",
        "import magic\n",
        "import pickle\n",
        "import ftfy"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O6YVmAjEQC1r"
      },
      "source": [
        "tsv_filepath = '/content/drive/MyDrive/UnivAI/AI-3 Project/data/Everyday Sexism Project/data_placeholders.tsv'\n",
        "data_write_path = '/content/drive/MyDrive/UnivAI/AI-3 Project/data/Everyday Sexism Project/data.tsv'\n",
        "\n",
        "\n",
        "def find_post_id(name_box):\n",
        "    classes = name_box['class']\n",
        "    for _class in classes:\n",
        "        if _class[:5] == \"post-\" and len(_class) > 5:\n",
        "            return int(_class[5:])\n",
        "    return -1\n",
        "\n",
        "\n",
        "def crawl_post(post_page, post_number):\n",
        "    crawl_page = post_page + str(post_number)\n",
        "\n",
        "    isCrawledSuccessfully = False\n",
        "    while not isCrawledSuccessfully:\n",
        "        try:\n",
        "            page = urllib2.urlopen(crawl_page).read()\n",
        "            isCrawledSuccessfully = True\n",
        "        except urllib2.HTTPError as e:\n",
        "            if e.code == 404:\n",
        "                print(\"\\t\\t>>> Found 404\")\n",
        "                return None\n",
        "            else:\n",
        "                print(\"\\t\\t>>> Waitiing for 30 seconds\")\n",
        "                time.sleep(30)\n",
        "        except Exception:\n",
        "            print(\"\\t\\t>>> Waitiing for 30 seconds\")\n",
        "            time.sleep(30)\n",
        "\n",
        "    soup = BeautifulSoup(page, 'html.parser')\n",
        "    name_boxes = soup.find_all('article', attrs={'class': 'post'})\n",
        "    assert len(name_boxes) == 1\n",
        "    for i, name_box in enumerate(name_boxes):\n",
        "        post_id = find_post_id(name_box)\n",
        "        if post_id == -1:\n",
        "            print(\"\\t\\t>>> ERROR: Error fetching post_id\")\n",
        "            return None\n",
        "        \n",
        "        #TODO: VERIFY IF THIS IS CORRECT\n",
        "        soup = BeautifulSoup(name_box.encode(\"utf-8\"), 'html.parser')\n",
        "        _name_box = soup.find('div', attrs={'class': 'entry-content'})\n",
        "        post = _name_box.get_text()\n",
        "        post = post.replace('\\n', '<br/>')\n",
        "    return post\n",
        "\n",
        "\n",
        "import re\n",
        "\n",
        "r_white = re.compile(r'\\s+')\n",
        "\n",
        "xml_csv = open(data_write_path, 'w')\n",
        "csv_writer = csv.writer(xml_csv, delimiter='\\t')\n",
        "csv_writer.writerow(['post', 'labels'])\n",
        "\n",
        "post_id_dict = {}\n",
        "\n",
        "with open(tsv_filepath, 'r') as csvfile:\n",
        "    spamreader = csv.reader(csvfile, delimiter='\\t')\n",
        "    for i, row in enumerate(spamreader):\n",
        "        if i == 0:\n",
        "            continue\n",
        "        labels = row[1]\n",
        "        parts = row[0].split(\"__\")\n",
        "        post_id__str = parts[0]\n",
        "        post_offsets__str = [offset.split(\"_\") for offset in parts[1:]]\n",
        "\n",
        "        if post_id__str in post_id_dict:\n",
        "            print(\"accessing \" + post_id__str)\n",
        "            post_text = post_id_dict[post_id__str]\n",
        "        else:\n",
        "            print(\"fetching \" + post_id__str)\n",
        "            post_text = crawl_post(\"https://everydaysexism.com/everyday-sexism/\", post_id__str)\n",
        "            if post_text == None:\n",
        "                continue\n",
        "            post_id_dict[post_id__str] = post_text\n",
        "\n",
        "        post_text = post_text.replace('<br/>', ' ')\n",
        "        post_text = re.sub(' +', ' ', post_text)\n",
        "        post_text = post_text.lstrip()\n",
        "        post_text__fixed = ftfy.fix_text(post_text)\n",
        "        text = \"\"\n",
        "        for index in range(len(post_offsets__str)):\n",
        "            text = text + post_text__fixed[int(post_offsets__str[index][0]):int(post_offsets__str[index][1]) + 1]\n",
        "            if len(post_offsets__str) > 1 and len(post_offsets__str) != index + 1:\n",
        "                text = text + \" \"\n",
        "        text = r_white.sub(' ', text)\n",
        "        text = text.strip()\n",
        "        csv_writer.writerow([text, labels])\n",
        "\n",
        "xml_csv.close()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_mdpyM9LQEhS"
      },
      "source": [
        "df_es = pd.read_csv(data_write_path, sep='\\t')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ah5cDwFaq734"
      },
      "source": [
        "path = '/content/drive/MyDrive/UnivAI/AI-3 Project/data/Everyday Sexism Project/data.tsv'\n",
        "data = pd.read_csv(path, sep='\\t')"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "vq07SePqrlpW",
        "outputId": "2feb0672-0cdb-45b3-e1ad-6eb5f276c43e"
      },
      "source": [
        "data.head()"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>post</th>\n",
              "      <th>labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>My cousin (female, and 14 years older than me)...</td>\n",
              "      <td>Attribute stereotyping, Body shaming, Denial o...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Big brother came to hang out with us toward th...</td>\n",
              "      <td>Sexual assault (excluding rape)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>The next morning, I told my friend. She didn't...</td>\n",
              "      <td>Denial or trivialization of sexist misconduct,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>My ex-boyfriend tried guilt tripping me into s...</td>\n",
              "      <td>Sexual harassment (excluding assault), Slut sh...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>While I was working at McDonald's there was th...</td>\n",
              "      <td>Hostile work environment (excluding pay gap), ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                post                                             labels\n",
              "0  My cousin (female, and 14 years older than me)...  Attribute stereotyping, Body shaming, Denial o...\n",
              "1  Big brother came to hang out with us toward th...                    Sexual assault (excluding rape)\n",
              "2  The next morning, I told my friend. She didn't...  Denial or trivialization of sexist misconduct,...\n",
              "3  My ex-boyfriend tried guilt tripping me into s...  Sexual harassment (excluding assault), Slut sh...\n",
              "4  While I was working at McDonald's there was th...  Hostile work environment (excluding pay gap), ..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kcLLaKzaBIfP",
        "outputId": "b5b97b3b-3436-49c8-82d2-9ed52076e1b7"
      },
      "source": [
        "'Moral policing' in data['labels'][0]"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-VlP5JXeAxSX"
      },
      "source": [
        "label_list = ['Attribute stereotyping', 'Role stereotyping', 'Pay gap', 'Hostile work environment', 'Tone policing', 'Moral policing', 'Mansplaining']"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i0oTEwyUBqkH"
      },
      "source": [
        "def transform(text, label_list):\n",
        "  for label in label_list:\n",
        "    if label in text:\n",
        "      return 1\n",
        "  return 0"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "riGjhfxQEEpn"
      },
      "source": [
        "data['labels'] = data['labels'].apply(transform,label_list=label_list)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "KPaRiZtbGJnV",
        "outputId": "6381ebfc-f9f7-4fce-c7a7-b6670a28b697"
      },
      "source": [
        "data.head()"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>post</th>\n",
              "      <th>labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>My cousin (female, and 14 years older than me)...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Big brother came to hang out with us toward th...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>The next morning, I told my friend. She didn't...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>My ex-boyfriend tried guilt tripping me into s...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>While I was working at McDonald's there was th...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                post  labels\n",
              "0  My cousin (female, and 14 years older than me)...       1\n",
              "1  Big brother came to hang out with us toward th...       0\n",
              "2  The next morning, I told my friend. She didn't...       0\n",
              "3  My ex-boyfriend tried guilt tripping me into s...       0\n",
              "4  While I was working at McDonald's there was th...       1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dOt4RR-jGLKZ"
      },
      "source": [
        "data = data[data['labels']==1]"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "Ul61p1UiGinI",
        "outputId": "3ae09f0c-f936-43bc-aba2-e690c0f1f4b5"
      },
      "source": [
        "data.head()"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>post</th>\n",
              "      <th>labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>My cousin (female, and 14 years older than me)...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>While I was working at McDonald's there was th...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>I told multiple supervisors and the head of th...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>I worked as a Project Engineer at a major comp...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>I wanted to go in the forces and was told \"Wom...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                post  labels\n",
              "0  My cousin (female, and 14 years older than me)...       1\n",
              "4  While I was working at McDonald's there was th...       1\n",
              "5  I told multiple supervisors and the head of th...       1\n",
              "6  I worked as a Project Engineer at a major comp...       1\n",
              "9  I wanted to go in the forces and was told \"Wom...       1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U72TzlqGGkUK"
      },
      "source": [
        "data.rename(columns = {'post':'Sentences', 'labels':'Label'}, inplace=True)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Sp_G_SvNHIbt",
        "outputId": "dd0ba32d-afce-4170-afe3-3ad0eee5fcbd"
      },
      "source": [
        "len(data)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "6326"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "4l0sUiJDHidY",
        "outputId": "66118009-092e-48ce-973c-cf55dda43375"
      },
      "source": [
        "data.head()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Sentences</th>\n",
              "      <th>Label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>My cousin (female, and 14 years older than me)...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>While I was working at McDonald's there was th...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>I told multiple supervisors and the head of th...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>I worked as a Project Engineer at a major comp...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>I wanted to go in the forces and was told \"Wom...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                           Sentences  Label\n",
              "0  My cousin (female, and 14 years older than me)...      1\n",
              "4  While I was working at McDonald's there was th...      1\n",
              "5  I told multiple supervisors and the head of th...      1\n",
              "6  I worked as a Project Engineer at a major comp...      1\n",
              "9  I wanted to go in the forces and was told \"Wom...      1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dk-rjKrLHea-"
      },
      "source": [
        "len_list = [len(x) for x in data.Sentences.values]"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FmHZmXqJIbrz",
        "outputId": "0689d4f7-f037-425b-89bd-7d7f3a7596ec"
      },
      "source": [
        "max(len_list)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "879"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ktBWIAXuIcjx",
        "outputId": "5fdced80-87fd-4717-c773-f8ea072eea73"
      },
      "source": [
        "min(len_list)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "14"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eCrCvAoaIghX"
      },
      "source": [
        "temp = pd.DataFrame(len_list)"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "of6xB0zhI1_u",
        "outputId": "428b0f8d-64c5-4583-d8ec-ef1c47fd6952"
      },
      "source": [
        "len(temp[temp[0]>500])"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "45"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u4C8ZzujI75l"
      },
      "source": [
        " max_len = data['Sentences'].apply(len).max()"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "g_KDT7UgJrDV",
        "outputId": "91e1346c-d370-4685-8df5-98a7cd414e2d"
      },
      "source": [
        "max_len"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "879"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MnAhQ1hSJtOX"
      },
      "source": [
        "data.to_csv('/content/drive/MyDrive/UnivAI/AI-3 Project/data/Preprocessed data/everyday_sexist_comment.csv')"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W0G8KTJcKmOS"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}