dataset_preparation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "## Newsela"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "8519ccd6d8c8ac6d"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "newsela_align = pd.read_csv('data/newsela_data_share-20150302/newsela_articles_20150302.aligned.sents.txt', sep='\\t', names=['DOC', 'V_normal', 'V_simple', 'normal_phrase', 'simple_phrase'], on_bad_lines='skip').dropna()\n",
    "newsela_align"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "d1794106997d3008"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "newsela_align = newsela_align.groupby(['V_normal','V_simple', 'DOC', 'normal_phrase']).agg(tuple).applymap(list).reset_index()\n",
    "newsela_align['simple_phrase'] = newsela_align['simple_phrase'].apply(lambda x: ' '.join(x))\n",
    "newsela_align = newsela_align[newsela_align.V_normal == 'V0']\n",
    "newsela_align"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "8efb9752c0959214"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "for _ in range(10):\n",
    "    index = random.randint(0, len(newsela_align))\n",
    "    print(\"**\")\n",
    "    print(\"Normal phrase:\", newsela_align.iloc[index]['normal_phrase'])\n",
    "    print(\"Simple phrase:\", newsela_align.iloc[index]['simple_phrase'])"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "fe8e1392e99080cf"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "newsela_align.to_csv('data/newsela_sent_aligned_V0.csv', index=False)"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "11666b9fe116e2a6"
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Mask NEs"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "9db927f37002d190"
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "newsela = pd.read_csv(\"data/newsela_sent_aligned_V0.csv\")\n",
    "newsela = newsela[['V_normal', 'V_simple', 'DOC', 'normal_phrase', 'simple_phrase']].copy()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-01T10:56:40.413253400Z",
     "start_time": "2024-02-01T10:56:39.675254100Z"
    }
   },
   "id": "93f033804607d7de"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "import spacy\n",
    "\n",
    "nlp = spacy.load(\"en_core_web_md\", disable=[\"tok2vec\", \"tagger\", \"parser\", \"attribute_ruler\", \"lemmatizer\"])\n",
    "def replace_ne(index: int, column: str, doc, phrase):\n",
    "    #for ent in doc.ents:\n",
    "        #print(ent.label_, ent.label_ in ['EVENT', 'GPE', 'LANGUAGE', 'LAW', 'LOC', 'NORP', 'ORG', 'PERSON', 'PRODUCT', 'WORK_OF_ART'])\n",
    "        #if ent.label_ in ['EVENT', 'GPE', 'LANGUAGE', 'LAW', 'LOC', 'NORP', 'ORG', 'PERSON', 'PRODUCT', 'WORK_OF_ART']:\n",
    "        #    phrase.replace(ent.text, 'NAME')\n",
    "        #    print(phrase)\n",
    "    newsela.at[index, column] = ' '.join(\"NAME\" if token.ent_type_ in ['EVENT', 'GPE', 'LANGUAGE', 'LAW', 'LOC', 'NORP', 'ORG', 'PERSON', 'PRODUCT', 'WORK_OF_ART'] else token.text for token in doc)\n",
    "\n",
    "for i, row in tqdm(newsela.iterrows(), total=len(newsela)):\n",
    "    doc = nlp(row.normal_phrase)\n",
    "    replace_ne(i, 'normal_phrase_ne', doc, row.normal_phrase)\n",
    "    doc = nlp(row.simple_phrase)\n",
    "    replace_ne(i, 'simple_phrase_ne', doc, row.simple_phrase)\n",
    "\n",
    "newsela"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "d96d458a50f53f9b"
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [],
   "source": [
    "newsela.to_csv(\"data/newsela_sent_aligned_entities_masked.csv\", index=False)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-01T11:57:24.953630900Z",
     "start_time": "2024-02-01T11:57:23.707635300Z"
    }
   },
   "id": "9f4d6ae462e4bc1b"
  },
  {
   "cell_type": "markdown",
   "source": [
    "## IT: Teacher/Terence"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "400b817390987e8a"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import xml.etree.ElementTree as ET\n",
    "with open('data/CORPORA_TEXT_SIMP/Teacher/1_anna_frank_last_senza_ann.txt', 'r', encoding='utf-8') as f:\n",
    "    content = f.read()\n",
    "tree = ET.fromstring('<foo>'+content+'</foo>')"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "af2faaff8c235bbc"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "frase_norm, frase_simp = tree.findall('doc')\n",
    "tmp = pd.DataFrame({'normal_phrase': \"\", 'simple_phrase': [f.text for f in frase_simp.findall('frase')]})\n",
    "for norm_fras in frase_norm.findall('frase'):\n",
    "    tmp.loc[int(norm_fras.get('frase_all')) -1, \"normal_phrase\"] += norm_fras.text\n",
    "tmp\n"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "725a4e73529ca374"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "from xml.etree.ElementTree import ParseError\n",
    "import pandas as pd\n",
    "import xml.etree.ElementTree as ET\n",
    "import os\n",
    "\n",
    "data_path = \"'data/CORPORA_TEXT_SIMP/Teacher/\"\n",
    "def load_data(data_path, align_phrase, source):\n",
    "    corpus = pd.DataFrame()\n",
    "    data_path += '/' if not data_path.endswith('/') else ''\n",
    "    print('*', data_path)\n",
    "    for filename in os.listdir(data_path):\n",
    "        if filename.endswith('.txt'):\n",
    "            print('**', filename)\n",
    "            with open(data_path + filename, 'r', encoding='utf-8') as f:\n",
    "                content = f.read()\n",
    "            try:\n",
    "                tree = ET.fromstring('<foo>'+content+'</foo>')\n",
    "            except ParseError:\n",
    "                print('\\tcould not parse')\n",
    "                continue\n",
    "            if len(tree.findall('doc')) > 1:\n",
    "                frase_norm, frase_simp = tree.findall('doc')\n",
    "            else:\n",
    "                frase_norm = tree.find('*/originale')\n",
    "                frase_simp = tree.find('*/semplificato')\n",
    "            tmp = pd.DataFrame({'normal_phrase': \"\", 'simple_phrase': [f.text for f in frase_simp.findall('frase')]})\n",
    "            for norm_fras in frase_norm.findall('frase'):\n",
    "                #print(norm_fras.get('frase_all'))\n",
    "                if len(norm_fras.get(align_phrase)) > 0:\n",
    "                    for simp_index in norm_fras.get(align_phrase).split(';'):\n",
    "                        tmp.loc[int(simp_index) -1 , \"normal_phrase\"] += norm_fras.text\n",
    "            tmp = tmp.groupby(['normal_phrase']).agg(tuple).applymap(list).reset_index()\n",
    "            tmp['simple_phrase'] = tmp['simple_phrase'].apply(lambda x: ' '.join(x))\n",
    "\n",
    "            with open(data_path + filename.replace('.txt', '.ann'), 'r', encoding='utf-8') as f:\n",
    "                annotations = [line for line in f if not line.startswith('#')]\n",
    "            #print(annotations)\n",
    "            tmp = tmp[(tmp.normal_phrase.str.len() > 0) & (tmp.simple_phrase.str.len() > 0)]\n",
    "            tmp['simp_ops'] = len(annotations) / len(tmp)\n",
    "            tmp['doc'] = filename\n",
    "            corpus = pd.concat([corpus, tmp])\n",
    "    corpus['source'] = source\n",
    "    return corpus\n",
    "\n",
    "corpus_text_simp = pd.concat([\n",
    "    load_data('data/CORPORA_TEXT_SIMP/Teacher/', 'frase_all', 'Teacher'),\n",
    "    pd.concat([load_data('data/CORPORA_TEXT_SIMP/Terence/' + c, 'frase_al', 'Terence') for c in os.listdir('data/CORPORA_TEXT_SIMP/Terence/')])\n",
    "])\n",
    "corpus_text_simp"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "c2a3a02a62d9265c"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "corpus_text_simp.dropna().to_csv('data/corpus_simp_it.csv', index=False)"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "b919262b25380c71"
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Simpitiki"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "c3772ffc27ab65b"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "tree = ET.parse('data/simpitiki-v2.xml')\n",
    "normal_phrases = []\n",
    "simple_phrases = []\n",
    "types = []\n",
    "for simplification in tree.findall('*/simplification'):\n",
    "    normal_phrases.append(simplification.find('before').text)\n",
    "    simple_phrases.append(simplification.find('after').text)\n",
    "    types.append(simplification.get('type').strip())\n",
    "simpitiki = pd.DataFrame({'normal_phrase': normal_phrases, 'simple_phrase': simple_phrases, 'type': types})\n",
    "simpitiki"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "427fd6c356d2f421"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "simpitiki.to_csv('data/simpitiki.csv', index=False)"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "33171bb7cbae041e"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "relevant_simplification_types = [\"1\",\"2\",\"3\",\"32\",\"33\",\"34\",\"35\",\"36\",\"37\"]\n",
    "simpitiki[simpitiki.type.isin(relevant_simplification_types)]"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "da601d0fd685c58f"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}