report_generation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import json\n",
    "import requests\n",
    "import re\n",
    "import numpy as np\n",
    "from datetime import datetime\n",
    "import pytz\n",
    "from urllib.parse import urlparse,unquote\n",
    "from urllib.parse import parse_qs\n",
    "from bs4 import BeautifulSoup\n",
    "from requests.exceptions import HTTPError\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "current_file = os.getcwd()\n",
    "myfile = os.path.join(current_file, 'result_all.zip')\n",
    "myUniquefile = os.path.join(current_file, 'uniqueResults')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "logData = pd.read_csv(myfile, compression='zip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# s = ss['all'][0].split('md:datetime')\n",
    "# s[1].split('T')[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "ss = logData.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "logData = ss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# logData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "logData = logData.set_index('Unnamed: 0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_title(x):\n",
    "    s = x.split('md:title')\n",
    "    return s[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_pub_date(x):\n",
    "    s = x.split('md:datetime')\n",
    "    return s[1].split('T')[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_author(x):\n",
    "    s = x.split('md:author')\n",
    "    return s[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_year(x):\n",
    "    s = x.split('md:year')\n",
    "    return s[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove(x):\n",
    "    s = x.split('md:title')\n",
    "    if len(s) < 2:\n",
    "        return True\n",
    "    else:\n",
    "        return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "logData['remove'] = logData['all'].apply(remove)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "logData = logData[logData['remove'] == False]\n",
    "# logData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "logData['title'] = logData['all'].apply(split_title)\n",
    "logData['year'] = logData['all'].apply(split_year)\n",
    "logData['author'] = logData['all'].apply(split_author)\n",
    "logData['pub_date'] = logData['all'].apply(split_pub_date)\n",
    "# logData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "logData = logData.drop(['all','remove'],axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "logData['title'] = logData['title'].apply(lambda x: re.sub(r'^>|</$', '', str(x)) )\n",
    "logData['year'] = logData['year'].apply(lambda x: re.sub(r'^>|</$', '', str(x)) )\n",
    "logData['author'] = logData['author'].apply(lambda x: re.sub(r'^>|</$', '', str(x)) )\n",
    "logData['pub_date'] = logData['pub_date'].apply(lambda x: re.sub(r'^>', '', str(x)) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# logData = logData.drop('remove',axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# logData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "DataA = pd.read_csv('sendData')\n",
    "DataA = DataA.set_index('Unnamed: 0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# xd = DataA[DataA['resource'] == '/10.1594/PANGAEA.908578?format=html']\n",
    "# xd['time'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_min_date(x):\n",
    "    datax = DataA[DataA['resource'] == x]\n",
    "    return datax['time'].min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_max_date(x):\n",
    "    datax = DataA[DataA['resource'] == x]\n",
    "    return datax['time'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "logData['min_date'] = logData['resource'].apply(get_min_date)\n",
    "logData['max_date'] = logData['resource'].apply(get_max_date)\n",
    "# logData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "logData['max_date'] = logData['max_date'].apply(lambda x: str(datetime.strptime(str(x).split(' ')[0], \"%Y-%m-%d\")).split(' ')[0])\n",
    "logData['min_date'] = logData['min_date'].apply(lambda x: str(datetime.strptime(str(x).split(' ')[0], \"%Y-%m-%d\")).split(' ')[0])\n",
    "# logData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "logData['count'] = logData['count'].apply(lambda x: str(x))\n",
    "logData['paper_id'] = logData['paper_id'].apply(lambda x: str(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "resource    object\n",
       "count       object\n",
       "dtype: object"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uniqueLog.dtypes\n",
    "# logData.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "start = logData['min_date'].min()\n",
    "end = logData['max_date'].max()\n",
    "# end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "uniqueLog = pd.read_csv(myUniquefile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "uniqueLog = uniqueLog.set_index('Unnamed: 0')\n",
    "# uniqueLog"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "uniqueLog['resource'] = uniqueLog['resource'].apply(lambda x: re.sub(r\"[\\[\\]']+\", '', str(x)) )\n",
    "# uniqueLog"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "uniqueLog['count'] = uniqueLog['count'].apply(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>resource</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>/10.1594/PANGAEA.908578?format=html</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       resource count\n",
       "Unnamed: 0                                           \n",
       "100         /10.1594/PANGAEA.908578?format=html     5"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datadd = uniqueLog[uniqueLog['resource'] == '/10.1594/PANGAEA.908578?format=html']\n",
    "datadd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getUniCount(x):\n",
    "    data = uniqueLog[uniqueLog['resource'] == x]\n",
    "    if data.size > 0:\n",
    "        return data['count']\n",
    "    return data['count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Wrong number of items passed 58651, placement implies 1",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   2896\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2897\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2898\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'unique_count'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/pandas/core/internals/managers.py\u001b[0m in \u001b[0;36mset\u001b[0;34m(self, item, value)\u001b[0m\n\u001b[1;32m   1068\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1069\u001b[0;31m             \u001b[0mloc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1070\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   2898\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2899\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_cast_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2900\u001b[0m         \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtolerance\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtolerance\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'unique_count'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-34-04476f3c519a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlogData\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'unique_count'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlogData\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'resource'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mgetUniCount\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m__setitem__\u001b[0;34m(self, key, value)\u001b[0m\n\u001b[1;32m   3470\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3471\u001b[0m             \u001b[0;31m# set column\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3472\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_set_item\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3473\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3474\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_setitem_slice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m_set_item\u001b[0;34m(self, key, value)\u001b[0m\n\u001b[1;32m   3548\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ensure_valid_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3549\u001b[0m         \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sanitize_column\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3550\u001b[0;31m         \u001b[0mNDFrame\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_set_item\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3551\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3552\u001b[0m         \u001b[0;31m# check if we are modifying a copy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m_set_item\u001b[0;34m(self, key, value)\u001b[0m\n\u001b[1;32m   3379\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3380\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_set_item\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3381\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3382\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_clear_item_cache\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3383\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/pandas/core/internals/managers.py\u001b[0m in \u001b[0;36mset\u001b[0;34m(self, item, value)\u001b[0m\n\u001b[1;32m   1070\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1071\u001b[0m             \u001b[0;31m# This item wasn't present, just insert at end\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1072\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minsert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mitem\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1073\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1074\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/pandas/core/internals/managers.py\u001b[0m in \u001b[0;36minsert\u001b[0;34m(self, loc, item, value, allow_duplicates)\u001b[0m\n\u001b[1;32m   1179\u001b[0m         \u001b[0mnew_axis\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minsert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mitem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1180\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1181\u001b[0;31m         \u001b[0mblock\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmake_block\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mndim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mplacement\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mslice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloc\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1182\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1183\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mblkno\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcount\u001b[0m \u001b[0;32min\u001b[0m \u001b[0m_fast_count_smallints\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_blknos\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/pandas/core/internals/blocks.py\u001b[0m in \u001b[0;36mmake_block\u001b[0;34m(values, placement, klass, ndim, dtype, fastpath)\u001b[0m\n\u001b[1;32m   3265\u001b[0m         \u001b[0mvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDatetimeArray\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_simple_new\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3266\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3267\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mklass\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mndim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mndim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mplacement\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mplacement\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3268\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3269\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/pandas/core/internals/blocks.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, values, placement, ndim)\u001b[0m\n\u001b[1;32m    126\u001b[0m             raise ValueError(\n\u001b[1;32m    127\u001b[0m                 \u001b[0;34m\"Wrong number of items passed {val}, placement implies \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 128\u001b[0;31m                 \u001b[0;34m\"{mgr}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmgr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmgr_locs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    129\u001b[0m             )\n\u001b[1;32m    130\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Wrong number of items passed 58651, placement implies 1"
     ]
    }
   ],
   "source": [
    "# logData['unique_count'] = logData['resource'].apply(lambda x: getUniCount(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def getUniCount(x):\n",
    "#     data = uniqueLog[uniqueLog['resource'] == x]\n",
    "#     return data['count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logData['unique_count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def getPaperId(x):\n",
    "#     if x in logData['resource'].values:\n",
    "#         data = logData[logData['resource'] == x]\n",
    "#         return data['paper_id']\n",
    "#     else:\n",
    "#         return 'remove'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# uniqueLog = pd.read_csv(myUniquefile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# uniqueLog = uniqueLog.set_index('Unnamed: 0')\n",
    "# uniqueLog"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# uniqueLog['resource'] = uniqueLog['resource'].apply(lambda x: re.sub(r\"[\\[\\]']+\", '', str(x)) )\n",
    "# uniqueLog"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# uniqueLog['paper_id'] = uniqueLog['resource'].apply(lambda x: getPaperId(x))\n",
    "# uniqueLog['paper_id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "metadata": {},
   "outputs": [],
   "source": [
    "r={}\n",
    "r['report'] = []\n",
    "r['report'].append({\n",
    "    \"id\":\"\",\n",
    "    \"report-header\":{\n",
    "        \"report-name\":\"pangaea_dsr\",\n",
    "        \"report-id\":\"pangea_\"+start+\"_\"+end,\n",
    "        \"release\":\"RD1\",\n",
    "        \"created-by\":\"pangaea\",\n",
    "        \"created\":\"2020-06-30\",\n",
    "        \"reporting-period\":{\n",
    "            \"end-date\":end,\n",
    "            \"begin-date\":start\n",
    "        },\n",
    "    }\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 292,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'report': [{'id': '',\n",
       "   'report-header': {'report-name': 'pangaea_dsr',\n",
       "    'report-id': 'pangea_2020-03-23_2020-04-26',\n",
       "    'release': 'RD1',\n",
       "    'created-by': 'pangaea',\n",
       "    'created': '2020-06-30',\n",
       "    'reporting-period': {'end-date': '2020-04-26',\n",
       "     'begin-date': '2020-03-23'}}}]}"
      ]
     },
     "execution_count": 292,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = {}\n",
    "data['report']=[]\n",
    "data['report'].append({\n",
    "    \"id\":\"\",\n",
    "    \"report-header\":{\n",
    "        \"report-name\":\"pangaea_dsr\",\n",
    "        \"report-id\":\"pangea_\"+start+\"_\"+end,\n",
    "        \"release\":\"RD1\",\n",
    "        \"created-by\":\"pangaea\",\n",
    "        \"created\":\"2020-06-30\",\n",
    "        \"reporting-period\":{\n",
    "            \"end-date\":end,\n",
    "            \"begin-date\":start\n",
    "        },\n",
    "    }\n",
    "})\n",
    "\n",
    "data['report-datasets'] = []\n",
    "data['report-datasets'].append({\n",
    "    \"uri\": \"https://doi.pangaea.de\"+logData['resource'][0],\n",
    "    \"yop\": logData['year'][0],\n",
    "    \"platform\": \"PANGAEA\",\n",
    "    \"data-type\": \"dataset\",\n",
    "    \"publisher\": \"PANGAEA\",\n",
    "    \"dataset-id\":[\n",
    "        {\n",
    "            \"type\": \"doi\",\n",
    "            \"value\": \"10.15146/\"+str(logData['paper_id'][0])\n",
    "        }\n",
    "    ],\n",
    "    \"performance\": [\n",
    "        {\n",
    "            \"period\":{\n",
    "                \"end-date\":logData['max_date'][0],\n",
    "                \"begin-date\":logData['min_date'][0]\n",
    "            },\n",
    "            \"instance\":[\n",
    "                {\n",
    "                    \"count\":logData['count'][0],\n",
    "                    \"metric-type\": \"Total_Dataset_Requests\",\n",
    "                    \"access-method\": \"regular\",\n",
    "                }\n",
    "            ]\n",
    "        }\n",
    "    ],\n",
    "    \"publisher-id\":[\n",
    "        {\n",
    "            \"type\":\"grid\",\n",
    "            \"value\":\"474422.3\"\n",
    "        }\n",
    "    ],\n",
    "    \"dataset-dates\":[\n",
    "        {\n",
    "            \"type\":\"pub-dates\",\n",
    "            \"value\":logData['pub_date'][0]\n",
    "        }\n",
    "    ],\n",
    "    \"dataset-title\":logData['title'][0],\n",
    "    \"dataset-contributors\":[\n",
    "        {\n",
    "            \"type\":\"name\",\n",
    "            \"value\":logData['author'][0]\n",
    "        }\n",
    "    ]\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "metadata": {},
   "outputs": [],
   "source": [
    "# final ={}\n",
    "# final.append(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 329,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'report': [{'report-header': {'report-name': 'pangaea_dsr',\n",
       "    'report-id': 'pangea_2020-03-23_2020-04-26',\n",
       "    'release': 'RD1',\n",
       "    'created-by': 'pangaea',\n",
       "    'created': '2020-06-30',\n",
       "    'reporting-period': {'end-date': '2020-04-26',\n",
       "     'begin-date': '2020-03-23'}}}],\n",
       " 'report-datasets': [{'uri': 'https://doi.pangaea.de/10.1594/PANGAEA.908578?format=html',\n",
       "   'yop': '2019',\n",
       "   'platform': 'PANGAEA',\n",
       "   'data-type': 'dataset',\n",
       "   'publisher': 'PANGAEA',\n",
       "   'dataset-id': [{'type': 'doi', 'value': '10.15146/908578'}],\n",
       "   'performance': [{'period': {'end-date': '2020-04-24',\n",
       "      'begin-date': '2020-03-23'},\n",
       "     'instance': [{'count': '80',\n",
       "       'metric-type': 'Total_Dataset_Requests',\n",
       "       'access-method': 'regular'}]}],\n",
       "   'publisher-id': [{'type': 'grid', 'value': '474422.3'}],\n",
       "   'dataset-dates': [{'type': 'pub-dates', 'value': '2019-11-15'}],\n",
       "   'dataset-title': 'Broad typology for rivers and lakes in Europe for large scale analysis',\n",
       "   'dataset-contributors': [{'type': 'name', 'value': 'Globevnik, L'}]}]}"
      ]
     },
     "execution_count": 329,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# data.to_json('usage-report-file')\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = json.dumps(data)\n",
    "with open('data.txt', 'w') as f:\n",
    "    f.write(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "metadata": {},
   "outputs": [],
   "source": [
    "Datafile = os.path.join(current_file, 'data.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {},
   "outputs": [],
   "source": [
    "import curl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "metadata": {},
   "outputs": [],
   "source": [
    "token = 'eyJhbGciOiJSUzI1NiJ9.eyJ1aWQiOiJQQU5HQUVBLlJFUE9TSVRPUlkiLCJuYW1lIjoiUEFOR0FFQSIsImVtYWlsIjoidXNjaGluZGxlckBwYW5nYWVhLmRlIiwicHJvdmlkZXJfaWQiOiJQQU5HQUVBIiwiY2xpZW50X2lkIjoiUEFOR0FFQS5SRVBPU0lUT1JZIiwicm9sZV9pZCI6ImNsaWVudF9hZG1pbiIsImF1ZCI6InN0YWdlIiwiaWF0IjoxNTkyMzIyNTEzLCJleHAiOjE2MDgxMzM3MTN9.YOhsbgTQsC3OUlx-0QhaQtAmFeLiZ9xw0QdinRrLMCPeO0sQZ9aokjQZUQtEqNI-96hUyF2r2b6NQq02AMBBSVqjVGUo3FrTJH7piUKGn-Ju_YiJLDHSjjCrG0aHwpyT3qzBmEqO_-Csc_h-b8NC13cwqJ3FgGAlBNAwDLfBbsXrvHlK7QboyAUm6juMtUolAZx7fEBHz-Or9E_fgzLXG39cuiwZy4R2CFhxVXrFQZAVojrDy_S8w4BxI-uiJNG9MsxEZjhvZCNpIyT0CaSqbDetAHiKnEPw2bbhAl21xe9hxa7tBzLYxjyNQfXQHENcP6RD2zeuq0t4arEjMzAcsw'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {},
   "outputs": [],
   "source": [
    "api_test = 'https://api.test.datacite.org/reports/'\n",
    "api_te = 'https://api.datacite.org/reports/'\n",
    "apii = 'https://api.datacite.org/reports/21fd2e8e-5481-4bbd-b2ef-742d8b270a66'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 350,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf = requests.put(api_test,headers={'Authorization': token}, data=json.dumps(data))\n",
    "# rff = requests.get(apii,headers={'Authorization': token})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "503"
      ]
     },
     "execution_count": 351,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf.status_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [503]>"
      ]
     },
     "execution_count": 352,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}