fine_tuning_model.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "fine-tuning-model.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "QyHvdR0m24I3"
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "df = pd.read_csv(\"metin_ozetleme.csv\")\n",
        "df = df[['Metin','Özet']]\n",
        "df.columns = ['text','summary']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 159
        },
        "id": "B12dEGGRlF4l",
        "outputId": "9a958067-2c9e-410f-a205-4b170c0a875b"
      },
      "source": [
        "df.iloc[1]['text']"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "\"Yıllar önce hayatımıza giren ve geçtiğimiz yıllarda da Facebook tarafından satın alınan dünyanın en popüler anlık mesajlaşma uygulaması WhatsApp, 2021 yılının başından beri tepkilerin odağındaydı. Çünkü şirket, kullanım koşullarını ve gizlilik ilkelerini değiştirecek, kullanıcılara ait verileri bizzat kullanıcının iznini alarak başta Facebook olmak üzere çeşitli şirketlerle paylaşabilecekti. Türkiye'de de uygulama konacak yeni koşullar için verilen son tarih ise 15 Mayıs'tı.Konuyla ilgili olarak yaşanan gelişmelerden bir tanesi, Rekabet Kurumu'nun açtığı soruşturmaydı. Kurum, kullanıcılara böyle bir şey yapılamayacağını söyleyerek, WhatsApp hakkında soruşturma başlatmıştı.\\xa0Görünen o ki bu soruşturma sonuç vermiş durumda. Rekabet Kurumu'nun yaptığı bir açıklamaya göre\\xa0WhatsApp'ın yeni gizlilik ilkeleri ve kullanım koşulları, Türkiye'de uygulanmayacak.Yeni Şafak'ın servis ettiği habere göre Rekabet Kurumu yetkilileriyle yapılan görüşmeler sonucunda şöyle bir açıklama yapılmış;WhatsApp tarafından Kurumumuza yapılan bilgilendirme çerçevesinde güncellemenin, 15 Mayıs 2021 tarihinde Türkiye’de yürürlüğe girmeyeceği ve tüm kullanıcıların WhatsApp’ı tam işlevsellikle kullanmaya devam edebilecekleri, bu kapsamda Türkiye’deki kullanıcıların, güncellemeye onay vermelerini talep eden bildirimler almayacakları anlaşılmıştır.\\xa0Dolayısıyla bu aşamada WhatsApp’ın güncellemeye yönelik küresel politikasından Türkiye’yi hariç tuttuğu söylenebilecektir. Bununla birlikte WhatsApp’ın ilerideki günlerde konuya ilişkin olarak Türkiye’deki kullanıcıları bilgilendirmesinin faydalı olacağını tahmin ediyoruz.Rekabet Kurumu'nun başlattığı soruşturma, WhatsApp'a yönelik geçici tedbir kararı alınmasına yol açmıştı. Şayet anlaşmaya varılamasa ve WhatsApp'ın yeni kullanım koşulları ülkemizde zorunlu olarak tutulmaya devam etseydi, anlık mesajlaşma uygulaması idari para cezası kesilecekti. Görünen o ki şu durumda, WhatsApp'a yönelik bir yaptırıma\\xa0gerek kalmamış. Bu arada, söz konusu gelişmeyle ilgili olarak konunun muhatabı olan WhatsApp'tan bir açıklamanın gelmediğini söylemek gerek.\\xa0Şayet konuyla ilgili yeni bir gelişme olur ve WhatsApp'tan konuyla ilgili bir açıklama yapılırsa, bu açıklamayı size aktaracağız.\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 106
        },
        "id": "Qc5E7S5_lKLG",
        "outputId": "8e045f0e-f379-4180-f11e-ba297903dd26"
      },
      "source": [
        "df.iloc[1]['summary']"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "\" Kurum, kullanıcılara böyle bir şey yapılamayacağını söyleyerek, WhatsApp hakkında soruşturma başlatmıştı. Görünen o ki bu soruşturma sonuç vermiş durumda. Dolayısıyla bu aşamada WhatsApp’ın güncellemeye yönelik küresel politikasından Türkiye’yi hariç tuttuğu söylenebilecektir. Görünen o ki şu durumda, WhatsApp'a yönelik bir yaptırıma\\xa0gerek kalmamış. Bu arada, söz konusu gelişmeyle ilgili olarak konunun muhatabı olan WhatsApp'tan bir açıklamanın gelmediğini söylemek gerek. Şayet konuyla ilgili yeni bir gelişme olur ve WhatsApp'tan konuyla ilgili bir açıklama yapılırsa, bu açıklamayı size aktaracağız.\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 402
        },
        "id": "oAukIgQ73XXx",
        "outputId": "75a64cd7-5d3e-47c6-d8d5-a3b04848afe0"
      },
      "source": [
        "df[9866:]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>summary</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>9866</th>\n",
              "      <td>Netflix'in Türk yapımı ilk orijinal dizisi ola...</td>\n",
              "      <td>Buna karşın Netflix'e yakın kaynaklar, dizini...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9867</th>\n",
              "      <td>Marvel'ın aldığı kararla yepyeni bir sinematik...</td>\n",
              "      <td>Marvel'ın aldığı kararla yepyeni bir sinemati...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9868</th>\n",
              "      <td>Kara deliklerin ne olduğunu herkes az çok bili...</td>\n",
              "      <td>Kara deliklerin ne olduğunu herkes az çok bil...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9869</th>\n",
              "      <td>Düşük ışık fotoğrafları söz konusu olduğunda S...</td>\n",
              "      <td>Zira şirketin f/1.5 diyafram açıklığına sahip...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9870</th>\n",
              "      <td>Organize İşler 2: Sazan Sarmalı vizyondan kalk...</td>\n",
              "      <td>Semercioğlu, \"Netflix, dünyada agresif bir şe...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16438</th>\n",
              "      <td>Dış Ekonomik İlişkiler Kurulu (DEİK), 1 Nisan’...</td>\n",
              "      <td>Online seminerde konuşma yapana Gökhan Hotamı...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16439</th>\n",
              "      <td>Bugün Koronavirüs Bilim Kurulu toplantısı sonr...</td>\n",
              "      <td>Sağlık Bakanı da grafiklerle birlikte açıklad...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16440</th>\n",
              "      <td>Neredeyse tüm dünya ülkelerine bulaşan koronav...</td>\n",
              "      <td>Neredeyse tüm dünya ülkelerine bulaşan korona...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16441</th>\n",
              "      <td>Çin Halk Cumhuriyeti'nin Wuhan kentinde ortaya...</td>\n",
              "      <td>Bununla birlikte yakın gelecekte farklı ülkel...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16442</th>\n",
              "      <td>Koronavirüsün ülkemizde görülmesi ve giderek y...</td>\n",
              "      <td>Katılımcıların geri kalanı ise koronavirüsün,...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>6577 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    text                                            summary\n",
              "9866   Netflix'in Türk yapımı ilk orijinal dizisi ola...   Buna karşın Netflix'e yakın kaynaklar, dizini...\n",
              "9867   Marvel'ın aldığı kararla yepyeni bir sinematik...   Marvel'ın aldığı kararla yepyeni bir sinemati...\n",
              "9868   Kara deliklerin ne olduğunu herkes az çok bili...   Kara deliklerin ne olduğunu herkes az çok bil...\n",
              "9869   Düşük ışık fotoğrafları söz konusu olduğunda S...   Zira şirketin f/1.5 diyafram açıklığına sahip...\n",
              "9870   Organize İşler 2: Sazan Sarmalı vizyondan kalk...   Semercioğlu, \"Netflix, dünyada agresif bir şe...\n",
              "...                                                  ...                                                ...\n",
              "16438  Dış Ekonomik İlişkiler Kurulu (DEİK), 1 Nisan’...   Online seminerde konuşma yapana Gökhan Hotamı...\n",
              "16439  Bugün Koronavirüs Bilim Kurulu toplantısı sonr...   Sağlık Bakanı da grafiklerle birlikte açıklad...\n",
              "16440  Neredeyse tüm dünya ülkelerine bulaşan koronav...   Neredeyse tüm dünya ülkelerine bulaşan korona...\n",
              "16441  Çin Halk Cumhuriyeti'nin Wuhan kentinde ortaya...   Bununla birlikte yakın gelecekte farklı ülkel...\n",
              "16442  Koronavirüsün ülkemizde görülmesi ve giderek y...   Katılımcıların geri kalanı ise koronavirüsün,...\n",
              "\n",
              "[6577 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aJGOUnu-2--z"
      },
      "source": [
        "train_df = df[:9866]\n",
        "test_df  = df[9866:]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IrjKsJxP3mI5"
      },
      "source": [
        "train_df.to_csv(\"train.csv\",index=False)\n",
        "test_df.to_csv(\"test.csv\",index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5ljO-eXs-29E"
      },
      "source": [
        "!git clone https://github.com/huggingface/transformers.git"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hpud-B5_B2eF"
      },
      "source": [
        "!pip install git+https://github.com/huggingface/transformers"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gTb5-s_D_h8t"
      },
      "source": [
        "!pip install -r /content/transformers/examples/pytorch/summarization/requirements.txt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lORNDS404T26"
      },
      "source": [
        "!rm -r /content/result\n",
        "!rm -r /content/runs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R4pHFHL3_QPI",
        "outputId": "f03764b4-c8e5-4967-8022-919e15b7a7df"
      },
      "source": [
        "!python3 /content/transformers/examples/pytorch/summarization/run_summarization.py \\\n",
        "--model_name_or_path google/mt5-small \\\n",
        "--do_train \\\n",
        "--do_eval \\\n",
        "--train_file /content/train.csv \\\n",
        "--validation_file /content/test.csv \\\n",
        "--source_prefix \"summarize: \" \\\n",
        "--output_dir /content/result/ \\\n",
        "--overwrite_output_dir \\\n",
        "--per_device_train_batch_size=4 \\\n",
        "--per_device_eval_batch_size=4 \\\n",
        "--predict_with_generate"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-05-26 10:42:27.019609: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcudart.so.11.0\n",
            "05/26/2021 10:42:28 - WARNING - __main__ -   Process rank: -1, device: cuda:0, n_gpu: 1distributed training: False, 16-bits training: False\n",
            "05/26/2021 10:42:28 - INFO - __main__ -   Training/evaluation parameters Seq2SeqTrainingArguments(output_dir='/content/result/', overwrite_output_dir=True, do_train=True, do_eval=True, do_predict=False, evaluation_strategy=<IntervalStrategy.NO: 'no'>, prediction_loss_only=False, per_device_train_batch_size=4, per_device_eval_batch_size=4, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=1, eval_accumulation_steps=None, learning_rate=5e-05, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=3.0, max_steps=-1, lr_scheduler_type=<SchedulerType.LINEAR: 'linear'>, warmup_ratio=0.0, warmup_steps=0, logging_dir='runs/May26_10-42-28_499e75e2c4c8', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=False, logging_steps=500, save_strategy=<IntervalStrategy.STEPS: 'steps'>, save_steps=500, save_total_limit=None, no_cuda=False, seed=42, fp16=False, fp16_opt_level='O1', fp16_backend='auto', fp16_full_eval=False, local_rank=-1, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=500, dataloader_num_workers=0, past_index=-1, run_name='/content/result/', disable_tqdm=False, remove_unused_columns=True, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=[], deepspeed=None, label_smoothing_factor=0.0, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=None, dataloader_pin_memory=True, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, log_on_each_node=True, mp_parameters='', sortish_sampler=False, predict_with_generate=True)\n",
            "05/26/2021 10:42:28 - WARNING - datasets.builder -   Using custom data configuration default-89e736806a3f23c2\n",
            "05/26/2021 10:42:28 - WARNING - datasets.builder -   Reusing dataset csv (/root/.cache/huggingface/datasets/csv/default-89e736806a3f23c2/0.0.0/2dc6629a9ff6b5697d82c25b73731dd440507a69cbce8b425db50b751e8fcfd0)\n",
            "https://huggingface.co/google/mt5-small/resolve/main/config.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpsy6dibyy\n",
            "Downloading: 100% 553/553 [00:00<00:00, 846kB/s]\n",
            "storing https://huggingface.co/google/mt5-small/resolve/main/config.json in cache at /root/.cache/huggingface/transformers/97693496c1a0cae463bd18428187f9e9924d2dfbadaa46e4d468634a0fc95a41.dadce13f8f85f4825168354a04675d4b177749f8f11b167e87676777695d4fe4\n",
            "creating metadata file for /root/.cache/huggingface/transformers/97693496c1a0cae463bd18428187f9e9924d2dfbadaa46e4d468634a0fc95a41.dadce13f8f85f4825168354a04675d4b177749f8f11b167e87676777695d4fe4\n",
            "loading configuration file https://huggingface.co/google/mt5-small/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/97693496c1a0cae463bd18428187f9e9924d2dfbadaa46e4d468634a0fc95a41.dadce13f8f85f4825168354a04675d4b177749f8f11b167e87676777695d4fe4\n",
            "Model config MT5Config {\n",
            "  \"architectures\": [\n",
            "    \"MT5ForConditionalGeneration\"\n",
            "  ],\n",
            "  \"d_ff\": 1024,\n",
            "  \"d_kv\": 64,\n",
            "  \"d_model\": 512,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"dropout_rate\": 0.1,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"feed_forward_proj\": \"gated-gelu\",\n",
            "  \"initializer_factor\": 1.0,\n",
            "  \"is_encoder_decoder\": true,\n",
            "  \"layer_norm_epsilon\": 1e-06,\n",
            "  \"model_type\": \"mt5\",\n",
            "  \"num_decoder_layers\": 8,\n",
            "  \"num_heads\": 6,\n",
            "  \"num_layers\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"relative_attention_num_buckets\": 32,\n",
            "  \"tie_word_embeddings\": false,\n",
            "  \"tokenizer_class\": \"T5Tokenizer\",\n",
            "  \"transformers_version\": \"4.7.0.dev0\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 250112\n",
            "}\n",
            "\n",
            "loading configuration file https://huggingface.co/google/mt5-small/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/97693496c1a0cae463bd18428187f9e9924d2dfbadaa46e4d468634a0fc95a41.dadce13f8f85f4825168354a04675d4b177749f8f11b167e87676777695d4fe4\n",
            "Model config MT5Config {\n",
            "  \"architectures\": [\n",
            "    \"MT5ForConditionalGeneration\"\n",
            "  ],\n",
            "  \"d_ff\": 1024,\n",
            "  \"d_kv\": 64,\n",
            "  \"d_model\": 512,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"dropout_rate\": 0.1,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"feed_forward_proj\": \"gated-gelu\",\n",
            "  \"initializer_factor\": 1.0,\n",
            "  \"is_encoder_decoder\": true,\n",
            "  \"layer_norm_epsilon\": 1e-06,\n",
            "  \"model_type\": \"mt5\",\n",
            "  \"num_decoder_layers\": 8,\n",
            "  \"num_heads\": 6,\n",
            "  \"num_layers\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"relative_attention_num_buckets\": 32,\n",
            "  \"tie_word_embeddings\": false,\n",
            "  \"tokenizer_class\": \"T5Tokenizer\",\n",
            "  \"transformers_version\": \"4.7.0.dev0\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 250112\n",
            "}\n",
            "\n",
            "https://huggingface.co/google/mt5-small/resolve/main/spiece.model not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmp4_kwxfkw\n",
            "Downloading: 100% 4.31M/4.31M [00:00<00:00, 56.8MB/s]\n",
            "storing https://huggingface.co/google/mt5-small/resolve/main/spiece.model in cache at /root/.cache/huggingface/transformers/37d0f67f084f8c5fc5589e0bba5ff3c6307af833bb0b7f4eb33fbfd8d4038a9d.84ea7af2df68dc8db434d3160aab65cce8ac63ce5b6f7743f8c9a4a14b4f77e2\n",
            "creating metadata file for /root/.cache/huggingface/transformers/37d0f67f084f8c5fc5589e0bba5ff3c6307af833bb0b7f4eb33fbfd8d4038a9d.84ea7af2df68dc8db434d3160aab65cce8ac63ce5b6f7743f8c9a4a14b4f77e2\n",
            "https://huggingface.co/google/mt5-small/resolve/main/special_tokens_map.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmp1wecztjw\n",
            "Downloading: 100% 99.0/99.0 [00:00<00:00, 123kB/s]\n",
            "storing https://huggingface.co/google/mt5-small/resolve/main/special_tokens_map.json in cache at /root/.cache/huggingface/transformers/685ac0ca8568ec593a48b61b0a3c272beee9bc194a3c7241d15dcadb5f875e53.f76030f3ec1b96a8199b2593390c610e76ca8028ef3d24680000619ffb646276\n",
            "creating metadata file for /root/.cache/huggingface/transformers/685ac0ca8568ec593a48b61b0a3c272beee9bc194a3c7241d15dcadb5f875e53.f76030f3ec1b96a8199b2593390c610e76ca8028ef3d24680000619ffb646276\n",
            "https://huggingface.co/google/mt5-small/resolve/main/tokenizer_config.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpe7nulfmj\n",
            "Downloading: 100% 82.0/82.0 [00:00<00:00, 121kB/s]\n",
            "storing https://huggingface.co/google/mt5-small/resolve/main/tokenizer_config.json in cache at /root/.cache/huggingface/transformers/6a9e52d6dd21568e37b65fc180ada927968e8f7124f0acd6efcaf90cd2e0f4bb.4b81e5d952ad810ca1de2b3e362b9a26a5cc77b4b75daf20caf69fb838751c32\n",
            "creating metadata file for /root/.cache/huggingface/transformers/6a9e52d6dd21568e37b65fc180ada927968e8f7124f0acd6efcaf90cd2e0f4bb.4b81e5d952ad810ca1de2b3e362b9a26a5cc77b4b75daf20caf69fb838751c32\n",
            "loading file https://huggingface.co/google/mt5-small/resolve/main/spiece.model from cache at /root/.cache/huggingface/transformers/37d0f67f084f8c5fc5589e0bba5ff3c6307af833bb0b7f4eb33fbfd8d4038a9d.84ea7af2df68dc8db434d3160aab65cce8ac63ce5b6f7743f8c9a4a14b4f77e2\n",
            "loading file https://huggingface.co/google/mt5-small/resolve/main/tokenizer.json from cache at None\n",
            "loading file https://huggingface.co/google/mt5-small/resolve/main/added_tokens.json from cache at None\n",
            "loading file https://huggingface.co/google/mt5-small/resolve/main/special_tokens_map.json from cache at /root/.cache/huggingface/transformers/685ac0ca8568ec593a48b61b0a3c272beee9bc194a3c7241d15dcadb5f875e53.f76030f3ec1b96a8199b2593390c610e76ca8028ef3d24680000619ffb646276\n",
            "loading file https://huggingface.co/google/mt5-small/resolve/main/tokenizer_config.json from cache at /root/.cache/huggingface/transformers/6a9e52d6dd21568e37b65fc180ada927968e8f7124f0acd6efcaf90cd2e0f4bb.4b81e5d952ad810ca1de2b3e362b9a26a5cc77b4b75daf20caf69fb838751c32\n",
            "https://huggingface.co/google/mt5-small/resolve/main/pytorch_model.bin not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpwc1dgr1q\n",
            "Downloading: 100% 1.20G/1.20G [00:20<00:00, 57.5MB/s]\n",
            "storing https://huggingface.co/google/mt5-small/resolve/main/pytorch_model.bin in cache at /root/.cache/huggingface/transformers/8e7b2a80ddcb5611b27d8c89e1e8e33a947e105415051402a22b9c8d7d1caeb0.e22331f3a065b885b30ae3dd1ff11ccaf7fbc444485f6eb07ef5e0138bca8b70\n",
            "creating metadata file for /root/.cache/huggingface/transformers/8e7b2a80ddcb5611b27d8c89e1e8e33a947e105415051402a22b9c8d7d1caeb0.e22331f3a065b885b30ae3dd1ff11ccaf7fbc444485f6eb07ef5e0138bca8b70\n",
            "loading weights file https://huggingface.co/google/mt5-small/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/8e7b2a80ddcb5611b27d8c89e1e8e33a947e105415051402a22b9c8d7d1caeb0.e22331f3a065b885b30ae3dd1ff11ccaf7fbc444485f6eb07ef5e0138bca8b70\n",
            "All model checkpoint weights were used when initializing MT5ForConditionalGeneration.\n",
            "\n",
            "All the weights of MT5ForConditionalGeneration were initialized from the model checkpoint at google/mt5-small.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use MT5ForConditionalGeneration for predictions without further training.\n",
            "100% 10/10 [00:12<00:00,  1.21s/ba]\n",
            "100% 7/7 [00:10<00:00,  1.45s/ba]\n",
            "***** Running training *****\n",
            "  Num examples = 9866\n",
            "  Num Epochs = 3\n",
            "  Instantaneous batch size per device = 4\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 4\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 7401\n",
            "{'loss': 5.1956, 'learning_rate': 4.662207809755439e-05, 'epoch': 0.2}\n",
            "  7% 500/7401 [04:12<57:59,  1.98it/s]Saving model checkpoint to /content/result/checkpoint-500\n",
            "Configuration saved in /content/result/checkpoint-500/config.json\n",
            "Model weights saved in /content/result/checkpoint-500/pytorch_model.bin\n",
            "tokenizer config file saved in /content/result/checkpoint-500/tokenizer_config.json\n",
            "Special tokens file saved in /content/result/checkpoint-500/special_tokens_map.json\n",
            "Copy vocab file to /content/result/checkpoint-500/spiece.model\n",
            "{'loss': 0.9947, 'learning_rate': 4.324415619510877e-05, 'epoch': 0.41}\n",
            " 14% 1000/7401 [09:21<47:11,  2.26it/s]Saving model checkpoint to /content/result/checkpoint-1000\n",
            "Configuration saved in /content/result/checkpoint-1000/config.json\n",
            "Model weights saved in /content/result/checkpoint-1000/pytorch_model.bin\n",
            "tokenizer config file saved in /content/result/checkpoint-1000/tokenizer_config.json\n",
            "Special tokens file saved in /content/result/checkpoint-1000/special_tokens_map.json\n",
            "Copy vocab file to /content/result/checkpoint-1000/spiece.model\n",
            "{'loss': 0.6817, 'learning_rate': 3.986623429266316e-05, 'epoch': 0.61}\n",
            " 20% 1500/7401 [14:33<45:17,  2.17it/s]Saving model checkpoint to /content/result/checkpoint-1500\n",
            "Configuration saved in /content/result/checkpoint-1500/config.json\n",
            "Model weights saved in /content/result/checkpoint-1500/pytorch_model.bin\n",
            "tokenizer config file saved in /content/result/checkpoint-1500/tokenizer_config.json\n",
            "Special tokens file saved in /content/result/checkpoint-1500/special_tokens_map.json\n",
            "Copy vocab file to /content/result/checkpoint-1500/spiece.model\n",
            "{'loss': 0.5605, 'learning_rate': 3.6488312390217535e-05, 'epoch': 0.81}\n",
            " 27% 2000/7401 [19:44<44:19,  2.03it/s]Saving model checkpoint to /content/result/checkpoint-2000\n",
            "Configuration saved in /content/result/checkpoint-2000/config.json\n",
            "Model weights saved in /content/result/checkpoint-2000/pytorch_model.bin\n",
            "tokenizer config file saved in /content/result/checkpoint-2000/tokenizer_config.json\n",
            "Special tokens file saved in /content/result/checkpoint-2000/special_tokens_map.json\n",
            "Copy vocab file to /content/result/checkpoint-2000/spiece.model\n",
            "{'loss': 0.493, 'learning_rate': 3.3110390487771925e-05, 'epoch': 1.01}\n",
            " 34% 2500/7401 [24:53<36:40,  2.23it/s]Saving model checkpoint to /content/result/checkpoint-2500\n",
            "Configuration saved in /content/result/checkpoint-2500/config.json\n",
            "Model weights saved in /content/result/checkpoint-2500/pytorch_model.bin\n",
            "tokenizer config file saved in /content/result/checkpoint-2500/tokenizer_config.json\n",
            "Special tokens file saved in /content/result/checkpoint-2500/special_tokens_map.json\n",
            "Copy vocab file to /content/result/checkpoint-2500/spiece.model\n",
            "{'loss': 0.4343, 'learning_rate': 2.9732468585326305e-05, 'epoch': 1.22}\n",
            " 41% 3000/7401 [30:06<34:18,  2.14it/s]Saving model checkpoint to /content/result/checkpoint-3000\n",
            "Configuration saved in /content/result/checkpoint-3000/config.json\n",
            "Model weights saved in /content/result/checkpoint-3000/pytorch_model.bin\n",
            "tokenizer config file saved in /content/result/checkpoint-3000/tokenizer_config.json\n",
            "Special tokens file saved in /content/result/checkpoint-3000/special_tokens_map.json\n",
            "Copy vocab file to /content/result/checkpoint-3000/spiece.model\n",
            "{'loss': 0.4105, 'learning_rate': 2.6354546682880692e-05, 'epoch': 1.42}\n",
            " 47% 3500/7401 [35:17<29:33,  2.20it/s]Saving model checkpoint to /content/result/checkpoint-3500\n",
            "Configuration saved in /content/result/checkpoint-3500/config.json\n",
            "Model weights saved in /content/result/checkpoint-3500/pytorch_model.bin\n",
            "tokenizer config file saved in /content/result/checkpoint-3500/tokenizer_config.json\n",
            "Special tokens file saved in /content/result/checkpoint-3500/special_tokens_map.json\n",
            "Copy vocab file to /content/result/checkpoint-3500/spiece.model\n",
            "{'loss': 0.3914, 'learning_rate': 2.297662478043508e-05, 'epoch': 1.62}\n",
            " 54% 4000/7401 [40:31<30:15,  1.87it/s]Saving model checkpoint to /content/result/checkpoint-4000\n",
            "Configuration saved in /content/result/checkpoint-4000/config.json\n",
            "Model weights saved in /content/result/checkpoint-4000/pytorch_model.bin\n",
            "tokenizer config file saved in /content/result/checkpoint-4000/tokenizer_config.json\n",
            "Special tokens file saved in /content/result/checkpoint-4000/special_tokens_map.json\n",
            "Copy vocab file to /content/result/checkpoint-4000/spiece.model\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/torch/serialization.py\", line 372, in save\n",
            "    _save(obj, opened_zipfile, pickle_module, pickle_protocol)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/torch/serialization.py\", line 491, in _save\n",
            "    zip_file.write_record(name, storage.data_ptr(), num_bytes)\n",
            "OSError: [Errno 28] No space left on device\n",
            "\n",
            "During handling of the above exception, another exception occurred:\n",
            "\n",
            "Traceback (most recent call last):\n",
            "  File \"/content/transformers/examples/pytorch/summarization/run_summarization.py\", line 606, in <module>\n",
            "    main()\n",
            "  File \"/content/transformers/examples/pytorch/summarization/run_summarization.py\", line 530, in main\n",
            "    train_result = trainer.train(resume_from_checkpoint=checkpoint)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/transformers/trainer.py\", line 1315, in train\n",
            "    self._maybe_log_save_evaluate(tr_loss, model, trial, epoch)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/transformers/trainer.py\", line 1408, in _maybe_log_save_evaluate\n",
            "    self._save_checkpoint(model, trial, metrics=metrics)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/transformers/trainer.py\", line 1498, in _save_checkpoint\n",
            "    torch.save(self.optimizer.state_dict(), os.path.join(output_dir, \"optimizer.pt\"))\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/torch/serialization.py\", line 373, in save\n",
            "    return\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/torch/serialization.py\", line 259, in __exit__\n",
            "    self.file_like.write_end_of_file()\n",
            "RuntimeError: [enforce fail at inline_container.cc:274] . unexpected pos 1376992704 vs 1376992592\n",
            "terminate called after throwing an instance of 'c10::Error'\n",
            "  what():  [enforce fail at inline_container.cc:274] . unexpected pos 1376992704 vs 1376992592\n",
            "frame #0: c10::ThrowEnforceNotMet(char const*, int, char const*, std::string const&, void const*) + 0x47 (0x7f79858380e7 in /usr/local/lib/python3.7/dist-packages/torch/lib/libc10.so)\n",
            "frame #1: <unknown function> + 0x20699b0 (0x7f79c38319b0 in /usr/local/lib/python3.7/dist-packages/torch/lib/libtorch_cpu.so)\n",
            "frame #2: <unknown function> + 0x2065b83 (0x7f79c382db83 in /usr/local/lib/python3.7/dist-packages/torch/lib/libtorch_cpu.so)\n",
            "frame #3: caffe2::serialize::PyTorchStreamWriter::writeRecord(std::string const&, void const*, unsigned long, bool) + 0xa9 (0x7f79c3832329 in /usr/local/lib/python3.7/dist-packages/torch/lib/libtorch_cpu.so)\n",
            "frame #4: caffe2::serialize::PyTorchStreamWriter::writeEndOfFile() + 0xe1 (0x7f79c3832e61 in /usr/local/lib/python3.7/dist-packages/torch/lib/libtorch_cpu.so)\n",
            "frame #5: caffe2::serialize::PyTorchStreamWriter::~PyTorchStreamWriter() + 0x115 (0x7f79c3833655 in /usr/local/lib/python3.7/dist-packages/torch/lib/libtorch_cpu.so)\n",
            "frame #6: <unknown function> + 0x7414a3 (0x7f79d44404a3 in /usr/local/lib/python3.7/dist-packages/torch/lib/libtorch_python.so)\n",
            "frame #7: <unknown function> + 0x36a1d8 (0x7f79d40691d8 in /usr/local/lib/python3.7/dist-packages/torch/lib/libtorch_python.so)\n",
            "frame #8: <unknown function> + 0x36b4de (0x7f79d406a4de in /usr/local/lib/python3.7/dist-packages/torch/lib/libtorch_python.so)\n",
            "<omitting python frames>\n",
            "frame #22: __libc_start_main + 0xe7 (0x7f7a084c7bf7 in /lib/x86_64-linux-gnu/libc.so.6)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}