arxiv_4_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ah1yQrPaW8yD"
   },
   "source": [
    "# Dython Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "KVsU3k-M4d2x",
    "outputId": "412a4de1-f506-4ccf-8c15-28582dba3798"
   },
   "outputs": [],
   "source": [
    "!pip install \"nougat-ocr[dataset]\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "gQvKV2pTSShw",
    "outputId": "68b08527-6b43-4d93-95fa-a2a622330ba4"
   },
   "outputs": [],
   "source": [
    "!pip install tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "2pzhd03MrRzf"
   },
   "source": [
    "# Drive Mount"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "J3PxInuaC6lA",
    "outputId": "8a83669f-c4ab-471d-f922-54a7cb4b1fef"
   },
   "outputs": [],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "lvK7dnOkGmEA",
    "outputId": "c014626e-86dc-4bea-d346-d283f4e3a452"
   },
   "outputs": [],
   "source": [
    "# Change to your specific path.\n",
    "%cd \"/content/drive/MyDrive/Workspace/nougat\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JlNVxdx-LB7I"
   },
   "source": [
    "# CUDA Version Verification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "w16ql85OLDk6",
    "outputId": "143d648b-7f8d-4b4a-f99f-1ec79dcb3bdc"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "print(torch.version.cuda)\n",
    "!nvcc --version"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AA5shakX-X9Y"
   },
   "source": [
    "# Lightning Module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "tthmGbnO-aFJ"
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Donut\n",
    "Copyright (c) 2022-present NAVER Corp.\n",
    "MIT License\n",
    "Copyright (c) Meta Platforms, Inc. and affiliates.\n",
    "\"\"\"\n",
    "import math\n",
    "import random\n",
    "from pathlib import Path\n",
    "\n",
    "import numpy as np\n",
    "import lightning.pytorch as pl\n",
    "import torch\n",
    "from lightning.pytorch.utilities import rank_zero_only\n",
    "from torch.nn.utils.rnn import pad_sequence\n",
    "from torch.optim.lr_scheduler import LambdaLR\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "from nougat import NougatConfig, NougatModel\n",
    "from nougat.metrics import get_metrics\n",
    "\n",
    "\n",
    "class NougatModelPLModule(pl.LightningModule):\n",
    "    def __init__(self, config):\n",
    "        super().__init__()\n",
    "        self.validation_step_outputs = []\n",
    "        self.config = config\n",
    "        if self.config.get(\"model_path\", False):\n",
    "            self.model = NougatModel.from_pretrained(\n",
    "                self.config.model_path,\n",
    "                input_size=self.config.input_size,\n",
    "                max_length=self.config.max_length,\n",
    "                align_long_axis=self.config.align_long_axis,\n",
    "                window_size=self.config.window_size,\n",
    "                encoder_layer=self.config.encoder_layer,\n",
    "                decoder_layer=self.config.decoder_layer,\n",
    "                patch_size=self.config.patch_size,\n",
    "                embed_dim=self.config.embed_dim,\n",
    "                num_heads=self.config.num_heads,\n",
    "                hidden_dimension=self.config.hidden_dimension,\n",
    "                ignore_mismatched_sizes=True,\n",
    "            )\n",
    "        else:\n",
    "            self.model = NougatModel(\n",
    "                config=NougatConfig(\n",
    "                    input_size=self.config.input_size,\n",
    "                    max_length=self.config.max_length,\n",
    "                    align_long_axis=self.config.align_long_axis,\n",
    "                    window_size=self.config.window_size,\n",
    "                    encoder_layer=self.config.encoder_layer,\n",
    "                    decoder_layer=self.config.decoder_layer,\n",
    "                    tokenizer_file=self.config.tokenizer,\n",
    "                    patch_size=self.config.patch_size,\n",
    "                    embed_dim=self.config.embed_dim,\n",
    "                    num_heads=self.config.num_heads,\n",
    "                    hidden_dimension=self.config.hidden_dimension,\n",
    "                )\n",
    "            )\n",
    "\n",
    "    def training_step(self, batch, batch_idx):\n",
    "        image_tensors, decoder_input_ids, attention_masks = list(), list(), list()\n",
    "        if batch is None:\n",
    "            return\n",
    "        for batch_data in batch:\n",
    "            if batch_data is None or batch_data[0] is None:\n",
    "                continue\n",
    "            image_tensors.append(batch_data[0])\n",
    "            decoder_input_ids.append(batch_data[1])\n",
    "            attention_masks.append(batch_data[2])\n",
    "        image_tensors = torch.cat(image_tensors)\n",
    "        decoder_input_ids = torch.cat(decoder_input_ids)\n",
    "        attention_masks = torch.cat(attention_masks)\n",
    "        loss = self.model(image_tensors, decoder_input_ids, attention_masks)[0]\n",
    "        if loss is not None:\n",
    "            self.log_dict({\"train/loss\": loss}, sync_dist=True)\n",
    "        return loss\n",
    "\n",
    "    def validation_step(self, batch, batch_idx, dataset_idx=0):\n",
    "        if batch is None:\n",
    "            return\n",
    "        image_tensors, decoder_input_ids, _ = batch\n",
    "        if image_tensors is None:\n",
    "            return\n",
    "        markdown = pad_sequence(\n",
    "            decoder_input_ids,\n",
    "            batch_first=True,\n",
    "        )\n",
    "        preds = self.model.inference(\n",
    "            image_tensors=image_tensors,\n",
    "            return_attentions=False,\n",
    "        )[\"predictions\"]\n",
    "        gts = self.model.decoder.tokenizer.batch_decode(\n",
    "            markdown, skip_special_tokens=True\n",
    "        )\n",
    "        metrics = get_metrics(gts, preds, pool=False)\n",
    "        scores = {\n",
    "            \"val/\" + key: sum(values) / len(values) for key, values in metrics.items()\n",
    "        }\n",
    "        self.validation_step_outputs.append(scores)\n",
    "        return scores\n",
    "\n",
    "    def on_validation_epoch_end(self):\n",
    "        if (\n",
    "            self.validation_step_outputs is not None\n",
    "            and len(self.validation_step_outputs) >= 1\n",
    "        ):\n",
    "            self.log_dict(self.validation_step_outputs[0], sync_dist=True)\n",
    "            self.validation_step_outputs.clear()\n",
    "\n",
    "    def configure_optimizers(self):\n",
    "        def _get_device_count():\n",
    "            if torch.cuda.is_available():\n",
    "                return torch.cuda.device_count()\n",
    "            elif torch.backends.mps.is_available():\n",
    "                # Can MPS have more than one device?\n",
    "                return 1\n",
    "            return 1\n",
    "\n",
    "        max_iter = None\n",
    "\n",
    "        if int(self.config.get(\"max_epochs\", -1)) > 0:\n",
    "            assert (\n",
    "                len(self.config.train_batch_sizes) == 1\n",
    "            ), \"Set max_epochs only if the number of datasets is 1\"\n",
    "            steps = self.config.num_training_samples_per_epoch\n",
    "            max_iter = (self.config.max_epochs * steps) / max(\n",
    "                1,\n",
    "                (\n",
    "                    self.config.train_batch_sizes[0]\n",
    "                    * _get_device_count()\n",
    "                    * self.config.get(\"num_nodes\", 1)\n",
    "                ),\n",
    "            )\n",
    "\n",
    "        if int(self.config.get(\"max_steps\", -1)) > 0:\n",
    "            max_iter = (\n",
    "                min(self.config.max_steps, max_iter)\n",
    "                if max_iter is not None\n",
    "                else self.config.max_steps\n",
    "            )\n",
    "\n",
    "        assert max_iter is not None\n",
    "        optimizer = torch.optim.AdamW(self.parameters(), lr=self.config.lr)\n",
    "        scheduler = {\n",
    "            \"scheduler\": self.exponential_scheduler(\n",
    "                optimizer,\n",
    "                self.config.warmup_steps,\n",
    "                self.config.lr,\n",
    "                self.config.get(\"min_lr\", 5e-5),\n",
    "                self.config.get(\"gamma\", 0.9996),\n",
    "            ),\n",
    "            \"name\": \"learning_rate\",\n",
    "            \"interval\": \"step\",\n",
    "            \"frequency\": self.config.get(\"lr_step\", 1),\n",
    "        }\n",
    "        return [optimizer], [scheduler]\n",
    "\n",
    "    @staticmethod\n",
    "    def cosine_scheduler(optimizer, training_steps, warmup_steps):\n",
    "        def lr_lambda(current_step):\n",
    "            if current_step < warmup_steps:\n",
    "                return current_step / max(1, warmup_steps)\n",
    "            progress = current_step - warmup_steps\n",
    "            progress /= max(1, training_steps - warmup_steps)\n",
    "            return max(0.0, 0.5 * (1.0 + math.cos(math.pi * progress)))\n",
    "\n",
    "        return LambdaLR(optimizer, lr_lambda)\n",
    "\n",
    "    @staticmethod\n",
    "    def exponential_scheduler(optimizer, warmup_steps, lr, min_lr=5e-5, gamma=0.9999):\n",
    "        def lr_lambda(x):\n",
    "            if x > warmup_steps or warmup_steps <= 0:\n",
    "                if lr * gamma ** (x - warmup_steps) > min_lr:\n",
    "                    return gamma ** (x - warmup_steps)\n",
    "                else:\n",
    "                    return min_lr / lr\n",
    "            else:\n",
    "                return x / warmup_steps\n",
    "\n",
    "        return LambdaLR(optimizer, lr_lambda=lr_lambda)\n",
    "\n",
    "    def get_progress_bar_dict(self):\n",
    "        items = super().get_progress_bar_dict()\n",
    "        items.pop(\"v_num\", None)\n",
    "        items[\"exp_name\"] = f\"{self.config.get('exp_name', '')}\"\n",
    "        items[\"exp_version\"] = f\"{self.config.get('exp_version', '')}\"\n",
    "        return items\n",
    "\n",
    "    @rank_zero_only\n",
    "    def on_save_checkpoint(self, checkpoint):\n",
    "        save_path = (\n",
    "            Path(self.config.result_path)\n",
    "            / self.config.exp_name\n",
    "            / self.config.exp_version\n",
    "        )\n",
    "        self.model.save_pretrained(save_path)\n",
    "        self.model.decoder.tokenizer.save_pretrained(save_path)\n",
    "\n",
    "\n",
    "class NougatDataPLModule(pl.LightningDataModule):\n",
    "    def __init__(self, config):\n",
    "        super().__init__()\n",
    "        self.config = config\n",
    "        self.train_batch_sizes = self.config.train_batch_sizes\n",
    "        self.val_batch_sizes = self.config.val_batch_sizes\n",
    "        self.train_datasets = []\n",
    "        self.val_datasets = []\n",
    "        self.g = torch.Generator()\n",
    "        self.g.manual_seed(self.config.seed)\n",
    "\n",
    "    def train_dataloader(self):\n",
    "        loaders = [\n",
    "            DataLoader(\n",
    "                torch.utils.data.ConcatDataset(self.train_datasets),\n",
    "                batch_size=self.train_batch_sizes[0],\n",
    "                num_workers=self.config.num_workers,\n",
    "                pin_memory=True,\n",
    "                worker_init_fn=self.seed_worker,\n",
    "                generator=self.g,\n",
    "                shuffle=True,\n",
    "                collate_fn=self.ignore_none_collate,\n",
    "            )\n",
    "        ]\n",
    "        return loaders\n",
    "\n",
    "    def val_dataloader(self):\n",
    "        loaders = [\n",
    "            DataLoader(\n",
    "                torch.utils.data.ConcatDataset(self.val_datasets),\n",
    "                batch_size=self.val_batch_sizes[0],\n",
    "                pin_memory=True,\n",
    "                shuffle=True,\n",
    "                collate_fn=self.ignore_none_collate,\n",
    "            )\n",
    "        ]\n",
    "        return loaders\n",
    "\n",
    "    @staticmethod\n",
    "    def seed_worker(wordker_id):\n",
    "        worker_seed = torch.initial_seed() % 2**32\n",
    "        np.random.seed(worker_seed)\n",
    "        random.seed(worker_seed)\n",
    "\n",
    "    @staticmethod\n",
    "    def ignore_none_collate(batch):\n",
    "        if batch is None:\n",
    "            return\n",
    "        try:\n",
    "            batch = [x for x in batch if x is not None and x[0] is not None]\n",
    "            if len(batch) == 0:\n",
    "                return\n",
    "            return torch.utils.data.dataloader.default_collate(batch)\n",
    "        except AttributeError:\n",
    "            pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "O93iNYSq-NER"
   },
   "source": [
    "# Test Script Definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Zvjx1vka-Me9"
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Donut\n",
    "Copyright (c) 2022-present NAVER Corp.\n",
    "MIT License\n",
    "Copyright (c) Meta Platforms, Inc. and affiliates.\n",
    "\"\"\"\n",
    "import argparse\n",
    "import json\n",
    "import os\n",
    "import logging\n",
    "from multiprocessing import Pool\n",
    "from collections import defaultdict\n",
    "from pathlib import Path\n",
    "\n",
    "import numpy as np\n",
    "import torch\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "from nougat import NougatModel\n",
    "from nougat.metrics import compute_metrics\n",
    "from nougat.utils.checkpoint import get_checkpoint\n",
    "from nougat.utils.dataset import NougatDataset\n",
    "from nougat.utils.device import move_to_device\n",
    "\n",
    "\n",
    "logging.basicConfig(level=logging.INFO)\n",
    "\n",
    "\n",
    "def test(args):\n",
    "    pretrained_model = NougatModel.from_pretrained(args.checkpoint)\n",
    "    pretrained_model = move_to_device(pretrained_model)\n",
    "\n",
    "    pretrained_model.eval()\n",
    "\n",
    "    if args.save_path:\n",
    "        os.makedirs(os.path.dirname(args.save_path), exist_ok=True)\n",
    "    else:\n",
    "        logging.warning(\"Results can not be saved. Please provide a -o/--save_path\")\n",
    "    predictions = []\n",
    "    ground_truths = []\n",
    "    metrics = defaultdict(list)\n",
    "    dataset = NougatDataset(\n",
    "        dataset_path=args.dataset,\n",
    "        nougat_model=pretrained_model,\n",
    "        max_length=pretrained_model.config.max_length,\n",
    "        split=args.split,\n",
    "        root_name=\"out\",\n",
    "    )\n",
    "\n",
    "    dataloader = torch.utils.data.DataLoader(\n",
    "        dataset,\n",
    "        batch_size=args.batch_size,\n",
    "        num_workers=4,\n",
    "        pin_memory=True,\n",
    "        shuffle=args.shuffle,\n",
    "        collate_fn=NougatDataPLModule.ignore_none_collate,\n",
    "    )\n",
    "\n",
    "    for idx, sample in tqdm(enumerate(dataloader), total=len(dataloader)):\n",
    "        if sample is None:\n",
    "            continue\n",
    "        image_tensors, decoder_input_ids, _ = sample\n",
    "        if image_tensors is None:\n",
    "            return\n",
    "        # if len(predictions) >= args.num_samples:\n",
    "        #     break\n",
    "        ground_truth = pretrained_model.decoder.tokenizer.batch_decode(\n",
    "            decoder_input_ids, skip_special_tokens=True\n",
    "        )\n",
    "        outputs = pretrained_model.inference(\n",
    "            image_tensors=image_tensors,\n",
    "            return_attentions=False,\n",
    "        )[\"predictions\"]\n",
    "        predictions.extend(outputs)\n",
    "        ground_truths.extend(ground_truth)\n",
    "        with Pool(args.batch_size) as p:\n",
    "            _metrics = p.starmap(compute_metrics, iterable=zip(outputs, ground_truth))\n",
    "            for m in _metrics:\n",
    "                for key, value in m.items():\n",
    "                    metrics[key].append(value)\n",
    "\n",
    "            print({key: sum(values) / len(values) for key, values in metrics.items()})\n",
    "\n",
    "    scores = {}\n",
    "    for metric, vals in metrics.items():\n",
    "        scores[f\"{metric}_accuracies\"] = vals\n",
    "        scores[f\"{metric}_accuracy\"] = np.mean(vals)\n",
    "    try:\n",
    "        print(\n",
    "            f\"Total number of samples: {len(vals)}, Edit Distance (ED) based accuracy score: {scores['edit_dist_accuracy']}, BLEU score: {scores['bleu_accuracy']}, METEOR score: {scores['meteor_accuracy']}\"\n",
    "        )\n",
    "    except:\n",
    "        pass\n",
    "    if args.save_path:\n",
    "        scores[\"predictions\"] = predictions\n",
    "        scores[\"ground_truths\"] = ground_truths\n",
    "        with open(args.save_path, \"w\") as f:\n",
    "            json.dump(scores, f)\n",
    "\n",
    "    return predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BFS8OXC-4fjM"
   },
   "source": [
    "# Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "S_2WxNLtSOkL"
   },
   "outputs": [],
   "source": [
    "def format_duration(total_seconds):\n",
    "    total_seconds = int(total_seconds)\n",
    "    hours = total_seconds // 3600\n",
    "    minutes = (total_seconds % 3600) // 60\n",
    "    seconds = total_seconds % 60\n",
    "    return f\"{hours:02d}h{minutes:02d}m{seconds:02d}s\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000,
     "referenced_widgets": [
      "5c11998ab470475da1560d78bd0c7744",
      "16557948718a44ebb3a6f88b6aa1560f",
      "111d6557c49f40c2981be0df68935c12",
      "189555d0a69c4aaea02e07ad8f81a659",
      "c39698aef91f4051840bd142b41a860c",
      "8687696407674c8aa1f8eb841497dcd5",
      "b2a35edf1c17481c94da66c1b191e835",
      "e40c568403dc4405a2e0910857cb4b64",
      "2ee5ec758a7c40baa6c97257fbe9ef1d",
      "e86e04eebdda4dc79f532253771505af",
      "6f9e8ed4fde44b5cad52e1f2d3b20753"
     ]
    },
    "id": "HwNEAa_F-RDH",
    "outputId": "96654dd0-e691-47d4-e7ee-87113fb15c2f"
   },
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "class Args:\n",
    "    pass\n",
    "\n",
    "args = Args()\n",
    "args.checkpoint = \"./checkpoint_base\"\n",
    "args.dataset = \"./dataset/arxiv_2001/test.jsonl\"\n",
    "args.split = \"test\"\n",
    "args.save_path = \"./results/arxiv_2001.json\"\n",
    "args.num_samples = -1\n",
    "args.shuffle = False\n",
    "args.batch_size = 10\n",
    "\n",
    "# Execution and duration computation.\n",
    "start_time = time.time()\n",
    "predictions = test(args)\n",
    "end_time = time.time()\n",
    "execution_time = end_time - start_time\n",
    "print(format_duration(execution_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ELYxvoHnEp2F"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "A100",
   "machine_shape": "hm",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "111d6557c49f40c2981be0df68935c12": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_e40c568403dc4405a2e0910857cb4b64",
      "max": 39,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_2ee5ec758a7c40baa6c97257fbe9ef1d",
      "value": 39
     }
    },
    "16557948718a44ebb3a6f88b6aa1560f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_8687696407674c8aa1f8eb841497dcd5",
      "placeholder": "​",
      "style": "IPY_MODEL_b2a35edf1c17481c94da66c1b191e835",
      "value": "100%"
     }
    },
    "189555d0a69c4aaea02e07ad8f81a659": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_e86e04eebdda4dc79f532253771505af",
      "placeholder": "​",
      "style": "IPY_MODEL_6f9e8ed4fde44b5cad52e1f2d3b20753",
      "value": " 39/39 [51:17&lt;00:00, 83.48s/it]"
     }
    },
    "2ee5ec758a7c40baa6c97257fbe9ef1d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "5c11998ab470475da1560d78bd0c7744": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_16557948718a44ebb3a6f88b6aa1560f",
       "IPY_MODEL_111d6557c49f40c2981be0df68935c12",
       "IPY_MODEL_189555d0a69c4aaea02e07ad8f81a659"
      ],
      "layout": "IPY_MODEL_c39698aef91f4051840bd142b41a860c"
     }
    },
    "6f9e8ed4fde44b5cad52e1f2d3b20753": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "8687696407674c8aa1f8eb841497dcd5": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "b2a35edf1c17481c94da66c1b191e835": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "c39698aef91f4051840bd142b41a860c": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e40c568403dc4405a2e0910857cb4b64": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e86e04eebdda4dc79f532253771505af": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}