Attention_Basic.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Dj1p0mGYYTD_"
   },
   "source": [
    "This part mainly contains the basic attention mechanism and a simple transformer-based model, the gpt2 model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0z6C6HjSUiDn"
   },
   "source": [
    "> some codes referenced from <a href='https://scholar.harvard.edu/binxuw/classes/machine-learning-scratch/materials/transformers'>MLFS</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "eRpBW_mW4gKX"
   },
   "source": [
    "# Self-Attention: Single Head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "nZFYcVC0-DES",
    "outputId": "bb4f1188-a1bb-46d2-8841-b886fd9a0d62"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting einops\n",
      "  Downloading einops-0.6.1-py3-none-any.whl (42 kB)\n",
      "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/42.2 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.2/42.2 kB\u001b[0m \u001b[31m3.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: einops\n",
      "Successfully installed einops-0.6.1\n"
     ]
    }
   ],
   "source": [
    "!pip install einops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "V1utzF5d4tlS"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "import numpy as np\n",
    "import math\n",
    "from einops import rearrange"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "ASLLQLVz4z-5"
   },
   "outputs": [],
   "source": [
    "seed = 42\n",
    "np.random.seed(seed)\n",
    "torch.manual_seed(seed)\n",
    "torch.cuda.manual_seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "id": "ToVdFlKK5Rf8"
   },
   "outputs": [],
   "source": [
    "class SingleHeadAttention:\n",
    "    def __init__(self, emb_dim, head_dim, head=1):\n",
    "        self.to_q = # your code here\n",
    "        self.to_k = # your code here\n",
    "        self.to_v = # your code here\n",
    "\n",
    "    def get_qkv(self, tokens):\n",
    "        # your code here\n",
    "        return q, k, v\n",
    "\n",
    "    def __call__(self, tokens):\n",
    "        # your code here\n",
    "        return val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "id": "-rq5j63M4xaC"
   },
   "outputs": [],
   "source": [
    "# TEST implementation\n",
    "emb_dim = 128\n",
    "head_dim = 64\n",
    "tokens = torch.rand(32, 12, emb_dim)\n",
    "\n",
    "m = SingleHeadAttention(emb_dim, head_dim)\n",
    "q, k, v = m.get_qkv(tokens)\n",
    "assert torch.allclose(m(tokens), F.scaled_dot_product_attention(q, k, v))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "gubtQoYY4kWp"
   },
   "source": [
    "# Self-Attention: Multi Head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "id": "aT2n9kHI8-0b"
   },
   "outputs": [],
   "source": [
    "class MultiHeadAttention:\n",
    "    def __init__(self, emb_dim, head_dim, head):\n",
    "        assert emb_dim == head_dim * head, \"required! emb_dim == head_dim * head\"\n",
    "\n",
    "        inner_dim = head_dim * head\n",
    "        self.head_dim = head_dim\n",
    "        self.head = head\n",
    "        self.to_qs = # your code here\n",
    "        self.to_ks = # your code here\n",
    "        self.to_vs = # your code here\n",
    "\n",
    "    def get_qkv(self, tokens):\n",
    "        bs, t, _ = tokens.shape\n",
    "        # your code here\n",
    "        # ....\n",
    "        \n",
    "        assert q.shape == (bs, self.head, t, self.head_dim)\n",
    "        assert k.shape == (bs, self.head, t, self.head_dim)\n",
    "        assert v.shape == (bs, self.head, t, self.head_dim)\n",
    "        return q, k, v\n",
    "\n",
    "    def get_att_score(self, tokens):\n",
    "        # your code here\n",
    "\n",
    "    def __call__(self, tokens):\n",
    "        # your code here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "id": "Or2Z_jqm89Vg"
   },
   "outputs": [],
   "source": [
    "# TEST implementation\n",
    "emb_dim = 768\n",
    "head_dim = 64\n",
    "head = 12\n",
    "\n",
    "tokens = torch.rand(32, 12, emb_dim)\n",
    "\n",
    "m = MultiHeadAttention(emb_dim, head_dim, head)\n",
    "mha = nn.MultiheadAttention(emb_dim, head, batch_first=True,)\n",
    "\n",
    "# load state dict\n",
    "tmp_dict = {\n",
    "    'in_proj_weight': torch.cat([m.to_qs.weight.data.T, m.to_ks.weight.data.T, m.to_vs.weight.data.T], dim=1).T\n",
    "}\n",
    "for k, v in mha.state_dict().items():\n",
    "    if k in tmp_dict:\n",
    "        continue\n",
    "    tmp_dict[k] = v\n",
    "mha.load_state_dict(tmp_dict)\n",
    "\n",
    "mha_att = mha(tokens, tokens, tokens, average_attn_weights=False)[1]\n",
    "\n",
    "assert torch.allclose(m.get_att_score(tokens), mha_att, atol=1e-6, rtol=1e-6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "oRN5VBQmqVZm"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "HAf44Qo148c8"
   },
   "source": [
    "# Multi Head Attention with mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "id": "HxzsK6355H0q"
   },
   "outputs": [],
   "source": [
    "class MultiHeadAttentionMask:\n",
    "    def __init__(self, emb_dim, head_dim, head, token_num):\n",
    "        assert emb_dim == head * head_dim\n",
    "        inner_dim = head * head_dim\n",
    "\n",
    "        self.head = head\n",
    "        self.head_dim = head_dim\n",
    "\n",
    "        self.to_qs = # your code here\n",
    "        self.to_ks = # your code here\n",
    "        self.to_vs = # your code here\n",
    "\n",
    "        self.mask = # your code here\n",
    "\n",
    "    def get_qkv(self, tokens):\n",
    "        bs, t, _ = tokens.shape\n",
    "        # your code here\n",
    "\n",
    "        assert q.shape == (bs, self.head, t, self.head_dim)\n",
    "        assert k.shape == (bs, self.head, t, self.head_dim)\n",
    "        assert v.shape == (bs, self.head, t, self.head_dim)\n",
    "        return q, k, v\n",
    "\n",
    "    def get_att(self, tokens):\n",
    "        # your code here\n",
    "\n",
    "    def __call__(self, tokens):\n",
    "        # your code here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "id": "7AfDUcSs4_8R"
   },
   "outputs": [],
   "source": [
    "# TEST implementation\n",
    "emb_dim = 768\n",
    "head_dim = 64\n",
    "head = emb_dim // head_dim\n",
    "token_num = 12\n",
    "\n",
    "tokens = torch.rand(32, token_num, emb_dim)\n",
    "\n",
    "m = MultiHeadAttentionMask(emb_dim, head_dim, head, token_num)\n",
    "mha = nn.MultiheadAttention(emb_dim, head, batch_first=True)\n",
    "\n",
    "# load state dict\n",
    "tmp_dict = {\n",
    "    'in_proj_weight': torch.cat([m.to_qs.weight.data.T, m.to_ks.weight.data.T, m.to_vs.weight.data.T], dim=1).T\n",
    "}\n",
    "for k, v in mha.state_dict().items():\n",
    "    if k in tmp_dict:\n",
    "        continue\n",
    "    tmp_dict[k] = v\n",
    "mha.load_state_dict(tmp_dict)\n",
    "\n",
    "mha_att = mha(tokens, tokens, tokens, average_attn_weights=False, attn_mask=m.mask)[1]\n",
    "m_att = m.get_att(tokens)\n",
    "\n",
    "assert torch.allclose(m_att, mha_att, atol=1e-6, rtol=1e-6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "p4AOGg6k_TQl"
   },
   "source": [
    "# Transformer Model\n",
    "\n",
    "Key Part in Transformer Model\n",
    "\n",
    "- Res connection\n",
    "- LayerNorm\n",
    "- Multi-Head Attention\n",
    "- FFN\n",
    "\n",
    "Using gpt2 as an exmaple to impelment a simple Transformer model. Arch:\n",
    "1. word token embedding (50257, 768)\n",
    "2. word position embedding (1024, 768)\n",
    "3. 12 * gpt2_transformer_block\n",
    "4. layer norm\n",
    "\n",
    "Arch of gpt2 transformer block\n",
    "```\n",
    "     x-----------\n",
    "layer_norm      |\n",
    "mul_head_att    |\n",
    "  res_add ------|\n",
    "     x-----------\n",
    "layer_norm      |\n",
    "   ffn          |\n",
    " res_add -------|\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "5uVAPc8kXKxM"
   },
   "outputs": [],
   "source": [
    "!pip install transformers[torch]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "id": "-ZL96O669BlJ"
   },
   "outputs": [],
   "source": [
    "class GPT2Block:\n",
    "    def __init__(self, emb_dim, head, drop_out=0.0):\n",
    "        self.ln1 = # your code here\n",
    "        self.attn = # your code here\n",
    "        self.ln2 = # your code here\n",
    "        self.ffn = # your code here\n",
    "\n",
    "    def get_causal_mask(self, token_num):\n",
    "        # your code here\n",
    "        return mask\n",
    "\n",
    "    def __call__(self, tokens, is_causal=True):\n",
    "        # your code here"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uTV4Sf2cW19o"
   },
   "source": [
    "**TEST implementation**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "cellView": "form",
    "id": "0XoABZi_UbxG"
   },
   "outputs": [],
   "source": [
    "# @title test utils\n",
    "\n",
    "from transformers.activations import NewGELUActivation\n",
    "\n",
    "def GPT2block_to_TransformerBlock_simple(tfmblock, gpt2block, ):\n",
    "    \"\"\"copy the weights from a GPT2 block to a TransformerBlock_simple\"\"\"\n",
    "    tfmblock.ln1.weight.data = gpt2block.ln_1.weight\n",
    "    tfmblock.ln1.bias.data = gpt2block.ln_1.bias\n",
    "    tfmblock.ln2.weight.data = gpt2block.ln_2.weight\n",
    "    tfmblock.ln2.bias.data = gpt2block.ln_2.bias\n",
    "    tfmblock.attn.in_proj_weight.data = gpt2block.attn.c_attn.weight.T\n",
    "    tfmblock.attn.in_proj_bias.data = gpt2block.attn.c_attn.bias\n",
    "    tfmblock.attn.out_proj.weight.data = gpt2block.attn.c_proj.weight.T\n",
    "    tfmblock.attn.out_proj.bias.data = gpt2block.attn.c_proj.bias\n",
    "    tfmblock.ffn[0].weight.data = gpt2block.mlp.c_fc.weight.T\n",
    "    tfmblock.ffn[0].bias.data = gpt2block.mlp.c_fc.bias\n",
    "    tfmblock.ffn[1] = NewGELUActivation()\n",
    "    # mlp in GPT2 and BERT used a new GELU activation, using nn.GeLU() will cause a small error around 1E-3\n",
    "    tfmblock.ffn[2].weight.data = gpt2block.mlp.c_proj.weight.T\n",
    "    tfmblock.ffn[2].bias.data = gpt2block.mlp.c_proj.bias\n",
    "    return tfmblock\n",
    "\n",
    "def GPT2Model_to_GPT2Model_simple(gpt2modelsimple, gpt2model, ):\n",
    "    \"\"\"copy the weights from a GPT2 model to a GPT2Model_simple\"\"\"\n",
    "    gpt2modelsimple.wte.weight.data = gpt2model.wte.weight\n",
    "    gpt2modelsimple.wpe.weight.data = gpt2model.wpe.weight\n",
    "    gpt2modelsimple.ln_f.weight.data = gpt2model.ln_f.weight\n",
    "    gpt2modelsimple.ln_f.bias.data = gpt2model.ln_f.bias\n",
    "    for i in range(12):\n",
    "        GPT2block_to_TransformerBlock_simple(gpt2modelsimple.blocks[i], gpt2model.h[i])\n",
    "    return gpt2modelsimple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 574,
     "referenced_widgets": [
      "d5d3f0e7574049b8a87994bee0bb91dc",
      "d0b20f9b03c74821a2ef2b21f93ffb0e",
      "4985a90175fd49118fa24d719d3b4168",
      "d0e2e986593945e59ced2bab64744e66",
      "2860fd39e4c94fb5934c5dbdaa6cb49d",
      "09ea0dc2cc8c409dbcb4967a10cae246",
      "8cd430e0404e4d27a349673d18f32996",
      "47a0795ff69b42a8883039d7868f4247",
      "f820dd319b9c41dc878d681bef6ac33f",
      "3d5a65fdd0c6416aa5c4104b5a801ce4",
      "13f0a71a62504879b85f36fd99a8bed4",
      "96b987a8a77848f5ab5f74c3d89b8fe5",
      "c60b5ecb0b5c4ce9b96386f7d95e0744",
      "d4171779654f4694bc5659b68b0d6465",
      "14f3da0f2a884248ae63a9177cc1dc95",
      "3e34c6b4a9364839a42b16f08f4bcfab",
      "797cc29cf5ca442982337b805896d349",
      "84162339a7b944148c31c9742073ba87",
      "1b6414daa7a04ee0836d06b6e67d957c",
      "bcb59724b1694e72b42fa3bc49e2d508",
      "b2bb6fdafbe2486e8c63c1debd42bb5a",
      "70b0a2b48feb4f2ba711d6abea889986",
      "d2ac2eb2d53c427295a4b6846da09d73",
      "d325d37eec88426fa1552f922ec5a145",
      "bacb64c8339240898bd3eb5cbcc51a54",
      "7d0492f41d514779a3f028f4f1d9f0c4",
      "193945c3e2ab4d21afcccbc013c3485a",
      "f2d22f32efd04c08bb9d733244ee759d",
      "0d77c36de8a5481399e59a24f4ddd656",
      "6d87953681df49ec94e447882458d3ab",
      "91175f7c20d94a0795a0262b70a8e240",
      "2cfa51de74044282a8a5fcf359b83823",
      "7445c8410c2d47b08577ee29fa9f553a",
      "cdc6d12c5da841e784baa6426e41270b",
      "1af5ede6a6374e66ad84d98db8ebee51",
      "92938c0a2f664e2a90565933f42b396c",
      "2cc3b6ea94814c358e3435266b3400af",
      "d596456328ff45daa46fecf66522da89",
      "12d544c6b5524c8a9193097200ab00ee",
      "c444790d4a2745bbab083948b7a2bd5e",
      "526da09d88a94d1088b872560bce6c3f",
      "5acd99da9f1b482f8623bed5f945b0a7",
      "14e79cfcf4334503a961c865df63d86c",
      "2f78ce1e5a09455582f0b1ab33a28eaf"
     ]
    },
    "id": "tsgR764-VTaF",
    "outputId": "b96a1adc-8405-42a0-df4b-963734512602"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d5d3f0e7574049b8a87994bee0bb91dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)olve/main/vocab.json:   0%|          | 0.00/1.04M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "96b987a8a77848f5ab5f74c3d89b8fe5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)olve/main/merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d2ac2eb2d53c427295a4b6846da09d73",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/665 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cdc6d12c5da841e784baa6426e41270b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading model.safetensors:   0%|          | 0.00/548M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "GPT2Model(\n",
       "  (wte): Embedding(50257, 768)\n",
       "  (wpe): Embedding(1024, 768)\n",
       "  (drop): Dropout(p=0.1, inplace=False)\n",
       "  (h): ModuleList(\n",
       "    (0-11): 12 x GPT2Block(\n",
       "      (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "      (attn): GPT2Attention(\n",
       "        (c_attn): Conv1D()\n",
       "        (c_proj): Conv1D()\n",
       "        (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "        (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "      )\n",
       "      (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "      (mlp): GPT2MLP(\n",
       "        (c_fc): Conv1D()\n",
       "        (c_proj): Conv1D()\n",
       "        (act): NewGELUActivation()\n",
       "        (dropout): Dropout(p=0.1, inplace=False)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       ")"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import GPT2Tokenizer, GPT2Model\n",
    "\n",
    "tokenizer = GPT2Tokenizer.from_pretrained(\"gpt2\")\n",
    "model = GPT2Model.from_pretrained(\"gpt2\")\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "id": "oM30NK3LVKqi"
   },
   "outputs": [],
   "source": [
    "embdim = 768\n",
    "headcnt = 12\n",
    "tfmblock = GPT2Block(embdim, headcnt)\n",
    "GPT2block_to_TransformerBlock_simple(tfmblock, model.h[0])\n",
    "\n",
    "tokens_embs = torch.randn(2, 5, 768)\n",
    "tfmblock_out = tfmblock(tokens_embs, is_causal=True)\n",
    "modelblock_out, = model.h[0](tokens_embs)\n",
    "assert torch.allclose(tfmblock_out, modelblock_out, atol=1e-5, rtol=1e-5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "E2cCXLzVXFNZ"
   },
   "source": [
    "**Simple GPT2 Model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "id": "ngGyXN1OSJAr"
   },
   "outputs": [],
   "source": [
    "class GPT2:\n",
    "    def __init__(self, emb_dim, head, drop_out=0.0):\n",
    "        self.wte = # your code here\n",
    "        self.wpe = # your code here\n",
    "        self.blocks = # your code here\n",
    "        self.ln_f = # your code here\n",
    "\n",
    "    def __call__(self, input_ids, is_causal=True):\n",
    "        # your code here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "id": "g5X22yKDWH9D"
   },
   "outputs": [],
   "source": [
    "embdim = 768\n",
    "headcnt = 12\n",
    "\n",
    "model_ours = GPT2(embdim, headcnt)\n",
    "GPT2Model_to_GPT2Model_simple(model_ours, model)\n",
    "\n",
    "inputs = tokenizer(\"I have a cat, her name is\", return_tensors=\"pt\")\n",
    "outputs = model(**inputs, )\n",
    "\n",
    "hidden_last_ours = model_ours(inputs['input_ids'])\n",
    "\n",
    "assert torch.allclose(outputs.last_hidden_state, hidden_last_ours, atol=1e-5, rtol=1e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Gm-pQBYdX8ms"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "T4",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "09ea0dc2cc8c409dbcb4967a10cae246": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0d77c36de8a5481399e59a24f4ddd656": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "12d544c6b5524c8a9193097200ab00ee": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "13f0a71a62504879b85f36fd99a8bed4": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "14e79cfcf4334503a961c865df63d86c": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "14f3da0f2a884248ae63a9177cc1dc95": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_b2bb6fdafbe2486e8c63c1debd42bb5a",
      "placeholder": "​",
      "style": "IPY_MODEL_70b0a2b48feb4f2ba711d6abea889986",
      "value": " 456k/456k [00:00&lt;00:00, 18.4MB/s]"
     }
    },
    "193945c3e2ab4d21afcccbc013c3485a": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1af5ede6a6374e66ad84d98db8ebee51": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_12d544c6b5524c8a9193097200ab00ee",
      "placeholder": "​",
      "style": "IPY_MODEL_c444790d4a2745bbab083948b7a2bd5e",
      "value": "Downloading model.safetensors: 100%"
     }
    },
    "1b6414daa7a04ee0836d06b6e67d957c": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2860fd39e4c94fb5934c5dbdaa6cb49d": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2cc3b6ea94814c358e3435266b3400af": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_14e79cfcf4334503a961c865df63d86c",
      "placeholder": "​",
      "style": "IPY_MODEL_2f78ce1e5a09455582f0b1ab33a28eaf",
      "value": " 548M/548M [00:01&lt;00:00, 300MB/s]"
     }
    },
    "2cfa51de74044282a8a5fcf359b83823": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2f78ce1e5a09455582f0b1ab33a28eaf": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "3d5a65fdd0c6416aa5c4104b5a801ce4": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3e34c6b4a9364839a42b16f08f4bcfab": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "47a0795ff69b42a8883039d7868f4247": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4985a90175fd49118fa24d719d3b4168": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_47a0795ff69b42a8883039d7868f4247",
      "max": 1042301,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_f820dd319b9c41dc878d681bef6ac33f",
      "value": 1042301
     }
    },
    "526da09d88a94d1088b872560bce6c3f": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "5acd99da9f1b482f8623bed5f945b0a7": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "6d87953681df49ec94e447882458d3ab": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "70b0a2b48feb4f2ba711d6abea889986": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "7445c8410c2d47b08577ee29fa9f553a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "797cc29cf5ca442982337b805896d349": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7d0492f41d514779a3f028f4f1d9f0c4": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_2cfa51de74044282a8a5fcf359b83823",
      "placeholder": "​",
      "style": "IPY_MODEL_7445c8410c2d47b08577ee29fa9f553a",
      "value": " 665/665 [00:00&lt;00:00, 40.1kB/s]"
     }
    },
    "84162339a7b944148c31c9742073ba87": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "8cd430e0404e4d27a349673d18f32996": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "91175f7c20d94a0795a0262b70a8e240": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "92938c0a2f664e2a90565933f42b396c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_526da09d88a94d1088b872560bce6c3f",
      "max": 548105171,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_5acd99da9f1b482f8623bed5f945b0a7",
      "value": 548105171
     }
    },
    "96b987a8a77848f5ab5f74c3d89b8fe5": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_c60b5ecb0b5c4ce9b96386f7d95e0744",
       "IPY_MODEL_d4171779654f4694bc5659b68b0d6465",
       "IPY_MODEL_14f3da0f2a884248ae63a9177cc1dc95"
      ],
      "layout": "IPY_MODEL_3e34c6b4a9364839a42b16f08f4bcfab"
     }
    },
    "b2bb6fdafbe2486e8c63c1debd42bb5a": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "bacb64c8339240898bd3eb5cbcc51a54": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_6d87953681df49ec94e447882458d3ab",
      "max": 665,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_91175f7c20d94a0795a0262b70a8e240",
      "value": 665
     }
    },
    "bcb59724b1694e72b42fa3bc49e2d508": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "c444790d4a2745bbab083948b7a2bd5e": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "c60b5ecb0b5c4ce9b96386f7d95e0744": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_797cc29cf5ca442982337b805896d349",
      "placeholder": "​",
      "style": "IPY_MODEL_84162339a7b944148c31c9742073ba87",
      "value": "Downloading (…)olve/main/merges.txt: 100%"
     }
    },
    "cdc6d12c5da841e784baa6426e41270b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_1af5ede6a6374e66ad84d98db8ebee51",
       "IPY_MODEL_92938c0a2f664e2a90565933f42b396c",
       "IPY_MODEL_2cc3b6ea94814c358e3435266b3400af"
      ],
      "layout": "IPY_MODEL_d596456328ff45daa46fecf66522da89"
     }
    },
    "d0b20f9b03c74821a2ef2b21f93ffb0e": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_09ea0dc2cc8c409dbcb4967a10cae246",
      "placeholder": "​",
      "style": "IPY_MODEL_8cd430e0404e4d27a349673d18f32996",
      "value": "Downloading (…)olve/main/vocab.json: 100%"
     }
    },
    "d0e2e986593945e59ced2bab64744e66": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_3d5a65fdd0c6416aa5c4104b5a801ce4",
      "placeholder": "​",
      "style": "IPY_MODEL_13f0a71a62504879b85f36fd99a8bed4",
      "value": " 1.04M/1.04M [00:00&lt;00:00, 3.15MB/s]"
     }
    },
    "d2ac2eb2d53c427295a4b6846da09d73": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_d325d37eec88426fa1552f922ec5a145",
       "IPY_MODEL_bacb64c8339240898bd3eb5cbcc51a54",
       "IPY_MODEL_7d0492f41d514779a3f028f4f1d9f0c4"
      ],
      "layout": "IPY_MODEL_193945c3e2ab4d21afcccbc013c3485a"
     }
    },
    "d325d37eec88426fa1552f922ec5a145": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_f2d22f32efd04c08bb9d733244ee759d",
      "placeholder": "​",
      "style": "IPY_MODEL_0d77c36de8a5481399e59a24f4ddd656",
      "value": "Downloading (…)lve/main/config.json: 100%"
     }
    },
    "d4171779654f4694bc5659b68b0d6465": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1b6414daa7a04ee0836d06b6e67d957c",
      "max": 456318,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_bcb59724b1694e72b42fa3bc49e2d508",
      "value": 456318
     }
    },
    "d596456328ff45daa46fecf66522da89": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d5d3f0e7574049b8a87994bee0bb91dc": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_d0b20f9b03c74821a2ef2b21f93ffb0e",
       "IPY_MODEL_4985a90175fd49118fa24d719d3b4168",
       "IPY_MODEL_d0e2e986593945e59ced2bab64744e66"
      ],
      "layout": "IPY_MODEL_2860fd39e4c94fb5934c5dbdaa6cb49d"
     }
    },
    "f2d22f32efd04c08bb9d733244ee759d": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f820dd319b9c41dc878d681bef6ac33f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}