FasterViT.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting fastervit\n",
      "  Downloading fastervit-0.9.8-py3-none-any.whl.metadata (9.6 kB)\n",
      "Requirement already satisfied: timm>=0.6.12 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from fastervit) (0.9.16)\n",
      "Requirement already satisfied: torchvision in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from fastervit) (0.17.0+cu121)\n",
      "Requirement already satisfied: pyyaml in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from fastervit) (6.0.1)\n",
      "Requirement already satisfied: torch in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from timm>=0.6.12->fastervit) (2.2.0+cu121)\n",
      "Requirement already satisfied: huggingface_hub in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from timm>=0.6.12->fastervit) (0.20.3)\n",
      "Requirement already satisfied: safetensors in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from timm>=0.6.12->fastervit) (0.4.2)\n",
      "Requirement already satisfied: numpy in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from torchvision->fastervit) (1.26.4)\n",
      "Requirement already satisfied: requests in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from torchvision->fastervit) (2.28.1)\n",
      "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from torchvision->fastervit) (10.2.0)\n",
      "Requirement already satisfied: filelock in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from torch->timm>=0.6.12->fastervit) (3.9.0)\n",
      "Requirement already satisfied: typing-extensions>=4.8.0 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from torch->timm>=0.6.12->fastervit) (4.8.0)\n",
      "Requirement already satisfied: sympy in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from torch->timm>=0.6.12->fastervit) (1.12)\n",
      "Requirement already satisfied: networkx in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from torch->timm>=0.6.12->fastervit) (3.2.1)\n",
      "Requirement already satisfied: jinja2 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from torch->timm>=0.6.12->fastervit) (3.1.2)\n",
      "Requirement already satisfied: fsspec in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from torch->timm>=0.6.12->fastervit) (2024.2.0)\n",
      "Requirement already satisfied: tqdm>=4.42.1 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from huggingface_hub->timm>=0.6.12->fastervit) (4.66.2)\n",
      "Requirement already satisfied: packaging>=20.9 in c:\\users\\admin\\appdata\\roaming\\python\\python310\\site-packages (from huggingface_hub->timm>=0.6.12->fastervit) (23.1)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests->torchvision->fastervit) (2.1.1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests->torchvision->fastervit) (3.4)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests->torchvision->fastervit) (1.26.13)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests->torchvision->fastervit) (2022.12.7)\n",
      "Requirement already satisfied: colorama in c:\\users\\admin\\appdata\\roaming\\python\\python310\\site-packages (from tqdm>=4.42.1->huggingface_hub->timm>=0.6.12->fastervit) (0.4.6)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from jinja2->torch->timm>=0.6.12->fastervit) (2.1.3)\n",
      "Requirement already satisfied: mpmath>=0.19 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from sympy->torch->timm>=0.6.12->fastervit) (1.3.0)\n",
      "Downloading fastervit-0.9.8-py3-none-any.whl (165 kB)\n",
      "   ---------------------------------------- 0.0/165.7 kB ? eta -:--:--\n",
      "   -------------- ------------------------- 61.4/165.7 kB 1.6 MB/s eta 0:00:01\n",
      "   ---------------------------------------- 165.7/165.7 kB 2.0 MB/s eta 0:00:00\n",
      "Installing collected packages: fastervit\n",
      "Successfully installed fastervit-0.9.8\n"
     ]
    }
   ],
   "source": [
    "!pip install fastervit\n",
    "# You just need to copy the backbone of the model. This shit is bug af"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importance\n",
    "\n",
    "Before using FasterVit, if this error shows up\n",
    "\n",
    "`cannot import name '_update_default_kwargs' from 'timm.models._builder'`\n",
    "\n",
    "Go to original `faster_vit.py` and `faster_vit_any_res.py` and change all the function from `_update_default_kwargs` to `_update_default_model_kwargs`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastervit import create_model\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torchvision.transforms import Compose, ToTensor, Lambda, Resize, Normalize\n",
    "from PIL import Image, ImageDraw\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.metrics import accuracy_score,f1_score\n",
    "\n",
    "from tqdm import tqdm\n",
    "from transformers import CLIPTokenizerFast"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **[FasterViT](https://arxiv.org/abs/2306.06189)** for Image Classification\n",
    "\n",
    "This notebook will use a suitable variance of **FasterViT**, *fastervit_2_224* for image classification\n",
    "\n",
    "Original paper: **[FasterViT: Fast Vision Transformers with Hierarchical Attention](https://arxiv.org/abs/2306.06189)**\n",
    "\n",
    "Github: **[FasterViT](https://github.com/NVlabs/FasterViT)**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DIRECTROY = 'data'\n",
    "MODEL_PATH = 'models'\n",
    "IMG_SIZE = 224\n",
    "BATCH_SIZE = 32\n",
    "EPOCHS = 100\n",
    "LR = 0.0001\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv(f'{DIRECTROY}/reduced_train.csv') \n",
    "df_test = pd.read_csv(f'{DIRECTROY}/reduced_test.csv') \n",
    "num_classes = len(df_train['newid'].unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preparing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_transforms = Compose([\n",
    "    Resize((IMG_SIZE, IMG_SIZE)),\n",
    "    ToTensor(), \n",
    "    Normalize(mean=[0.48145466, 0.4578275, 0.40821073], std=[0.26862954, 0.26130258, 0.27577711])\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomDataset(Dataset):\n",
    "    def __init__(self, df, transforms, directory):\n",
    "        self.tokenizer =  CLIPTokenizerFast.from_pretrained(\"openai/clip-vit-base-patch16\")\n",
    "        self.df = df\n",
    "        self.transforms = transforms\n",
    "        self.directory = directory\n",
    "        self.labels = torch.Tensor(df['newid'].values).long()\n",
    "        self.imgs = torch.cat([ self.transforms(self.resize_img(Image.open(f'{DIRECTROY}/{self.directory}/{x}')).convert('RGB')).half().reshape(1,3,IMG_SIZE,IMG_SIZE) for x in tqdm(df['name'].values)])\n",
    "        self.tokenized = self.tokenizer(df['label'].tolist(), padding=True, truncation=True, return_tensors=\"pt\")\n",
    "        self.input_ids = self.tokenized['input_ids']\n",
    "        self.attention_mask = self.tokenized['attention_mask']\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.df)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        img = self.imgs[idx]\n",
    "        label = self.labels[idx]\n",
    "        input_ids = self.input_ids[idx]\n",
    "        attention_mask = self.attention_mask[idx]\n",
    "        return img, label, input_ids, attention_mask\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load FasterViT\n",
    "Model parameters: 75.9M"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\Admin\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\torch\\functional.py:507: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at ..\\aten\\src\\ATen\\native\\TensorShape.cpp:3550.)\n",
      "  return _VF.meshgrid(tensors, **kwargs)  # type: ignore[attr-defined]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "FasterViT(\n",
       "  (patch_embed): PatchEmbed(\n",
       "    (proj): Identity()\n",
       "    (conv_down): Sequential(\n",
       "      (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (1): BatchNorm2d(64, eps=0.0001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): ReLU()\n",
       "      (3): Conv2d(64, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (4): BatchNorm2d(96, eps=0.0001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): ReLU()\n",
       "    )\n",
       "  )\n",
       "  (levels): ModuleList(\n",
       "    (0): FasterViTLayer(\n",
       "      (blocks): ModuleList(\n",
       "        (0): ConvBlock(\n",
       "          (conv1): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "          (norm1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (act1): GELU(approximate='none')\n",
       "          (conv2): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "          (norm2): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (drop_path): Identity()\n",
       "        )\n",
       "        (1): ConvBlock(\n",
       "          (conv1): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "          (norm1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (act1): GELU(approximate='none')\n",
       "          (conv2): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "          (norm2): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (drop_path): DropPath(drop_prob=0.011)\n",
       "        )\n",
       "        (2): ConvBlock(\n",
       "          (conv1): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "          (norm1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (act1): GELU(approximate='none')\n",
       "          (conv2): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "          (norm2): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (drop_path): DropPath(drop_prob=0.022)\n",
       "        )\n",
       "      )\n",
       "      (downsample): Downsample(\n",
       "        (norm): LayerNorm2d((96,), eps=1e-06, elementwise_affine=True)\n",
       "        (reduction): Sequential(\n",
       "          (0): Conv2d(96, 192, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (1): FasterViTLayer(\n",
       "      (blocks): ModuleList(\n",
       "        (0): ConvBlock(\n",
       "          (conv1): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "          (norm1): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (act1): GELU(approximate='none')\n",
       "          (conv2): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "          (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (drop_path): DropPath(drop_prob=0.033)\n",
       "        )\n",
       "        (1): ConvBlock(\n",
       "          (conv1): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "          (norm1): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (act1): GELU(approximate='none')\n",
       "          (conv2): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "          (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (drop_path): DropPath(drop_prob=0.044)\n",
       "        )\n",
       "        (2): ConvBlock(\n",
       "          (conv1): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "          (norm1): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (act1): GELU(approximate='none')\n",
       "          (conv2): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "          (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (drop_path): DropPath(drop_prob=0.056)\n",
       "        )\n",
       "      )\n",
       "      (downsample): Downsample(\n",
       "        (norm): LayerNorm2d((192,), eps=1e-06, elementwise_affine=True)\n",
       "        (reduction): Sequential(\n",
       "          (0): Conv2d(192, 384, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (2): FasterViTLayer(\n",
       "      (blocks): ModuleList(\n",
       "        (0): HAT(\n",
       "          (pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (drop_path): DropPath(drop_prob=0.067)\n",
       "          (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (hat_mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_drop_path): DropPath(drop_prob=0.067)\n",
       "          (hat_pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (upsampler): Upsample(size=7, mode='nearest')\n",
       "        )\n",
       "        (1): HAT(\n",
       "          (pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (drop_path): DropPath(drop_prob=0.078)\n",
       "          (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (hat_mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_drop_path): DropPath(drop_prob=0.078)\n",
       "          (hat_pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (upsampler): Upsample(size=7, mode='nearest')\n",
       "        )\n",
       "        (2): HAT(\n",
       "          (pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (drop_path): DropPath(drop_prob=0.089)\n",
       "          (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (hat_mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_drop_path): DropPath(drop_prob=0.089)\n",
       "          (hat_pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (upsampler): Upsample(size=7, mode='nearest')\n",
       "        )\n",
       "        (3): HAT(\n",
       "          (pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (drop_path): DropPath(drop_prob=0.100)\n",
       "          (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (hat_mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_drop_path): DropPath(drop_prob=0.100)\n",
       "          (hat_pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (upsampler): Upsample(size=7, mode='nearest')\n",
       "        )\n",
       "        (4): HAT(\n",
       "          (pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (drop_path): DropPath(drop_prob=0.111)\n",
       "          (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (hat_mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_drop_path): DropPath(drop_prob=0.111)\n",
       "          (hat_pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (upsampler): Upsample(size=7, mode='nearest')\n",
       "        )\n",
       "        (5): HAT(\n",
       "          (pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (drop_path): DropPath(drop_prob=0.122)\n",
       "          (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (hat_mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_drop_path): DropPath(drop_prob=0.122)\n",
       "          (hat_pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (upsampler): Upsample(size=7, mode='nearest')\n",
       "        )\n",
       "        (6): HAT(\n",
       "          (pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (drop_path): DropPath(drop_prob=0.133)\n",
       "          (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (hat_mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_drop_path): DropPath(drop_prob=0.133)\n",
       "          (hat_pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (upsampler): Upsample(size=7, mode='nearest')\n",
       "        )\n",
       "        (7): HAT(\n",
       "          (pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (drop_path): DropPath(drop_prob=0.144)\n",
       "          (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (hat_mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_drop_path): DropPath(drop_prob=0.144)\n",
       "          (hat_pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (upsampler): Upsample(size=7, mode='nearest')\n",
       "        )\n",
       "      )\n",
       "      (downsample): Downsample(\n",
       "        (norm): LayerNorm2d((384,), eps=1e-06, elementwise_affine=True)\n",
       "        (reduction): Sequential(\n",
       "          (0): Conv2d(384, 768, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "        )\n",
       "      )\n",
       "      (global_tokenizer): TokenInitializer(\n",
       "        (pos_embed): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384)\n",
       "        (to_global_feature): Sequential(\n",
       "          (pos): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384)\n",
       "          (pool): AvgPool2d(kernel_size=5, stride=3, padding=0)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (3): FasterViTLayer(\n",
       "      (blocks): ModuleList(\n",
       "        (0): HAT(\n",
       "          (pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=768, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (norm1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=16, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (drop_path): DropPath(drop_prob=0.156)\n",
       "          (norm2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (1): HAT(\n",
       "          (pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=768, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (norm1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=16, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (drop_path): DropPath(drop_prob=0.167)\n",
       "          (norm2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (2): HAT(\n",
       "          (pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=768, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (norm1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=16, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (drop_path): DropPath(drop_prob=0.178)\n",
       "          (norm2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (3): HAT(\n",
       "          (pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=768, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (norm1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=16, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (drop_path): DropPath(drop_prob=0.189)\n",
       "          (norm2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (4): HAT(\n",
       "          (pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=768, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (norm1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=16, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (drop_path): DropPath(drop_prob=0.200)\n",
       "          (norm2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (norm): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "  (head): Linear(in_features=768, out_features=1000, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_name = 'faster_vit_2_224'\n",
    "model = create_model(model_name)\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load pretrained weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'models/download\\\\fastervit_2_224_1k.pth.tar'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# from huggingface_hub import hf_hub_download\n",
    "# hf_hub_download(repo_id=\"ahatamiz/FasterViT\", filename='fastervit_2_224_1k.pth.tar', local_dir='models/download')\n",
    "# model.load_state_dict(torch.load(f'{MODEL_PATH}/download/fastervit_2_224_1k.pth.tar'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Change output layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.head = nn.Linear(model.head.in_features, num_classes).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.load_state_dict(torch.load(f'{MODEL_PATH}/cp_faster_vit_reduced_model_99.pth'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method Module.parameters of FasterViT(\n",
       "  (patch_embed): PatchEmbed(\n",
       "    (proj): Identity()\n",
       "    (conv_down): Sequential(\n",
       "      (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (1): BatchNorm2d(64, eps=0.0001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): ReLU()\n",
       "      (3): Conv2d(64, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (4): BatchNorm2d(96, eps=0.0001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): ReLU()\n",
       "    )\n",
       "  )\n",
       "  (levels): ModuleList(\n",
       "    (0): FasterViTLayer(\n",
       "      (blocks): ModuleList(\n",
       "        (0): ConvBlock(\n",
       "          (conv1): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "          (norm1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (act1): GELU(approximate='none')\n",
       "          (conv2): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "          (norm2): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (drop_path): Identity()\n",
       "        )\n",
       "        (1): ConvBlock(\n",
       "          (conv1): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "          (norm1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (act1): GELU(approximate='none')\n",
       "          (conv2): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "          (norm2): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (drop_path): DropPath(drop_prob=0.011)\n",
       "        )\n",
       "        (2): ConvBlock(\n",
       "          (conv1): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "          (norm1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (act1): GELU(approximate='none')\n",
       "          (conv2): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "          (norm2): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (drop_path): DropPath(drop_prob=0.022)\n",
       "        )\n",
       "      )\n",
       "      (downsample): Downsample(\n",
       "        (norm): LayerNorm2d((96,), eps=1e-06, elementwise_affine=True)\n",
       "        (reduction): Sequential(\n",
       "          (0): Conv2d(96, 192, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (1): FasterViTLayer(\n",
       "      (blocks): ModuleList(\n",
       "        (0): ConvBlock(\n",
       "          (conv1): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "          (norm1): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (act1): GELU(approximate='none')\n",
       "          (conv2): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "          (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (drop_path): DropPath(drop_prob=0.033)\n",
       "        )\n",
       "        (1): ConvBlock(\n",
       "          (conv1): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "          (norm1): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (act1): GELU(approximate='none')\n",
       "          (conv2): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "          (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (drop_path): DropPath(drop_prob=0.044)\n",
       "        )\n",
       "        (2): ConvBlock(\n",
       "          (conv1): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "          (norm1): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (act1): GELU(approximate='none')\n",
       "          (conv2): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "          (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (drop_path): DropPath(drop_prob=0.056)\n",
       "        )\n",
       "      )\n",
       "      (downsample): Downsample(\n",
       "        (norm): LayerNorm2d((192,), eps=1e-06, elementwise_affine=True)\n",
       "        (reduction): Sequential(\n",
       "          (0): Conv2d(192, 384, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (2): FasterViTLayer(\n",
       "      (blocks): ModuleList(\n",
       "        (0): HAT(\n",
       "          (pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (drop_path): DropPath(drop_prob=0.067)\n",
       "          (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (hat_mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_drop_path): DropPath(drop_prob=0.067)\n",
       "          (hat_pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (upsampler): Upsample(size=7, mode='nearest')\n",
       "        )\n",
       "        (1): HAT(\n",
       "          (pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (drop_path): DropPath(drop_prob=0.078)\n",
       "          (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (hat_mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_drop_path): DropPath(drop_prob=0.078)\n",
       "          (hat_pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (upsampler): Upsample(size=7, mode='nearest')\n",
       "        )\n",
       "        (2): HAT(\n",
       "          (pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (drop_path): DropPath(drop_prob=0.089)\n",
       "          (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (hat_mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_drop_path): DropPath(drop_prob=0.089)\n",
       "          (hat_pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (upsampler): Upsample(size=7, mode='nearest')\n",
       "        )\n",
       "        (3): HAT(\n",
       "          (pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (drop_path): DropPath(drop_prob=0.100)\n",
       "          (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (hat_mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_drop_path): DropPath(drop_prob=0.100)\n",
       "          (hat_pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (upsampler): Upsample(size=7, mode='nearest')\n",
       "        )\n",
       "        (4): HAT(\n",
       "          (pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (drop_path): DropPath(drop_prob=0.111)\n",
       "          (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (hat_mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_drop_path): DropPath(drop_prob=0.111)\n",
       "          (hat_pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (upsampler): Upsample(size=7, mode='nearest')\n",
       "        )\n",
       "        (5): HAT(\n",
       "          (pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (drop_path): DropPath(drop_prob=0.122)\n",
       "          (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (hat_mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_drop_path): DropPath(drop_prob=0.122)\n",
       "          (hat_pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (upsampler): Upsample(size=7, mode='nearest')\n",
       "        )\n",
       "        (6): HAT(\n",
       "          (pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (drop_path): DropPath(drop_prob=0.133)\n",
       "          (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (hat_mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_drop_path): DropPath(drop_prob=0.133)\n",
       "          (hat_pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (upsampler): Upsample(size=7, mode='nearest')\n",
       "        )\n",
       "        (7): HAT(\n",
       "          (pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (drop_path): DropPath(drop_prob=0.144)\n",
       "          (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n",
       "          (hat_attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=384, out_features=1152, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=384, out_features=384, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=8, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (hat_mlp): Mlp(\n",
       "            (fc1): Linear(in_features=384, out_features=1536, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=1536, out_features=384, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (hat_drop_path): DropPath(drop_prob=0.144)\n",
       "          (hat_pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=384, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (upsampler): Upsample(size=7, mode='nearest')\n",
       "        )\n",
       "      )\n",
       "      (downsample): Downsample(\n",
       "        (norm): LayerNorm2d((384,), eps=1e-06, elementwise_affine=True)\n",
       "        (reduction): Sequential(\n",
       "          (0): Conv2d(384, 768, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "        )\n",
       "      )\n",
       "      (global_tokenizer): TokenInitializer(\n",
       "        (pos_embed): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384)\n",
       "        (to_global_feature): Sequential(\n",
       "          (pos): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384)\n",
       "          (pool): AvgPool2d(kernel_size=5, stride=3, padding=0)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (3): FasterViTLayer(\n",
       "      (blocks): ModuleList(\n",
       "        (0): HAT(\n",
       "          (pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=768, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (norm1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=16, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (drop_path): DropPath(drop_prob=0.156)\n",
       "          (norm2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (1): HAT(\n",
       "          (pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=768, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (norm1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=16, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (drop_path): DropPath(drop_prob=0.167)\n",
       "          (norm2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (2): HAT(\n",
       "          (pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=768, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (norm1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=16, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (drop_path): DropPath(drop_prob=0.178)\n",
       "          (norm2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (3): HAT(\n",
       "          (pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=768, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (norm1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=16, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (drop_path): DropPath(drop_prob=0.189)\n",
       "          (norm2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (4): HAT(\n",
       "          (pos_embed): PosEmbMLPSwinv1D(\n",
       "            (cpb_mlp): Sequential(\n",
       "              (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "              (1): ReLU()\n",
       "              (2): Linear(in_features=512, out_features=768, bias=False)\n",
       "            )\n",
       "          )\n",
       "          (norm1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (attn): WindowAttention(\n",
       "            (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
       "            (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "            (proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "            (pos_emb_funct): PosEmbMLPSwinv2D(\n",
       "              (cpb_mlp): Sequential(\n",
       "                (0): Linear(in_features=2, out_features=512, bias=True)\n",
       "                (1): ReLU(inplace=True)\n",
       "                (2): Linear(in_features=512, out_features=16, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (drop_path): DropPath(drop_prob=0.200)\n",
       "          (norm2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (act): GELU(approximate='none')\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (norm): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "  (head): Linear(in_features=768, out_features=649, bias=True)\n",
       ")>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "optimizer = optim.Adam(model.parameters(), lr = LR)\n",
    "scheduler = optim.lr_scheduler.LinearLR(optimizer, start_factor=1.0, end_factor=0.1, total_iters=EPOCHS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer.load_state_dict(torch.load(f'{MODEL_PATH}/optimizer/faster_vit_reduced_optimizer_99.pth'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = torch.load(f'{DIRECTROY}/train_dataset/train_dataset_reduced_all.pth')\n",
    "train_dataloader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n",
    "\n",
    "test_dataset = torch.load(f'{DIRECTROY}/test_public_dataset/test_public_reduced_dataset_0.pth')\n",
    "test_dataloader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training epoch: 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:54<00:00,  5.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100, Loss: 2.4189698501147678e-05\n",
      "Evaluating epoch: 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:10<00:00, 18.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100, Loss: 0.019151645977855167\n",
      "Accuracy: 0.9060546232313261\n",
      "F1 Score Weighted: 0.9049886150131257\n",
      "F1 Score Macro: 0.9022118751911821\n",
      "Training epoch: 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:53<00:00,  5.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2/100, Loss: 2.6278606693317317e-05\n",
      "Evaluating epoch: 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:10<00:00, 18.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2/100, Loss: 0.019119120535636722\n",
      "Accuracy: 0.9040802895689372\n",
      "F1 Score Weighted: 0.9029683590869729\n",
      "F1 Score Macro: 0.9001279544537402\n",
      "Training epoch: 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:53<00:00,  5.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3/100, Loss: 3.696225058398297e-05\n",
      "Evaluating epoch: 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:10<00:00, 18.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3/100, Loss: 0.01921060277517136\n",
      "Accuracy: 0.905890095426127\n",
      "F1 Score Weighted: 0.9047550744633539\n",
      "F1 Score Macro: 0.9018194829082689\n",
      "Training epoch: 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:53<00:00,  5.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4/100, Loss: 2.2733185700404876e-05\n",
      "Evaluating epoch: 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:10<00:00, 18.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4/100, Loss: 0.01918424259840867\n",
      "Accuracy: 0.9073708456729187\n",
      "F1 Score Weighted: 0.9061048960473981\n",
      "F1 Score Macro: 0.9040015968870226\n",
      "Training epoch: 5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:53<00:00,  5.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5/100, Loss: 2.5802104268103244e-05\n",
      "Evaluating epoch: 5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:10<00:00, 18.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5/100, Loss: 0.019607907353044823\n",
      "Accuracy: 0.9047384007897334\n",
      "F1 Score Weighted: 0.9035699584722532\n",
      "F1 Score Macro: 0.9018059337478138\n",
      "Training epoch: 6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:53<00:00,  5.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 6/100, Loss: 2.4636137107250445e-05\n",
      "Evaluating epoch: 6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 6/100, Loss: 0.019395218193908555\n",
      "Accuracy: 0.9060546232313261\n",
      "F1 Score Weighted: 0.9047366667296302\n",
      "F1 Score Macro: 0.9019194466389283\n",
      "Training epoch: 7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:53<00:00,  5.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 7/100, Loss: 2.2204619605760104e-05\n",
      "Evaluating epoch: 7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 7/100, Loss: 0.019130336199603817\n",
      "Accuracy: 0.9052319842053307\n",
      "F1 Score Weighted: 0.9041570678084535\n",
      "F1 Score Macro: 0.9008578670932008\n",
      "Training epoch: 8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:53<00:00,  5.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 8/100, Loss: 2.496238400235514e-05\n",
      "Evaluating epoch: 8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 8/100, Loss: 0.01949671411117566\n",
      "Accuracy: 0.9053965120105297\n",
      "F1 Score Weighted: 0.9041274453617452\n",
      "F1 Score Macro: 0.9007756724409426\n",
      "Training epoch: 9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:53<00:00,  5.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9/100, Loss: 2.3850512938920695e-05\n",
      "Evaluating epoch: 9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9/100, Loss: 0.019357217593499246\n",
      "Accuracy: 0.9063836788417242\n",
      "F1 Score Weighted: 0.9051103473610791\n",
      "F1 Score Macro: 0.9026398507721662\n",
      "Training epoch: 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:53<00:00,  5.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10/100, Loss: 2.3482999902568062e-05\n",
      "Evaluating epoch: 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10/100, Loss: 0.019595300752918033\n",
      "Accuracy: 0.9039157617637381\n",
      "F1 Score Weighted: 0.9022895265047217\n",
      "F1 Score Macro: 0.8996888177886928\n",
      "Training epoch: 11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:53<00:00,  5.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 11/100, Loss: 2.2810339322174607e-05\n",
      "Evaluating epoch: 11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 11/100, Loss: 0.019308254989686326\n",
      "Accuracy: 0.9060546232313261\n",
      "F1 Score Weighted: 0.9044899445801986\n",
      "F1 Score Macro: 0.9025874839120719\n",
      "Training epoch: 12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 12/100, Loss: 2.6778735117650766e-05\n",
      "Evaluating epoch: 12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 12/100, Loss: 0.019590750106924053\n",
      "Accuracy: 0.9068772622573215\n",
      "F1 Score Weighted: 0.9054580705257079\n",
      "F1 Score Macro: 0.9035275392709073\n",
      "Training epoch: 13\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 13/100, Loss: 2.318505142945666e-05\n",
      "Evaluating epoch: 13\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 13/100, Loss: 0.019360083686136533\n",
      "Accuracy: 0.9072063178677197\n",
      "F1 Score Weighted: 0.9055699080170452\n",
      "F1 Score Macro: 0.9028014726290848\n",
      "Training epoch: 14\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 14/100, Loss: 1.8300944212817543e-05\n",
      "Evaluating epoch: 14\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 14/100, Loss: 0.019315748964645125\n",
      "Accuracy: 0.9063836788417242\n",
      "F1 Score Weighted: 0.9046517569184037\n",
      "F1 Score Macro: 0.9024793214931471\n",
      "Training epoch: 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 15/100, Loss: 2.106483917846194e-05\n",
      "Evaluating epoch: 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 15/100, Loss: 0.01925585458746429\n",
      "Accuracy: 0.9050674564001316\n",
      "F1 Score Weighted: 0.9038289180395143\n",
      "F1 Score Macro: 0.9010341593570199\n",
      "Training epoch: 16\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 16/100, Loss: 2.337303228380203e-05\n",
      "Evaluating epoch: 16\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 16/100, Loss: 0.01893883467928462\n",
      "Accuracy: 0.9062191510365252\n",
      "F1 Score Weighted: 0.9047971584543518\n",
      "F1 Score Macro: 0.9022476295763847\n",
      "Training epoch: 17\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 17/100, Loss: 1.942613011045135e-05\n",
      "Evaluating epoch: 17\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 17/100, Loss: 0.01907246434430773\n",
      "Accuracy: 0.9072063178677197\n",
      "F1 Score Weighted: 0.9054630684962085\n",
      "F1 Score Macro: 0.9030879153530286\n",
      "Training epoch: 18\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 18/100, Loss: 2.3343399421460357e-05\n",
      "Evaluating epoch: 18\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 18/100, Loss: 0.019290334736735967\n",
      "Accuracy: 0.9063836788417242\n",
      "F1 Score Weighted: 0.904657013633665\n",
      "F1 Score Macro: 0.9015988694408174\n",
      "Training epoch: 19\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 19/100, Loss: 1.9090974706430973e-05\n",
      "Evaluating epoch: 19\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 19/100, Loss: 0.019425754420391926\n",
      "Accuracy: 0.9047384007897334\n",
      "F1 Score Weighted: 0.9030678999773826\n",
      "F1 Score Macro: 0.8993842560111768\n",
      "Training epoch: 20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 20/100, Loss: 2.6409580606495415e-05\n",
      "Evaluating epoch: 20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 20/100, Loss: 0.019146566607467014\n",
      "Accuracy: 0.9065482066469234\n",
      "F1 Score Weighted: 0.9053304810295201\n",
      "F1 Score Macro: 0.9023097267747926\n",
      "Training epoch: 21\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 21/100, Loss: 2.7795131767974194e-05\n",
      "Evaluating epoch: 21\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 21/100, Loss: 0.019421962065971264\n",
      "Accuracy: 0.9052319842053307\n",
      "F1 Score Weighted: 0.9040289885701921\n",
      "F1 Score Macro: 0.9012653127456746\n",
      "Training epoch: 22\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 22/100, Loss: 2.1958899312271345e-05\n",
      "Evaluating epoch: 22\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 22/100, Loss: 0.01936163947154903\n",
      "Accuracy: 0.9052319842053307\n",
      "F1 Score Weighted: 0.9038899599068859\n",
      "F1 Score Macro: 0.9005214022873218\n",
      "Training epoch: 23\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 23/100, Loss: 2.8436540189179873e-05\n",
      "Evaluating epoch: 23\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 23/100, Loss: 0.019319645020323257\n",
      "Accuracy: 0.9068772622573215\n",
      "F1 Score Weighted: 0.9056158602814836\n",
      "F1 Score Macro: 0.9022133073842857\n",
      "Training epoch: 24\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 24/100, Loss: 1.9090100761439684e-05\n",
      "Evaluating epoch: 24\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 24/100, Loss: 0.019414901068243274\n",
      "Accuracy: 0.9072063178677197\n",
      "F1 Score Weighted: 0.9059774900941099\n",
      "F1 Score Macro: 0.9017104101180248\n",
      "Training epoch: 25\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 25/100, Loss: 1.569211187883352e-05\n",
      "Evaluating epoch: 25\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 25/100, Loss: 0.01949382014817786\n",
      "Accuracy: 0.9050674564001316\n",
      "F1 Score Weighted: 0.9035976251924271\n",
      "F1 Score Macro: 0.8998827393243743\n",
      "Training epoch: 26\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 26/100, Loss: 2.263885263520384e-05\n",
      "Evaluating epoch: 26\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 26/100, Loss: 0.019408670931264013\n",
      "Accuracy: 0.9073708456729187\n",
      "F1 Score Weighted: 0.9058667271928482\n",
      "F1 Score Macro: 0.9022895027172038\n",
      "Training epoch: 27\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 27/100, Loss: 1.750462056826155e-05\n",
      "Evaluating epoch: 27\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 27/100, Loss: 0.019345534515316058\n",
      "Accuracy: 0.907864429088516\n",
      "F1 Score Weighted: 0.906331491058598\n",
      "F1 Score Macro: 0.9026058592522911\n",
      "Training epoch: 28\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 28/100, Loss: 1.6573623335716673e-05\n",
      "Evaluating epoch: 28\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 28/100, Loss: 0.01963107471361431\n",
      "Accuracy: 0.9085225403093122\n",
      "F1 Score Weighted: 0.9069660671032571\n",
      "F1 Score Macro: 0.9037047796073691\n",
      "Training epoch: 29\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 29/100, Loss: 1.9639862025694796e-05\n",
      "Evaluating epoch: 29\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 29/100, Loss: 0.0194909998284343\n",
      "Accuracy: 0.9090161237249095\n",
      "F1 Score Weighted: 0.9074523119747103\n",
      "F1 Score Macro: 0.9035408062543026\n",
      "Training epoch: 30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 30/100, Loss: 1.646799065810447e-05\n",
      "Evaluating epoch: 30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 30/100, Loss: 0.019218062989160106\n",
      "Accuracy: 0.9090161237249095\n",
      "F1 Score Weighted: 0.9077758148562146\n",
      "F1 Score Macro: 0.9037914133036705\n",
      "Training epoch: 31\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 31/100, Loss: 2.1633988936161234e-05\n",
      "Evaluating epoch: 31\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 31/100, Loss: 0.019413587137210916\n",
      "Accuracy: 0.9073708456729187\n",
      "F1 Score Weighted: 0.9061258558953038\n",
      "F1 Score Macro: 0.9024724517747764\n",
      "Training epoch: 32\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 32/100, Loss: 1.8033079345420044e-05\n",
      "Evaluating epoch: 32\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 32/100, Loss: 0.019551511322790822\n",
      "Accuracy: 0.9081934846989141\n",
      "F1 Score Weighted: 0.9066829542073397\n",
      "F1 Score Macro: 0.9033581378392486\n",
      "Training epoch: 33\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 33/100, Loss: 2.3522179020938654e-05\n",
      "Evaluating epoch: 33\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 33/100, Loss: 0.019790345540358183\n",
      "Accuracy: 0.908028956893715\n",
      "F1 Score Weighted: 0.9066855117618376\n",
      "F1 Score Macro: 0.9032907110749329\n",
      "Training epoch: 34\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 34/100, Loss: 2.3184936282888185e-05\n",
      "Evaluating epoch: 34\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 34/100, Loss: 0.019610292035432512\n",
      "Accuracy: 0.9075353734781179\n",
      "F1 Score Weighted: 0.9063988408137823\n",
      "F1 Score Macro: 0.903862221408181\n",
      "Training epoch: 35\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 35/100, Loss: 1.8848705071894898e-05\n",
      "Evaluating epoch: 35\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 35/100, Loss: 0.0194055793930609\n",
      "Accuracy: 0.9073708456729187\n",
      "F1 Score Weighted: 0.9062391621332324\n",
      "F1 Score Macro: 0.9029707344320523\n",
      "Training epoch: 36\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 36/100, Loss: 1.9284044521750317e-05\n",
      "Evaluating epoch: 36\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 36/100, Loss: 0.01937724517010158\n",
      "Accuracy: 0.9075353734781179\n",
      "F1 Score Weighted: 0.9064420601092934\n",
      "F1 Score Macro: 0.902491447109689\n",
      "Training epoch: 37\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 37/100, Loss: 2.4825744261174795e-05\n",
      "Evaluating epoch: 37\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 37/100, Loss: 0.019138384916445258\n",
      "Accuracy: 0.9090161237249095\n",
      "F1 Score Weighted: 0.9079681284463321\n",
      "F1 Score Macro: 0.9044797616829037\n",
      "Training epoch: 38\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 38/100, Loss: 1.7431880963592203e-05\n",
      "Evaluating epoch: 38\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 38/100, Loss: 0.019138262923068822\n",
      "Accuracy: 0.9095097071405067\n",
      "F1 Score Weighted: 0.9081470605229546\n",
      "F1 Score Macro: 0.9044794778403145\n",
      "Training epoch: 39\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 39/100, Loss: 1.7384378796112192e-05\n",
      "Evaluating epoch: 39\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 39/100, Loss: 0.018989262406063096\n",
      "Accuracy: 0.907864429088516\n",
      "F1 Score Weighted: 0.9063180310903187\n",
      "F1 Score Macro: 0.9019002989352405\n",
      "Training epoch: 40\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 40/100, Loss: 2.7332962036253735e-05\n",
      "Evaluating epoch: 40\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 40/100, Loss: 0.019041602649726663\n",
      "Accuracy: 0.9086870681145114\n",
      "F1 Score Weighted: 0.9073914566790576\n",
      "F1 Score Macro: 0.9038162041142193\n",
      "Training epoch: 41\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 41/100, Loss: 1.5337719352580662e-05\n",
      "Evaluating epoch: 41\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 41/100, Loss: 0.018962397895343094\n",
      "Accuracy: 0.9095097071405067\n",
      "F1 Score Weighted: 0.9082332243048081\n",
      "F1 Score Macro: 0.9038668359913852\n",
      "Training epoch: 42\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 42/100, Loss: 3.0196634080627976e-05\n",
      "Evaluating epoch: 42\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 42/100, Loss: 0.018932883939992325\n",
      "Accuracy: 0.907864429088516\n",
      "F1 Score Weighted: 0.9065904821868561\n",
      "F1 Score Macro: 0.903324561047955\n",
      "Training epoch: 43\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 43/100, Loss: 1.545104411299875e-05\n",
      "Evaluating epoch: 43\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 43/100, Loss: 0.019221437673844297\n",
      "Accuracy: 0.9063836788417242\n",
      "F1 Score Weighted: 0.9050406420083675\n",
      "F1 Score Macro: 0.9014418415250651\n",
      "Training epoch: 44\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 44/100, Loss: 2.1749127493779844e-05\n",
      "Evaluating epoch: 44\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 44/100, Loss: 0.019193647042713327\n",
      "Accuracy: 0.9073708456729187\n",
      "F1 Score Weighted: 0.9060412673785582\n",
      "F1 Score Macro: 0.9026687377603634\n",
      "Training epoch: 45\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 45/100, Loss: 1.932295515028638e-05\n",
      "Evaluating epoch: 45\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 45/100, Loss: 0.019509272587625585\n",
      "Accuracy: 0.9068772622573215\n",
      "F1 Score Weighted: 0.9056800558389286\n",
      "F1 Score Macro: 0.9019384157968744\n",
      "Training epoch: 46\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 46/100, Loss: 2.9749394022280528e-05\n",
      "Evaluating epoch: 46\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 46/100, Loss: 0.019619540061099828\n",
      "Accuracy: 0.9075353734781179\n",
      "F1 Score Weighted: 0.9061756869010786\n",
      "F1 Score Macro: 0.9024246462218847\n",
      "Training epoch: 47\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 47/100, Loss: 2.1533774368802248e-05\n",
      "Evaluating epoch: 47\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 47/100, Loss: 0.019408777756222486\n",
      "Accuracy: 0.908028956893715\n",
      "F1 Score Weighted: 0.9067669236036361\n",
      "F1 Score Macro: 0.9035730482047741\n",
      "Training epoch: 48\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 48/100, Loss: 1.7680054171312757e-05\n",
      "Evaluating epoch: 48\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 48/100, Loss: 0.019346581908395798\n",
      "Accuracy: 0.9081934846989141\n",
      "F1 Score Weighted: 0.9070572016016292\n",
      "F1 Score Macro: 0.903601455658539\n",
      "Training epoch: 49\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 49/100, Loss: 2.042439164249073e-05\n",
      "Evaluating epoch: 49\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 49/100, Loss: 0.01914163092036267\n",
      "Accuracy: 0.9090161237249095\n",
      "F1 Score Weighted: 0.9077453987721574\n",
      "F1 Score Macro: 0.9041952264035084\n",
      "Training epoch: 50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 50/100, Loss: 1.753525716728486e-05\n",
      "Evaluating epoch: 50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 50/100, Loss: 0.01952357396534504\n",
      "Accuracy: 0.9076999012833169\n",
      "F1 Score Weighted: 0.9066302980293265\n",
      "F1 Score Macro: 0.9034094270816544\n",
      "Training epoch: 51\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 51/100, Loss: 1.863534651961389e-05\n",
      "Evaluating epoch: 51\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 51/100, Loss: 0.01961904279920812\n",
      "Accuracy: 0.908028956893715\n",
      "F1 Score Weighted: 0.9067164875058618\n",
      "F1 Score Macro: 0.9033177569122993\n",
      "Training epoch: 52\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 52/100, Loss: 2.018373310577273e-05\n",
      "Evaluating epoch: 52\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 52/100, Loss: 0.01974881837782566\n",
      "Accuracy: 0.9075353734781179\n",
      "F1 Score Weighted: 0.9062219266357534\n",
      "F1 Score Macro: 0.9026867818630899\n",
      "Training epoch: 53\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 53/100, Loss: 1.6707025255919976e-05\n",
      "Evaluating epoch: 53\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 53/100, Loss: 0.019782134662966105\n",
      "Accuracy: 0.9068772622573215\n",
      "F1 Score Weighted: 0.9055537341104364\n",
      "F1 Score Macro: 0.9023061700854268\n",
      "Training epoch: 54\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 54/100, Loss: 1.6187847368465098e-05\n",
      "Evaluating epoch: 54\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 54/100, Loss: 0.019946650940374008\n",
      "Accuracy: 0.9072063178677197\n",
      "F1 Score Weighted: 0.9057551383195499\n",
      "F1 Score Macro: 0.9023179116807554\n",
      "Training epoch: 55\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 55/100, Loss: 2.7228950231727665e-05\n",
      "Evaluating epoch: 55\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 55/100, Loss: 0.01983795128221634\n",
      "Accuracy: 0.9075353734781179\n",
      "F1 Score Weighted: 0.9059614646133806\n",
      "F1 Score Macro: 0.9027611010220692\n",
      "Training epoch: 56\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 56/100, Loss: 2.5805550929915666e-05\n",
      "Evaluating epoch: 56\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 56/100, Loss: 0.01979095170445715\n",
      "Accuracy: 0.9060546232313261\n",
      "F1 Score Weighted: 0.9044087105702104\n",
      "F1 Score Macro: 0.9013752298866549\n",
      "Training epoch: 57\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 57/100, Loss: 1.9914754011783555e-05\n",
      "Evaluating epoch: 57\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 57/100, Loss: 0.019757833921072967\n",
      "Accuracy: 0.9073708456729187\n",
      "F1 Score Weighted: 0.90571941976525\n",
      "F1 Score Macro: 0.901892929260789\n",
      "Training epoch: 58\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 58/100, Loss: 1.809261971851211e-05\n",
      "Evaluating epoch: 58\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 58/100, Loss: 0.019659074816974473\n",
      "Accuracy: 0.9073708456729187\n",
      "F1 Score Weighted: 0.9058124471074757\n",
      "F1 Score Macro: 0.9026944211702952\n",
      "Training epoch: 59\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 59/100, Loss: 1.6467056602924828e-05\n",
      "Evaluating epoch: 59\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 59/100, Loss: 0.019643090949129107\n",
      "Accuracy: 0.9081934846989141\n",
      "F1 Score Weighted: 0.9067913003536255\n",
      "F1 Score Macro: 0.9035986901436478\n",
      "Training epoch: 60\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 60/100, Loss: 1.5356329096096565e-05\n",
      "Evaluating epoch: 60\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 60/100, Loss: 0.019733743003166496\n",
      "Accuracy: 0.9083580125041132\n",
      "F1 Score Weighted: 0.9070796257531859\n",
      "F1 Score Macro: 0.9039551793289763\n",
      "Training epoch: 61\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 61/100, Loss: 1.960041906735248e-05\n",
      "Evaluating epoch: 61\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 61/100, Loss: 0.01966996466556833\n",
      "Accuracy: 0.9083580125041132\n",
      "F1 Score Weighted: 0.9067518132822813\n",
      "F1 Score Macro: 0.90303390274601\n",
      "Training epoch: 62\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 62/100, Loss: 1.6995886870348335e-05\n",
      "Evaluating epoch: 62\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 62/100, Loss: 0.01934126941266257\n",
      "Accuracy: 0.9091806515301086\n",
      "F1 Score Weighted: 0.90769386591292\n",
      "F1 Score Macro: 0.9041866991604713\n",
      "Training epoch: 63\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 63/100, Loss: 1.4562038986683218e-05\n",
      "Evaluating epoch: 63\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 63/100, Loss: 0.019439431800269198\n",
      "Accuracy: 0.9090161237249095\n",
      "F1 Score Weighted: 0.9074380968729658\n",
      "F1 Score Macro: 0.9033853046360313\n",
      "Training epoch: 64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 64/100, Loss: 1.318025269071024e-05\n",
      "Evaluating epoch: 64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 64/100, Loss: 0.01948367417926417\n",
      "Accuracy: 0.9067127344521224\n",
      "F1 Score Weighted: 0.905236929077039\n",
      "F1 Score Macro: 0.9012899231861293\n",
      "Training epoch: 65\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 65/100, Loss: 1.7186995494242088e-05\n",
      "Evaluating epoch: 65\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 65/100, Loss: 0.019321299347237943\n",
      "Accuracy: 0.9091806515301086\n",
      "F1 Score Weighted: 0.9076840057182765\n",
      "F1 Score Macro: 0.9037667103254398\n",
      "Training epoch: 66\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 66/100, Loss: 1.5679536455748144e-05\n",
      "Evaluating epoch: 66\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 66/100, Loss: 0.019703199577366073\n",
      "Accuracy: 0.9083580125041132\n",
      "F1 Score Weighted: 0.9067283185943922\n",
      "F1 Score Macro: 0.9027859632995654\n",
      "Training epoch: 67\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 67/100, Loss: 1.295260166652473e-05\n",
      "Evaluating epoch: 67\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 67/100, Loss: 0.019949413883748577\n",
      "Accuracy: 0.9076999012833169\n",
      "F1 Score Weighted: 0.9063443018380478\n",
      "F1 Score Macro: 0.9020413683776896\n",
      "Training epoch: 68\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 68/100, Loss: 1.539217824585872e-05\n",
      "Evaluating epoch: 68\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 68/100, Loss: 0.019822059345714755\n",
      "Accuracy: 0.907864429088516\n",
      "F1 Score Weighted: 0.9062019162648297\n",
      "F1 Score Macro: 0.9023138622048509\n",
      "Training epoch: 69\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 69/100, Loss: 1.728500301297494e-05\n",
      "Evaluating epoch: 69\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 69/100, Loss: 0.019930233808439793\n",
      "Accuracy: 0.9076999012833169\n",
      "F1 Score Weighted: 0.9063234185278475\n",
      "F1 Score Macro: 0.9018173008897336\n",
      "Training epoch: 70\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 70/100, Loss: 1.76470174339385e-05\n",
      "Evaluating epoch: 70\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 70/100, Loss: 0.01982632716092497\n",
      "Accuracy: 0.9076999012833169\n",
      "F1 Score Weighted: 0.9063440381095093\n",
      "F1 Score Macro: 0.902078862876736\n",
      "Training epoch: 71\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 71/100, Loss: 2.556698081261963e-05\n",
      "Evaluating epoch: 71\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 71/100, Loss: 0.019426101630322285\n",
      "Accuracy: 0.908028956893715\n",
      "F1 Score Weighted: 0.9065972018126083\n",
      "F1 Score Macro: 0.9024980738835015\n",
      "Training epoch: 72\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 72/100, Loss: 1.6273104784272806e-05\n",
      "Evaluating epoch: 72\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 72/100, Loss: 0.019384365792944714\n",
      "Accuracy: 0.9086870681145114\n",
      "F1 Score Weighted: 0.9072473886335042\n",
      "F1 Score Macro: 0.9034299736348393\n",
      "Training epoch: 73\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 73/100, Loss: 1.873087095979308e-05\n",
      "Evaluating epoch: 73\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 73/100, Loss: 0.019505961510735928\n",
      "Accuracy: 0.9103323461665022\n",
      "F1 Score Weighted: 0.9088919421761203\n",
      "F1 Score Macro: 0.9052848868483682\n",
      "Training epoch: 74\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 74/100, Loss: 1.440272971351411e-05\n",
      "Evaluating epoch: 74\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 74/100, Loss: 0.019371363086886594\n",
      "Accuracy: 0.910167818361303\n",
      "F1 Score Weighted: 0.9086612183091856\n",
      "F1 Score Macro: 0.9049599728335659\n",
      "Training epoch: 75\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 75/100, Loss: 1.5649602034814368e-05\n",
      "Evaluating epoch: 75\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 75/100, Loss: 0.019525162707917428\n",
      "Accuracy: 0.9091806515301086\n",
      "F1 Score Weighted: 0.9078008506525355\n",
      "F1 Score Macro: 0.9041712539674145\n",
      "Training epoch: 76\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 76/100, Loss: 1.6973747969064074e-05\n",
      "Evaluating epoch: 76\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 76/100, Loss: 0.019628619695045944\n",
      "Accuracy: 0.910003290556104\n",
      "F1 Score Weighted: 0.9085463952741805\n",
      "F1 Score Macro: 0.904906384610905\n",
      "Training epoch: 77\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 77/100, Loss: 2.3571634312965568e-05\n",
      "Evaluating epoch: 77\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 77/100, Loss: 0.01957946904517989\n",
      "Accuracy: 0.9083580125041132\n",
      "F1 Score Weighted: 0.9068884179145654\n",
      "F1 Score Macro: 0.9027736001878008\n",
      "Training epoch: 78\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 78/100, Loss: 1.7452167736010474e-05\n",
      "Evaluating epoch: 78\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 78/100, Loss: 0.019338200805117514\n",
      "Accuracy: 0.9083580125041132\n",
      "F1 Score Weighted: 0.9068630971305718\n",
      "F1 Score Macro: 0.9027531183469565\n",
      "Training epoch: 79\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 79/100, Loss: 2.6723541289797992e-05\n",
      "Evaluating epoch: 79\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 79/100, Loss: 0.019160909532393633\n",
      "Accuracy: 0.907864429088516\n",
      "F1 Score Weighted: 0.9062465875770976\n",
      "F1 Score Macro: 0.9032361833162276\n",
      "Training epoch: 80\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 80/100, Loss: 1.4774165496311316e-05\n",
      "Evaluating epoch: 80\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 80/100, Loss: 0.019357892562625552\n",
      "Accuracy: 0.9075353734781179\n",
      "F1 Score Weighted: 0.9057344359255199\n",
      "F1 Score Macro: 0.9024629704195728\n",
      "Training epoch: 81\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 81/100, Loss: 1.508602677435324e-05\n",
      "Evaluating epoch: 81\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 81/100, Loss: 0.01957803579163412\n",
      "Accuracy: 0.9081934846989141\n",
      "F1 Score Weighted: 0.906563337679023\n",
      "F1 Score Macro: 0.9029394940764895\n",
      "Training epoch: 82\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 82/100, Loss: 1.8516938561125544e-05\n",
      "Evaluating epoch: 82\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 82/100, Loss: 0.019430470707985464\n",
      "Accuracy: 0.908028956893715\n",
      "F1 Score Weighted: 0.906328285264145\n",
      "F1 Score Macro: 0.9024250376886894\n",
      "Training epoch: 83\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 83/100, Loss: 1.319409330968106e-05\n",
      "Evaluating epoch: 83\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 83/100, Loss: 0.01925582835834098\n",
      "Accuracy: 0.9098387627509049\n",
      "F1 Score Weighted: 0.9081642051210846\n",
      "F1 Score Macro: 0.9048651975274634\n",
      "Training epoch: 84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 84/100, Loss: 1.26315745096178e-05\n",
      "Evaluating epoch: 84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 84/100, Loss: 0.019328839936167467\n",
      "Accuracy: 0.9086870681145114\n",
      "F1 Score Weighted: 0.9071153901994736\n",
      "F1 Score Macro: 0.9031240662207307\n",
      "Training epoch: 85\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:54<00:00,  5.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 85/100, Loss: 1.4603545754837382e-05\n",
      "Evaluating epoch: 85\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 85/100, Loss: 0.019556551902415318\n",
      "Accuracy: 0.9083580125041132\n",
      "F1 Score Weighted: 0.9068789249271823\n",
      "F1 Score Macro: 0.9032029817813254\n",
      "Training epoch: 86\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:54<00:00,  5.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 86/100, Loss: 1.3526080497102999e-05\n",
      "Evaluating epoch: 86\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 86/100, Loss: 0.01951609931982237\n",
      "Accuracy: 0.9085225403093122\n",
      "F1 Score Weighted: 0.9069610376432767\n",
      "F1 Score Macro: 0.9031866276731146\n",
      "Training epoch: 87\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:54<00:00,  5.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 87/100, Loss: 1.720062721484016e-05\n",
      "Evaluating epoch: 87\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:10<00:00, 18.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 87/100, Loss: 0.01967149295540276\n",
      "Accuracy: 0.9073708456729187\n",
      "F1 Score Weighted: 0.9056439603700489\n",
      "F1 Score Macro: 0.9018170804109534\n",
      "Training epoch: 88\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:54<00:00,  5.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 88/100, Loss: 1.3484190323181252e-05\n",
      "Evaluating epoch: 88\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 88/100, Loss: 0.01941351274775323\n",
      "Accuracy: 0.9076999012833169\n",
      "F1 Score Weighted: 0.9061429262659013\n",
      "F1 Score Macro: 0.9026178517327664\n",
      "Training epoch: 89\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:54<00:00,  5.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 89/100, Loss: 1.955376120536431e-05\n",
      "Evaluating epoch: 89\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:10<00:00, 18.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 89/100, Loss: 0.019502913313684908\n",
      "Accuracy: 0.9093451793353077\n",
      "F1 Score Weighted: 0.9078423417436221\n",
      "F1 Score Macro: 0.9039177863665012\n",
      "Training epoch: 90\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:57<00:00,  5.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 90/100, Loss: 1.63392682837144e-05\n",
      "Evaluating epoch: 90\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:10<00:00, 18.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 90/100, Loss: 0.019119442531125026\n",
      "Accuracy: 0.9086870681145114\n",
      "F1 Score Weighted: 0.9070517560486553\n",
      "F1 Score Macro: 0.9034953629835755\n",
      "Training epoch: 91\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:54<00:00,  5.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 91/100, Loss: 3.160878788541065e-05\n",
      "Evaluating epoch: 91\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 91/100, Loss: 0.01924530363834996\n",
      "Accuracy: 0.9086870681145114\n",
      "F1 Score Weighted: 0.9072590183424399\n",
      "F1 Score Macro: 0.9036704522180296\n",
      "Training epoch: 92\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:53<00:00,  5.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 92/100, Loss: 1.4031699044679606e-05\n",
      "Evaluating epoch: 92\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:10<00:00, 18.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 92/100, Loss: 0.019379908644964527\n",
      "Accuracy: 0.9070417900625206\n",
      "F1 Score Weighted: 0.9054785325193185\n",
      "F1 Score Macro: 0.9018865248363255\n",
      "Training epoch: 93\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:54<00:00,  5.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 93/100, Loss: 1.4972396931972457e-05\n",
      "Evaluating epoch: 93\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:10<00:00, 18.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 93/100, Loss: 0.019304319132930303\n",
      "Accuracy: 0.9086870681145114\n",
      "F1 Score Weighted: 0.9070263275004495\n",
      "F1 Score Macro: 0.9036049772735417\n",
      "Training epoch: 94\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:54<00:00,  5.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 94/100, Loss: 1.2917584464946515e-05\n",
      "Evaluating epoch: 94\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 94/100, Loss: 0.019044100866561588\n",
      "Accuracy: 0.9070417900625206\n",
      "F1 Score Weighted: 0.9057855431577353\n",
      "F1 Score Macro: 0.9018367493073425\n",
      "Training epoch: 95\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 95/100, Loss: 2.1897088124930395e-05\n",
      "Evaluating epoch: 95\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 95/100, Loss: 0.019284020097498446\n",
      "Accuracy: 0.9081934846989141\n",
      "F1 Score Weighted: 0.9068224655233743\n",
      "F1 Score Macro: 0.9027599506373906\n",
      "Training epoch: 96\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 96/100, Loss: 1.4550608419327775e-05\n",
      "Evaluating epoch: 96\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 96/100, Loss: 0.01946303548229019\n",
      "Accuracy: 0.9083580125041132\n",
      "F1 Score Weighted: 0.9069759641020023\n",
      "F1 Score Macro: 0.9034879119467809\n",
      "Training epoch: 97\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 97/100, Loss: 1.565465767523093e-05\n",
      "Evaluating epoch: 97\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 97/100, Loss: 0.019239003110220518\n",
      "Accuracy: 0.9081934846989141\n",
      "F1 Score Weighted: 0.9068284603689073\n",
      "F1 Score Macro: 0.903176505353788\n",
      "Training epoch: 98\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 98/100, Loss: 1.2994578026817979e-05\n",
      "Evaluating epoch: 98\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 98/100, Loss: 0.019189284824164106\n",
      "Accuracy: 0.9085225403093122\n",
      "F1 Score Weighted: 0.9071177411972315\n",
      "F1 Score Macro: 0.9032629523767345\n",
      "Training epoch: 99\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 99/100, Loss: 1.5031200114305833e-05\n",
      "Evaluating epoch: 99\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 99/100, Loss: 0.019283990544177093\n",
      "Accuracy: 0.9086870681145114\n",
      "F1 Score Weighted: 0.9072313614815976\n",
      "F1 Score Macro: 0.9034028575609419\n",
      "Training epoch: 100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914/914 [02:52<00:00,  5.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 100/100, Loss: 1.5163623648771589e-05\n",
      "Evaluating epoch: 100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 190/190 [00:09<00:00, 19.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 100/100, Loss: 0.01954697220215073\n",
      "Accuracy: 0.9090161237249095\n",
      "F1 Score Weighted: 0.90765540537648\n",
      "F1 Score Macro: 0.9038241286546793\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "ename": "",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31mThe Kernel crashed while executing code in the current cell or a previous cell. \n",
      "\u001b[1;31mPlease review the code in the cell(s) to identify a possible cause of the failure. \n",
      "\u001b[1;31mClick <a href='https://aka.ms/vscodeJupyterKernelCrash'>here</a> for more info. \n",
      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
     ]
    }
   ],
   "source": [
    "max_accuracy = 0.9076\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "    model.train()\n",
    "    train_loss = 0.0\n",
    "    \n",
    "    # Training loop\n",
    "    print('Training epoch:', epoch+1)\n",
    "    len_train = 0\n",
    " \n",
    "    \n",
    "\n",
    "    \n",
    "    for inputs, labels, input_ids, attention_mask in tqdm(train_dataloader):\n",
    "        optimizer.zero_grad()\n",
    "        inputs = inputs.to(device).type(torch.cuda.FloatTensor)\n",
    "        labels = labels.to(device)\n",
    "\n",
    "        outputs = model(inputs)\n",
    "        \n",
    "\n",
    "        loss = criterion(outputs, labels)\n",
    "\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        train_loss += loss.item()\n",
    "    len_train += len(train_dataset)\n",
    "   \n",
    "        \n",
    "    scheduler.step()    \n",
    "    train_loss/=len_train\n",
    "    print(f'Epoch {epoch+1}/{EPOCHS}, Loss: {train_loss}')\n",
    "    \n",
    "    eval_loss = 0.0\n",
    "    model.eval()\n",
    "    \n",
    "    true_labels = []\n",
    "    pred_labels = []\n",
    "    \n",
    "    print('Evaluating epoch:', epoch+1)\n",
    "    with torch.no_grad():\n",
    "        len_test = 0\n",
    "        \n",
    "        \n",
    "        for inputs, labels, input_ids, attention_mask in tqdm(test_dataloader):\n",
    "            inputs = inputs.to(device).type(torch.cuda.FloatTensor)\n",
    "            labels = labels.to(device)\n",
    "            \n",
    "            outputs = model(inputs)\n",
    "            \n",
    "            loss = criterion(outputs, labels).to(device)\n",
    "            eval_loss += loss.item()\n",
    "            \n",
    "            outputs = torch.argmax(outputs, 1).flatten().cpu().numpy()\n",
    "            labels = labels.flatten().cpu().numpy()\n",
    "            \n",
    "            true_labels.extend(labels)\n",
    "            pred_labels.extend(outputs)\n",
    "        \n",
    "        len_test += len(test_dataset)\n",
    "        \n",
    "        \n",
    "    print(f'Epoch {epoch+1}/{EPOCHS}, Loss: {eval_loss/len_test}')\n",
    "    print(f'Accuracy: {accuracy_score(true_labels, pred_labels)}')\n",
    "    print(f'F1 Score Weighted: {f1_score(true_labels, pred_labels, average=\"weighted\")}')\n",
    "    print(f'F1 Score Macro: {f1_score(true_labels, pred_labels, average=\"macro\")}')\n",
    "\n",
    "    if accuracy_score(true_labels, pred_labels) > max_accuracy:\n",
    "        max_accuracy = accuracy_score(true_labels, pred_labels)\n",
    "        torch.save(model.state_dict(), f'{MODEL_PATH}/faster_vit_reduced_model_{epoch+1}.pth')\n",
    "        torch.save(optimizer.state_dict(), f'{MODEL_PATH}/optimizer/faster_vit_reduced_optimizer_{epoch+1}.pth')\n",
    "            "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}