shopee-pytorch-siamese-triplet-img.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.9","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!ls ../input/shopee-generate-data-for-triplet-loss/train_triplets_imgs.csv\n!ls ../input/shopee-efficientnetb3-128-siamese-triplet/efficientnet_b3_128_fold0_min_val_loss.pth","metadata":{"trusted":true},"execution_count":2,"outputs":[{"name":"stdout","text":"../input/shopee-generate-data-for-triplet-loss/train_triplets_imgs.csv\nefficientnet_b3_128_fold0.pth\nefficientnet_b3_128_fold0_max_val_acc.pth\nefficientnet_b3_128_fold0_min_val_loss.pth\n","output_type":"stream"}]},{"cell_type":"code","source":"import sys\nsys.path.append ('/kaggle/input/timm-pytorch-image-models/pytorch-image-models-master')\n\nimport os\nimport gc\nimport time\nimport math\nimport random\nimport datetime\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\n\nimport torch\nimport torch.nn as nn\nfrom   torch.nn import init\nfrom   torch.nn import CrossEntropyLoss, MSELoss\nfrom   torch.nn.modules.loss import _WeightedLoss\nimport torch.nn.functional as F\nfrom   torch.nn import Parameter\nfrom   torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler\nfrom   transformers import AdamW, get_cosine_schedule_with_warmup\nfrom   torch.cuda.amp import autocast, GradScaler\n\nfrom   fastai.losses import LabelSmoothingCrossEntropy\n\n# from warmup_scheduler import GradualWarmupScheduler\nfrom   sklearn import preprocessing\nfrom   sklearn.metrics import accuracy_score\nfrom   sklearn.model_selection import StratifiedKFold, GroupKFold\nfrom   sklearn.metrics import accuracy_score, precision_recall_fscore_support, matthews_corrcoef, roc_auc_score\nfrom   sklearn.metrics import mean_squared_error, mean_absolute_error, explained_variance_score, r2_score\n\nimport timm\nimport albumentations as A\nfrom   albumentations import *\nfrom   albumentations.pytorch import ToTensorV2\nfrom   albumentations.core.transforms_interface import DualTransform\nfrom   albumentations.augmentations import functional as AF\nimport cv2\n\nfrom   tqdm import tqdm\nfrom   pprint import pprint\nfrom   functools import partial\nimport matplotlib.pyplot as plt\n# from GPUtil import showUtilization as gpu_usage\nfrom   numba import cuda\nimport warnings\nwarnings.filterwarnings (\"ignore\")","metadata":{"papermill":{"duration":6.115871,"end_time":"2021-03-29T06:13:02.423924","exception":false,"start_time":"2021-03-29T06:12:56.308053","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    device       = torch.device ('cuda' if torch.cuda.is_available () else 'cpu')\n    num_workers  = 8\n    use_fc       = False    \n    model_name   = 'efficientnet_b3'      # ['deit_base_patch16_224', 'vit_base_patch16_384', 'resnext50_32x4d', 'tf_efficientnet_b7_ns']\n    n_classes    = 512                    # 11014\n    margin       = 1.0\n    fc_dim       = 0\n    dropout      = 0.2\n    ls_eps       = 0.0\n    pretrained   = True        \n    size         = 128                     # [64, 128, 224, 384, 512, 640, 720]\n    isTrain      = True\n    isFreeze     = True\n    lr           = 5e-4\n    epochs       = 3\n    warmup_steps = 0.50                    # if float: these many epochs are with frozen model at the beginning, if int = actual steps\n    eval_steps   = 0                       # if float: these many epochs are with frozen model at the beginning, if int = actual steps \n    lr_num_cycles= 2.5\n    epochsNx     = 1\n    # criterion  = 'dice_bce'              # ['dice_bce', CrossEntropyLoss', 'BCEWithLogitsLoss', 'SmoothBCEwithLogits']\n    weight_decay = 1e-6\n    max_grad_norm= 1000.0\n    seed         = 42\n    n_fold       = 10\n    train_fold   = [0]                      # [0, 1, 2, 3, 4]\n    print_every  = 100\n    adam_epsilon = 1e-8\n    train_batch_size = 12\n    eval_batch_size  = 12    \n    model_infer_path_prefix = \".\"\n    model_train_path_prefix = \".\"\n    img_triplets_csv = \"../input/shopee-generate-data-for-triplet-loss/train_triplets_imgs.csv\"\n    train_path       = '../input/shopee-product-matching/train_images'\n    # train_csv        = '../input/vinbigdata-chest-xray-abnormalities-detection/train.csv'\n    # test_path        = '../input/vinbigdata-chest-xray-resized-png-1024x1024/test'\n    output_dir       = './results'        # output directory    \n    max_steps        = 0\n    MODEL            = None","metadata":{"papermill":{"duration":0.051644,"end_time":"2021-03-29T06:13:02.512003","exception":false,"start_time":"2021-03-29T06:13:02.460359","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def init_logger (log_file=CFG.output_dir+'train.log'):\n    \n    from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler\n    logger = getLogger (__name__)\n    logger.setLevel (INFO)\n    handler1 = StreamHandler ()\n    handler1.setFormatter (Formatter (\"%(message)s\"))\n    handler2 = FileHandler (filename=log_file)\n    handler2.setFormatter (Formatter (\"%(message)s\"))\n    logger.addHandler (handler1)\n    logger.addHandler (handler2)\n    return logger","metadata":{"papermill":{"duration":0.046379,"end_time":"2021-03-29T06:13:02.594867","exception":false,"start_time":"2021-03-29T06:13:02.548488","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything (seed):\n    \n    random.seed (seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed (seed)\n    torch.manual_seed (seed)\n    torch.cuda.manual_seed (seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    return","metadata":{"papermill":{"duration":0.047319,"end_time":"2021-03-29T06:13:02.67797","exception":false,"start_time":"2021-03-29T06:13:02.630651","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Image Augmentations","metadata":{}},{"cell_type":"code","source":"def get_transforms (data='train'):\n    \n    light_transforms = A.Compose ([\n        A.Resize (CFG.size, CFG.size),\n        # A.HorizontalFlip (p=0.5),\n        A.OneOf([\n            A.CLAHE (clip_limit=(1,4), p=1),\n            A.HueSaturationValue (hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=1),\n            A.RandomBrightnessContrast (brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=1),\n            A.augmentations.transforms.RGBShift (r_shift_limit=20, g_shift_limit=20, b_shift_limit=20, always_apply=False, p=0.5),\n        ], p=0.5),\n        # A.augmentations.transforms.ChannelDropout (channel_drop_range=(1, 1), fill_value=0, always_apply=False, p=0.5),\n        A.augmentations.transforms.GridDistortion (num_steps=5, distort_limit=0.3, interpolation=1, border_mode=4, value=None, mask_value=None, always_apply=False, p=0.5),\n        A.CoarseDropout(p=0.5),\n        # A.Cutout (max_h_size=int(CFG.size * 0.2), max_w_size=int(CFG.size * 0.2), num_holes=2, p=0.75),\n        A.Normalize (),\n        ToTensorV2 (p=1.0),\n    ])\n    \n    valid_transforms = A.Compose ([\n        A.Resize (CFG.size, CFG.size),\n        A.Normalize (),\n        ToTensorV2 (p=1.0),\n    ])\n    \n    if 'train' in data:\n        return light_transforms\n    elif 'valid' in data:\n        return valid_transforms\n    return valid_transforms","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"# see image triplets\ndf = pd.read_csv (CFG.img_triplets_csv)\nTRAIN_DF = df.iloc[:(df.shape[0]*9//10)]\nTEST_DF  = df.iloc[(df.shape[0]*9//10):]\ndel df\ngc.collect ()\nTEST_DF.head ()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TripletImageDataset (Dataset):\n    \n    def __init__(self, df=TRAIN_DF, \n                 transform=get_transforms ('train')):\n        self.df = df # pd.read_csv (img_triplets_csv).reset_index (drop=True)\n        self.transform = transform\n        \n    def __getitem__(self, index):\n        \n        triplet  = self.df.iloc[index]\n        anchor   = self.transform (image=cv2.imread (f\"{CFG.train_path}/{triplet['anchor']}\"))['image']\n        positive = self.transform (image=cv2.imread (f\"{CFG.train_path}/{triplet['positive']}\"))['image']\n        negative = self.transform (image=cv2.imread (f\"{CFG.train_path}/{triplet['negative']}\"))['image']\n        return (anchor, positive, negative)\n    \n    def __len__(self):\n        return self.df.shape[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_imgs (dataset_show):\n    \n    from pylab import rcParams\n    rcParams['figure.figsize'] = 20,10\n    for i in range (2):\n        f, axarr = plt.subplots (1,3)\n        for p in [0]:\n            idx = np.random.randint (0, len (dataset_show))\n            img1, img2, img3 = dataset_show[idx]                         # ;print (img.size()) ;print (label)\n            img1 = img1.byte ()\n            img2 = img2.byte ()\n            img3 = img3.byte ()\n            axarr[p].imshow (img1.permute(1, 2, 0))\n            axarr[p].set_title ('Anchor')\n            axarr[p+1].imshow (img2.permute(1, 2, 0))\n            axarr[p+1].set_title ('+ve')\n            axarr[p+2].imshow (img3.permute(1, 2, 0))\n            axarr[p+2].set_title ('-ve')\n    return\n\nTR_DATASET = TripletImageDataset ()\nplot_imgs (TR_DATASET)\ndel TR_DATASET\ngc.collect ()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class ShopeeNet (nn.Module):\n\n    def __init__(self,\n                 n_classes=CFG.n_classes,\n                 model_name=CFG.model_name,\n                 use_fc=CFG.use_fc,\n                 fc_dim=CFG.fc_dim,\n                 dropout=CFG.dropout,\n                 pretrained=CFG.pretrained):\n        \"\"\"\n        :param n_classes:\n        :param model_name: name of model from pretrainedmodels\n            e.g. resnet50, resnext101_32x4d, pnasnet5large\n        \"\"\"\n        super(ShopeeNet, self).__init__()\n        print('Building Model Backbone for {} model'.format(model_name))\n        \n        self.backbone = timm.create_model (model_name, pretrained=pretrained)\n        final_in_features = self.backbone.classifier.in_features\n        \n        self.backbone.classifier = nn.Identity()\n        self.backbone.global_pool = nn.Identity()\n        \n        self.pooling =  nn.AdaptiveAvgPool2d(1)\n        \n        self.use_fc = use_fc\n        if use_fc:\n            self.dropout = nn.Dropout(p=dropout)\n            self.fc = nn.Linear(final_in_features, fc_dim)\n            self.bn = nn.BatchNorm1d(fc_dim)\n            self._init_params()\n            final_in_features = fc_dim\n        \n        self.final = nn.Linear (final_in_features, n_classes)\n    \n    def freeze (self):\n        \n        for param in self.parameters ():\n            param.requires_grad = False        \n        for param in self.backbone.parameters ():\n            param.requires_grad = False\n        \n        for param in self.final.parameters ():\n            param.requires_grad = True\n        if self.use_fc:\n            for param in self.fc.parameters ():\n                param.requires_grad = True\n            for param in self.bn.parameters ():\n                param.requires_grad = True\n        return\n    \n    def unfreeze (self):\n        \n        for param in self.backbone.parameters ():\n            param.requires_grad = True\n        for param in self.parameters ():\n            param.requires_grad = True\n            \n        for param in self.final.parameters ():\n            param.requires_grad = True\n        if self.use_fc:\n            for param in self.fc.parameters ():\n                param.requires_grad = True\n            for param in self.bn.parameters ():\n                param.requires_grad = True\n        return\n    \n    def _init_params (self):\n        nn.init.xavier_normal_(self.fc.weight)\n        nn.init.constant_(self.fc.bias, 0)\n        nn.init.constant_(self.bn.weight, 1)\n        nn.init.constant_(self.bn.bias, 0)\n    \n    def forward(self, x):\n        feature = self.extract_feat (x)\n        logits = self.final (feature)\n        return logits\n    \n    def extract_feat (self, x):\n        batch_size = x.shape[0]\n        x = self.backbone (x)\n        x = self.pooling (x).view (batch_size, -1)\n\n        if self.use_fc:\n            x = self.dropout (x)\n            x = self.fc (x)\n            x = self.bn (x)\n\n        return x","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TripletModel (nn.Module):\n    \n    def __init__(self, embeddingModel):\n        \n        super (TripletModel, self).__init__()\n        self.embeddingModel = embeddingModel\n    \n    def forward (self, i1, i2, i3):\n        \n        E1 = self.embeddingModel (i1)\n        E2 = self.embeddingModel (i2)\n        E3 = self.embeddingModel (i3)\n        return E1, E2, E3\n    \n    def freeze (self):        \n        self.embeddingModel.freeze ()\n        return\n    \n    def unfreeze (self):\n        self.embeddingModel.unfreeze ()\n        return","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # Loss Functions","metadata":{"papermill":{"duration":0.052832,"end_time":"2021-03-29T06:13:08.081563","exception":false,"start_time":"2021-03-29T06:13:08.028731","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_criterion (margin):\n    \n    criterion = nn.MarginRankingLoss (margin=margin)\n    return criterion","metadata":{"papermill":{"duration":0.062338,"end_time":"2021-03-29T06:13:08.439577","exception":false,"start_time":"2021-03-29T06:13:08.377239","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_state (model_path, model):\n    \n    state_dict = None\n    try:  # single GPU model_file\n        state_dict = torch.load (model_path, map_location=torch.device ('cpu'))['state_dict']\n        # print (state_dict)\n        state_dict = fix_state_dict (state_dict)\n        model.load_state_dict (state_dict) # (torch.load (model_path, map_location=torch.device ('cpu')), strict=True)\n    except:  # multi GPU model_file\n        state_dict = torch.load (model_path, map_location=torch.device ('cpu'))\n        state_dict = {k[7:] if k.startswith ('module.') else k: state_dict[k] for k in state_dict.keys ()}\n        model.load_state_dict (state_dict)\n    return state_dict","metadata":{"papermill":{"duration":0.061166,"end_time":"2021-03-29T06:13:09.161427","exception":false,"start_time":"2021-03-29T06:13:09.100261","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getModel (fold, isTrain=True):\n    \n    model = ShopeeNet ()\n    if isTrain:\n        \n        # model_path = f'{CFG.model_train_path_prefix}/{CFG.model_name}_fold{fold}_start.pth'\n        # print (\"loading\", model_path)\n        # load_state (model_path, model)\n        pass\n    else:\n        \n        # model_path = f'{CFG.model_infer_path_prefix}/{CFG.model_name}_fold{fold}_maxacc.pth'\n        # load_state (model_path, model)\n        pass\n    \n    # torch.save (model.state_dict (), model_path)\n    \n    # Warp as model for triples\n    model = TripletModel (model)\n    \n    if CFG.isFreeze:        \n        model.freeze ()\n    else:\n        model.unfreeze ()\n    \n    return model","metadata":{"papermill":{"duration":0.061008,"end_time":"2021-03-29T06:13:09.273719","exception":false,"start_time":"2021-03-29T06:13:09.212711","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Trainer Helpers","metadata":{"papermill":{"duration":0.052281,"end_time":"2021-03-29T06:13:09.38012","exception":false,"start_time":"2021-03-29T06:13:09.327839","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def format_time (elapsed):\n    '''\n    Takes a time in seconds and returns a string hh:mm:ss\n    '''\n    \n    # Round to the nearest second.\n    elapsed_rounded = int(round((elapsed)))\n    \n    # Format as hh:mm:ss\n    return str (datetime.timedelta (seconds=elapsed_rounded))","metadata":{"papermill":{"duration":0.059421,"end_time":"2021-03-29T06:13:09.49086","exception":false,"start_time":"2021-03-29T06:13:09.431439","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Trainer","metadata":{"papermill":{"duration":0.050869,"end_time":"2021-03-29T06:13:09.718636","exception":false,"start_time":"2021-03-29T06:13:09.667767","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class MyTrainer:\n    \n    def __init__(self, fold, model, train_dataset, eval_dataset, criterion, \n                 checkpoint_path=None, isResume=False):\n        \n        self.fold             = fold\n        self.start_epoch      = 0\n        self.model            = model\n        # load checkpoint\n        if checkpoint_path is not None:\n            if isResume:\n                self.start_epoch = self.load_checkpoint (checkpoint_path, isResume=True) + 1\n            else:\n                self.load_checkpoint (checkpoint_path, isResume=False)\n        self.model            = self.model.to (CFG.device)\n        if CFG.isFreeze:\n            self.model.freeze ()\n        else:\n            self.model.unfreeze ()\n        CFG.MODEL             = self.model\n        self.train_dataset    = train_dataset\n        self.eval_dataset     = eval_dataset\n        self.criterion        = criterion\n        self.isTrained        = False\n        self.device           = CFG.device\n        self.optimizer        = AdamW (self.model.parameters (), lr=CFG.lr, eps=CFG.adam_epsilon, weight_decay=CFG.weight_decay)\n        self.epochs           = CFG.epochs\n        self.set_dataLoaders ()\n        self.training_stats   = []\n        self.modelFile        = f\"{CFG.model_train_path_prefix}/{CFG.model_name}_{CFG.size}_fold{self.fold}.pth\"\n        if eval_dataset is not None:\n            self.minLossModelFile = f\"{CFG.model_train_path_prefix}/{CFG.model_name}_{CFG.size}_fold{self.fold}_min_val_loss.pth\"\n            self.maxAccModelFile  = f\"{CFG.model_train_path_prefix}/{CFG.model_name}_{CFG.size}_fold{self.fold}_max_val_acc.pth\"\n        else:\n            self.minLossModelFile = f\"{CFG.model_train_path_prefix}/{CFG.model_name}_{CFG.size}_fold{self.fold}_min_tr_loss.pth\"\n            self.maxAccModelFile  = f\"{CFG.model_train_path_prefix}/{CFG.model_name}_{CFG.size}_fold{self.fold}_max_tr_acc.pth\"\n        \n        self.min_val_loss         = 9999\n        self.min_train_loss       = 9999\n        self.max_val_acc          = -1    \n        return\n    \n    def set_dataLoaders (self):\n        # Create the DataLoaders for our training and validation sets.\n        \n        if isinstance (self.train_dataset, torch.utils.data.IterableDataset):\n            train_sampler = None\n        else:\n            train_sampler = RandomSampler (self.train_dataset)           # Better use RandomSampler\n        train_dataloader  = DataLoader (\n                    self.train_dataset,                                  # The training samples.\n                    sampler     = train_sampler,                           \n                    batch_size  = CFG.train_batch_size,\n                    num_workers = CFG.num_workers,\n                    pin_memory  = True\n        )\n        # train_dataloader  = DataLoader (self.train_dataset, batch_size=CFG.train_batch_size) # TODO: comment this\n        validation_dataloader = None\n        if self.eval_dataset:\n            validation_dataloader = DataLoader (\n                        self.eval_dataset, \n                        sampler     = SequentialSampler (self.eval_dataset),\n                        batch_size  = CFG.eval_batch_size,\n                        num_workers = CFG.num_workers,\n                        pin_memory  = False\n            )\n            # validation_dataloader  = DataLoader (self.eval_dataset, batch_size=CFG.eval_batch_size) # TODO: comment this\n        \n        if type (CFG.warmup_steps) is float:\n            CFG.warmup_steps = int (CFG.warmup_steps * len (train_dataloader))\n        # Total number of training steps is [number of batches] x [number of epochs]\n        num_training_steps = len (train_dataloader) * self.epochs        \n        lr_scheduler = get_cosine_schedule_with_warmup (self.optimizer, num_cycles=CFG.lr_num_cycles,\n                        num_warmup_steps=CFG.warmup_steps, num_training_steps=num_training_steps)\n        \n        if type (CFG.eval_steps) is float:\n            CFG.eval_steps = int (CFG.eval_steps * len (train_dataloader))\n        self.train_dataloader, self.validation_dataloader, self.lr_scheduler, self.num_training_steps=train_dataloader, validation_dataloader, lr_scheduler, num_training_steps\n        return\n            \n    def test_iterate_dataloader (self):\n        \n        for step, batch in enumerate (self.train_dataloader):\n            print (step)\n            print (batch)\n            break\n        return\n    \n    def save_checkpoint (self, epoch, path):\n        \n        checkpoint = {\n            'epoch'               : epoch,\n            'model_state_dict'    : self.model.state_dict (),\n            'optimizer_state_dict': self.optimizer.state_dict (),\n            'lr_sched_state_dict' : self.lr_scheduler.state_dict (),\n            'training_stats'      : self.training_stats,\n            'max_val_acc'         : self.max_val_acc,\n            'min_train_loss'      : self.min_train_loss,\n            'min_val_loss'        : self.min_val_loss,\n        }\n        torch.save (checkpoint, path)\n        gc.collect (); torch.cuda.empty_cache ()\n        print (\"saved checkpoint\", path)\n        return\n    \n    def load_checkpoint (self, path, isResume=False):\n        \n        epoch      = 0\n        checkpoint = torch.load (path, map_location=torch.device ('cpu'))\n        self.model.load_state_dict (checkpoint['model_state_dict'])\n        if isResume:\n            \n            self.optimizer.load_state_dict (checkpoint['optimizer_state_dict'])\n            self.lr_scheduler.load_state_dict (checkpoint['lr_sched_state_dict'])\n            epoch = checkpoint['epoch']\n            self.training_stats  = checkpoint['training_stats']\n            self.min_val_loss    = checkpoint['min_val_loss']\n            self.min_train_loss  = checkpoint['min_train_loss']\n            self.max_val_acc     = checkpoint['max_val_acc']\n            print (\"Loaded model, optimizer, and lr_scheduler from -\", path)\n        else:\n            print (\"Loaded model from -\", path)\n            \n        self.model.train ()\n        return epoch\n    \n    def train (self):\n        \n        seed_everything (seed=CFG.seed)\n        step             = 0\n        total_t0         = time.time ()\n        scaler           = GradScaler()\n        for epoch_i in range (self.start_epoch, self.epochs):\n            \n            avg_epoch_train_loss   = 0\n            total_epoch_train_loss = 0\n            print('======== Epoch {:} / {:} ========'.format (epoch_i + 1, self.epochs))\n            t0 = time.time ()\n            self.model.train ()\n            for stp, batch in tqdm (enumerate (self.train_dataloader), total=len(self.train_dataloader)):\n                \n                # Print Stats\n                # if step % CFG.print_every == 0:\n                #     elapsed = format_time (time.time() - t0)\n                #     print ('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format (step, len (self.train_dataloader), elapsed))                \n                if ((CFG.max_steps > 0 and CFG.max_steps < step) or \n                    (CFG.eval_steps>0 and stp==CFG.eval_steps)): # or step==0):   # TODO: rm this comment\n                    \n                    self.save_checkpoint (epoch_i, self.modelFile)\n                    training_time = format_time (time.time () - t0)            \n                    if self.validation_dataloader:\n                        \n                        avg_val_loss, avg_val_accuracy, validation_time = self.evaluate (epoch_i, avg_epoch_train_loss, training_time)\n                        # save this model if the eval loss decreases from the minimum so far\n                        checkpoint_epoch = epoch_i\n                        if stp==CFG.eval_steps:\n                            # don't count this epoch in the checkpoint since this epoch \n                            # has not completed. Hence, checkpoint at prev epoch\n                            checkpoint_epoch = epoch_i-1\n                        if avg_val_loss < self.min_val_loss:                             \n                            self.min_val_loss = avg_val_loss\n                            self.save_checkpoint (checkpoint_epoch, self.minLossModelFile)\n                        if avg_val_accuracy > self.max_val_acc:\n                            self.max_val_acc = avg_val_accuracy\n                            self.save_checkpoint (checkpoint_epoch, self.maxAccModelFile)\n                    if CFG.max_steps > 0 and CFG.max_steps < step:\n                        \n                        print (\"\")\n                        print (\"Training complete!\")\n                        print (\"Total training took {:} (h:mm:ss)\".format (format_time (time.time ()-total_t0)))\n                        self.isTrained = True\n                        self.model.cpu ()\n                        self.model.eval ()\n                        self.save_checkpoint (epoch_i, self.modelFile)\n                        try:\n                            torch.cuda.empty_cache ()\n                            self.plot_train_stats (self.training_stats)\n                        except:\n                            pass\n                        return pd.DataFrame (self.training_stats)\n                \n                ########################################################\n                # Train\n                ########################################################\n                # self.model.zero_grad ()\n                anc_imgs = batch[0].to (self.device)\n                pos_imgs = batch[1].to (self.device)\n                neg_imgs = batch[2].to (self.device)\n                with autocast():\n                    \n                    E1, E2, E3 = self.model (anc_imgs, pos_imgs, neg_imgs)\n                    dist_E1_E2 = F.pairwise_distance (E1, E2, 2)\n                    dist_E1_E3 = F.pairwise_distance (E1, E3, 2)\n                    target = torch.FloatTensor (dist_E1_E2.size()).fill_ (-1)\n                    target = target.to (CFG.device)\n                    loss = self.criterion (dist_E1_E2, dist_E1_E3, target)\n                    scaler.scale (loss).backward ()\n                    torch.nn.utils.clip_grad_norm_ (self.model.parameters (), CFG.max_grad_norm)\n                    scaler.step (self.optimizer)\n                    scaler.update ()\n                    self.optimizer.zero_grad ()\n                    self.lr_scheduler.step ()\n                    \n                total_epoch_train_loss += loss.cpu ().item ()\n                avg_epoch_train_loss    = total_epoch_train_loss / (stp+1)\n                step += 1\n            # all steps of an epoch end\n            \n            # Measure how long this epoch took.\n            training_time = format_time (time.time () - t0)            \n            print (\"  Average training loss: {0:.4f}\".format (avg_epoch_train_loss))\n            print (\"  Training epcoh took: {:}\".format (training_time))            \n            if self.validation_dataloader:    \n                \n                avg_val_loss, avg_val_accuracy, validation_time = self.evaluate (epoch_i, avg_epoch_train_loss, training_time)\n                # save this epoch's model if the eval loss decreases from the minimum so far\n                if avg_val_loss < self.min_val_loss:                    \n                    self.min_val_loss = avg_val_loss\n                    self.save_checkpoint (epoch_i, self.minLossModelFile)\n                if avg_val_accuracy > self.max_val_acc:\n                    self.max_val_acc = avg_val_accuracy\n                    self.save_checkpoint (epoch_i, self.maxAccModelFile)\n            else:                \n                training_stats.append ({\n                    'epoch'         : epoch_i + 1,\n                    'training_loss' : avg_epoch_train_loss,\n                    'training_time' : training_time,\n                })\n                if avg_train_loss < self.min_train_loss:                     \n                    self.min_train_loss = avg_train_loss\n                    self.save_checkpoint (epoch_i, self.minLossModelFile)\n            self.save_checkpoint (epoch_i, self.modelFile)\n            # 1 epoch end\n        # all epochs end\n        \n        # just get the best class thresholds at the end\n        if self.validation_dataloader:\n            print ('At training end, threshold Adjustment (last row of the train summary DF)')\n            print (self.evaluate (epoch_i, avg_epoch_train_loss, training_time, isThreshAdjust=True))\n            print ('<: avg_val_loss, avg_val_accuracy, validation_time')\n        \n        print (\"***** Training complete! *****\")\n        print (\"Total training took {:} (h:mm:ss)\".format (format_time (time.time ()-total_t0)))\n        self.isTrained = True\n        self.model.cpu ()\n        self.model.eval ()\n        try:\n            torch.cuda.empty_cache ()\n            self.plot_train_stats (self.training_stats)\n        except:\n            pass\n        return pd.DataFrame (self.training_stats)\n    \n    def evaluate (self, epoch_i, avg_train_loss=999, training_time=999, isThreshAdjust=False):\n        \n        t0           = time.time ()\n        all_labels   = []\n        all_pred_prs = []\n        # Put the model in evaluation mode--the dropout layers behave differently\n        # during evaluation.\n        self.model.eval ()\n        \n        # Tracking variables\n        total_eval_accuracy  = 0\n        total_eval_loss      = 0\n        nb_eval_steps        = 0\n        correct_pred_count   = 0\n        total_pred_count     = 0\n        # Evaluate data for one epoch\n        for batch in self.validation_dataloader:\n            with torch.no_grad ():\n                \n                anc_imgs = batch[0].to (self.device)\n                pos_imgs = batch[1].to (self.device)\n                neg_imgs = batch[2].to (self.device)                \n                E1, E2, E3 = self.model (anc_imgs, pos_imgs, neg_imgs)\n                dist_E1_E2 = F.pairwise_distance (E1, E2, 2)\n                dist_E1_E3 = F.pairwise_distance (E1, E3, 2)\n                target     = torch.FloatTensor (dist_E1_E2.size()).fill_ (-1).to (CFG.device)\n                loss       = self.criterion (dist_E1_E2, dist_E1_E3, target).cpu ().detach ()\n                \n            correct_pred_count += np.sum ((dist_E1_E2 < dist_E1_E3).detach().cpu().numpy() + 0.0)\n            total_pred_count   += len (dist_E1_E2) \n            total_eval_loss    += loss.item ()     \n        \n        avg_val_loss     = total_eval_loss / len (self.validation_dataloader)\n        avg_val_accuracy = correct_pred_count / total_pred_count\n        print (\"Val Loss: {0:.4f}\".format (avg_val_loss))\n        print (\"Val Accuracy: {0:.4f}\".format (avg_val_accuracy))\n        validation_time = format_time (time.time () - t0)\n        self.training_stats.append ({\n                'epoch'         : epoch_i + 1,\n                'training_loss' : avg_train_loss,\n                'eval_loss'     : avg_val_loss,\n                'eval_accuracy' : avg_val_accuracy,\n                'training_time' : training_time,\n                'eval_time'     : validation_time                   \n        })\n        self.model.train ()\n        print (\"Validation took {:} (h:mm:ss)\".format (format_time (time.time () - t0)))\n        return avg_val_loss, avg_val_accuracy, validation_time\n        \n        \n    def plot_train_stats (self, training_stats):\n        \"\"\"\n        Draw Classification Report curve\n        \"\"\"\n        \n        accuracies = eval_losses = tr_losses = epochs = -1\n        epochs = len (training_stats)\n        if 'eval_accuracy' in training_stats[0]:\n            accuracies = [e['eval_accuracy'] for e in training_stats]\n            sns.lineplot (x=np.arange(1, epochs + 1), y=accuracies, label='val_accuracy')\n        if 'eval_loss' in training_stats[0]:\n            eval_losses= [e['eval_loss'] for e in training_stats]\n        if 'training_loss'  in training_stats[0]:\n            tr_losses  = [e['training_loss'] for e in training_stats]\n            sns.lineplot (x=np.arange(1, epochs + 1), y=tr_losses,  label='tr_losses')\n            \n        plt.show ()\n        print ('accuracies :', accuracies)        \n        print ('eval_losses:', eval_losses)\n        print ('tr_losses  :', tr_losses)\n        return\n    \n    def get_trained_model (self):\n        \n        if self.isTrained:\n            return self.model.eval ()\n        return None","metadata":{"papermill":{"duration":0.126789,"end_time":"2021-03-29T06:13:09.896626","exception":false,"start_time":"2021-03-29T06:13:09.769837","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def free_gpu_cache ():\n    \n    # print(\"Initial GPU Usage\")\n    # gpu_usage()                             \n\n    torch.cuda.empty_cache ()\n\n    # cuda.select_device(0)\n    # cuda.close()\n    # cuda.select_device(0)\n\n    # print(\"GPU Usage after emptying the cache\")\n    # gpu_usage()\n    return\n\n# free_gpu_cache()           ","metadata":{"papermill":{"duration":0.065307,"end_time":"2021-03-29T06:13:10.015641","exception":false,"start_time":"2021-03-29T06:13:09.950334","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train","metadata":{"papermill":{"duration":0.051834,"end_time":"2021-03-29T06:13:10.119642","exception":false,"start_time":"2021-03-29T06:13:10.067808","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def train_fold_loop (checkpoint_path=None, isResume=False):\n\n    print (f\"========== training ==========\")\n    fold = 0 # fold is not used\n    criterion      = get_criterion (CFG.margin)\n    model          = CFG.MODEL\n    if model is None and checkpoint_path is None:\n        print (\"CFG.MODEL is None\")\n        model      = getModel (fold, isTrain=True)\n        model      = model.float()\n    elif model is not None and checkpoint_path is not None:\n        checkpoint_path = None\n    elif model is None and checkpoint_path is not None:\n        print (\"CFG.MODEL is None\")\n        model      = getModel (fold, isTrain=False)\n        model      = model.float()\n        \n    train_dataset  = TripletImageDataset ()\n    valid_dataset  = TripletImageDataset (df=TEST_DF, transform=get_transforms ('valid'))\n    trainer        = MyTrainer (\n        fold            = fold,\n        model           = model,\n        train_dataset   = train_dataset,\n        eval_dataset    = valid_dataset,\n        criterion       = criterion,\n        checkpoint_path = checkpoint_path,\n        isResume        = isResume\n    )\n    metrics = trainer.train ()\n    return metrics\n    \n    # To plot lr uncomment this\n    # lrs = []\n    # for i in range (CFG.epochs*len (trainer.train_dataloader)):\n    #     trainer.lr_scheduler.step ()\n    #     lrs.append (trainer.optimizer.param_groups[0][\"lr\"])\n    # print (lrs)\n    # plt.plot (lrs)\n    # plt.show ()","metadata":{"papermill":{"duration":0.069432,"end_time":"2021-03-29T06:13:10.24463","exception":false,"start_time":"2021-03-29T06:13:10.175198","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_main (checkpoint_path=None, isResume=False):\n    \n    print (f\"========== train_main() ==========\")\n    if CFG.isTrain:        \n        valid_scores_df = pd.DataFrame ()\n        for fold in range (CFG.n_fold):\n            if fold in CFG.train_fold:\n                \n                valid_scores_fold_df = train_fold_loop (checkpoint_path, isResume)\n                # valid_scores_fold = np.array (valid_scores_fold).reshape ((1, -1))\n                valid_scores_df = valid_scores_df.append (valid_scores_fold_df)\n                \n        print (f\"========== CV ==========\")\n        # print (valid_scores_df)\n        # valid_scores = np.vstack (valid_scores)\n        # valid_scores = np.mean (valid_scores, axis=0)\n        valid_scores = valid_scores_df.iloc[-1, :]  #.mean ()\n        print (\"CV Scores :-\");  print (valid_scores)\n    return valid_scores_df","metadata":{"papermill":{"duration":0.061261,"end_time":"2021-03-29T06:13:10.357409","exception":false,"start_time":"2021-03-29T06:13:10.296148","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Single config training","metadata":{"papermill":{"duration":0.050971,"end_time":"2021-03-29T06:13:10.459841","exception":false,"start_time":"2021-03-29T06:13:10.40887","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"!mkdir -p /kaggle/working/Output/\n!touch /kaggle/working/Output/train.log\ngc.collect ()\nmodel_names = timm.list_models (pretrained=True)\nmodel_names = timm.list_models ('*resnet*', pretrained=True)\npprint (model_names)\nLOGGER = init_logger ()\nseed_everything (seed=CFG.seed)","metadata":{"papermill":{"duration":0.051612,"end_time":"2021-03-29T06:13:10.563247","exception":false,"start_time":"2021-03-29T06:13:10.511635","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"valid_scores_df = train_main ()\n\nvalid_scores_df","metadata":{"papermill":{"duration":0.051001,"end_time":"2021-03-29T06:13:10.665748","exception":false,"start_time":"2021-03-29T06:13:10.614747","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# To train, uncomment these","metadata":{"papermill":{"duration":0.052916,"end_time":"2021-03-29T06:13:10.770432","exception":false,"start_time":"2021-03-29T06:13:10.717516","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"gc.collect (); torch.cuda.empty_cache ()\nCFG.warmup_steps = 0.5\n# CFG.eval_steps   = 0.5\nCFG.num_workers  = 8\nCFG.size = 128         # [128, 224, 384, 512, 640, 1024]\nCFG.train_batch_size = 88\nCFG.eval_batch_size  = 88\nCFG.freeze = True\nCFG.epochs = 1\nprint (f\"***** Training for size={CFG.size} freeze={CFG.isFreeze} *****\")\nvalid_scores_df = train_main () # (checkpoint_path=\"../input/vin-manual-torch-trainer-fp16-cbamresnet50-gradcam/cbam_resnet50_1024_fold0.pth\")\nvalid_scores_df","metadata":{}},{"cell_type":"markdown","source":"gc.collect (); torch.cuda.empty_cache ()\nCFG.warmup_steps = 0.5\n# CFG.eval_steps   = 0.5\nCFG.num_workers = 8\nCFG.size = 128\nCFG.train_batch_size = 16\nCFG.eval_batch_size  = 16\nCFG.isFreeze = False\nCFG.epochs = 5\nprint (f\"***** Training for size={CFG.size} freeze={CFG.isFreeze} *****\")\nvalid_scores_df = train_main ()\nvalid_scores_df","metadata":{}},{"cell_type":"code","source":"gc.collect (); torch.cuda.empty_cache ()\nCFG.num_workers      = 4\nCFG.warmup_steps     = 0.5\n# CFG.eval_steps       = 0.5\nCFG.size             = 224\nCFG.train_batch_size = 16\nCFG.eval_batch_size  = 16\nCFG.isFreeze         = True\nCFG.epochs           = 1\nprint (f\"***** Training for size={CFG.size} freeze={CFG.isFreeze} *****\")\nvalid_scores_df      = train_main (checkpoint_path=\"../input/shopee-efficientnetb3-128-siamese-triplet/efficientnet_b3_128_fold0_min_val_loss.pth\", isResume=False)\nvalid_scores_df","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect (); torch.cuda.empty_cache ()\nCFG.num_workers      = 8\nCFG.warmup_steps     = 0.5\n# CFG.eval_steps       = 0.5\nCFG.size             = 224\nCFG.train_batch_size = 4\nCFG.eval_batch_size  = 4\nCFG.isFreeze         = False\nCFG.epochs           = 5\nprint (f\"***** Training for size={CFG.size} freeze={CFG.isFreeze} *****\")\nvalid_scores_df      = train_main ()\nvalid_scores_df","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"gc.collect (); torch.cuda.empty_cache ()\nCFG.num_workers      = 4\nCFG.warmup_steps     = 0.5\n# CFG.eval_steps       = 0.5\nCFG.size             = 384\nCFG.train_batch_size = 228 \nCFG.eval_batch_size  = 228 \nCFG.isFreeze         = True\nCFG.epochs           = 2\nprint (f\"***** Training for size={CFG.size} freeze={CFG.isFreeze} *****\")\nvalid_scores_df      = train_main () # (checkpoint_path=\"../input/vin-manual-torch-trainer-fp16-cbamresnet50-gradcam/cbam_resnet50_384_fold0.pth\")\nvalid_scores_df","metadata":{}},{"cell_type":"markdown","source":"gc.collect (); torch.cuda.empty_cache ()\nCFG.num_workers      = 8\nCFG.warmup_steps     = 0.5\n# CFG.eval_steps       = 0.5\nCFG.size             = 384\nCFG.train_batch_size = 21 \nCFG.eval_batch_size  = 21 \nCFG.isFreeze         = False\nCFG.epochs           = 4\nprint (f\"***** Training for size={CFG.size} freeze={CFG.isFreeze} *****\")\nvalid_scores_df      = train_main ()\nvalid_scores_df","metadata":{}},{"cell_type":"markdown","source":"gc.collect (); torch.cuda.empty_cache ()\nCFG.num_workers      = 8\nCFG.warmup_steps     = 0.5\n# CFG.eval_steps       = 0.5\nCFG.size             = 512\nCFG.train_batch_size = 80 \nCFG.eval_batch_size  = 80 \nCFG.isFreeze         = True\nCFG.epochs           = 1\nprint (f\"***** Training for size={CFG.size} freeze={CFG.isFreeze} *****\")\nvalid_scores_df      = train_main () # (checkpoint_path=\"../input/vin-cbamresnet50-384-fold0/cbam_resnet50_384_fold0.pth\") \nvalid_scores_df","metadata":{"papermill":{"duration":0.052836,"end_time":"2021-03-29T06:13:11.505308","exception":false,"start_time":"2021-03-29T06:13:11.452472","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"gc.collect (); torch.cuda.empty_cache ()\nCFG.num_workers      = 8\nCFG.warmup_steps     = 0.5\n# CFG.eval_steps       = 0.5\nCFG.size             = 512\nCFG.train_batch_size = 17 \nCFG.eval_batch_size  = 17 \nCFG.isFreeze         = False\nCFG.epochs           = 1\nprint (f\"***** Training for size={CFG.size} freeze={CFG.isFreeze} *****\")\nvalid_scores_df      = train_main () # (checkpoint_path=\"../input/vin-manual-torch-trainer-fp16-cbamresnet50-gradcam/cbam_resnet50_384_fold0.pth\")\nvalid_scores_df","metadata":{"papermill":{"duration":0.051632,"end_time":"2021-03-29T06:13:11.608811","exception":false,"start_time":"2021-03-29T06:13:11.557179","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print ('Done !')","metadata":{"papermill":{"duration":0.109165,"end_time":"2021-03-29T06:13:39.505776","exception":false,"start_time":"2021-03-29T06:13:39.396611","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]}]}