quick_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/dyMEAN/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import json\n",
    "from pytorch_lightning.trainer import Trainer\n",
    "from model_interface import MInterface\n",
    "from data_interface import DInterface"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "args = {\n",
    "    'res_dir': './results',\n",
    "    'ex_name': 'debug',\n",
    "    'dataset': 'PTM',\n",
    "    'model_name': 'MeToken',\n",
    "    'batch_size': 16,\n",
    "    'num_workers': 16,\n",
    "    'pad': 1024,\n",
    "    'min_length': 40,\n",
    "    'path': './data_test/generalization/qPTM_dataset/',\n",
    "    'with_null_ptm': 0,\n",
    "    'final_tau': 1e-4,\n",
    "    'test_only': 1,\n",
    "    'inference_pos': None,\n",
    "    'ckpt_path': \"pretrained_model/lightning_checkpoint.ckpt\",\n",
    "    'gpus': [0],\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### qPTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_module = DInterface(**args)\n",
    "data_module.setup(stage=\"test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/dyMEAN/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/accelerator_connector.py:478: LightningDeprecationWarning: Setting `Trainer(gpus=[0])` is deprecated in v1.7 and will be removed in v2.0. Please use `Trainer(accelerator='gpu', devices=[0])` instead.\n",
      "  rank_zero_deprecation(\n",
      "/root/anaconda3/envs/dyMEAN/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/checkpoint_connector.py:55: LightningDeprecationWarning: Setting `Trainer(resume_from_checkpoint=)` is deprecated in v1.5 and will be removed in v2.0. Please pass `Trainer.fit(ckpt_path=)` directly instead.\n",
      "  rank_zero_deprecation(\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using a CUDA device ('NVIDIA A100-SXM4-80GB') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing DataLoader 0: 100%|██████████| 27/27 [00:10<00:00,  2.48it/s]\n",
      "accuracy: 0.9021, precision: 0.7778, recall: 0.6775, f1 score: 0.7125, mcc score: 0.8040, auroc: 0.9707, auprc: 0.7468\n"
     ]
    }
   ],
   "source": [
    "trainer_config = {\n",
    "    'gpus': args['gpus'],\n",
    "    'accelerator': 'gpu',\n",
    "    'resume_from_checkpoint': args['ckpt_path']\n",
    "}\n",
    "\n",
    "trainer = Trainer(**trainer_config)\n",
    "model = MInterface.load_from_checkpoint(trainer_config[\"resume_from_checkpoint\"], strict=False, model_name=args[\"model_name\"])\n",
    "trainer.test(model,data_module)\n",
    "metrics = model.cal_metric(path=args[\"path\"])\n",
    "with open(os.path.join(args[\"res_dir\"], args[\"ex_name\"], 'qptm_metrics.json'), 'w') as file_obj:\n",
    "    json.dump(metrics, file_obj)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PTMint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "args.update({\n",
    "    'path': './data_test/generalization/PTMint_dataset',\n",
    "})\n",
    "\n",
    "data_module = DInterface(**args)\n",
    "data_module.setup(stage=\"test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using a CUDA device ('NVIDIA A100-SXM4-80GB') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing DataLoader 0: 100%|██████████| 19/19 [00:06<00:00,  2.78it/s]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/dyMEAN/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.9244, precision: 0.6382, recall: 0.5285, f1 score: 0.5511, mcc score: 0.8007, auroc: 0.9758, auprc: 0.7170\n"
     ]
    }
   ],
   "source": [
    "trainer.test(model,data_module)\n",
    "metrics = model.cal_metric(path=args[\"path\"])\n",
    "with open(os.path.join(args[\"res_dir\"], args[\"ex_name\"], 'ptmint_metrics.json'), 'w') as file_obj:\n",
    "    json.dump(metrics, file_obj)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Our Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "args.update({\n",
    "    'path': './data_test/large_scale_dataset',\n",
    "})\n",
    "\n",
    "data_module = DInterface(**args)\n",
    "data_module.setup(stage=\"test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using a CUDA device ('NVIDIA A100-SXM4-80GB') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing DataLoader 0: 100%|██████████| 1537/1537 [08:24<00:00,  3.05it/s]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/dyMEAN/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/root/anaconda3/envs/dyMEAN/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/root/anaconda3/envs/dyMEAN/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:43: UserWarning: No positive samples in targets, true positive value should be meaningless. Returning zero tensor in true positive score\n",
      "  warnings.warn(*args, **kwargs)  # noqa: B028\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.9035, precision: 0.5843, recall: 0.4849, f1 score: 0.5009, mcc score: 0.7707, auroc: 0.9286, auprc: 0.5220\n"
     ]
    }
   ],
   "source": [
    "trainer.test(model,data_module)\n",
    "metrics = model.cal_metric(path=args[\"path\"])\n",
    "with open(os.path.join(args[\"res_dir\"], args[\"ex_name\"], 'large_scale_metrics.json'), 'w') as file_obj:\n",
    "    json.dump(metrics, file_obj)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dyMEAN",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}