2_model_inference.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**This notebook provides examples of how to verify the performance of GCNNs on the validation set (function: reverify_sigopt_models), select the top-performing models accordingly (function: keep_the_best_few_models), compute the prediction on the test and holdout sets (function: get_all_model_predictions), and extract the latent embeddings of CGCNN and e3nn after all message passing and graph convolution layers (function: get_all_embeddings).**\n",
    "\n",
    "Parameters:\n",
    "- struct_type: the structure representation to use (options: unrelaxed, relaxed, M3Gnet_relaxed)\n",
    "- model_type: the model architechture to use (options: CGCNN, e3nn, Painn)\n",
    "- gpu_num: the GPU to use\n",
    "- training_fraction: if not trained on the entire training set, the fraction of the training set to use\n",
    "- num_best_models: the number of top-performing models to use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from inference.select_best_models import reverify_sigopt_models, keep_the_best_few_models\n",
    "from inference.test_model_prediction import get_all_model_predictions\n",
    "from inference.embedding_extraction import get_all_embeddings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CGCNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded data\n",
      "Completed data processing\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:01<00:00, 3673.91it/s]\n",
      "100%|██████████| 1277/1277 [00:00<00:00, 3896.71it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n",
      "Reverifying sigopt model #0\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [03:00<00:00, 34.75it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #1\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 9963.99it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #2\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 9905.31it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #3\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 9445.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #4\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10096.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #5\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 9807.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #6\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10205.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #7\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10148.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #8\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10167.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #9\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 9964.14it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #10\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10290.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #11\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10160.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #12\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10109.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #13\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10191.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #14\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10187.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #15\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10256.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #16\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10143.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #17\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10212.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #18\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10172.68it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #19\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10256.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #20\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10142.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #21\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10119.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #22\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10181.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #23\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10129.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #24\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10117.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #25\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10634.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #26\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10572.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #27\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10608.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #28\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10231.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #29\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 9874.62it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #30\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10132.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #31\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10436.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #32\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10162.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #33\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10482.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #34\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 9998.55it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #35\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 9856.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #36\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 9766.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #37\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 9834.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #38\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 9692.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #39\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 9813.59it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #40\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 9810.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #41\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 9844.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #42\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 9809.30it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #43\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 9888.15it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #44\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 9762.74it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #45\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 9697.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #46\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 9680.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #47\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 9832.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #48\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10371.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #49\n",
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 9855.00it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    }
   ],
   "source": [
    "reverify_sigopt_models(\n",
    "    model_params={\n",
    "        \"struct_type\": \"unrelaxed\",\n",
    "        \"model_type\": \"CGCNN\",\n",
    "        \"training_fraction\":1.0,\n",
    "    },\n",
    "    gpu_num=1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n",
      "Best models #0 found and saved\n",
      "Best models #1 found and saved\n",
      "Best models #2 found and saved\n"
     ]
    }
   ],
   "source": [
    "keep_the_best_few_models(\n",
    "    model_params={\n",
    "        \"struct_type\": \"unrelaxed\",\n",
    "        \"model_type\": \"CGCNN\",\n",
    "        \"training_fraction\":1.0,\n",
    "    },\n",
    "    num_best_models=3\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded data\n",
      "Completed data processing\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:01<00:00, 3688.76it/s]\n",
      "100%|██████████| 1261/1261 [00:00<00:00, 3857.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [02:52<00:00, 36.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timing...\n",
      "38.74220252037048\n",
      "39.582425117492676\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10611.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timing...\n",
      "1.7987136840820312\n",
      "2.6389362812042236\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10400.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timing...\n",
      "1.8962302207946777\n",
      "2.73645281791687\n",
      "Completed model prediction for test_set\n",
      "Loaded data\n",
      "Completed data processing\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:01<00:00, 3603.58it/s]\n",
      "100%|██████████| 600/600 [00:00<00:00, 3996.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [02:59<00:00, 34.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timing...\n",
      "19.097195386886597\n",
      "19.49189281463623\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 10086.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timing...\n",
      "0.8781490325927734\n",
      "1.2728464603424072\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 9547.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timing...\n",
      "0.9408743381500244\n",
      "1.3355717658996582\n",
      "Completed model prediction for holdout_set_B_sites\n",
      "Loaded data\n",
      "Completed data processing\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:01<00:00, 3720.38it/s]\n",
      "100%|██████████| 863/863 [00:00<00:00, 3358.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [04:03<00:00, 25.75it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timing...\n",
      "38.92133092880249\n",
      "39.53340554237366\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 6635.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timing...\n",
      "1.7603375911712646\n",
      "2.3724122047424316\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:00<00:00, 7622.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timing...\n",
      "1.8842463493347168\n",
      "2.496320962905884\n",
      "Completed model prediction for holdout_set_series\n"
     ]
    }
   ],
   "source": [
    "get_all_model_predictions(\n",
    "    model_params={\n",
    "        \"struct_type\": \"unrelaxed\",\n",
    "        \"model_type\": \"CGCNN\",\n",
    "        \"training_fraction\":1.0,\n",
    "    },\n",
    "    gpu_num=1,\n",
    "    num_best_models=3\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded data\n",
      "Completed data processing\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:02<00:00, 2587.64it/s]\n",
      "100%|██████████| 1261/1261 [00:00<00:00, 2642.57it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [03:42<00:00, 28.27it/s]\n",
      "100%|██████████| 6276/6276 [00:00<00:00, 10066.29it/s]\n",
      "100%|██████████| 6276/6276 [00:00<00:00, 10180.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Completed embedding extraction for test_set\n",
      "Loaded data\n",
      "Completed data processing\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:01<00:00, 3942.30it/s]\n",
      "100%|██████████| 600/600 [00:00<00:00, 3911.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [03:02<00:00, 34.40it/s]\n",
      "100%|██████████| 6276/6276 [00:00<00:00, 8054.76it/s]\n",
      "100%|██████████| 6276/6276 [00:00<00:00, 9514.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Completed embedding extraction for holdout_set_B_sites\n",
      "Loaded data\n",
      "Completed data processing\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:01<00:00, 3659.18it/s]\n",
      "100%|██████████| 863/863 [00:00<00:00, 3883.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dft_e_hull_htvs_data_unrelaxed_CGCNN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [02:58<00:00, 35.14it/s]\n",
      "100%|██████████| 6276/6276 [00:00<00:00, 10307.28it/s]\n",
      "100%|██████████| 6276/6276 [00:00<00:00, 10143.75it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Completed embedding extraction for holdout_set_series\n"
     ]
    }
   ],
   "source": [
    "get_all_embeddings(\n",
    "    model_params={\n",
    "        \"struct_type\": \"unrelaxed\",\n",
    "        \"model_type\": \"CGCNN\",\n",
    "        \"training_fraction\":1.0,\n",
    "    },\n",
    "    gpu_num=1,\n",
    "    num_best_models=3\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# e3nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded data\n",
      "Completed data processing\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4467/4467 [00:01<00:00, 2909.58it/s]\n",
      "100%|██████████| 4467/4467 [07:21<00:00, 10.11it/s]\n",
      "100%|██████████| 1277/1277 [00:00<00:00, 3144.33it/s]\n",
      "100%|██████████| 1277/1277 [02:10<00:00,  9.82it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n",
      "Reverifying sigopt model #0\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #1\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #2\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #3\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #4\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #5\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #6\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #7\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #8\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #9\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #10\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #11\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #12\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #13\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #14\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #15\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #16\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #17\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #18\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #19\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #20\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #21\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #22\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #23\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #24\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #25\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #26\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #27\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #28\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #29\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #30\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #31\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #32\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #33\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #34\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #35\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #36\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #37\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #38\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #39\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #40\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #41\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #42\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #43\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #44\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #45\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #46\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #47\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #48\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #49\n",
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    }
   ],
   "source": [
    "reverify_sigopt_models(\n",
    "    model_params={\n",
    "        \"struct_type\": \"relaxed\",\n",
    "        \"model_type\": \"e3nn\",\n",
    "        \"training_fraction\":0.5,\n",
    "    },\n",
    "    gpu_num=1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n",
      "Best models #0 found and saved\n",
      "Best models #1 found and saved\n",
      "Best models #2 found and saved\n"
     ]
    }
   ],
   "source": [
    "keep_the_best_few_models(\n",
    "    model_params={\n",
    "        \"struct_type\": \"relaxed\",\n",
    "        \"model_type\": \"e3nn\",\n",
    "        \"training_fraction\":0.5,\n",
    "    },\n",
    "    num_best_models=3\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded data\n",
      "Completed data processing\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4467/4467 [00:01<00:00, 3264.45it/s]\n",
      "100%|██████████| 4467/4467 [07:07<00:00, 10.46it/s]\n",
      "100%|██████████| 1261/1261 [00:00<00:00, 3285.96it/s]\n",
      "100%|██████████| 1261/1261 [02:08<00:00,  9.78it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timing...\n",
      "16.113953590393066\n",
      "145.44184708595276\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timing...\n",
      "15.490482330322266\n",
      "144.81837582588196\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timing...\n",
      "9.801529884338379\n",
      "139.12942337989807\n",
      "Completed model prediction for test_set\n",
      "Loaded data\n",
      "Completed data processing\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4467/4467 [00:01<00:00, 3379.61it/s]\n",
      "100%|██████████| 4467/4467 [06:53<00:00, 10.79it/s]\n",
      "100%|██████████| 600/600 [00:00<00:00, 3369.61it/s]\n",
      "100%|██████████| 600/600 [00:59<00:00, 10.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timing...\n",
      "7.313486099243164\n",
      "67.485910654068\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timing...\n",
      "7.317779541015625\n",
      "67.49020409584045\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timing...\n",
      "4.4580771923065186\n",
      "64.63050174713135\n",
      "Completed model prediction for holdout_set_B_sites\n",
      "Loaded data\n",
      "Completed data processing\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4467/4467 [00:01<00:00, 3467.94it/s]\n",
      "100%|██████████| 4467/4467 [06:55<00:00, 10.75it/s]\n",
      "100%|██████████| 863/863 [00:00<00:00, 3395.37it/s]\n",
      "100%|██████████| 863/863 [01:26<00:00, 10.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timing...\n",
      "10.51332974433899\n",
      "97.07377457618713\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timing...\n",
      "10.311720848083496\n",
      "96.87216567993164\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timing...\n",
      "6.410120725631714\n",
      "92.97056555747986\n",
      "Completed model prediction for holdout_set_series\n"
     ]
    }
   ],
   "source": [
    "get_all_model_predictions(\n",
    "    model_params={\n",
    "        \"struct_type\": \"relaxed\",\n",
    "        \"model_type\": \"e3nn\",\n",
    "        \"training_fraction\":0.5,\n",
    "    },\n",
    "    gpu_num=1,\n",
    "    num_best_models=3\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded data\n",
      "Completed data processing\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4467/4467 [00:01<00:00, 3472.17it/s]\n",
      "100%|██████████| 4467/4467 [06:51<00:00, 10.86it/s]\n",
      "100%|██████████| 1261/1261 [00:00<00:00, 3393.57it/s]\n",
      "100%|██████████| 1261/1261 [02:03<00:00, 10.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n",
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n",
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Completed embedding extraction for test_set\n",
      "Loaded data\n",
      "Completed data processing\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4467/4467 [00:01<00:00, 3450.89it/s]\n",
      "100%|██████████| 4467/4467 [06:54<00:00, 10.79it/s]\n",
      "100%|██████████| 600/600 [00:00<00:00, 3319.14it/s]\n",
      "100%|██████████| 600/600 [01:00<00:00,  9.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n",
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n",
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Completed embedding extraction for holdout_set_B_sites\n",
      "Loaded data\n",
      "Completed data processing\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4467/4467 [00:01<00:00, 3418.59it/s]\n",
      "100%|██████████| 4467/4467 [06:51<00:00, 10.85it/s]\n",
      "100%|██████████| 863/863 [00:00<00:00, 3359.71it/s]\n",
      "100%|██████████| 863/863 [01:26<00:00, 10.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dft_e_hull_htvs_data_relaxed_e3nn_TrainingFraction0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n",
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n",
      "/home/jypeng/miniconda3/envs/Perovskite_ML_Environment/lib/python3.10/site-packages/torch/jit/_check.py:181: UserWarning: The TorchScript type system doesn't support instance-level annotations on empty non-base types in `__init__`. Instead, either 1) use a type annotation in the class body, or 2) wrap the type in `torch.jit.Attribute`.\n",
      "  warnings.warn(\"The TorchScript type system doesn't support \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Completed embedding extraction for holdout_set_series\n"
     ]
    }
   ],
   "source": [
    "get_all_embeddings(\n",
    "    model_params={\n",
    "        \"struct_type\": \"relaxed\",\n",
    "        \"model_type\": \"e3nn\",\n",
    "        \"training_fraction\":0.5,\n",
    "    },\n",
    "    gpu_num=1,\n",
    "    num_best_models=3\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Painn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded data\n",
      "Completed data processing\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:01<00:00, 3921.85it/s]\n",
      "100%|██████████| 6276/6276 [10:18<00:00, 10.15it/s]\n",
      "100%|██████████| 6276/6276 [00:00<00:00, 12287.37it/s]\n",
      "100%|██████████| 1277/1277 [00:00<00:00, 3917.27it/s]\n",
      "100%|██████████| 1277/1277 [02:05<00:00, 10.14it/s]\n",
      "100%|██████████| 1277/1277 [00:00<00:00, 11028.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dft_e_hull_htvs_data_unrelaxed_Painn\n",
      "Reverifying sigopt model #0\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:10<00:00, 122.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #1\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:03<00:00, 336.79it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #2\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:07<00:00, 175.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #3\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:05<00:00, 240.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #4\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:08<00:00, 146.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #5\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:12<00:00, 99.20it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #6\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:08<00:00, 142.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #7\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:13<00:00, 96.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #8\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:03<00:00, 365.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #9\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:09<00:00, 133.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #10\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:07<00:00, 173.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #11\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:05<00:00, 252.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #12\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:05<00:00, 243.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #13\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:12<00:00, 105.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #14\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:13<00:00, 97.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #15\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:06<00:00, 186.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #16\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:03<00:00, 372.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #17\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:03<00:00, 329.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #18\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:05<00:00, 251.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #19\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:03<00:00, 376.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #20\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:05<00:00, 245.75it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #21\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:12<00:00, 105.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #22\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:10<00:00, 121.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #23\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:09<00:00, 138.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #24\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:13<00:00, 94.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #25\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:11<00:00, 107.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #26\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:11<00:00, 107.79it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #27\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:06<00:00, 182.71it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #28\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:12<00:00, 106.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #29\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:12<00:00, 106.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #30\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:13<00:00, 95.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #31\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:08<00:00, 149.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #32\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:06<00:00, 186.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #33\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:03<00:00, 367.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #34\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:11<00:00, 107.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #35\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:13<00:00, 94.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #36\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:03<00:00, 366.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #37\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:05<00:00, 232.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #38\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:07<00:00, 160.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #39\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:08<00:00, 148.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #40\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:10<00:00, 125.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #41\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:05<00:00, 251.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #42\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:11<00:00, 108.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #43\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:10<00:00, 125.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #44\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:05<00:00, 245.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #45\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:07<00:00, 180.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #46\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:08<00:00, 151.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #47\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:05<00:00, 248.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #48\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:03<00:00, 361.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #49\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:09<00:00, 141.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reverifying sigopt model #50\n",
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1277/1277 [00:05<00:00, 241.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    }
   ],
   "source": [
    "reverify_sigopt_models(\n",
    "    model_params={\n",
    "        \"struct_type\": \"unrelaxed\",\n",
    "        \"model_type\": \"Painn\",\n",
    "        \"training_fraction\":1.0,\n",
    "    },\n",
    "    gpu_num=1\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dft_e_hull_htvs_data_unrelaxed_Painn\n",
      "Best models #0 found and saved\n",
      "Best models #1 found and saved\n",
      "Best models #2 found and saved\n"
     ]
    }
   ],
   "source": [
    "keep_the_best_few_models(\n",
    "    model_params={\n",
    "        \"struct_type\": \"unrelaxed\",\n",
    "        \"model_type\": \"Painn\",\n",
    "        \"training_fraction\":1.0,\n",
    "    },\n",
    "    num_best_models=3\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded data\n",
      "Completed data processing\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:01<00:00, 3937.55it/s]\n",
      "100%|██████████| 6276/6276 [10:17<00:00, 10.17it/s]\n",
      "100%|██████████| 6276/6276 [00:00<00:00, 12324.16it/s]\n",
      "100%|██████████| 1261/1261 [00:00<00:00, 3923.31it/s]\n",
      "100%|██████████| 1261/1261 [02:03<00:00, 10.25it/s]\n",
      "100%|██████████| 1261/1261 [00:00<00:00, 11580.82it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1261/1261 [00:12<00:00, 100.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timing...\n",
      "12.63401198387146\n",
      "136.1617124080658\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1261/1261 [00:09<00:00, 126.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timing...\n",
      "10.007908821105957\n",
      "133.5356092453003\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1261/1261 [00:05<00:00, 248.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timing...\n",
      "5.0994532108306885\n",
      "128.62715363502502\n",
      "Completed model prediction for test_set\n",
      "Loaded data\n",
      "Completed data processing\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:01<00:00, 3867.79it/s]\n",
      "100%|██████████| 6276/6276 [10:11<00:00, 10.26it/s]\n",
      "100%|██████████| 6276/6276 [00:00<00:00, 12096.91it/s]\n",
      "100%|██████████| 600/600 [00:00<00:00, 3872.45it/s]\n",
      "100%|██████████| 600/600 [00:59<00:00, 10.04it/s]\n",
      "100%|██████████| 600/600 [00:00<00:00, 11353.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 600/600 [00:06<00:00, 98.54it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timing...\n",
      "6.106281280517578\n",
      "66.12156367301941\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 600/600 [00:04<00:00, 127.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timing...\n",
      "4.715498685836792\n",
      "64.73078107833862\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 600/600 [00:02<00:00, 240.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timing...\n",
      "2.505908250808716\n",
      "62.52119064331055\n",
      "Completed model prediction for holdout_set_B_sites\n",
      "Loaded data\n",
      "Completed data processing\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6276/6276 [00:01<00:00, 3980.47it/s]\n",
      "100%|██████████| 6276/6276 [10:19<00:00, 10.13it/s]\n",
      "100%|██████████| 6276/6276 [00:00<00:00, 12397.23it/s]\n",
      "100%|██████████| 863/863 [00:00<00:00, 3897.42it/s]\n",
      "100%|██████████| 863/863 [01:24<00:00, 10.17it/s]\n",
      "100%|██████████| 863/863 [00:00<00:00, 10568.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dft_e_hull_htvs_data_unrelaxed_Painn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 863/863 [00:09<00:00, 95.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timing...\n",
      "9.083925724029541\n",
      "94.29788994789124\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 863/863 [00:06<00:00, 123.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timing...\n",
      "7.008980751037598\n",
      "92.22294497489929\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 863/863 [00:03<00:00, 243.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timing...\n",
      "3.5661513805389404\n",
      "88.78011560440063\n",
      "Completed model prediction for holdout_set_series\n"
     ]
    }
   ],
   "source": [
    "get_all_model_predictions(\n",
    "    model_params={\n",
    "        \"struct_type\": \"unrelaxed\",\n",
    "        \"model_type\": \"Painn\",\n",
    "        \"training_fraction\":1.0,\n",
    "    },\n",
    "    gpu_num=1,\n",
    "    num_best_models=3\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Perovskite_ML_Environment",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}