diff --git a/.github/workflows/install_and_test.yml b/.github/workflows/install_and_test.yml index a4949fe..37c0c22 100644 --- a/.github/workflows/install_and_test.yml +++ b/.github/workflows/install_and_test.yml @@ -36,10 +36,13 @@ jobs: pytest -v tests/unit/test_loss.py - name: Run integration tests run: | - pytest -v tests/integration/test_non_xc_energy.py - pytest -v tests/integration/test_functional_implementations.py - pytest -v tests/integration/test_Harris.py - pytest -v tests/integration/test_predict_B88.py - pytest -v tests/integration/test_predict_B3LYP.py - pytest -v tests/integration/test_training.py + pytest -v tests/integration/molecules/test_non_xc_energy.py + pytest -v tests/integration/molecules/test_functional_implementations.py + pytest -v tests/integration/molecules/test_Harris.py + pytest -v tests/integration/molecules/test_predict_B88.py + pytest -v tests/integration/molecules/test_training.py + pytest -v tests/integration/solids/test_training.py + pytest -v tests/integration/solids/test_non_xc_energy.py + pytest -v tests/integration/solids/test_functional_implementations.py + diff --git a/examples/intermediate_notebooks/periodic_systems_bz_sampling_05.ipynb b/examples/intermediate_notebooks/periodic_systems_bz_sampling_05.ipynb new file mode 100644 index 0000000..060c590 --- /dev/null +++ b/examples/intermediate_notebooks/periodic_systems_bz_sampling_05.ipynb @@ -0,0 +1,632 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/XanaduAI/GradDFT/blob/main/examples/intermediate_notebooks/periodic_systems_bz_sampling_05.ipynb)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# In colab run\n", + "# !pip install git+https://github.com/XanaduAI/GradDFT.git" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Periodic systems with k-point sampling in Grad DFT\n", + "\n", + "In this tutorial, you will learn how to train a simple neural functional for solids with full integration over the 1BZ. Calculations in this tutorial are certainly not converged with respect to k-point sampling or basis sets, so please consider this when adapting to your own calculations.\n", + "\n", + "## Perform solid-state calculations with PySCF\n", + "\n", + "PySCF implements DFT and some wavefunction methods in periodic boundary conditions with integration over the 1BZ. To begin, we need:\n", + "\n", + "(1) A DFT starting point to prime Grad DFT. We'll use the PBE functional.\n", + "\n", + "(2) Accurate training and validation data. We'll use the periodic CCSD solver implemented in PySCF.\n", + "\n", + "Our calculations will be run using Sodium Chloride (NaCl) in the rock salt structure.\n", + "\n", + "Let's import the modules required for the PySCF pre-computations." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/pablo.casares/miniforge3/envs/graddft/lib/python3.10/site-packages/pyscf/dft/libxc.py:772: UserWarning: Since PySCF-2.3, B3LYP (and B3P86) are changed to the VWN-RPA variant, the same to the B3LYP functional in Gaussian and ORCA (issue 1480). To restore the VWN5 definition, you can put the setting \"B3LYP_WITH_VWN5 = True\" in pyscf_conf.py\n", + " warnings.warn('Since PySCF-2.3, B3LYP (and B3P86) are changed to the VWN-RPA variant, '\n" + ] + } + ], + "source": [ + "from pyscf.pbc import gto, scf, mp, cc\n", + "import numpy as np" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we define the primitive cell for NaCl in the Rocksalt struture. This structural data was taken from [The Materials Project](https://next-gen.materialsproject.org/materials/mp-22862). \n", + "\n", + "We will train using the pristine geometry and validate using a slightly expanded cell." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# Training geometry: pristine NaCl\n", + "\n", + "param = 5.272336\n", + "lat_vec = np.array(\n", + " [\n", + " [0.0, param, param],\n", + " [param, 0.0, param],\n", + " [param, param, 0.0]\n", + " ]\n", + ")\n", + "\n", + "cell_tr = gto.M(\n", + " a = lat_vec,\n", + " atom = \"\"\"Na 0.0 0.0 0.0\n", + " Cl %.5f %.5f %.5f\"\"\" % (param, param, param),\n", + " basis = 'sto-3g',\n", + ")\n", + "cell_tr.exp_to_discard=0.1\n", + "\n", + "# Validation geometry: NaCl with a 5% larger lattice parameter\n", + "\n", + "param_strain = param*1.05\n", + "lat_vec_strain = np.array(\n", + " [\n", + " [0.0, param_strain, param_strain],\n", + " [param_strain, 0.0, param_strain],\n", + " [param_strain, param_strain, 0.0]\n", + " ]\n", + ")\n", + "cell_val = gto.M(\n", + " a = lat_vec_strain,\n", + " atom = \"\"\"Na 0.0 0.0 0.0 \n", + " Cl %.5f %.5f %.5f\"\"\" % (param_strain, param_strain, param_strain),\n", + " basis = 'sto-3g'\n", + ")\n", + "cell_val.exp_to_discard=0.1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we get the DFT starting point to prime Grad DFT. We use PBE and a small number of k-points. We also use Gaussian density fitting for the electronic coulomb terms. [All other PBC density fitting approaches](https://pyscf.org/user/pbc/df.html) in PySCF are also compatible with Grad DFT." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " does not have attributes nlcgrids nlc\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "WARN: HOMO 0.133342844043 == LUMO 0.133342844043\n", + "\n", + "\n", + "WARN: HOMO 0.129696510302 == LUMO 0.129696796143\n", + "\n", + "\n", + "WARN: HOMO 0.123485609714 == LUMO 0.123533794981\n", + "\n", + "\n", + "WARN: HOMO 0.122470342913 == LUMO 0.122549417589\n", + "\n", + "\n", + "WARN: HOMO 0.121977200489 == LUMO 0.121984450283\n", + "\n", + "\n", + "WARN: HOMO 0.127199590342 == LUMO 0.127215372837\n", + "\n", + "\n", + "WARN: HOMO 0.127429516524 == LUMO 0.127576695046\n", + "\n", + "SCF not converged.\n", + "SCF energy = -613.999831522497\n", + "\n", + "WARN: HOMO 0.136411628811 == LUMO 0.136411628812\n", + "\n", + "\n", + "WARN: HOMO 0.113400677255 == LUMO 0.113620213224\n", + "\n", + "\n", + "WARN: HOMO 0.114848903144 == LUMO 0.115097636252\n", + "\n", + "\n", + "WARN: HOMO 0.178814138743 == LUMO 0.17892358234\n", + "\n", + "\n", + "WARN: HOMO 0.180008068011 == LUMO 0.180066156279\n", + "\n", + "\n", + "WARN: HOMO 0.179951129106 == LUMO 0.179972904236\n", + "\n", + "SCF not converged.\n", + "SCF energy = -613.984277821783\n" + ] + } + ], + "source": [ + "# Run training DFT starting point\n", + "kmf_tr = scf.KRKS(cell_tr, kpts=cell_tr.make_kpts([2,2,2])).density_fit()\n", + "kmf_tr.xc = \"PBE\"\n", + "kmf_tr.max_cycle = 10\n", + "kmf_tr = kmf_tr.run()\n", + "\n", + "# Run validation DFT starting point\n", + "kmf_val = scf.KRKS(cell_val, kpts=cell_val.make_kpts([2,2,2])).density_fit()\n", + "kmf_val.xc = \"PBE\"\n", + "kmf_val.max_cycle = 10\n", + "kmf_val = kmf_val.run()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we can perform the CCSD calculations which will be used for truth values in training." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "converged SCF energy = -614.443314677291\n", + "E(RCCSD) = -614.4825393595149 E_corr = -0.03922468222342719\n", + "converged SCF energy = -614.434597523731\n", + "E(RCCSD) = -614.4738196894269 E_corr = -0.03922216569629406\n" + ] + } + ], + "source": [ + "# Make one training data-point and one validation using CCSD\n", + "\n", + "# Training\n", + "khf_tr = scf.KRHF(cell_tr, kpts=cell_tr.make_kpts([2,2,2])).density_fit()\n", + "khf_tr = khf_tr.run()\n", + "ccsd_tr = cc.KCCSD(khf_tr)\n", + "ccsd_tr = ccsd_tr.run()\n", + "E_tr = ccsd_tr.e_tot\n", + "\n", + "\n", + "# Validation\n", + "khf_val = scf.KRHF(cell_val, kpts=cell_val.make_kpts([2,2,2])).density_fit()\n", + "khf_val = khf_val.run()\n", + "ccsd_val = cc.KCCSD(khf_val)\n", + "ccsd_val = ccsd_val.run()\n", + "E_val = ccsd_val.e_tot\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loading into Grad DFT\n", + "\n", + "The DFT starting points from PySCF can be loaded into Grad DFT with the convenience function `solid_from_pyscf`. This mirrors `molecule_from_pyscf` but now many arrays have an additional k-points dimension." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "WARNING:tensorflow:From /Users/pablo.casares/miniforge3/envs/graddft/lib/python3.10/site-packages/tensorflow/python/compat/v2_compat.py:107: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "non-resource variables are not supported in the long term\n" + ] + } + ], + "source": [ + "from jax.config import config\n", + "config.update(\"jax_enable_x64\", True)\n", + "import grad_dft as gd\n", + "\n", + "gd_sol_tr = gd.solid_from_pyscf(kmf_tr)\n", + "gd_sol_val = gd.solid_from_pyscf(kmf_val)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Make a simple neural functional\n", + "\n", + "Like in `~/examples/intermediate_notebooks/training_methods_03.ipynb`, we create a scaled down version of the net used the original Grad DFT reference article" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "from grad_dft.functional import canonicalize_inputs, dm21_coefficient_inputs, dm21_densities\n", + "from jax.nn import gelu\n", + "from functools import partial\n", + "import jax.numpy as jnp\n", + "from jax.random import PRNGKey\n", + "\n", + "seed = 1984 # Random seed used throughout this notebok for reproducibility reasons.\n", + "key = PRNGKey(seed)\n", + "\n", + "squash_offset = 1e-4\n", + "layer_widths = [6] * 2\n", + "\n", + "out_features = 2\n", + "sigmoid_scale_factor = 2.0\n", + "activation = gelu\n", + "\n", + "def nn_coefficients(instance, rhoinputs, *_, **__):\n", + " x = canonicalize_inputs(rhoinputs) # Making sure dimensions are correct\n", + " # Initial layer: log -> dense -> tanh\n", + " x = jnp.log(jnp.abs(x) + squash_offset) # squash_offset = 1e-4\n", + " instance.sow(\"intermediates\", \"log\", x)\n", + " x = instance.dense(features=layer_widths[0])(x) # features = 256\n", + " instance.sow(\"intermediates\", \"initial_dense\", x)\n", + " x = jnp.tanh(x)\n", + " instance.sow(\"intermediates\", \"norm\", x)\n", + " # 2 Residual blocks with 6-features dense layer and layer norm\n", + " for features, i in zip(layer_widths, range(len(layer_widths))): # layer_widths = [256]*6\n", + " res = x\n", + " x = instance.dense(features=features)(x)\n", + " instance.sow(\"intermediates\", \"residual_dense_\" + str(i), x)\n", + " x = x + res # nn.Dense + Residual connection\n", + " instance.sow(\"intermediates\", \"residual_residual_\" + str(i), x)\n", + " x = instance.layer_norm()(x) # + res # nn.LayerNorm\n", + " instance.sow(\"intermediates\", \"residual_layernorm_\" + str(i), x)\n", + " x = activation(x) # activation = jax.nn.gelu\n", + " instance.sow(\"intermediates\", \"residual_elu_\" + str(i), x)\n", + " return instance.head(x, out_features, sigmoid_scale_factor)\n", + " \n", + "functional = gd.NeuralFunctional(\n", + " coefficients=nn_coefficients,\n", + " coefficient_inputs=dm21_coefficient_inputs,\n", + " energy_densities=partial(dm21_densities, functional_type=\"GGA\"),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Non self-consistent training using the energy only\n", + "\n", + "Once again, borrowing from `~/examples/intermediate_notebooks/training_methods_03.ipynb`, we define a training and validation regime" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "from optax import adam, apply_updates\n", + "from tqdm import tqdm\n", + "from jax import value_and_grad\n", + "\n", + "def train_neural_functional(train_recipe: tuple, validate_recipe: tuple) -> None:\n", + " r\"\"\"Minimize a Grad DFT loss function using 50 epochs of the Adam optimizer.\n", + "\n", + " Args:\n", + " train_recipe (tuple):train_recipe (tuple): information regarding the loss, its arguments and the predictor.\n", + " validate_recipe (tuple):train_recipe (tuple): the same information as train_recipe, but for the validation calculation.\n", + " Returns:\n", + " tuple: the training and validation loss history over the number of training epochs\n", + " \"\"\"\n", + " \n", + " loss_func, loss_args = train_recipe\n", + " val_func, val_args = validate_recipe\n", + " \n", + " tr_params = functional.init(key, dm21_coefficient_inputs(loss_args[2][0]))\n", + " loss_args[0] = tr_params\n", + " val_args[0] = tr_params\n", + " \n", + " tx = adam(learning_rate=0.01, b1=0.9)\n", + " opt_state = tx.init(tr_params)\n", + " loss_and_grad = value_and_grad(loss_func)\n", + " tr_loss_history = []\n", + " val_loss_history = []\n", + " for i in tqdm(range(10), desc=\"Training epoch\"):\n", + " tr_loss_value, grads = loss_and_grad(*loss_args)\n", + " val_loss_value = val_func(*val_args)\n", + " tr_loss_history.append(tr_loss_value)\n", + " val_loss_history.append(val_loss_value)\n", + " updates, opt_state = tx.update(grads, opt_state, tr_params)\n", + " tr_params = apply_updates(tr_params, updates)\n", + " loss_args[0] = tr_params\n", + " val_args[0] = tr_params\n", + " if (i + 1) % 5 == 0:\n", + " print(f\"At epoch {i+1} training loss = {tr_loss_value}, validation loss = {val_loss_value}\")\n", + " return tr_loss_history, val_loss_history\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "non_sc_en_train_recipe = (\n", + " gd.mse_energy_loss, \n", + " [None, gd.non_scf_predictor(functional), [gd_sol_tr], [E_tr], True]\n", + ")\n", + "non_sc_en_validate_recipe = (\n", + " gd.mse_energy_loss, \n", + " [None, gd.non_scf_predictor(functional), [gd_sol_val], [E_val], True]\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we can perform the non-self consistent training." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Training epoch: 50%|█████ | 5/10 [01:11<01:08, 13.70s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "At epoch 5 training loss = 0.00021681184278852755, validation loss = 0.00022157386004632528\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Training epoch: 100%|██████████| 10/10 [02:17<00:00, 13.78s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "At epoch 10 training loss = 3.611312949052283e-07, validation loss = 1.7956748122185697e-07\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "# Begin training\n", + "tr_loss_his_non_sc_en, val_loss_his_non_sc_en = train_neural_functional(non_sc_en_train_recipe, non_sc_en_validate_recipe)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and check out the loss as a function of epochs" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkgAAAHHCAYAAABEEKc/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABxCklEQVR4nO3dd3wUdf7H8dfsJtn0hFASAiG0SC9SRVTgpAkWkCYWihz87hRQsaLSz8OKqCiop6AnHIiF4xSRgKgnRKooSBGVDgk9gdTN7vz+CNlzSYAQkkzK+/l47IPd73znO5+Z78J+mPnOdwzTNE1ERERExMNmdQAiIiIipY0SJBEREZHzKEESEREROY8SJBEREZHzKEESEREROY8SJBEREZHzKEESEREROY8SJBEREZHzKEESEREROY8SJBEpEXv37sUwDObNm1eo9Q3DYPLkyUUak5Rfw4YNo3bt2laHIWWYEiQp1+bNm4dhGPj7+3Po0KE8yzt37kzTpk0tiCzH1q1b6d+/P7Gxsfj7+1OjRg26devGa6+9lqeuy+Vi7ty5dO7cmYiICBwOB7Vr12b48OFs3LjRUy93n/N7PfHEExeNZ8GCBcycObOod7Nc+uNx/u677/IsN02TmJgYDMPg5ptv9lp29uxZJk2aRNOmTQkKCqJy5cq0bNmSBx54gMOHD3vqTZ48+YJ9aRgGiYmJxb6fIhWVj9UBiJSEzMxMnn322XwTD6usXbuWLl26UKtWLUaOHElUVBQHDhzg+++/55VXXmHMmDGeuunp6dx+++0sX76cG264gSeffJKIiAj27t3Lhx9+yHvvvcf+/fupWbOmZ52pU6dSp04dr21eKhlcsGAB27Zt48EHHyzSfQWIjY0lPT0dX1/fQq2fnp6Oj0/p+yfL39+fBQsWcN1113mVf/PNNxw8eBCHw+FV7nQ6ueGGG9i5cydDhw5lzJgxnD17lp9//pkFCxbQt29foqOjvdaZPXs2wcHBebYdHh5e5PsjIjlK3782IsWgZcuWvP3224wfPz7Pj49VnnnmGcLCwtiwYUOeH7qjR496fX700UdZvnw5L7/8cp7kZdKkSbz88st52r/pppto06ZNUYftkZGRgZ+fHzZbwU5E557JK6wrWbc49erVi8WLF/Pqq696JXALFiygdevWHD9+3Kv+kiVL+OGHH5g/fz533nmn17KMjAyysrLybKN///5UqVKleHaghLjdbrKyskptP4qcT5fYpEJ48skncblcPPvss5esm52dzbRp06hXr57nMtaTTz5JZmamV73atWtz8803891339GuXTv8/f2pW7cu77//foFi+u2332jSpEm+ZwGqVavmeX/w4EHefPNNunXrlu+ZHbvdziOPPOJ19qgwOnfuzOeff86+ffs8l3Byx3B8/fXXGIbBwoULefrpp6lRowaBgYGkpKRw8uRJHnnkEZo1a0ZwcDChoaHcdNNN/Pjjj17t5zcGadiwYQQHB3Po0CH69OlDcHAwVatW5ZFHHsHlcnmtf/4YpNzLT7/++ivDhg0jPDycsLAwhg8fTlpamte66enpjB07lipVqhASEsKtt97KoUOH8h3XtHPnTvbv31/g4zZ48GBOnDhBfHy8pywrK4uPPvooTwIEOf0O0LFjxzzL/P39CQ0NLfC2C+KDDz6gdevWBAQEEBERwR133MGBAwe86uReat6+fTtdunQhMDCQGjVq8Pzzz+dpLzMzk0mTJlG/fn0cDgcxMTE89thjef5+GIbB6NGjmT9/Pk2aNMHhcLB8+XIAfvrpJzp16kRAQAA1a9bkb3/7G3PnzsUwDPbu3QvA0KFDqVKlCk6nM08M3bt3p0GDBpd9LFJTU3n44YeJiYnB4XDQoEEDXnzxRUzT9KoXHx/PddddR3h4OMHBwTRo0IAnn3zSq85rr71GkyZNCAwMpFKlSrRp04YFCxZcdkxSeukMklQIderUYciQIbz99ts88cQTFz2L9Oc//5n33nuP/v378/DDD7Nu3TqmT5/Ojh07+PTTT73q/vrrr/Tv358RI0YwdOhQ3n33XYYNG0br1q1p0qTJRWOKjY0lISGBbdu2XfTS1xdffEF2djb33HPPZe1zcnJynrMXFzsL8dRTT5GcnMzBgwc9Z6TOv6wzbdo0/Pz8eOSRR8jMzMTPz4/t27ezZMkSBgwYQJ06dUhKSuLNN9+kU6dObN++/ZJn7FwuFz169KB9+/a8+OKLrFy5kpdeeol69erx17/+9ZL7OXDgQOrUqcP06dPZvHkz//jHP6hWrRrPPfecp86wYcP48MMPueeee7jmmmv45ptv6N27d77tNWrUiE6dOvH1119fctuQkyh36NCBf/3rX9x0001ATp8lJydzxx138Oqrr3rVj42NBeD999/n6aefxjCMS27j5MmTecp8fHwueYntmWeeYcKECQwcOJA///nPHDt2jNdee40bbriBH374wWv9U6dO0bNnT26//XYGDhzIRx99xOOPP06zZs08++V2u7n11lv57rvvGDVqFI0aNWLr1q28/PLL/PLLLyxZssRr+1999RUffvgho0ePpkqVKtSuXZtDhw7RpUsXDMNg/PjxBAUF8Y9//CPPpch77rmH999/ny+//NJrDFdiYiJfffUVkyZNuuRx+yPTNLn11ltZvXo1I0aMoGXLlnz55Zc8+uijHDp0yPOd//nnn7n55ptp3rw5U6dOxeFw8Ouvv7JmzRpPW2+//TZjx46lf//+PPDAA2RkZPDTTz+xbt26fJNiKaNMkXJs7ty5JmBu2LDB/O2330wfHx9z7NixnuWdOnUymzRp4vm8ZcsWEzD//Oc/e7XzyCOPmID51VdfecpiY2NNwPz22289ZUePHjUdDof58MMPXzK2FStWmHa73bTb7WaHDh3Mxx57zPzyyy/NrKwsr3oPPfSQCZg//PDDZe1zfq9L6d27txkbG5unfPXq1SZg1q1b10xLS/NalpGRYbpcLq+yPXv2mA6Hw5w6dapXGWDOnTvXUzZ06FAT8KpnmqZ59dVXm61bt/YqA8xJkyZ5Pk+aNMkEzHvvvderXt++fc3KlSt7Pm/atMkEzAcffNCr3rBhw/K0mbudTp065TkG5/vjd2vWrFlmSEiI59gMGDDA7NKli2maOd+T3r17e9ZLS0szGzRoYAJmbGysOWzYMPOdd94xk5KS8mwjdx/zezVo0OCi8e3du9e02+3mM88841W+detW08fHx6u8U6dOJmC+//77nrLMzEwzKirK7Nevn6fsn//8p2mz2cz//ve/Xm3OmTPHBMw1a9Z4ygDTZrOZP//8s1fdMWPGmIZheH2fT5w4YUZERJiAuWfPHtM0TdPlcpk1a9Y0Bw0a5LX+jBkzTMMwzN9///2i+z906FCv7/KSJUtMwPzb3/7mVa9///6mYRjmr7/+apqmab788ssmYB47duyCbd92221e/25I+aRLbFJh1K1bl3vuuYe33nqLI0eO5Ftn2bJlAIwbN86r/OGHHwbg888/9ypv3Lgx119/vedz1apVadCgAb///vsl4+nWrRsJCQnceuut/Pjjjzz//PP06NGDGjVqsHTpUk+9lJQUAEJCQgqwl//z+uuvEx8f7/W6UkOHDiUgIMCrzOFweMYhuVwuTpw44bkssXnz5gK1+5e//MXr8/XXX1+gY3ihdU+cOOE5brmXde677z6ven8cBP9HpmkW+OxRroEDB5Kens5nn33GmTNn+Oyzzy54JiEgIIB169bx6KOPAjl3w40YMYLq1aszZsyYPJeqAD7++OM8fTl37tyLxvTJJ5/gdrsZOHAgx48f97yioqKIi4tj9erVXvWDg4O5++67PZ/9/Pxo166dVz8sXryYRo0a0bBhQ682//SnPwHkabNTp040btzYq2z58uV06NCBli1besoiIiK46667vOrZbDbuuusuli5dypkzZzzl8+fP59prr81zA8KlLFu2DLvdztixY73KH374YUzT5IsvvgD+N/D93//+N263O9+2wsPDOXjwIBs2bLisGKRsUYIkFcrTTz9Ndnb2Bcci7du3D5vNRv369b3Ko6KiCA8PZ9++fV7ltWrVytNGpUqVOHXqFJCTMCQmJnq9/jgIt23btnzyySecOnWK9evXM378eM6cOUP//v3Zvn07gGdMyh9/JAqiXbt2dO3a1et1pfL7UXK73bz88svExcXhcDioUqUKVatW5aeffiI5OfmSbfr7+1O1alWvsj8ew0s5vw8qVaoE4Fk/t0/Pj/38Pr4SVatWpWvXrixYsIBPPvkEl8tF//79L1g/LCyM559/nr1797J3717eeecdGjRowKxZs5g2bVqe+jfccEOevuzQocNFY9q9ezemaRIXF0fVqlW9Xjt27MhzI0DNmjXzXO47vx92797Nzz//nKe9q666Csh7c0F+35d9+/ble+zzKxsyZAjp6emeS9u7du1i06ZNl325OXe70dHRef6j0ahRI89ygEGDBtGxY0f+/Oc/ExkZyR133MGHH37olSw9/vjjBAcH065dO+Li4rj//vu9LsFJ+aAxSFKh1K1bl7vvvpu33nrronMCFWRcCOQMkM6PeW7Q54EDB/L8SKxevZrOnTt7lfn5+dG2bVvatm3LVVddxfDhw1m8eDGTJk2iYcOGQM6cSX/8X7cVzj97BPD3v/+dCRMmcO+99zJt2jQiIiKw2Ww8+OCDF/wf+B9d6BgW1KX6oKTceeedjBw5ksTERG666aYC34IfGxvLvffeS9++falbty7z58/nb3/72xXH43a7MQyDL774It9jdP74soIcR7fbTbNmzZgxY0a+dWNiYrw+5/d9uRyNGzemdevWfPDBBwwZMoQPPvgAPz8/Bg4ceEXtXkxAQADffvstq1ev5vPPP2f58uUsWrSIP/3pT6xYsQK73U6jRo3YtWsXn332GcuXL+fjjz/mjTfeYOLEiUyZMqXYYpOSpQRJKpynn36aDz74wGsQb67Y2Fjcbje7d+/2/M8SICkpidOnT3sG2BZUVFRUnktbLVq0uOg6ubfm514GvOmmm7Db7XzwwQeF+p/z5ShoYvhHH330EV26dOGdd97xKj99+nSpuDU9t0/37NlDXFycp/zXX38t0u307duX//u//+P7779n0aJFl71+pUqVqFevHtu2bSuSeOrVq4dpmtSpU8dzhqco2vzxxx+58cYbC/VdgZz+yO/YX6g/hgwZwrhx4zhy5AgLFiygd+/enrOEl7vdlStXcubMGa+zSDt37vQsz2Wz2bjxxhu58cYbmTFjBn//+9956qmnWL16tedMbFBQEIMGDWLQoEFkZWVx++2388wzzzB+/HhNZVBO6BKbVDj16tXj7rvv5s0338wzE3GvXr0A8swmnfs/5gvd+XQh/v7+eS6N5P7jvnr16nzPcuSOg8q9jTkmJoaRI0eyYsWKfCe6dLvdvPTSSxw8ePCyYstPUFBQgS6L/ZHdbs+zH4sXL8535nIr9OjRA4A33njDq/xCk4Ze7m3+uYKDg5k9ezaTJ0/mlltuuWC9H3/8Mc/dhZBziWf79u2Fun09P7fffjt2u50pU6bk6R/TNDlx4sRltzlw4EAOHTrE22+/nWdZeno6qampl2yjR48eJCQksGXLFk/ZyZMnmT9/fr71Bw8ejGEYPPDAA/z+++9e46QuR69evXC5XMyaNcur/OWXX8YwDM+devndMZh75jZ3fNj5x87Pz4/GjRtjmma+0xJI2aQzSFIhPfXUU/zzn/9k165dXrfjt2jRgqFDh/LWW29x+vRpOnXqxPr163nvvffo06cPXbp0KbIYxowZQ1paGn379qVhw4ZkZWWxdu1aFi1a5HmESK6XXnqJ3377jbFjx/LJJ59w8803U6lSJfbv38/ixYvZuXMnd9xxxxXH1Lp1axYtWsS4ceNo27YtwcHBF/2xB7j55puZOnUqw4cP59prr2Xr1q3Mnz+funXrXnE8RaF169b069ePmTNncuLECc9t/r/88guQ96zZ5d7m/0dDhw69ZJ34+HgmTZrErbfeyjXXXENwcDC///477777LpmZmfk+b+6jjz7Kdybtbt26ERkZme926tWrx9/+9jfGjx/P3r176dOnDyEhIezZs4dPP/2UUaNG8cgjj1zW/t1zzz18+OGH/OUvf2H16tV07NgRl8vFzp07+fDDD/nyyy8vOTnpY489xgcffEC3bt0YM2aM5zb/WrVqcfLkyTz9UbVqVXr27MnixYsJDw+/7P+k5Lrlllvo0qULTz31FHv37qVFixasWLGCf//73zz44IPUq1cPyJmB/ttvv6V3797ExsZy9OhR3njjDWrWrOmZLb179+5ERUXRsWNHIiMj2bFjB7NmzaJ3796XfTOFlGLW3DwnUjL+eCv2+XJvMT//dl2n02lOmTLFrFOnjunr62vGxMSY48ePNzMyMrzqnX/7dq5OnToV6DbxL774wrz33nvNhg0bmsHBwaafn59Zv359c8yYMfne8p2dnW3+4x//MK+//nozLCzM9PX1NWNjY83hw4d73TJ9sX2+lLNnz5p33nmnGR4e7rkN3TT/d5v/4sWL86yTkZFhPvzww2b16tXNgIAAs2PHjmZCQkKe43Ch2/yDgoLytJl7e/sfcYHb/M+/HTt3/3NvFzdN00xNTTXvv/9+MyIiwgwODjb79Olj7tq1ywTMZ599Ns92Lvc2/4s5/3vy+++/mxMnTjSvueYas1q1aqaPj49ZtWpVs3fv3l7TSPxxHy/0Wr169SXj/Pjjj83rrrvODAoKMoOCgsyGDRua999/v7lr1y5PnfOnu8h1/q3ypmmaWVlZ5nPPPWc2adLEdDgcZqVKlczWrVubU6ZMMZOTkz31APP+++/PN6YffvjBvP76602Hw2HWrFnTnD59uvnqq6+agJmYmJin/ocffmgC5qhRoy65vxeL/cyZM+ZDDz1kRkdHm76+vmZcXJz5wgsvmG6321Nn1apV5m233WZGR0ebfn5+ZnR0tDl48GDzl19+8dR58803zRtuuMGsXLmy6XA4zHr16pmPPvqo1/5L2WeYZgmPZBQRKQW2bNnC1VdfzQcffJDnFnMpeQ8++CBvvvkmZ8+ezTNg/N///jd9+vTh22+/9ZpWQ6Q4aQySiJR76enpecpmzpyJzWbjhhtusCCiiu38/jhx4gT//Oc/ue666/K9m+7tt9+mbt26eR4ILFKcNAZJRMq9559/nk2bNtGlSxd8fHz44osv+OKLLxg1alSeW9Ol+HXo0IHOnTvTqFEjkpKSeOedd0hJSWHChAle9RYuXMhPP/3E559/ziuvvFLoO+dECkOX2ESk3IuPj2fKlCls376ds2fPUqtWLe655x6eeuopfHz0/8SS9uSTT/LRRx9x8OBBDMOgVatWTJo0Kc9kpoZhEBwczKBBg5gzZ476SkqUEiQRERGR82gMkoiIiMh5lCCJiIiInEcXdAvJ7XZz+PBhQkJCNHBQRESkjDBNkzNnzhAdHY3NduHzREqQCunw4cO6+0VERKSMOnDgADVr1rzgciVIhZQ7nfyBAwcIDQ0tsnadTicrVqyge/fu+Pr6Flm7Unjqk9JF/VG6qD9KF/XHpaWkpBATE3PJx8IoQSqk3MtqoaGhRZ4gBQYGEhoaqi93KaE+KV3UH6WL+qN0UX8U3KWGx2iQtoiIiMh5lCCJiIiInEcJkoiIiMh5NAZJRERKBZfLhdPptDqMMs3pdOLj40NGRgYul8vqcCzh6+ub70OPL5cSJBERsZRpmiQmJnL69GmrQynzTNMkKiqKAwcOVOg5+sLDw4mKirqiY6AESURELJWbHFWrVo3AwMAK/cN+pdxuN2fPniU4OPiikyCWV6ZpkpaWxtGjRwGoXr16odtSgiQiIpZxuVye5Khy5cpWh1Pmud1usrKy8Pf3r5AJEkBAQAAAR48epVq1aoW+3FYxj56IiJQKuWOOAgMDLY5EypPc79OVjGlTgiQiIpbTZTUpSkXxfVKCJCIiInIeJUgiIiKlQO3atZk5c2aB63/99dcYhlHsd//NmzeP8PDwYt1GaaQESURE5DIYhnHR1+TJkwvV7oYNGxg1alSB61977bUcOXKEsLCwQm1PLk53sZU27mxC0g9A5hnwjbA6GhEROc+RI0c87xctWsTEiRPZtWuXpyw4ONjz3jRNXC4XPj6X/rmtWrXqZcXh5+dHVFTUZa0jBaczSKWM7d0e/GnnU6Tv/tbqUEREJB9RUVGeV1hYGIZheD7v3LmTkJAQvvjiC1q3bo3D4eC7777jt99+47bbbiMyMpLg4GDatm3LypUrvdo9/xKbYRj84x//oG/fvgQGBhIXF8fSpUs9y8+/xDZv3jwiIiJYtWoVTZo0ITg4mJ49e3oldNnZ2YwdO5bw8HAqV67M448/ztChQ+nTp89lHYPZs2dTr149/Pz8aNCgAf/85z89y0zTZPLkydSqVQuHw0F0dDRjx471LH/jjTeIi4vD39+fyMhI+vfvf1nbLilKkEqZr07mzAOStGudxZGIiFjDNE3SsrJL/GWaZpHtwxNPPMGzzz7Ljh07aN68OWfPnqVXr16sWrWKH374gZ49e3LLLbewf//+i7YzZcoUBg4cyE8//USvXr246667OHny5AXrp6WlMWvWLN577z2+/fZb9u/fzyOPPOJZ/txzzzF//nzmzp3LmjVrSElJYcmSJZe1b59++ikPPPAADz/8MNu2beP//u//GD58OKtXrwbg448/5uWXX+bNN99k9+7dLFmyhGbNmgGwceNGxo4dy9SpU9m1axfLly/nhhtuuKztlxRdYitlzkY0gaSvMJJ+sjoUERFLpDtdNJ74ZYlvd/vUHgT6Fc3P4tSpU+nWrZvnc0REBC1atPB8njZtGp9++ilLly5l9OjRF2xn2LBhDB48GIC///3vvPrqq6xfv56ePXvmW9/pdDJjxgxatGiBzWZj9OjRTJ061bP8tddeY/z48fTt2xeAWbNmsWzZssvatxdffJFhw4Zx3333ATBu3Di+//57XnzxRbp06cL+/fuJioqia9eu+Pr6UqtWLdq1awfA/v37CQoK4uabbyYkJITY2Fiuvvrqy9p+SdEZpFLGt2bOF6Vyyg6LIxERkcJq06aN1+ezZ8/yyCOP0KhRI8LDwwkODmbHjh2XPIPUvHlzz/ugoCBCQ0M9j9HIT2BgIHXq1PF8rl69uqd+cnIySUlJnmQFwG6307p168vatx07dtCxY0evso4dO7JjR87v1oABA0hPT6du3bqMHDmSTz/9lOzsbAC6detGbGwsdevW5Z577mH+/PmkpaVd1vZLis4glTKRca1xbzSo5DoBZ5IgJNLqkERESlSAr53tU3tYst2iEhQU5PX5kUceIT4+nhdffJH69esTEBBA//79ycrKumg7vr6+Xp8Nw8Dtdl9W/aK8dFgQMTEx7Nq1i5UrVxIfH899993HCy+8wDfffENISAibN2/m66+/ZsWKFUycOJHJkyezYcOGUjeVgM4glTINYqL4zcx5uN6p3zZYHI2ISMkzDINAP58SfxXnbN5r1qxh2LBh9O3bl2bNmhEVFcXevXuLbXv5CQsLIzIykg0b/vfb4nK52Lx582W106hRI9asWeNVtmbNGho3buz5HBAQwC233MKrr77K119/TUJCAlu3bgXAx8eHrl278vzzz/PTTz+xd+9evvrqqyvYs+KhM0ilTICfnd9sdYjjMCd/3UClljdbHZKIiFyhuLg4PvnkE2655RYMw2DChAkXPRNUXMaMGcP06dOpX78+DRs25LXXXuPUqVOXlRw++uijDBw4kKuvvpquXbvyn//8h08++cRzV968efNwuVy0b9+ewMBAPvjgAwICAoiNjeWzzz7j999/54YbbqBSpUosW7YMt9tNgwYNimuXC00JUimU6FcbstZgHt5idSgiIlIEZsyYwb333su1115LlSpVePzxx0lJSSnxOB5//HESExMZMmQIdrudUaNG0aNHj8t64n2fPn145ZVXePHFF3nggQeoU6cOc+fOpXPnzgCEh4fz7LPPMm7cOFwuF82aNeM///kPlStXJjw8nE8++YTJkyeTkZFBXFwc//rXv2jSpEkx7fEVMEuBWbNmmbGxsabD4TDbtWtnrlu37qL1P/zwQ7NBgwamw+EwmzZtan7++eeeZVlZWeZjjz1mNm3a1AwMDDSrV69u3nPPPeahQ4e82oiNjTUBr9f06dMLHHNycrIJmMnJyZe3s5eQlZVlvvbKs6Y5KdQ8PrV+kbYthZOVlWUuWbLEzMrKsjoUMdUfpc2V9kd6erq5fft2Mz09vYgjq5hcLpd56tQp0+VyFbj+VVddZT799NPFHFnJutj3qqC/35aPQVq0aBHjxo1j0qRJbN68mRYtWtCjR48LjtJfu3YtgwcPZsSIEfzwww/06dOHPn36sG3bNiBnDojNmzczYcIENm/ezCeffMKuXbu49dZb87Q1depUjhw54nmNGTOmWPe1wMJiAajsOgqpJywORkREyot9+/bx9ttv88svv7B161b++te/smfPHu68806rQyt1LE+QZsyYwciRIxk+fDiNGzdmzpw5BAYG8u677+Zb/5VXXqFnz548+uijNGrUiGnTptGqVStmzZoF5AxCi4+PZ+DAgTRo0IBrrrmGWbNmsWnTpjy3U4aEhHjNiHr+XQdWqRYSwB53zvTxp3/XQG0RESkaNpuNefPm0bZtWzp27MjWrVtZuXIljRo1sjq0UsfSMUhZWVls2rSJ8ePHe8psNhtdu3YlISEh33USEhIYN26cV1mPHj0uOhNocnIyhmHkuYXw2WefZdq0adSqVYs777yThx566ILPy8nMzCQzM9PzOffasdPpxOl0Xmw3L4vT6cRhhz1+cdTJTuTYru8JanhjkbUvly+3f4uyn6Xw1B+ly5X2h9PpxDRN3G63JYOWyxvz3C39ucf0fDVq1OC///1vnvLyduzdbjemaeJ0OvOMryrod9XSBOn48eO4XC4iI73n+omMjGTnzp35rpOYmJhv/cTExHzrZ2Rk8PjjjzN48GBCQ0M95WPHjqVVq1ZERESwdu1axo8fz5EjR5gxY0a+7UyfPp0pU6bkKV+xYgWBgYEX3c/COGirAcCZX7677FlOpXjEx8dbHYL8gfqjdClsf/j4+BAVFcXZs2cvOSeQFNyZM2esDsFSWVlZpKen8+2333omqcxV0Ikpy/VdbE6nk4EDB2KaJrNnz/Za9sezUM2bN8fPz4//+7//Y/r06TgcjjxtjR8/3mudlJQUYmJi6N69u1fiVRQxx8fHE9bgeti6kBj3AZr16lVk7cvly+2Tbt265ZmETUqe+qN0udL+yMjI4MCBAwQHB+Pv718MEVYspmly5swZQkJCinVep9IuIyODgIAAbrjhhjzfq4LePWhpglSlShXsdjtJSUle5UlJSURFReW7TlRUVIHq5yZH+/bt46uvvrpkEtO+fXuys7PZu3dvvvMxOByOfBMnX1/fYvlHulqDdrAVqmYnQvZZCKhU5NuQy1NcfS2Fo/4oXQrbHy6XC8MwsNls2GyWD4st83IvleUe04rKZrNhGEa+38uCfk8tPXp+fn60bt2aVatWecrcbjerVq2iQ4cO+a7ToUMHr/qQc2r3j/Vzk6Pdu3ezcuVKKleufMlYtmzZgs1mo1q1aoXcm6LVILYm+9w5sST/vtHiaERERCoWyy+xjRs3jqFDh9KmTRvatWvHzJkzSU1NZfjw4QAMGTKEGjVqMH36dAAeeOABOnXqxEsvvUTv3r1ZuHAhGzdu5K233gJykqP+/fuzefNmPvvsM1wul2d8UkREBH5+fiQkJLBu3Tq6dOlCSEgICQkJPPTQQ9x9991UqlQ6ztSE+Pvyk299Yl1HOb57A2FNul16JRERESkSlidIgwYN4tixY0ycOJHExERatmzJ8uXLPQOx9+/f73Wa8Nprr2XBggU8/fTTPPnkk8TFxbFkyRKaNm0KwKFDh1i6dCkALVu29NrW6tWr6dy5Mw6Hg4ULFzJ58mQyMzOpU6cODz30UJ6746yWUqkJHF+L+9APVociIiJSoVieIAGMHj2a0aNH57vs66+/zlM2YMAABgwYkG/92rVrX/LJxa1ateL777+/7DhLmr1GSzgOoae3Wx2KiIgUsc6dO9OyZUtmzpwJ5Px+Pfjggzz44IMXXMcwDD799FP69OlzRdsuqnYuZvLkySxZsoQtW7YU2zaKU8UdwVUGVI1rB0Ck8yBkJFscjYiIANxyyy307Nkz32X//e9/MQyDn3766bLb3bBhA6NGjbrS8LxMnjw5z9UUgCNHjnDTTTcV6bbKGyVIpViDunU4aFYB4MzezRZHIyIiACNGjCA+Pp6DBw/mWTZ37lzatGlD8+bNL7vdqlWrFsu8evmJiorK985s+R8lSKVYWKAvv/vUB+DYL+stjkZERABuvvlmqlatyrx587zKz549y+LFixkxYgQnTpxg8ODB1KhRg8DAQJo1a8a//vWvi7Zbu3Ztz+U2gN27d3vm8WncuHG+k3E+/vjjXHXVVQQGBlK3bl0mTpzomSl63rx5TJkyhR9//BHDMDAMwxOzYRheT6DYunUrf/rTnwgICKBy5cqMGjWKs2fPepYPGzaMPn368OKLL1K9enUqV67M/ffff1kzqLvdbqZOnUrNmjVxOByeMce5srKyGD16NNWrV8ff35/Y2FjPDVqmaTJ58mRq1aqFw+EgOjqasWPHFnjbhVEqxiDJhSWHN4YT35N9UAO1RaSCME1wFmy24yLlGwgFmFzRx8eHIUOGMG/ePJ566inPhIyLFy/G5XIxePBgzp49S+vWrXn88ccJDQ3l888/55577qFevXq0a9fukttwu93cfvvtREZGsm7dOpKTk/MdmxQSEsK8efOIjo5m69atjBw5El9fXyZMmMCgQYPYtm0by5cvZ+XKlUDO80rPl5qaSo8ePejQoQMbNmzg6NGj/PnPf2b06NFeSeDq1aupXr06q1ev5tdff2XQoEG0bNmSkSNHXnJ/IOdZqi+99BJvvvkmV199Ne+++y633norP//8M3Fxcbz66qssXbqUDz/8kFq1anHgwAEOHDgAwMcff8zLL7/MwoULadKkCYmJifz4448F2m5hKUEq5YzolnACQk//bHUoIiIlw5kGf48u+e0+eRj8CvbQ8nvvvZcXXniBb775hs6dOwM5l9f69etHWFgYYWFhPPLII576Y8aM4csvv+TDDz8sUIK0cuVKdu7cyZdffkl0dM6x+Pvf/55n3NDTTz/teV+7dm0efvhhFixYwIQJEwgICCA4ONjzOJcLWbBgARkZGbz//vueh7bPmjWLW265heeee85zV3mlSpWYNWsWdrudhg0b0rt3b1atWlXgBOnFF1/k8ccf54477gDgueeeY/Xq1cycOZPXX3+d/fv3ExcXx3XXXYdhGMTGxnrW3b9/P1FRUXTt2hVfX19q1apVoON4JXSJrZSrcm6gdrWsA5B59hK1RUSkJDRs2JBrr72Wd999F4Bff/2V//73v4wYMQLImSF82rRpNGvWjIiICIKDg/nyyy/Zv39/gdrfsWMHMTExnuQIyHcC5UWLFtGxY0eioqIIDg5mwoQJ+Y6NutS2WrRo4UmOADp27Ijb7WbXrl2esiZNmng9+LV69eocPXq0QNtISUnh8OHDdOzY0au8Y8eO7NixA8i5jLdlyxYaNGjA2LFjWbFihafegAEDSE9Pp27duowcOZJPP/00zzPWiprOIJVyDerV44gZQXXjJGf3/0Bw3PVWhyQiUrx8A3PO5lix3cswYsQIxowZw+uvv87cuXOpV68enTp1AuCFF17glVdeYebMmTRr1oygoCAefPDBIn0gb0JCAnfddRdTpkyhR48ehIWF8a9//YuXXnqpyLbxR+c/osMwDM+jTYpCq1at2LNnD1988QUrV65k4MCBdO3alY8++oiYmBh27drFypUriY+P57777vOcwSuuRw7pDFIpVynIj9/s9QA4umudxdGIiJQAw8i51FXSr8t8uOvAgQOx2WwsWLCA999/n3vvvdczHmnNmjXcdttt3H333bRo0YK6devyyy+/FLjtRo0aceDAAY4cOeIpO3/+vrVr1xIbG8tTTz1FmzZtiIuLY9++fV51/Pz8cLlcl9zWjz/+SGpqqqdszZo12Gy2fJ9NWhihoaFER0ezZs0ar/I1a9bQuHFjr3qDBg3i7bffZtGiRXz88cecPHkSgICAAG655RZeffVVvv76axISEti6dWuRxJcfnUEqA06FNYZTGzRQW0SkFAkODmbQoEGMHz+elJQUhg0b5lkWFxfHRx99xNq1a6lUqRIzZswgKSnJKxm4mK5du3LVVVcxdOhQXnjhBVJSUnjqqae86sTFxbF//34WLlxI27Zt+fzzz73uTIOccUl79uxhy5Yt1KxZk5CQkDy39991111MmjSJoUOHMnnyZI4dO8aYMWO45557POOPisKjjz7KpEmTqFevHi1btmTu3Lls2bKF+fPnAzBjxgyqV6/O1Vdfjc1mY/HixURFRREeHs68efNwuVy0b9+ewMBAPvjgAwICArzGKRU1nUEqA4zoFgAEn9RAbRGR0mTEiBGcOnWKHj16eI0Xevrpp2nVqhU9evSgc+fOREVFXdas1TabjU8//ZT09HTatWvHn//8Z5555hmvOrfeeisPPfQQo0ePpmXLlqxdu9Zr0DZAv3796NmzJ126dKFq1ar5TjUQGBjIl19+ycmTJ2nbti39+/fnxhtvZNasWZd3MC5h7NixjBs3jocffphmzZqxfPlyli5dSlxcHJBzR97zzz9PmzZtaNu2LXv37mXZsmXYbDbCw8N5++236dixI82bN2flypX85z//KdDD6AvLMC/1XA7JV0pKCmFhYSQnJxMaGlpk7TqdTpYtW0avXr0811XX/rCVa/99HS5s2J88BH4lM5GY5MivT8Q66o/S5Ur7IyMjgz179lCnTh38/f2LIcKKxe12k5KSQmhoqNdzTCuai32vCvr7XXGPXhlyVf04jplh2HGTeqB4530QERERJUhlQpUQf345N1D72K7S/5BdERGRsk4JUhlxMjRnYF/mAQ3UFhERKW5KkMoIo3pLQAO1RURESoISpDKiUv22AERm7gFnhsXRiIgULd0vJEWpKL5PSpDKiLi4BpwwQ/DBRfrBn6wOR0SkSOTe+ZaWZsHDaaXcyv0+XcmdrpoosoyoFhpAgq0eHcwtJP2yntp1ivchfSIiJcFutxMeHu55pldgYKBnNmq5fG63m6ysLDIyMirkbf6maZKWlsbRo0cJDw/3enbc5VKCVIacDG0EyVvI3L/J6lBERIpM7pPmC/rgU7kw0zRJT08nICCgQiea4eHhnu9VYSlBKkPcUS0h+V8EntBAbREpPwzDoHr16lSrVg2n02l1OGWa0+nk22+/5YYbbqiwE6n6+vpe0ZmjXEqQypDwem1gF0Rm/AbZWeDjZ3VIIiJFxm63F8kPW0Vmt9vJzs7G39+/wiZIRaXiXaAsw+KuasJpMwg/ssk4rLNIIiIixUUJUhkSGebPLltdAJI0o7aIiEixUYJUhhiGwfGQRgBk7N9scTQiIiLllxKkMsaMagFAwPGtFkciIiJSfilBKmNC6+bMqB2V/iu4dLeHiIhIcVCCVMbUa9CUFDMAP5xkHtlhdTgiIiLlkhKkMiY6PJBdRu5A7XUWRyMiIlI+KUEqYwzD4FhwQwDS9mlGbRERkeKgBKkMcp0bqO2vgdoiIiLFQglSGRRWpw0AUWm7we2yOBoREZHyRwlSGVSnYQvOmv74k0lW0k6rwxERESl3lCCVQTUjgvjFqA1A0k4N1BYRESlqSpDKIMMwSDo3UDtVA7VFRESKnBKkMiq7WnMAHMc0UFtERKSoKUEqo0Lq5gzUjkz7Bdxui6MREREpX5QglVG1r2pJuulHoJmO89huq8MREREpV5QglVGxVUPZdW6gdqJm1BYRESlSSpDKKMMwSApsAEDano0WRyMiIlK+KEEqw7LODdT21UBtERGRIqUEqQwLrtMagMjUXWCaFkcjIiJSfihBKsNqNWhFpulLkJlK9vHfrQ5HRESk3FCCVIbVqRbOL9QCIGnX9xZHIyIiUn4oQSrDbDaDI0E5M2qf2asZtUVERIqKEqQyLrNKMwB8k36yOBIREZHyQwlSGRd0bqB2tbM7NVBbRESkiChBKuNqNWxFlmknxDyD69Q+q8MREREpF5QglXF1Iiuz2zNQWzNqi4iIFAUlSGWc3WZw5NyM2im/a0ZtERGRoqAEqRzIODdQ20cDtUVERIqEEqRyICC2FQBVz+7QQG0REZEioASpHIhp2JZs00aYOxl38iGrwxERESnzlCCVA3WrV+ZXagIaqC0iIlIUlCCVAz52G4cCcgdqb7A4GhERkbKvVCRIr7/+OrVr18bf35/27duzfv36i9ZfvHgxDRs2xN/fn2bNmrFs2TLPMqfTyeOPP06zZs0ICgoiOjqaIUOGcPjwYa82Tp48yV133UVoaCjh4eGMGDGCs2fPFsv+lYT0Kk0BsCX+aHEkIiIiZZ/lCdKiRYsYN24ckyZNYvPmzbRo0YIePXpw9OjRfOuvXbuWwYMHM2LECH744Qf69OlDnz592LZtGwBpaWls3ryZCRMmsHnzZj755BN27drFrbfe6tXOXXfdxc8//0x8fDyfffYZ3377LaNGjSr2/S0uAbVyZtSucmaHxZGIiIiUfZYnSDNmzGDkyJEMHz6cxo0bM2fOHAIDA3n33Xfzrf/KK6/Qs2dPHn30URo1asS0adNo1aoVs2bNAiAsLIz4+HgGDhxIgwYNuOaaa5g1axabNm1i//79AOzYsYPly5fzj3/8g/bt23Pdddfx2muvsXDhwjxnmsqK6IbtcJkGldyncCcfsTocERGRMs3Hyo1nZWWxadMmxo8f7ymz2Wx07dqVhISEfNdJSEhg3LhxXmU9evRgyZIlF9xOcnIyhmEQHh7uaSM8PJw2bdp46nTt2hWbzca6devo27dvnjYyMzPJzMz0fE5JSQFyLuk5nc5L7mtB5bZ1uW3GVgvnN2pwFQdJ3LmWqq1uvfRKUiCF7RMpHuqP0kX9UbqoPy6toMfG0gTp+PHjuFwuIiMjvcojIyPZuXNnvuskJibmWz8xMTHf+hkZGTz++OMMHjyY0NBQTxvVqlXzqufj40NERMQF25k+fTpTpkzJU75ixQoCAwPz38ErEB8ff9nrOGy1uco8yM41n7Eh0dKuLZcK0ydSfNQfpYv6o3RRf1xYWlpageqV619Rp9PJwIEDMU2T2bNnX1Fb48eP9zpzlZKSQkxMDN27d/ckXkXB6XQSHx9Pt27d8PX1vax1lydugCPfUdN2lDq9ehVZTBXdlfSJFD31R+mi/ihd1B+XlnsF6FIsTZCqVKmC3W4nKSnJqzwpKYmoqKh814mKiipQ/dzkaN++fXz11VdeSUxUVFSeQeDZ2dmcPHnygtt1OBw4HI485b6+vsXyJSxMuwGxreEIVD6zU38xikFx9bUUjvqjdFF/lC7qjwsr6HGxdJC2n58frVu3ZtWqVZ4yt9vNqlWr6NChQ77rdOjQwas+5JxK/GP93ORo9+7drFy5ksqVK+dp4/Tp02zatMlT9tVXX+F2u2nfvn1R7Jolohu0w20aVHYdxzyb/12AIiIicmmW38U2btw43n77bd577z127NjBX//6V1JTUxk+fDgAQ4YM8RrE/cADD7B8+XJeeukldu7cyeTJk9m4cSOjR48GcpKj/v37s3HjRubPn4/L5SIxMZHExESysrIAaNSoET179mTkyJGsX7+eNWvWMHr0aO644w6io6NL/iAUkfox1dlDdQCO/3LxuaRERETkwiwfgzRo0CCOHTvGxIkTSUxMpGXLlixfvtwzEHv//v3YbP/L46699loWLFjA008/zZNPPklcXBxLliyhadOciRIPHTrE0qVLAWjZsqXXtlavXk3nzp0BmD9/PqNHj+bGG2/EZrPRr18/Xn311eLf4WLk52PjgCOOelmHOfnbBqq2utnqkERERMokyxMkgNGjR3vOAJ3v66+/zlM2YMAABgwYkG/92rVrYxbgifYREREsWLDgsuIsC1IrN4Uj38CRLVaHIiIiUmZZfolNipZfzVYARCRrRm0REZHCUoJUzkQ1bAdAVVcSZuoJi6MREREpm5QglTNxtWqw18yZquDErxssjkZERKRsUoJUzvj72tnnFwfAid26k01ERKQwlCCVQ2cjmuS80UBtERGRQlGCVA75xVwNQLgGaouIiBSKEqRyqFqDnNnAI7MPY6afsjgaERGRskcJUjnUoHYtDphVATj520aLoxERESl7lCCVQ/6+dvb6nhuo/YvuZBMREblcSpDKqTOVcgZquw9vsTYQERGRMkgJUjnlU/PcQO3TP1sciYiISNmjBKmcqtYgZ0btqOyDkJFicTQiIiJlixKkcuqqunU4bFYG4NTvmyyORkREpGxRglROBfr5sMe3PgDHNKO2iIjIZVGCVI4lh+cM1HYd/MHiSERERMoWJUjlmE/NlgCEnt5ubSAiIiJljBKkcqxKXM6M2lHOA5CVanE0IiIiZYcSpHLsqvr1STLDsePm9J7NVocjIiJSZihBKseCHT787pMzo/axX9ZZHI2IiEjZoQSpnDsd3hiA7INbrA1ERESkDFGCVM7ZolsCEHJqm7WBiIiIlCFKkMq5KnHnZtTO2gfOdIujERERKRuUIJVzcXFXccwMxQc3KXu3WB2OiIhImaAEqZwLDfDjd5+cGbWPaqC2iIhIgShBqgBOhuUM1M46oFv9RURECkIJUgVgi24BQPCpny2OREREpGxQglQBVK6fM6N29cy9kJ1pbTAiIiJlgBKkCqB+XENOmcH4ks2Z/T9ZHY6IiEippwSpAggPcrDbXg+ApF3fWxyNiIhI6acEqYI4FdYIgKwDP1gciYiISOmnBKmCMKu3BCDohGbUFhERuRQlSBVERL22AFTP/A1cToujERERKd2UIFUQ9Rs0I8UMxI9szh7canU4IiIipZoSpAoiItjBbntdABJ3akZtERGRi1GCVIGcCDk3o/Z+zagtIiJyMUqQKhB3VHMAAjRQW0RE5KKUIFUg4fXbAVA941dwZVscjYiISOmlBKkCqdegOWfMAPzJIu3IDqvDERERKbWUIFUgVUMD2G2rA0DiDs2oLSIiciFKkCqYEyE5M2qn799kcSQiIiKllxKkCsaVO1D7uAZqi4iIXIgSpAomvO65GbXTd4PbZXE0IiIipZMSpAqmbsOWpJkOAsggPXGn1eGIiIiUSkqQKphq4UHsttUGNKO2iIjIhShBqoCOBecM1E7bq4HaIiIi+VGCVAFlR+YM1PbXQG0REZF8KUGqgELrtgEgKm0XuN0WRyMiIlL6KEGqgOo0akWG6UsQ6WQe+9XqcEREREodJUgVUFR4MLuN2gAc3pFgbTAiIiKlkBKkCsgwDI4GNwQgbe9mi6MREREpfZQgVVDOajkDtR3HfrI4EhERkdJHCVIFFVKnNQCRabvANC2ORkREpHRRglRBxTZqTabpQ4iZSuax360OR0REpFRRglRB1agcxm9GLQASd31vcTQiIiKli+UJ0uuvv07t2rXx9/enffv2rF+//qL1Fy9eTMOGDfH396dZs2YsW7bMa/knn3xC9+7dqVy5MoZhsGXLljxtdO7cGcMwvF5/+ctfinK3Sj3DMEgMyhmofXaPZtQWERH5I0sTpEWLFjFu3DgmTZrE5s2badGiBT169ODo0aP51l+7di2DBw9mxIgR/PDDD/Tp04c+ffqwbdv/ZoROTU3luuuu47nnnrvotkeOHMmRI0c8r+eff75I960syKrWDADfo1stjkRERKR0sTRBmjFjBiNHjmT48OE0btyYOXPmEBgYyLvvvptv/VdeeYWePXvy6KOP0qhRI6ZNm0arVq2YNWuWp84999zDxIkT6dq160W3HRgYSFRUlOcVGhpapPtWFgTXPjejdupODdQWERH5Ax+rNpyVlcWmTZsYP368p8xms9G1a1cSEvKfvDAhIYFx48Z5lfXo0YMlS5Zc9vbnz5/PBx98QFRUFLfccgsTJkwgMDDwgvUzMzPJzMz0fE5JSQHA6XTidDove/sXkttWUbZ5IdH1W+D8yk4oKaQd/R3fiFrFvs2yqCT7RC5N/VG6qD9KF/XHpRX02FiWIB0/fhyXy0VkZKRXeWRkJDt37sx3ncTExHzrJyYmXta277zzTmJjY4mOjuann37i8ccfZ9euXXzyyScXXGf69OlMmTIlT/mKFSsumlgVVnx8fJG3eT7ThAxq0ph9fL1kHs6oNsW+zbKsJPpECk79UbqoP0oX9ceFpaWlFaieZQmSlUaNGuV536xZM6pXr86NN97Ib7/9Rr169fJdZ/z48V5nr1JSUoiJiaF79+5FennO6XQSHx9Pt27d8PX1LbJ2L+S7Xz6gcdo+YgLSadirV7Fvrywq6T6Ri1N/lC7qj9JF/XFpuVeALsWyBKlKlSrY7XaSkpK8ypOSkoiKisp3naioqMuqX1Dt27cH4Ndff71gguRwOHA4HHnKfX19i+VLWFztni+ranPYtwK/Y1v1l+kSSqpPpGDUH6WL+qN0UX9cWEGPi2WDtP38/GjdujWrVq3ylLndblatWkWHDh3yXadDhw5e9SHnNOKF6hdU7lQA1atXv6J2yqLA2udm1D6rgdoiIiK5LL3ENm7cOIYOHUqbNm1o164dM2fOJDU1leHDhwMwZMgQatSowfTp0wF44IEH6NSpEy+99BK9e/dm4cKFbNy4kbfeesvT5smTJ9m/fz+HDx8GYNeuXQCeu9V+++03FixYQK9evahcuTI//fQTDz30EDfccAPNmzcv4SNgvZhG7cj+2kY4p3GePoRvpZpWhyQiImI5S2/zHzRoEC+++CITJ06kZcuWbNmyheXLl3sGYu/fv58jR4546l977bUsWLCAt956ixYtWvDRRx+xZMkSmjZt6qmzdOlSrr76anr37g3AHXfcwdVXX82cOXOAnDNXK1eupHv37jRs2JCHH36Yfv368Z///KcE97z0qFWtMnuoAUDiznUWRyMiIlI6WD5Ie/To0YwePTrfZV9//XWesgEDBjBgwIALtjds2DCGDRt2weUxMTF88803lxtmuWWzGRwKbEBc+gFSft8AHfpZHZKIiIjlLH/UiFgvo0rOpUV70k8WRyIiIlI6KEESgmq3AqDq2fznnxIREalolCAJ0Q3b4zYNKrtPkJ185NIriIiIlHNKkIQ61auyh2gAknZpoLaIiIgSJMkZqB3QAIDTv220OBoRERHrKUESANKr5EyVYEv80eJIRERErKcESQDwj82ZUbvKGQ3UFhERUYIkANRo2A6Aqu6juM4etzgaERERaxUqQTpw4AAHDx70fF6/fj0PPvig1yM/pGypU6M6e8ycZ9El7vze4mhERESsVagE6c4772T16tUAJCYm0q1bN9avX89TTz3F1KlTizRAKRl2m8Eh/6sAOP3bBoujERERsVahEqRt27bRrl3OJZkPP/yQpk2bsnbtWubPn8+8efOKMj4pQamVcwZqGxqoLSIiFVyhEiSn04nD4QBg5cqV3HrrrQA0bNjQ6+GyUrY4auXMqF05ZYfFkYiIiFirUAlSkyZNmDNnDv/973+Jj4+nZ8+eABw+fJjKlSsXaYBScqIbtQcg0pWIO/WUxdGIiIhYp1AJ0nPPPcebb75J586dGTx4MC1atABg6dKlnktvUvbUrVmD/WY1ABJ3aaC2iIhUXD6FWalz584cP36clJQUKlWq5CkfNWoUgYGBRRaclCwfu40D/ldRK/Mop37bSHSrm6wOSURExBKFOoOUnp5OZmamJznat28fM2fOZNeuXVSrVq1IA5SSlRrRBADjyBZrAxEREbFQoRKk2267jffffx+A06dP0759e1566SX69OnD7NmzizRAKVl+MTkDtSsla6C2iIhUXIVKkDZv3sz1118PwEcffURkZCT79u3j/fff59VXXy3SAKVkRTW8BoDqrkOY6aetDUZERMQihUqQ0tLSCAkJAWDFihXcfvvt2Gw2rrnmGvbt21ekAUrJqlurFgfNKgAk7d5ocTQiIiLWKFSCVL9+fZYsWcKBAwf48ssv6d69OwBHjx4lNDS0SAOUkuXnY+OAIw6Ak7vXWxyNiIiINQqVIE2cOJFHHnmE2rVr065dOzp06ADknE26+uqrizRAKXlnInJm1HYf1ozaIiJSMRXqNv/+/ftz3XXXceTIEc8cSAA33ngjffv2LbLgxBq+Na+GRKiU/LPVoYiIiFiiUAkSQFRUFFFRURw8eBCAmjVrapLIciLqqvawEapnH8TMPIPhCLE6JBERkRJVqEtsbrebqVOnEhYWRmxsLLGxsYSHhzNt2jTcbndRxyglrG7dOiSaEdgwOaqB2iIiUgEV6gzSU089xTvvvMOzzz5Lx44dAfjuu++YPHkyGRkZPPPMM0UapJQsh4+dvX5xRDnXcWL3eiKbdrE6JBERkRJVqATpvffe4x//+Ae33nqrp6x58+bUqFGD++67TwlSOXCmUhM4ug734S1WhyIiIlLiCnWJ7eTJkzRs2DBPecOGDTl58uQVByXWs9fMuRsx7PR2iyMREREpeYVKkFq0aMGsWbPylM+aNYvmzZtfcVBivWpXtQcg2rkfMyvV4mhERERKVqEusT3//PP07t2blStXeuZASkhI4MCBAyxbtqxIAxRr1K9Xn6NmONWM0xz77QeqNrrO6pBERERKTKHOIHXq1IlffvmFvn37cvr0aU6fPs3tt9/Ozz//zD//+c+ijlEs4O9rZ69vfQCO/bLO4mhERERKVqHnQYqOjs4zGPvHH3/knXfe4a233rriwMR6yZWawLGNuA/9YHUoIiIiJapQZ5CkYrDXaAlA6CnNqC0iIhWLEiS5oCpxuQO192E60y2ORkREpOQoQZILiqvfgBNmCD64OLlni9XhiIiIlJjLGoN0++23X3T56dOnryQWKWUCHD5s861P5ewfOLprPZWv6mB1SCIiIiXishKksLCwSy4fMmTIFQUkpUtyWGM48QPZBzdbHYqIiEiJuawEae7cucUVh5RSthpXw4n5hJzSjNoiIlJxaAySXFTluHYARGf9DtlZFkcjIiJSMpQgyUXVj2vMaTMIP7I5ufdHq8MREREpEUqQ5KKC/H35zaceAEd3aUZtERGpGJQgySWdDmsCQNZBzagtIiIVgxIkuSQjugUAISe3WRyJiIhIyVCCJJdUqf65gdqZv4HLaXE0IiIixU8JklxS/QZNSTEDcODk9P6tVocjIiJS7JQgySWFBDg8A7UTd2qgtoiIlH9KkKRAToU2BiDrgAZqi4hI+acESQrErJ4zUDvopC6xiYhI+acESQokol5bAGpk/Apul8XRiIiIFC8lSFIgdRu15Kzpjz9ZpBz82epwREREipUSJCmQsEAHv9vrAHBkhwZqi4hI+aYESQrs+LmB2pn7N1kciYiISPFSgiQFZkblDNQOPKlLbCIiUr4pQZICq3RuoHZ0+i/gdlscjYiISPFRgiQFVqdhS9JNPwLJ4MzhnVaHIyIiUmwsT5Bef/11ateujb+/P+3bt2f9+vUXrb948WIaNmyIv78/zZo1Y9myZV7LP/nkE7p3707lypUxDIMtW7bkaSMjI4P777+fypUrExwcTL9+/UhKSirK3SqXKoUE8mvuQO2d31scjYiISPGxNEFatGgR48aNY9KkSWzevJkWLVrQo0cPjh49mm/9tWvXMnjwYEaMGMEPP/xAnz596NOnD9u2/e8p86mpqVx33XU899xzF9zuQw89xH/+8x8WL17MN998w+HDh7n99tuLfP/KoxPBDQFI37fZ4khERESKj6UJ0owZMxg5ciTDhw+ncePGzJkzh8DAQN59991867/yyiv07NmTRx99lEaNGjFt2jRatWrFrFmzPHXuueceJk6cSNeuXfNtIzk5mXfeeYcZM2bwpz/9idatWzN37lzWrl3L99/rrMiluKq3BCDwuGbUFhGR8svHqg1nZWWxadMmxo8f7ymz2Wx07dqVhISEfNdJSEhg3LhxXmU9evRgyZIlBd7upk2bcDqdXglUw4YNqVWrFgkJCVxzzTX5rpeZmUlmZqbnc0pKCgBOpxOn01ng7V9KbltF2WZRCom9GnZB9fRfcGZlgmH5VdpiV9r7pKJRf5Qu6o/SRf1xaQU9NpYlSMePH8flchEZGelVHhkZyc6d+Q8ATkxMzLd+YmJigbebmJiIn58f4eHhl9XO9OnTmTJlSp7yFStWEBgYWODtF1R8fHyRt1kUUrOyaWH6Emyk8flH75EdFHnplcqJ0tonFZX6o3RRf5Qu6o8LS0tLK1A9yxKksmb8+PFeZ69SUlKIiYmhe/fuhIaGFtl2nE4n8fHxdOvWDV9f3yJrtyjt3v53Gpu/Uq+aL/U69bI6nGJXFvqkIlF/lC7qj9JF/XFpuVeALsWyBKlKlSrY7fY8d48lJSURFRWV7zpRUVGXVf9CbWRlZXH69Gmvs0iXasfhcOBwOPKU+/r6FsuXsLjaLQpHgxvR+MyvZB7Ygq/vcKvDKTGluU8qIvVH6aL+KF3UHxdW0ONi2QASPz8/WrduzapVqzxlbrebVatW0aFDh3zX6dChg1d9yDmNeKH6+WndujW+vr5e7ezatYv9+/dfVjsVmSuyOQD+xzRQW0REyidLL7GNGzeOoUOH0qZNG9q1a8fMmTNJTU1l+PCcsxJDhgyhRo0aTJ8+HYAHHniATp068dJLL9G7d28WLlzIxo0beeuttzxtnjx5kv3793P48GEgJ/mBnDNHUVFRhIWFMWLECMaNG0dERAShoaGMGTOGDh06XHCAtngLrdsWfoXqabvANMEwrA5JRESkSFmaIA0aNIhjx44xceJEEhMTadmyJcuXL/cMxN6/fz822/9Ocl177bUsWLCAp59+mieffJK4uDiWLFlC06ZNPXWWLl3qSbAA7rjjDgAmTZrE5MmTAXj55Zex2Wz069ePzMxMevTowRtvvFECe1w+xDZqRdaXdkKNs6Qd3UNgZF2rQxIRESlSlg/SHj16NKNHj8532ddff52nbMCAAQwYMOCC7Q0bNoxhw4ZddJv+/v68/vrrvP7665cTqpxTrVIYO22xNDR/5/COBOorQRIRkXKm/E9iI8XiaFADAFL3bbI4EhERkaKnBEkKxVmtBQCOoxqoLSIi5Y8SJCmUkDqtAYjKHagtIiJSjihBkkKJbdKObNNGuJlMxon9VocjIiJSpJQgSaFUqxTG70YMAAe36yG/IiJSvihBkkIxDIOkoIYApO7daHE0IiIiRUsJkhRaVrVmAPhqoLaIiJQzSpCk0ILqtAEgKnWnxZGIiIgULSVIUmixjdrhMg0izFNknDxkdTgiIiJFRgmSFFpUlQj2GDUBOLxDA7VFRKT8UIIkhWYYBomBOTNqn9mjgdoiIlJ+KEGSK5KZO1A76SeLIxERESk6SpDkigTF5syoXVUDtUVEpBxRgiRXpGaj9rhNg6ru42QmJ1odjoiISJFQgiRXpEZkVfYZ1QE4vH2dxdGIiIgUDSVIckUMw+BwQM5A7ZQ9GyyORkREpGgoQZIrllk1Z6C2XQO1RUSknFCCJFcsIHeg9pkdFkciIiJSNJQgyRWr0ag9AJHuozjPHLc4GhERkSunBEmuWEz1KPYRBcChHQkWRyMiInLllCDJFTMMg8P+VwGQ/Ltm1BYRkbJPCZIUifTcgdqJGqgtIiJlnxIkKRL+MVcDUDlFA7VFRKTsU4IkRSK60TUAVHcfITv1lMXRiIiIXBklSFIkatWoyUGzKgCHd2pGbRERKduUIEmRsNkMDp6bUfvUr+stjkZEROTKKEGSIpNWuQkANg3UFhGRMk4JkhQZ/1o5M2pHpGy3OBIREZErowRJikxUw5wZtWu4DuFKT7Y4GhERkcJTgiRFJjYmliNmZQAO79Q4JBERKbuUIEmRsdsMDpybUfvkrxssjkZERKTwlCBJkTob0RQA48iPFkciIiJSeEqQpEg5zs2oXUkDtUVEpAxTgiRFKrJhzoza0c4DuDPOWhyNiIhI4ShBkiJVO7YOR81K2A2Tw79oHJKIiJRNSpCkSPnYbexzxAFwcrfuZBMRkbJJCZIUubMROTNqmxqoLSIiZZQSJClyvjVbARB++meLIxERESkcJUhS5KpddW5Gbed+3JlpFkcjIiJy+ZQgSZGrU7c+x80wfAw3ibs3WR2OiIjIZVOCJEXO18fOPr/6AJz94WOLoxEREbl8SpCkWByMuRWAq36by/Fv37E4GhERkcujBEmKRddBo/kwYCAA4V89QvJPyyyOSEREpOCUIEmxCHL40OWvr/GFvTM+uHF8Moz0vZo4UkREygYlSFJsqob603DUPBJojj+ZZL3fH+ex36wOS0RE5JKUIEmxqhNZiaB7/sV2szZh7tOcfvsWzLPHrA5LRETkopQgSbFrXq8mJ/vM54BZlapZh0iccxtkpVodloiIyAUpQZIScd3VTfmp0zucMoOpfvZnDr59B7iyrQ5LREQkX0qQpMT0/lMn4lu8QobpS81j33Lgg7+CaVodloiISB5KkKREDejbj8W1J+MyDWL2fMihf0+xOiQREZE8lCBJiTIMg8FD7mNBlbEA1NjyMklfv2VxVCIiIt6UIEmJ87Hb6P9/k1gcOAiAyl8/zqktn1kclYiIyP8oQRJLBPjZufG+1/jSpws+uAlYci9n96yzOiwRERFACZJYKCLYQeP/e4+1Rkv8ycT1zwFkHt1tdVgiIiKlI0F6/fXXqV27Nv7+/rRv357169dftP7ixYtp2LAh/v7+NGvWjGXLvJ/zZZomEydOpHr16gQEBNC1a1d27/b+4a1duzaGYXi9nn322SLfN7m4mKphhA9ZwM9mHcLcyaS8fSvuM0etDktERCo4yxOkRYsWMW7cOCZNmsTmzZtp0aIFPXr04OjR/H8k165dy+DBgxkxYgQ//PADffr0oU+fPmzbts1T5/nnn+fVV19lzpw5rFu3jqCgIHr06EFGRoZXW1OnTuXIkSOe15gxY4p1XyV/jevU4Ey/BTkTSToPayJJERGxnOUJ0owZMxg5ciTDhw+ncePGzJkzh8DAQN59991867/yyiv07NmTRx99lEaNGjFt2jRatWrFrFmzgJyzRzNnzuTpp5/mtttuo3nz5rz//vscPnyYJUuWeLUVEhJCVFSU5xUUFFTcuysXcE3zxuy4cS4nzWCiU7dz4M2BmkhSREQs42PlxrOysti0aRPjx4/3lNlsNrp27UpCQkK+6yQkJDBu3Divsh49eniSnz179pCYmEjXrl09y8PCwmjfvj0JCQnccccdnvJnn32WadOmUatWLe68804eeughfHzyPySZmZlkZmZ6PqekpADgdDpxOp2Xt+MXkdtWUbZZVnTpcA1LTrzKzVv+QsyJ79j73khq3P0mGIalcVXkPimN1B+li/qjdFF/XFpBj42lCdLx48dxuVxERkZ6lUdGRrJz585810lMTMy3fmJiomd5btmF6gCMHTuWVq1aERERwdq1axk/fjxHjhxhxowZ+W53+vTpTJmSd1LDFStWEBgYeIk9vXzx8fFF3mZZ4Gv4MDv4fh44O5Pa+z9h7Rsmx+r0tTosoOL2SWml/ihd1B+li/rjwtLS0gpUz9IEyUp/PAvVvHlz/Pz8+L//+z+mT5+Ow+HIU3/8+PFe66SkpBATE0P37t0JDQ0tsricTifx8fF069YNX1/fImu3LHG5b+Jfb5vcfXwm157+lENB7ajWaaRl8ahPShf1R+mi/ihd1B+XlnsF6FIsTZCqVKmC3W4nKSnJqzwpKYmoqKh814mKirpo/dw/k5KSqF69uledli1bXjCW9u3bk52dzd69e2nQoEGe5Q6HI9/EydfXt1i+hMXVblngC/T/v4l8PDOJfqn/IvK7p0iOqEHlVrdZG1cF7pPSSP1Ruqg/Shf1x4UV9LhYOkjbz8+P1q1bs2rVKk+Z2+1m1apVdOjQId91OnTo4FUfck4l5tavU6cOUVFRXnVSUlJYt27dBdsE2LJlCzabjWrVql3JLkkR8fe10/W+V/nS90Z8cBO09M+c+TX/cWkiIiJFzfJLbOPGjWPo0KG0adOGdu3aMXPmTFJTUxk+fDgAQ4YMoUaNGkyfPh2ABx54gE6dOvHSSy/Ru3dvFi5cyMaNG3nrrZzneRmGwYMPPsjf/vY34uLiqFOnDhMmTCA6Opo+ffoAOQO9161bR5cuXQgJCSEhIYGHHnqIu+++m0qVKllyHCSvsCA/mv1lHgmzbqGDuYWs+QPJ/MsqHJFXWR2aiIiUc5YnSIMGDeLYsWNMnDiRxMREWrZsyfLlyz2DrPfv34/N9r8TXddeey0LFizg6aef5sknnyQuLo4lS5bQtGlTT53HHnuM1NRURo0axenTp7nuuutYvnw5/v7+QM7lsoULFzJ58mQyMzOpU6cODz30UJ6748R60ZVDOXPvQn5+pxdNzN859vatRIz9Bnto5KVXFhERKSTDNE3T6iDKopSUFMLCwkhOTi7yQdrLli2jV69eun78Bxu37SRy8S3EGEc5FNiQ6AdWYjhCSmTb6pPSRf1Ruqg/Shf1x6UV9Pfb8okiRQqiTdOG/Np9HifNYGqk7WT/m4PApXk+RESkeChBkjKjS8eOfNfuDdJNP2JPrmHve6NAJ0BFRKQYKEGSMuXW3rfxn6uewWUa1N7/Cfs+ftrqkEREpBxSgiRlTv/BI1kclTOgPnbbLA6tfMPiiEREpLxRgiRljs1m0HfkU3wScicAUd89xbGNS6wNSkREyhUlSFImOXzsdLvvFVb4dcWOm5DPRpK8WxNJiohI0VCCJGVWSIAfLf46jwRbK/zJggUDST+S/0OORURELocSJCnTIiuFUG3Ev/iZuoSZKZx95zaykxOtDktERMo4JUhS5tWrEYVz0EL2m9Womp1I4pxbMTPPWB2WiIiUYUqQpFxo2agB+276JyfMEGqm72LfnAGaSFJERApNCZKUG9dfcw3rO8whzXRQ+1QCv787QhNJiohIoShBknLlpp43s7zR33GZBnUP/Zs9i5+0OiQRESmDlCBJudN30Ag+iX4EgDrb32D/ilkWRyQiImWNEiQpdwzDoM+fn+TT0HsAqLF2AonrP7E4KhERKUuUIEm55Gu30f2+l1nh6I4dN+HL/o+Tu/5rdVgiIlJGKEGScivI35er/zqXBHtr/MnCvvBOUg9rIkkREbk0JUhSrlUNDyb6z4vYRn3CzBRS37kNZ/IRq8MSEZFSTgmSlHux1avCnYvYZ0ZSzZVI4uxbMTNSrA5LRERKMSVIUiE0vao+R26ezwkzlJiMX9g7u78mkhQRkQtSgiQVxjVt2/LDdW+SZjqok7yO394ZrokkRUQkX0qQpELp2q0XK5s+R7Zpo97h//DboiesDklEREohJUhS4dzSfxj/jnkMgHo757B3+asWRyQiIqWNEiSpcAzD4Lbhj/Pv8KEAxHw/kcPfL7Y4KhERKU2UIEmF5GO30f2vM4j374Edk8rL7+PEDk0kKSIiOZQgSYUV4PChzf3zSLC3wUEWvh8O5uyh7VaHJSIipYASJKnQKoUEEjNqEduMOELNM6S924fMU4esDktERCymBEkqvJqRVbDftYh9ZhTVXEkcnXMb7nRNJCkiUpEpQRIBGtWvx7HbFuRMJJm5mz2z+0N2ltVhiYiIRZQgiZzTplVrtnZ6mzTTQb2Udez+xzBNJCkiUkEpQRL5g85/6snXLV4g27QRl/g5v/zrMatDEhERCyhBEjnPTX2H8HlszgzbV/3yFnu/1ESSIiIVjRIkkfMYhsHNwx7jPxHDAai3YRpVds8n68jPFkcmIiIlRQmSSD7sNoNuf3mR+MBe2AyTjme/JOjdThx4/hoOxL+BmX7a6hBFRKQYKUESuQB/Px86jH2fTxu8wFdmG5ymnZi0HcSsGU/mc3HsnnMnydu/Arfb6lBFRKSI+VgdgEhpFuzvy839h/PZ55Gsr12PY2s+oOnRpdQ3DhGX+Dl8+DnHfKqT3HAQsX8agW9ELatDFhGRIqAESaQAbAa0a9IA35bPkpw2lS++/RJjywd0TP+GqtlHqLptJu5tr/BbWHsC2w2levt+4OOwOmwRESkkJUgilyks0I+bet4CPW/hlwNJfPXVB9TY8zFt+Jl6yd9D/PekrHyMI7VuocafRhIc28rqkEVE5DIpQRK5AlfFRHLV0Idxuh7i240bSU54jzanllGdk4TuWwBzF3DQEUdW8zup3XkYtqAIq0MWEZECUIIkUgR87TZuaN8O2rfjaHIqy776mKCfF9LB+T01M3fDhilkbXiGPVW6EHHdvVRt3gNsdqvDFhGRC1CCJFLEqoUF0avvEMw+97B19+/s+/o9rjq0hAbGPhocj4cl8Rz/T1VO1O9P7RtH4qhWz+qQRUTkPEqQRIqJYRg0v6oeza+aSnrmRL5a+xXZG9+n/dlVVHEdo8qu2bBrNr8Ft8an9T3U6jgIwy/Q6rBFRAQlSCIlIsDhw5+6dIcu3Tlw9CTfrVxA1V8/pI3rJ+qd3QTfbOLsN09xoMZNVO88kvD67cEwrA5bRKTCUoIkUsJiqkUQc+do3O772fTjTxxbM5fmxz6jpnGMRoc+gvkfcci3Dmcb30HdG+/FN7Sa1SGLiFQ4SpBELGKzGbS9ugVcPZPktOeIX/1vfH+azzUZa6jh3AM/Tif7x+fZWel6QjoMp0brm8Guv7IiIiVB/9qKlAJhgQ669R4IvQeye99Bdq+aR+z+T2jCbzQ89TUs+5qTyyM4UrsvsV1HEhzdyOqQRUTKNSVIIqVMXGxN4u59GqfrSdZ8/y1p696ndfKXRLhPEvH7O/DWO/we0Ax3y7up2+kubP4hVocsIlLuKEESKaV87TY6duwMHTtz9HQK8SsXErpzEW2cm6ibvhUSHictYRJ7IrtT9YYRVGvcSQO7RUSKiBIkkTKgWngo3fqPwjRH8vPOnRz8Zi6NjvybWCORJklLYfFSjvjU5ORVA6nX9c/4R9SwOmQRkTJNCZJIGWIYBk0bNaJpo+dJz/w73/73c8zN/6Rt6rdUzz5I9e0zyN4+kx2hHfBvO5TaHfpi+PhZHbaISJmjBEmkjApw+HBD19ug620cTDzKdyvfI+q3j2hu7qRRyhpYtYbTXz3K/phbifnTSCrVbmF1yCIiZYYSJJFyoGZUNWre/Shu9yNs2rye02vfpfmJL6hKMuH7/wnz/snvjoacDW8IAREYQRH4BEXgCKlCQFhVgsKrEhxeLedhunZfq3dHRMRySpBEyhGbzaB1m/bQpj3JZ9JYvfpDArb9izaZ66mbuROSdl6yjVQCOGsLJc0nlEzfcJx+Ybj9K0FgBLbACHyDK+MIrUJgWFWCKlUlKKwahn8Y2GwlsIciIiVDCZJIORUWEkiXW4fBrcP49fff2Lv2Y4yziRgZp/HLOo3DeZrA7DMEu1MI4wyhpGEzTIJIJ8idDllJkAWkXnpbLmycNYI4awsl3SeMLN8wsh3huP0jILAS9qDK+IZUxj+kMoHh1QipVA3/0CoYfkG6805ESiUlSCIVQP269ahf97ELLs/MdnE8NYOU0ydIO3WUtJRjOFNO4Ew9gZl2EiP9JPZziZV/djKBrhSC3WcI5wxBRiZ23ISZZwhznQHXIcgEzl46rix8OGOEkmoPJcMnlCy/cLId4Zj+lTACI7AHV8YvuDL+YVUICqtGcEQ17AFhYLrB5QRcYLrAdGO6szFdblxuFy5XNqbbjdvtwu3KxnS7ct5nu3CbrpzPuXVcLtzubDhXP2dZzp+mmbMcM6dtt+kCtwvTfW57phvT/b8y3C5M89z7c9sx3G5PmXGuDNN97uUCk/99xjz33vR8Nrw+5743MfhfueFZ9w/lf1g3Txkmxh/KDdwYucv/+N50Y/yhXQPvcuNcO81dJge3TcVl88Nt+Ob8affFbfPDtPnhtjvA7odp9wMfP7A7wMcPm92B4evA8HFgO/en3deBzdcfHz8HPr7+2Pz88fHzx9fXgY/DHz8/f3wdAdh9HTnt2H2VZEuxKBUJ0uuvv84LL7xAYmIiLVq04LXXXqNdu3YXrL948WImTJjA3r17iYuL47nnnqNXr16e5aZpMmnSJN5++21Onz5Nx44dmT17NnFxcZ46J0+eZMyYMfznP//BZrPRr18/XnnlFYKDg4t1X0VKI4ePnWphQVQLC4LYWgVeL8PpIjHlLGdOHSX19DEyU46ReeYkrtTjOYlVxil8Mk/jyErGP/s0Qa4zhJg5iZWf4cKPbCqbJ6mcfRKygYyCbfc2gC3eZca5lw3QKCoLOK3bdCa+ZOODE1+chi/ZnpcfLpvvH5I2P9y2c4mb3ZGTsOUmbnY/DF9/DN8ADN8AbH7+2PwCsfsGYHcE4OsIzHn557wc/kE4AoKx+wWAj0NJWjlkeYK0aNEixo0bx5w5c2jfvj0zZ86kR48e7Nq1i2rV8j6kc+3atQwePJjp06dz8803s2DBAvr06cPmzZtp2rQpAM8//zyvvvoq7733HnXq1GHChAn06NGD7du34+/vD8Bdd93FkSNHiI+Px+l0Mnz4cEaNGsWCBQtKdP9FyjJ/XztRlcOIqhwGxF2yPuT8ByYtM5tjyac5e+ooacnHyDxznKwzJ3CnnoC0kxgZp/HNPIXDmYx/dgrB7hRCzTOEkYrNMC+5DbeZc77DhQ3z3J9/fO8+72UaueX2nHMoxrlybF7vzdz3xrkX//uMcW793PeGDdOwAwamYfesg/G/OnjWNcAwMDlXfu5zTh3jD/Vzy+3nMsKcNjx1c9c9145hy2nT+EObhqeeLaeqYf/D+gaGzY6R24bNlvPe+F8bhu3cuuS073Kb7P5lJ/ViY8CdjTs7EzM7C9OZiZmdCa5McGVhZOf8iSsLmzsLW+6fbid2d857H9OJ/dzL13Tic+5PX5z44cSPbHzJxtdwefW3AycOnED6uTNyBf4KFwm3aZBl+JKFH1mGH1mGA6fhR7bNQbbNn2y7A5fNgdvHH7fdPyc58/EH3wAMX3/wCcDml/Oy+wVg8wvExxGAj19OYubjCMQvIAiHfyB+/kH4+gdi2P2UlBUzwzTNEv4qeWvfvj1t27Zl1qxZALjdbmJiYhgzZgxPPPFEnvqDBg0iNTWVzz77zFN2zTXX0LJlS+bMmYNpmkRHR/Pwww/zyCOPAJCcnExkZCTz5s3jjjvuYMeOHTRu3JgNGzbQpk0bAJYvX06vXr04ePAg0dHRl4w7JSWFsLAwkpOTCQ0NLYpDAYDT6WTZsmX06tULX1/9P7g0UJ+UDm63yZn0TE4eT+K7//6X62+4Hj9HADabDZvNjmH3wW63Y7f5YNgN7IaBzTCw2fjDe/2gFLWS+vthmiZOl0mWy01WlhNnVkbOKzPnz+ysDFzOTFx//DM7E9OZkZO0OTNxZ2dhujIhOxOys8CVieHKwsjOwnDnvLe7MrG7M7C7MvFxZ+JrZuLrzsTPzMKPTBxmFn448ScLH8NdbPt7KS7TINNwkHkuKXMafjgNB06bHxnZBj5+DkzD53+JumHDbfjAuc+mYcO02c8ty/nTtNnPJcx2sNkwDZ9zSfK5ZTYfjNw65z5j2DHsOYm/YfPBsNvByPnTsPnk/N089/fTsOW0Z7P5YLPbwe7jeW/YfLDZc9/bsdt9sdntVKpWA/+AoCI9dgX9/bb0DFJWVhabNm1i/PjxnjKbzUbXrl1JSEjId52EhATGjRvnVdajRw+WLFkCwJ49e0hMTKRr166e5WFhYbRv356EhATuuOMOEhISCA8P9yRHAF27dsVms7Fu3Tr69u2bZ7uZmZlkZmZ6PqekpAA5/zg4nUV3bjm3raJsU66M+qT0CPSz41u1KqEhIVSrUvUSP8jnTiXkDKHBBbhcF6kuhVKSfz8MwGEDh78P+AcD1gyJcLlN0rLdZGRmkJmRhjMjFWdGOs7MNJyZabiy0nFlpuFyZuDKSsfMSsfMTgdnBjjTITsDIzsDmysDW3YmNnfmuYQsIycpc2fha2biZ2biRxYOMwsHWThwes6g2g2TQDIIJMP7rFnudzwzv8jLni03vE2T6/P+Jl+Jgn5XLU2Qjh8/jsvlIjIy0qs8MjKSnTvzvx05MTEx3/qJiYme5bllF6tz/uU7Hx8fIiIiPHXON336dKZMmZKnfMWKFQQGBl5oFwstPj6+yNuUK6M+KV3UH6WL+uN8fjkvIxQc5LyugGlCttvE5crG7crCzHZinrtkiSsL3NkYrixwZ2G4s88N0HefG3Sf+96VMzjfU55zw4DNM1Df5Rmkb+DGdu6z7Vx927m2DDP34rTLs67tXH0buXVNbKYbO67/LceN/Y/vzy3/wwVtfP5wEdyOm527f2PfmWVF0SEeaWlpBapn+RiksmL8+PFeZ65SUlKIiYmhe/fuRX6JLT4+nm7duulyTimhPild1B+li/qjdClv/dGvGNrMvQJ0KZYmSFWqVMFut5OUlORVnpSURFRUVL7rREVFXbR+7p9JSUlUr17dq07Lli09dY4ePerVRnZ2NidPnrzgdh0OBw5H3v8C+Pr6FsuXsLjalcJTn5Qu6o/SRf1Ruqg/Lqygx8XSqW/9/Pxo3bo1q1at8pS53W5WrVpFhw4d8l2nQ4cOXvUh59Rubv06deoQFRXlVSclJYV169Z56nTo0IHTp0+zadMmT52vvvoKt9tN+/bti2z/REREpGyy/BLbuHHjGDp0KG3atKFdu3bMnDmT1NRUhg8fDsCQIUOoUaMG06dPB+CBBx6gU6dOvPTSS/Tu3ZuFCxeyceNG3nrrLSDnaecPPvggf/vb34iLi/Pc5h8dHU2fPn0AaNSoET179mTkyJHMmTMHp9PJ6NGjueOOOwp0B5uIiIiUb5YnSIMGDeLYsWNMnDiRxMREWrZsyfLlyz2DrPfv34/tD894uvbaa1mwYAFPP/00Tz75JHFxcSxZssQzBxLAY489RmpqKqNGjeL06dNcd911LF++3DMHEsD8+fMZPXo0N954o2eiyFdffbXkdlxERERKLcsTJIDRo0czevTofJd9/fXXecoGDBjAgAEDLtieYRhMnTqVqVOnXrBORESEJoUUERGRfOnx2yIiIiLnUYIkIiIich4lSCIiIiLnUYIkIiIich4lSCIiIiLnUYIkIiIich4lSCIiIiLnUYIkIiIich4lSCIiIiLnKRUzaZdFpmkCOQ/CLUpOp5O0tDRSUlL0JOZSQn1Suqg/Shf1R+mi/ri03N/t3N/xC1GCVEhnzpwBICYmxuJIRERE5HKdOXOGsLCwCy43zEulUJIvt9vN4cOHCQkJwTCMIms3JSWFmJgYDhw4QGhoaJG1K4WnPild1B+li/qjdFF/XJppmpw5c4bo6GhstguPNNIZpEKy2WzUrFmz2NoPDQ3Vl7uUUZ+ULuqP0kX9UbqoPy7uYmeOcmmQtoiIiMh5lCCJiIiInEcJUinjcDiYNGkSDofD6lDkHPVJ6aL+KF3UH6WL+qPoaJC2iIiIyHl0BklERETkPEqQRERERM6jBElERETkPEqQRERERM6jBKmUef3116lduzb+/v60b9+e9evXWx1ShTR9+nTatm1LSEgI1apVo0+fPuzatcvqsOScZ599FsMwePDBB60OpcI6dOgQd999N5UrVyYgIIBmzZqxceNGq8OqsFwuFxMmTKBOnToEBARQr149pk2bdsnnjcmFKUEqRRYtWsS4ceOYNGkSmzdvpkWLFvTo0YOjR49aHVqF880333D//ffz/fffEx8fj9PppHv37qSmplodWoW3YcMG3nzzTZo3b251KBXWqVOn6NixI76+vnzxxRds376dl156iUqVKlkdWoX13HPPMXv2bGbNmsWOHTt47rnneP7553nttdesDq3M0m3+pUj79u1p27Yts2bNAnKe9xYTE8OYMWN44oknLI6uYjt27BjVqlXjm2++4YYbbrA6nArr7NmztGrVijfeeIO//e1vtGzZkpkzZ1odVoXzxBNPsGbNGv773/9aHYqcc/PNNxMZGck777zjKevXrx8BAQF88MEHFkZWdukMUimRlZXFpk2b6Nq1q6fMZrPRtWtXEhISLIxMAJKTkwGIiIiwOJKK7f7776d3795ef0+k5C1dupQ2bdowYMAAqlWrxtVXX83bb79tdVgV2rXXXsuqVav45ZdfAPjxxx/57rvvuOmmmyyOrOzSw2pLiePHj+NyuYiMjPQqj4yMZOfOnRZFJZBzJu/BBx+kY8eONG3a1OpwKqyFCxeyefNmNmzYYHUoFd7vv//O7NmzGTduHE8++SQbNmxg7Nix+Pn5MXToUKvDq5CeeOIJUlJSaNiwIXa7HZfLxTPPPMNdd91ldWhllhIkkUu4//772bZtG999953VoVRYBw4c4IEHHiA+Ph5/f3+rw6nw3G43bdq04e9//zsAV199Ndu2bWPOnDlKkCzy4YcfMn/+fBYsWECTJk3YsmULDz74INHR0eqTQlKCVEpUqVIFu91OUlKSV3lSUhJRUVEWRSWjR4/ms88+49tvv6VmzZpWh1Nhbdq0iaNHj9KqVStPmcvl4ttvv2XWrFlkZmZit9stjLBiqV69Oo0bN/Yqa9SoER9//LFFEcmjjz7KE088wR133AFAs2bN2LdvH9OnT1eCVEgag1RK+Pn50bp1a1atWuUpc7vdrFq1ig4dOlgYWcVkmiajR4/m008/5auvvqJOnTpWh1Sh3XjjjWzdupUtW7Z4Xm3atOGuu+5iy5YtSo5KWMeOHfNMe/HLL78QGxtrUUSSlpaGzeb9k26323G73RZFVPbpDFIpMm7cOIYOHUqbNm1o164dM2fOJDU1leHDh1sdWoVz//33s2DBAv79738TEhJCYmIiAGFhYQQEBFgcXcUTEhKSZ/xXUFAQlStX1rgwCzz00ENce+21/P3vf2fgwIGsX7+et956i7feesvq0CqsW265hWeeeYZatWrRpEkTfvjhB2bMmMG9995rdWhllm7zL2VmzZrFCy+8QGJiIi1btuTVV1+lffv2VodV4RiGkW/53LlzGTZsWMkGI/nq3LmzbvO30Geffcb48ePZvXs3derUYdy4cYwcOdLqsCqsM2fOMGHCBD799FOOHj1KdHQ0gwcPZuLEifj5+VkdXpmkBElERETkPBqDJCIiInIeJUgiIiIi51GCJCIiInIeJUgiIiIi51GCJCIiInIeJUgiIiIi51GCJCIiInIeJUgiIoVkGAZLliyxOgwRKQZKkESkTBo2bBiGYeR59ezZ0+rQRKQc0LPYRKTM6tmzJ3PnzvUqczgcFkUjIuWJziCJSJnlcDiIioryelWqVAnIufw1e/ZsbrrpJgICAqhbty4fffSR1/pbt27lT3/6EwEBAVSuXJlRo0Zx9uxZrzrvvvsuTZo0weFwUL16dUaPHu21/Pjx4/Tt25fAwEDi4uJYunSpZ9mpU6e46667qFq1KgEBAcTFxeVJ6ESkdFKCJCLl1oQJE+jXrx8//vgjd911F3fccQc7duwAIDU1lR49elCpUiU2bNjA4sWLWblypVcCNHv2bO6//35GjRrF1q1bWbp0KfXr1/faxpQpUxg4cCA//fQTvXr14q677uLkyZOe7W/fvp0vvviCHTt2MHv2bKpUqVJyB0BECs8UESmDhg4datrtdjMoKMjr9cwzz5imaZqA+Ze//MVrnfbt25t//etfTdM0zbfeesusVKmSefbsWc/yzz//3LTZbGZiYqJpmqYZHR1tPvXUUxeMATCffvppz+ezZ8+agPnFF1+Ypmmat9xyizl8+PCi2WERKVEagyQiZVaXLl2YPXu2V1lERITnfYcOHbyWdejQgS1btgCwY8cOWrRoQVBQkGd5x44dcbvd7Nq1C8MwOHz4MDfeeONFY2jevLnnfVBQEKGhoRw9ehSAv/71r/Tr14/NmzfTvXt3+vTpw7XXXluofRWRkqUESUTKrKCgoDyXvIpKQEBAger5+vp6fTYMA7fbDcBNN93Evn37WLZsGfHx8dx4443cf//9vPjii0Uer4gULY1BEpFy6/vvv8/zuVGjRgA0atSIH3/8kdTUVM/yNWvWYLPZaNCgASEhIdSuXZtVq1ZdUQxVq1Zl6NChfPDBB8ycOZO33nrritoTkZKhM0giUmZlZmaSmJjoVebj4+MZCL148WLatGnDddddx/z581m/fj3vvPMOAHfddReTJk1i6NChTJ48mWPHjjFmzBjuueceIiMjAZg8eTJ/+ctfqFatGjfddBNnzpxhzZo1jBkzpkDxTZw4kdatW9OkSRMyMzP57LPPPAmaiJRuSpBEpMxavnw51atX9ypr0KABO3fuBHLuMFu4cCH33Xcf1atX51//+heNGzcGIDAwkC+//JIHHniAtm3bEhgYSL9+/ZgxY4anraFDh5KRkcHLL7/MI488QpUqVejfv3+B4/Pz82P8+PHs3buXgIAArr/+ehYuXFgEey4ixc0wTdO0OggRkaJmGAaffvopffr0sToUESmDNAZJRERE5DxKkERERETOozFIIlIuafSAiFwJnUESEREROY8SJBEREZHzKEESEREROY8SJBEREZHzKEESEREROY8SJBEREZHzKEESEREROY8SJBEREZHzKEESEREROc//AwbqiZkAp0StAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "plt.plot(tr_loss_his_non_sc_en, label=\"Training loss\")\n", + "plt.plot(val_loss_his_non_sc_en, label=\"Validation loss\")\n", + "plt.ylabel(\"Loss\")\n", + "plt.xlabel(\"Epochs\")\n", + "plt.title(\"Non-SCF training: MSE energy loss\")\n", + "plt.grid()\n", + "plt.legend()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Self-consistent training using the energy only\n", + "\n", + "Training can also be performed in self consistent mode. Solids are presently supported in the linear mixing code: `gd.diff_simple_scf_loop`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "sc_en_train_recipe = (\n", + " gd.mse_energy_loss, \n", + " [None, gd.diff_simple_scf_loop(functional, cycles=5), [gd_sol_tr], [E_tr], True]\n", + ")\n", + "sc_en_validate_recipe = (\n", + " gd.mse_energy_loss, \n", + " [None, gd.diff_simple_scf_loop(functional, cycles=5), [gd_sol_val], [E_val], True]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Training epoch: 50%|█████ | 5/10 [10:38<10:36, 127.31s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "At epoch 5 training loss = 0.00012283809515136498, validation loss = 0.00012489890222459223\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Training epoch: 100%|██████████| 10/10 [21:38<00:00, 129.80s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "At epoch 10 training loss = 0.00016523055544214686, validation loss = 0.00016298261790833693\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "# Begin training\n", + "tr_loss_his_sc_en, val_loss_his_sc_en = train_neural_functional(sc_en_train_recipe, sc_en_validate_recipe)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkgAAAHHCAYAAABEEKc/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB2/klEQVR4nO3dd3wUdf7H8dfsZrObHmpC6E06REGQoliAIFhiQURPynHw0xMEc+KJIiDqcSogIipW1FMOxMJxipEQ8byT2GiCAqL0kgAihPTN7vz+CKwuCZBAwmyS9/PxyCPs7He+85n9ru47M/PdMUzTNBERERERH5vVBYiIiIgEGgUkERERkZMoIImIiIicRAFJRERE5CQKSCIiIiInUUASEREROYkCkoiIiMhJFJBERERETqKAJCIiInISBSQROaUdO3ZgGAavv/76Wa1vGAZTp04t15qk6ho+fDhNmjSxugwRQAFJpMw2bNjAzTffTOPGjXG5XNSvX5++ffvy7LPPFmvr8XiYP38+l19+OTVr1sTpdNKkSRNGjBjBt99+62v3+uuvYxhGiT8PPPDAaetZsGABs2fPLu/drJJ+/zr/73//K/a8aZo0bNgQwzC45ppr/J7LyspiypQptG/fnrCwMGrVqkV8fDzjxo1j3759vnZTp0495VgahkF6enqF76eInLsgqwsQqUxWrVrFFVdcQaNGjRg1ahSxsbHs3r2bL7/8kmeeeYaxY8f62ubm5nLjjTeSnJzMZZddxoMPPkjNmjXZsWMH77zzDm+88Qa7du2iQYMGvnWmTZtG06ZN/bbZvn3709a0YMECNm7cyPjx48t1XwEaN25Mbm4uDofjrNbPzc0lKCjw/jfjcrlYsGABvXr18lv+n//8hz179uB0Ov2Wu91uLrvsMjZv3sywYcMYO3YsWVlZfP/99yxYsIAbbriBuLg4v3VeeOEFwsPDi207Ojq63PdHRMpf4P2fSySAPf7440RFRfHNN98U+6A7cOCA3+MJEyaQnJzM008/XSy8TJkyhaeffrpY/1dffTVdunQp77J98vLyCA4OxmYr3cFjwzBwuVxnvb1zWbciDRgwgMWLFzNnzhy/ALdgwQI6d+7MoUOH/NovWbKEtWvX8vbbb3Pbbbf5PZeXl0dBQUGxbdx8883Url27YnbgPPF6vRQUFATsOIpUJJ1iEymDn3/+mXbt2pV4FKBu3bq+f+/Zs4cXX3yRvn37lnhkx263c9999/kdPTobl19+OR999BE7d+70ncI5cQ3HZ599hmEYLFy4kEmTJlG/fn1CQ0PJzMzk8OHD3HfffXTo0IHw8HAiIyO5+uqrWb9+vV//JV2DNHz4cMLDw9m7dy+JiYmEh4dTp04d7rvvPjwej9/6J1+DdOL0008//cTw4cOJjo4mKiqKESNGkJOT47dubm4u99xzD7Vr1yYiIoLrrruOvXv3lnhd0+bNm9m1a1epX7chQ4bwyy+/kJKS4ltWUFDAu+++WywAQdG4A/Ts2bPYcy6Xi8jIyFJvuzTeeustOnfuTEhICDVr1uTWW29l9+7dfm0uv/xy2rdvzw8//MAVV1xBaGgo9evX58knnyzWX35+PlOmTKFFixY4nU4aNmzI/fffT35+vl87wzAYM2YMb7/9Nu3atcPpdJKcnAzAd999R+/evQkJCaFBgwY89thjzJ8/H8Mw2LFjBwDDhg2jdu3auN3uYjX069ePVq1alfm1yM7O5i9/+QsNGzbE6XTSqlUrZsyYgWmafu1SUlLo1asX0dHRhIeH06pVKx588EG/Ns8++yzt2rUjNDSUGjVq0KVLFxYsWFDmmqR60BEkkTJo3LgxaWlpbNy48bSnvj7++GMKCwu54447ytT/0aNHix29ON1RiIceeoijR4+yZ88e3xGpk0/rPProowQHB3PfffeRn59PcHAwP/zwA0uWLGHQoEE0bdqUjIwMXnzxRXr37s0PP/xQ7HTRyTweDwkJCXTr1o0ZM2awYsUKZs6cSfPmzbnrrrvOuJ+33HILTZs2Zfr06axZs4ZXXnmFunXr8sQTT/jaDB8+nHfeeYc77riDSy65hP/85z8MHDiwxP7atGlD7969+eyzz864bYAmTZrQvXt3/vnPf3L11VcDRWN29OhRbr31VubMmePXvnHjxgC8+eabTJo0CcMwzriNw4cPF1sWFBR0xlNsjz/+OA8//DC33HILf/rTnzh48CDPPvssl112GWvXrvVb/9dff6V///7ceOON3HLLLbz77rv89a9/pUOHDr798nq9XHfddfzvf/9j9OjRtGnThg0bNvD000/z448/smTJEr/tf/rpp7zzzjuMGTOG2rVr06RJE/bu3csVV1yBYRhMnDiRsLAwXnnllWKnIu+44w7efPNNPvnkE79ruNLT0/n000+ZMmXKGV+33zNNk+uuu46VK1cycuRI4uPj+eSTT5gwYQJ79+71vee///57rrnmGjp27Mi0adNwOp389NNPfPHFF76+Xn75Ze655x5uvvlmxo0bR15eHt999x1fffVViaFYBFNESm358uWm3W437Xa72b17d/P+++83P/nkE7OgoMCv3b333msC5tq1a0vV7/z5802gxJ8zGThwoNm4ceNiy1euXGkCZrNmzcycnBy/5/Ly8kyPx+O3bPv27abT6TSnTZvmtwww58+f71s2bNgwE/BrZ5qmeeGFF5qdO3f2WwaYU6ZM8T2eMmWKCZh//OMf/drdcMMNZq1atXyPV69ebQLm+PHj/doNHz68WJ8nttO7d+9ir8HJTrzO33zzjTl37lwzIiLC99oMGjTIvOKKK0zTNM3GjRubAwcO9K2Xk5NjtmrVygTMxo0bm8OHDzdfffVVMyMjo9g2TuxjST+tWrU6bX07duww7Xa7+fjjj/st37BhgxkUFOS3vHfv3iZgvvnmm75l+fn5ZmxsrHnTTTf5lv3jH/8wbTab+d///tevz3nz5pmA+cUXX/iWAabNZjO///57v7Zjx441DcPwez//8ssvZs2aNU3A3L59u2mapunxeMwGDRqYgwcP9lt/1qxZpmEY5rZt2067/8OGDfN7Ly9ZssQEzMcee8yv3c0332wahmH+9NNPpmma5tNPP20C5sGDB0/Z9/XXX2+2a9futNsX+T2dYhMpg759+5KWlsZ1113H+vXrefLJJ0lISKB+/fosXbrU1y4zMxOAiIiIMvX/3HPPkZKS4vdzroYNG0ZISIjfMqfT6bsOyePx8Msvv/hOS6xZs6ZU/d55551+jy+99FK2bdt21uv+8ssvvtftxGmdP//5z37tfn8R/O+Zplnqo0cn3HLLLeTm5vLhhx9y7NgxPvzww1MeSQgJCeGrr75iwoQJQNFsuJEjR1KvXj3Gjh1b7FQVwHvvvVdsLOfPn3/amt5//328Xi+33HILhw4d8v3ExsbSsmVLVq5c6dc+PDycP/zhD77HwcHBdO3a1W8cFi9eTJs2bWjdurVfn1deeSVAsT579+5N27Zt/ZYlJyfTvXt34uPjfctq1qzJ7bff7tfOZrNx++23s3TpUo4dO+Zb/vbbb9OjR49iExDOZNmyZdjtdu655x6/5X/5y18wTZOPP/4Y+O3C93/96194vd4S+4qOjmbPnj188803ZapBqi+dYhMpo4svvpj333+fgoIC1q9fzwcffMDTTz/NzTffzLp162jbtq3vmpTff0iURteuXcv9Iu2SPpS8Xi/PPPMMzz//PNu3b/e7dqhWrVpn7NPlclGnTh2/ZTVq1ODXX38tVU2NGjUqti4UnTKKjIxk586d2Gy2YrW3aNGiVP2XRp06dejTpw8LFiwgJycHj8fDzTfffMr2UVFRPPnkkzz55JPs3LmT1NRUZsyYwdy5c4mKiuKxxx7za3/ZZZeV+SLtrVu3YpomLVu2LPH5k2cTNmjQoNjpvho1avDdd9/59blp06Zi43XCyZMLSnq/7Ny5k+7duxdbXtJ4DB06lCeeeIIPPviAoUOHsmXLFlavXs28efNK3P7p7Ny5k7i4uGJ/aLRp08b3PMDgwYN55ZVX+NOf/sQDDzzAVVddxY033sjNN9/s+0Pgr3/9KytWrKBr1660aNGCfv36cdttt5V4XZkIKCCJnLXg4GAuvvhiLr74Yi644AJGjBjB4sWLmTJlCq1btwaKvjPp9391W+Hko0cAf/vb33j44Yf54x//yKOPPkrNmjWx2WyMHz/+lH+B/57dbj+nmk61vnnShbcV7bbbbmPUqFGkp6dz9dVXl3oKfuPGjfnjH//IDTfcQLNmzXj77beLBaSz4fV6MQyDjz/+uMTX6OTry0rzOnq9Xjp06MCsWbNKbNuwYUO/xyW9X8qibdu2dO7cmbfeeouhQ4fy1ltvERwczC233HJO/Z5OSEgIn3/+OStXruSjjz4iOTmZRYsWceWVV7J8+XLsdjtt2rRhy5YtfPjhhyQnJ/Pee+/x/PPPM3nyZB555JEKq00qLwUkkXJw4qjP/v37gaLp+na7nbfeeqvMF2qXVWkuGD7Zu+++yxVXXMGrr77qt/zIkSMBMTW9cePGeL1etm/f7nc05aeffirX7dxwww383//9H19++SWLFi0q8/o1atSgefPmbNy4sVzqad68OaZp0rRpUy644IJy63P9+vVcddVVZ/VegaLxKOm1P9V4DB06lKSkJPbv38+CBQsYOHCg7yhhWbe7YsUKjh075ncUafPmzb7nT7DZbFx11VVcddVVzJo1i7/97W889NBDrFy5kj59+gAQFhbG4MGDGTx4MAUFBdx44408/vjjTJw4UV9lIMXoGiSRMli5cmWJRzmWLVsG4JvG3LBhQ0aNGsXy5ctL/IZtr9fLzJkz2bNnzznXFBYWxtGjR8u0jt1uL7YfixcvZu/evedcT3lISEgA4Pnnn/dbXtJrCWWf5n9CeHg4L7zwAlOnTuXaa689Zbv169cXm10IRad4fvjhh7Oavl6SG2+8EbvdziOPPFJsfEzT5Jdffilzn7fccgt79+7l5ZdfLvZcbm4u2dnZZ+wjISGBtLQ01q1b51t2+PBh3n777RLbDxkyBMMwGDduHNu2bfO7TqosBgwYgMfjYe7cuX7Ln376aQzD8M3UK2nG4IkjtyeuDzv5tQsODqZt27aYplni1xKI6AiSSBmMHTuWnJwcbrjhBlq3bk1BQQGrVq1i0aJFvluInDBz5kx+/vln7rnnHt5//32uueYaatSowa5du1i8eDGbN2/m1ltvPeeaOnfuzKJFi0hKSuLiiy8mPDz8tB/2ANdccw3Tpk1jxIgR9OjRgw0bNvD222/TrFmzc66nPHTu3JmbbrqJ2bNn88svv/im+f/4449A8aNmZZ3m/3vDhg07Y5uUlBSmTJnCddddxyWXXEJ4eDjbtm3jtddeIz8/v8T7zb377rslfpN23759iYmJKXE7zZs357HHHmPixIns2LGDxMREIiIi2L59Ox988AGjR4/mvvvuK9P+3XHHHbzzzjvceeedrFy5kp49e+LxeNi8eTPvvPMOn3zyyRmve7v//vt566236Nu3L2PHjvVN82/UqBGHDx8uNh516tShf//+LF68mOjo6FN+PcOZXHvttVxxxRU89NBD7Nixg06dOrF8+XL+9a9/MX78eJo3bw4UfQP9559/zsCBA2ncuDEHDhzg+eefp0GDBr5vS+/Xrx+xsbH07NmTmJgYNm3axNy5cxk4cGCZJ1NINWHR7DmRSunjjz82//jHP5qtW7c2w8PDzeDgYLNFixbm2LFjS5zyXVhYaL7yyivmpZdeakZFRZkOh8Ns3LixOWLECL8p07+ffl5WWVlZ5m233WZGR0f7pqGb5m/T/BcvXlxsnby8PPMvf/mLWa9ePTMkJMTs2bOnmZaWZvbu3dtvuvyppvmHhYUV6/PE9Pbf4xTT/E+ejn1i/09MFzdN08zOzjbvvvtus2bNmmZ4eLiZmJhobtmyxQTMv//978W2U9Zp/qdz8jT/bdu2mZMnTzYvueQSs27dumZQUJBZp04dc+DAgeann35a4utwqp+VK1eesc733nvP7NWrlxkWFmaGhYWZrVu3Nu+++25zy5Ytvja9e/cucdr6yVPlTdM0CwoKzCeeeMJs166d6XQ6zRo1apidO3c2H3nkEfPo0aO+doB59913l1jT2rVrzUsvvdR0Op1mgwYNzOnTp5tz5swxATM9Pb1Y+3feeccEzNGjR59xf09X+7Fjx8x7773XjIuLMx0Oh9myZUvzqaeeMr1er69Namqqef3115txcXFmcHCwGRcXZw4ZMsT88ccffW1efPFF87LLLjNr1aplOp1Os3nz5uaECRP89l/k9wzTPM9XRYqInKV169Zx4YUX8tZbbxWbYi7n3/jx43nxxRfJysoqdsH4v/71LxITE/n888+59NJLLapQ5OzpGiQRCUi5ubnFls2ePRubzcZll11mQUXV28nj8csvv/CPf/yDXr16lTib7uWXX6ZZs2bFbggsUlnoGiQRCUhPPvkkq1ev5oorriAoKIiPP/6Yjz/+mNGjRxebmi4Vr3v37lx++eW0adOGjIwMXn31VTIzM3n44Yf92i1cuJDvvvuOjz76iGeeeeasZ86JWE2n2EQkIKWkpPDII4/www8/kJWVRaNGjbjjjjt46KGHCArS33bn24MPPsi7777Lnj17MAyDiy66iClTpvim0J9gGAbh4eEMHjyYefPmaayk0lJAEhERETmJrkESEREROYkCkoiIiMhJdHL4LHm9Xvbt20dERIQuQhQREakkTNPk2LFjxMXF+W5mXBIFpLO0b98+zaQRERGppHbv3k2DBg1O+bwC0lk68dX0u3fvJjIystz6dbvdLF++nH79+uFwOMqtXzl7GpPAovEILBqPwKLxOLPMzEwaNmx4xlvMKCCdpROn1SIjI8s9IIWGhhIZGak3d4DQmAQWjUdg0XgEFo1H6Z3p8hhdpC0iIiJyEgUkERERkZMoIImIiIicRNcgiYhIQPB4PLjdbqvLqNTcbjdBQUHk5eXh8XisLscSDoejxBsol5UCkoiIWMo0TdLT0zly5IjVpVR6pmkSGxvL7t27q/V39EVHRxMbG3tOr4ECkoiIWOpEOKpbty6hoaHV+oP9XHm9XrKysggPDz/tlyBWVaZpkpOTw4EDBwCoV6/eWfelgCQiIpbxeDy+cFSrVi2ry6n0vF4vBQUFuFyuahmQAEJCQgA4cOAAdevWPevTbdXz1RMRkYBw4pqj0NBQiyuRquTE++lcrmlTQBIREcvptJqUp/J4PykgiYiIiJxEAUlERCQANGnShNmzZ5e6/WeffYZhGBU+++/1118nOjq6QrcRiBSQREREysAwjNP+TJ069az6/eabbxg9enSp2/fo0YP9+/cTFRV1VtuT09MstgDj9njZnwPH8gqpqRsNiogEnP379/v+vWjRIiZPnsyWLVt8y8LDw33/Nk0Tj8dDUNCZP27r1KlTpjqCg4OJjY0t0zpSejqCFGAGv/w1f18fxFfbD1tdioiIlCA2Ntb3ExUVhWEYvsebN28mIiKCjz/+mM6dO+N0Ovnf//7Hzz//zPXXX09MTAzh4eFcfPHFrFixwq/fk0+xGYbBK6+8wg033EBoaCgtW7Zk6dKlvudPPsX2+uuvU7NmTVJTU2nXrh3h4eH079/fL9AVFhZyzz33EB0dTa1atfjrX//KsGHDSExMLNNr8MILL9C8eXOCg4Np1aoV//jHP3zPmabJ1KlTadSoEU6nk7i4OO655x7f888//zwtW7bE5XIRExPDzTffXKZtny8KSAHmL975pARPIHvr51aXIiJiCdM0ySkoPO8/pmmW2z488MAD/P3vf2fTpk107NiRrKwsBgwYQGpqKmvXrqV///5ce+217Nq167T9PPLII9xyyy189913DBgwgNtvv53Dh0/9B3ROTg5z587ljTfe4PPPP2fXrl3cd999vuefeOIJ3n77bebPn88XX3xBZmYmS5YsKdO+ffDBB4wbN46//OUvbNy4kf/7v/9jxIgRrFy5EoD33nuPp59+mhdffJGtW7eyZMkSOnToAMC3337LPffcw7Rp09iyZQvJyclcdtllZdr++aJTbAGmkXGQZra9rE3/3upSREQskev20HbyJ+d9uz9MSyA0uHw+FqdNm0bfvn19j2vWrEmnTp18jx999FE++OADli5dypgxY07Zz/DhwxkyZAgAf/vb35gzZw5ff/01/fv3L7G92+1m1qxZdOrUCZvNxpgxY5g2bZrv+WeffZaJEydyww03ADB37lyWLVtWpn2bMWMGw4cP589//jMASUlJfPnll8yYMYMrrriCXbt2ERsbS58+fXA4HDRq1IiuXbsCsGvXLsLCwrjmmmuIiIigcePGXHjhhWXa/vmiI0iBpk5rAMKObrW4EBEROVtdunTxe5yVlcV9991HmzZtiI6OJjw8nE2bNp3xCFLHjh19/w4LCyMyMtJ3G42ShIaG0rRpU9/jevXq+dofPXqUjIwMX1gBsNvtdO7cuUz7tmnTJnr27Om3rGfPnmzatAmAQYMGkZubS7NmzRg1ahQffPABhYWFAPTt25fGjRvTrFkz7rjjDt5++21ycnLKtP3zRUeQAkx4w/awBWLytuP2eHHYlWFFpHoJcdj5YVqCJdstL2FhYX6P77vvPlJSUpgxYwYtWrQgJCSEm2++mYKCgtP24zhpso5hGHi93jK1L89Th6XRsGFDtmzZwooVK0hJSeHPf/4zTz31FP/5z3+IiIhgzZo1fPbZZyxfvpzJkyczdepUvvnmm4D7KgF9+gaY6MZFfy1cYOxm56Esi6sRETn/DMMgNDjovP9U5Ld5f/HFFwwfPpwbbriBDh06EBsby44dOypseyWJiooiJiaGb775xrfM4/GwZs2aMvXTpk0bvvjiC79lX3zxBW3btvU9DgkJ4dprr2XOnDl89tlnpKWlsWHDBgCCgoLo06cPTz75JN999x07duzg008/PYc9qxg6ghRgjNotKcRGpJHDN9t/pkVMYJ6bFRGR0mvZsiXvv/8+1157LYZh8PDDD5/2SFBFGTt2LNOnT6dFixa0bt2aZ599ll9//bVM4XDChAnccsstXHjhhfTp04d///vfvP/++75Zea+//joej4du3boRGhrKW2+9RUhICI0bN+bDDz9k27ZtXHbZZdSoUYNly5bh9Xpp1apVRe3yWdMRpEAT5CTdVvS9Fkd2fWdxMSIiUh5mzZpFjRo16NGjB9deey0JCQlcdNFF572Ov/71rwwZMoShQ4fSvXt3wsPDSUhIwOVylbqPxMREnnnmGWbMmEG7du148cUXmT9/PpdffjkA0dHRvPzyy/Ts2ZOOHTuyYsUK/v3vf1OrVi2io6N5//33ufLKK2nTpg3z5s3jn//8J+3ataugPT57hnm+T05WEZmZmURFRXH06FEiIyPLrV+3283GGQO4MP9r3q11JzePfaLc+paz43a7WbZsGQMGDCh2fl/OP41HYDnX8cjLy2P79u00bdq0TB/SUjKv10tmZiaRkZHYbGc+BuL1emnTpg233HILjz766Hmo8Pw43fuqtJ/fOsUWgLJCGkD+14Qf/dHqUkREpArZuXMny5cvp3fv3uTn5zN37ly2b9/ObbfdZnVpAUen2AKQJ6I+AHEFO8hzeyyuRkREqgqbzcbrr7/OxRdfTM+ePdmwYQMrVqygTZs2VpcWcHQEKQAVhjUAoKWxh20HjtG2frS1BYmISJXQsGHDYjPQpGQ6ghSAclx1KcBBiFHAnu2brC5HRESk2lFACkCmYeeQqwkAx3ZtsLYYERGRakgBKUDlRLcs+scBHUESERE53xSQApQ9tugbSSOP6Z5sIiIi55sCUoCKbtQBgAbuHWTnF1pcjYiISPWigBSgwhsWBaTmxj62pv9qcTUiIiLViwJSoIpqQK4RQrDhIX3b91ZXIyIi5ezyyy9n/PjxvsdNmjRh9uzZp13HMAyWLFlyztsur35OZ+rUqcTHx1foNiqS5QHpueeeo0mTJrhcLrp168bXX3992vaLFy+mdevWuFwuOnTowLJly/yef//99+nXrx+1atXCMAzWrVt3yr5M0+Tqq68+L2+UMjNs/BLSDIDs3bonm4hIoLj22mvp379/ic/997//xTAMvvuu7P/f/uabbxg9evS5lufnVCFl//79XH311eW6rarG0oC0aNEikpKSmDJlCmvWrKFTp04kJCRw4MCBEtuvWrWKIUOGMHLkSNauXUtiYiKJiYls3LjR1yY7O5tevXrxxBNnvofZ7Nmzy3QH4/Mtv2bR3Y1th7ZYXImIiJwwcuRIUlJS2LNnT7Hn5s+fT5cuXejYsWOZ+61Tpw6hoaHlUeIZxcbG4nQ6z8u2KitLA9KsWbMYNWoUI0aMoG3btsybN4/Q0FBee+21Ets/88wz9O/fnwkTJtCmTRseffRRLrroIubOnetrc8cddzB58mT69Olz2m2vW7eOmTNnnnJbgSA4rujuxtHHfrK4EhEROeGaa66hTp06vP76637Ls7KyWLx4MSNHjuSXX35hyJAh1K9fn9DQUDp06MA///nP0/Z78im2rVu3ctlll+FyuWjbti0pKSnF1vnrX//KBRdcQGhoKM2aNWPy5Mm43W4AXn/9dR555BHWr1+PYRgYhuGr+eQzJxs2bODKK68kJCSEWrVqMXr0aLKysnzPDx8+nMTERGbMmEG9evWoVasWd999t29bpeH1epk2bRoNGjTA6XQSHx9PcnKy7/mCggLGjBlDvXr1cLlcNG7cmOnTpwNFZ3ymTp1Ko0aNcDqdxMXFcc8995R622fDsluNFBQUsHr1aiZOnOhbZrPZ6NOnD2lpaSWuk5aWRlJSkt+yhISEMp8ey8nJ4bbbbuO5554jNja2zLWfLzWbxcPX0Mizk6M5bqJCdedyEakGTBPcOed/u45QKMVZhaCgIIYOHcrrr7/OQw895DsTsXjxYjweD0OGDCErK4vOnTvz17/+lcjISD766CPuuOMOmjdvTteuXc+4Da/Xy4033khMTAxfffUVR48e9bte6YSIiAhef/114uLi2LBhA6NGjcLhcPDwww8zePBgNm7cSHJyMitWrAAgKiqqWB/Z2dkkJCTQvXt3vvnmGw4cOMCf/vQnxowZ4xcCV65cSb169Vi5ciU//fQTgwcPJj4+nlGjRp1xf6DoIMfMmTN58cUXufDCC3nttde47rrr+P7772nZsiVz5sxh6dKlvPPOOzRq1Ijdu3eze/duAN577z2efvppFi5cSLt27UhPT2f9+vWl2u7ZsiwgHTp0CI/HQ0xMjN/ymJgYNm/eXOI66enpJbZPT08v07bvvfdeevTowfXXX1/qdfLz88nPz/c9zszMBMDtdpcpQZ/Jib7cbjfBdYtOsTUx0vl2134ual6v3LYjpff7MRHraTwCy7mOh9vtxjRNvF4vXq+3aGFBNra/NyivEkvN+8AeCA4rVdvhw4fz1FNPsXLlSi6//HKg6PTajTfeSEREBBEREX5/0N99990kJyezaNEiunTp4lt+Yt9Pfrx8+XI2b97Mxx9/TFxcHACPPfYYAwcO9HutHnzwQd+6jRo1IikpiX/+859MmjQJp9NJWFgYQUFB1K1b97f9PL7uiX7eeust8vLyeP311wkLC6Nt27bMmTOH66+/nunTpxMTE4NpmtSoUYM5c+Zgt9u54IILGDBgACtWrGDkyJElvkamafptb8aMGdx///3ccsstAEyfPp2VK1fy9NNPM3fuXHbu3EnLli3p0aMHhmHQsGFD3/o7d+4kNjaWK6+8EofDQYMGDejSpYvfa/d7Xq8X0zRxu93Y7Xa/50r7Xq12N6tdunQpn376KWvXri3TetOnT+eRRx4ptnz58uUVcs44JSUFTJMrCCfSyCItdQnpWxqX+3ak9Eo6vC3W0XgElrMdj6CgIGJjY8nKyqKgoKBooTuH6PIrrdQyjx0Dh6dUbePi4ujatSsvvfQSF110Edu2beO///0v//73v8nMzMTj8TBr1iw++OAD9u/fj9vtJj8/n+DgYN8f2IWFhRQUFPgee71e8vLyyMzMZN26ddSvX5/w8HDf8+3aFV12kZub61v2/vvv8+KLL7Jjxw6ys7MpLCwkIiKCY8eOAUV/3Hs8Hl/73zvRz3fffUe7du382nXo0AGv18uaNWvo2bMnbrebCy64gOzsbN/6tWrV4ocffiix75O3nZmZyb59+4iPj/dr36VLFzZu3EhmZiY333wzN9xwA61ateKqq64iISGBK6+8Eig6W/T000/TrFkz+vTpQ9++fenfvz9BQSXHmIKCAnJzc/n8888pLPT/LsGcnNIdnbQsINWuXRu73U5GRobf8oyMjFOe9oqNjS1T+5J8+umn/Pzzz0RHR/stv+mmm7j00kv57LPPSlxv4sSJfn8NZGZm0rBhQ/r160dkZGSpt38mbreblJQU+vbti8PhYN9PTxOZtY4GrjwGDBhQbtuR0jt5TMRaGo/Acq7jkZeXx+7duwkPD8flchUtNCOKjuacZ5GlPMV2wqhRoxg3bhwvvvgi7777Ls2bN/fNjH7iiSd48cUXmTVrFh06dCAsLIx7770Xr9fr+8wICgoiODjY99hms+FyuYiMjMTlcmGz2fw+X04ckQkJCSEyMpK0tDRGjx7N1KlT6devH1FRUSxcuJBZs2YRERGBYRg4nU7sdnuJn1Mn+gkODiYoKKjEbYWFhREZGYnD4fC1P8HpdBar8fdK2nZoaKjf499v+9JLL2Xbtm18/PHHpKam8sc//pGrrrqKxYsX07ZtW7Zs2cKKFStYsWIFEyZM4Pnnn2flypUlvu/y8vIICQnxXcP1e6cKdCezLCAFBwfTuXNnUlNTSUxMBIrSc2pqKmPGjClxne7du5Oamup3HjYlJYXu3buXersPPPAAf/rTn/yWdejQgaeffpprr732lOs5nc4Sr/h3OBwV8j/pE/26a7eGrHUE//qjPgwsVlFjLWdH4xFYznY8PB4PhmFgs9mw2X43b8geUY7VVYxbb72Ve++9l4ULF/KPf/yDu+66y3c6Z9WqVVx//fUMHToUKPp827p1K23btvXbzxP7fvLjtm3bsnv3bjIyMqhXr+jyihNfg3Pitfryyy9p3LgxkyZN8q2/a9cuv36cTicej8f/tT3uRD9t27bljTfeIDc3l7CwolOMaWlp2Gw22rRpg81m813kfXKtJ/opye+fj46OJi4ujrS0NK644gpfm1WrVtG1a1dfH9HR0QwZMoQhQ4YwaNAg+vfvz5EjR6hZsyZhYWFcf/31XH/99YwZM4bWrVvz/fffc9FFF5W4b4ZhlPi+LO371NJTbElJSQwbNowuXbrQtWtXZs+eTXZ2NiNGjABg6NCh1K9f33cV+7hx4+jduzczZ85k4MCBLFy4kG+//ZaXXnrJ1+fhw4fZtWsX+/btA2DLlqIp8rGxsX4/J2vUqBFNmzat6F0us5C4DrBjITWyfra6FBER+Z3w8HAGDx7MxIkTyczMZPjw4b7nWrZsybvvvsuqVauoUaMGs2bNIiMjg7Zt25aq7z59+nDBBRcwbNgwnnrqKTIzM3nooYf82rRs2ZJdu3axcOFCLr74Yj766KNik5aaNGnC9u3bWbduHQ0aNCAiIqLYH/u33347U6ZMYdiwYUydOpWDBw8yduxY7rjjjmLX/Z6LCRMmMGXKFJo3b058fDzz589n3bp1vP3220DRzPZ69epx4YUXYrPZWLx4MbGxsURHR/P666/j8Xjo1q0boaGhvPXWW4SEhNC4ccVdemLpNP/BgwczY8YMJk+eTHx8POvWrSM5Odk3ILt27WL//v2+9j169GDBggW89NJLdOrUiXfffZclS5bQvn17X5ulS5dy4YUXMnDgQKAo4V944YXMmzfv/O5cOandLB6ApuYuDmXln76xiIicVyNHjuTXX38lISHBdzE1wKRJk7joootISEjg8ssvJzY21ne2pDRsNhsffPABubm5dO3alT/96U88/vjjfm2uu+467r33XsaMGUN8fDyrVq3yO5oERZeP9O/fnyuuuII6deqU+FUDoaGhfPLJJxw+fJiLL76Ym2++mauuusrvK3TKwz333ENSUhJ/+ctf6NChA8nJySxdupSWLVsCRTPynnzySbp06cLFF1/Mjh07WLZsme8I1Msvv0zPnj3p2LEjK1as4N///je1atUq1xp/zzBPnGiUMsnMzCQqKoqjR4+W+zVIy5YtY8CAAUWHAXMOw5NFR7a+umUd3doG3lGuqq7YmIilNB6B5VzHIy8vj+3bt9O0adNi14pI2Xm9XjIzM4mMjDzlqa/q4HTvq9J+flffV6+yCK3JEXtRQj64XbccEREROR8UkCqBI+EtAMjfp5vWioiInA8KSJVAYe3WADh/LfkLNEVERKR8KSBVAmENiy5Cr5W9DV0yJiIiUvEUkCqB2k0vBKA5u0nPzLO4GhGR8qc//qQ8lcf7SQGpEnDEtgGgrnGEbTt3WlyNiEj5OTHzrbS3fxApjRPvp3OZ6Vrt7sVWKTnDORRUj9qF+zm8/Tvo2NrqikREyoXdbic6OpoDBw4ARd/JY5Thdh/iz+v1UlBQQF5eXrWc5m+aJjk5ORw4cIDo6OhiN6otCwWkSiIzogW1f92Pe79msolI1XLi7gYnQpKcPdM0yc3NJSQkpFoHzejo6DLdp7UkCkiVRd3W8Ot/CTnyo9WViIiUK8MwqFevHnXr1sXtdltdTqXmdrv5/PPPueyyy6rtF6k6HI5zOnJ0ggJSJRHWsCNsgTq5P+P1mths1fcvAxGpmux2e7l8sFVndrudwsJCXC5XtQ1I5aX6naCspGo17QRAC/aw+3C2xdWIiIhUbQpIlURQ3VZ4sBFtZLNz5zaryxEREanSFJAqC4eLQ8ENADiyY73FxYiIiFRtCkiVyLGolgB40n+wuBIREZGqTQGpErHVLfrCyNCjmskmIiJSkRSQKpHIRkUXatfL247b47W4GhERkapLAakSqXliJpuxh52HjllcjYiISNWlgFSJ2Go1x00QoUY+u7frNJuIiEhFUUCqTOxBHHQ1ASBzp2ayiYiIVBQFpEomJ/oCAMyMTRZXIiIiUnUpIFUy9piimWzhmTrFJiIiUlEUkCqZ6CZFF2rXL9hBnttjcTUiIiJVkwJSJRPduCMAzYx9bMs4Ym0xIiIiVZQCUiVjRDcmz3DhNArZt+17q8sRERGpkhSQKhubjYMhTQHI2r3B4mJERESqJgWkSiivRisAbAc1k01ERKQiKCBVQo7YdgBEHvvJ4kpERESqJgWkSqhm06ILtRu6d5BTUGhxNSIiIlWPAlIldOKmtU2MdH7ae8jiakRERKoeBaTKKCKWLCMcu2GSvk0XaouIiJQ3BaTKyDD4Jaw5ADl7N1pcjIiISNWjgFRJFdRsDYD90GaLKxEREal6FJAqKWdc0Uy2GprJJiIiUu4UkCqpWs3iAWjs2cnRXLe1xYiIiFQxlgek5557jiZNmuByuejWrRtff/31adsvXryY1q1b43K56NChA8uWLfN7/v3336dfv37UqlULwzBYt26d3/OHDx9m7NixtGrVipCQEBo1asQ999zD0aNHy3vXKlRYgw4ANLQd5Oc9+y2uRkREpGqxNCAtWrSIpKQkpkyZwpo1a+jUqRMJCQkcOHCgxParVq1iyJAhjBw5krVr15KYmEhiYiIbN/52oXJ2dja9evXiiSeeKLGPffv2sW/fPmbMmMHGjRt5/fXXSU5OZuTIkRWyjxUmtCZHbDUBOLBtvcXFiIiIVC1BVm581qxZjBo1ihEjRgAwb948PvroI1577TUeeOCBYu2feeYZ+vfvz4QJEwB49NFHSUlJYe7cucybNw+AO+64A4AdO3aUuM327dvz3nvv+R43b96cxx9/nD/84Q8UFhYSFGTpS1Imv4Y3JzrzMPl7NwLXWl2OiIhIlWHZEaSCggJWr15Nnz59fivGZqNPnz6kpaWVuE5aWppfe4CEhIRTti+to0ePEhkZWanCEUBhraKZbEG//GhxJSIiIlWLZYng0KFDeDweYmJi/JbHxMSweXPJU9fT09NLbJ+enn5OdTz66KOMHj36tO3y8/PJz8/3Pc7MzATA7XbjdpffRdIn+ipNn8H12sJ2qJn9U7nWIP7KMiZS8TQegUXjEVg0HmdW2temch0yKWeZmZkMHDiQtm3bMnXq1NO2nT59Oo888kix5cuXLyc0NLTca0tJSTljm/BjeTQGmpm7WPSvZUQ4yr0M+Z3SjImcPxqPwKLxCCwaj1PLyckpVTvLAlLt2rWx2+1kZGT4Lc/IyCA2NrbEdWJjY8vU/nSOHTtG//79iYiI4IMPPsDhOH26mDhxIklJSb7HmZmZNGzYkH79+hEZGVnm7Z+K2+0mJSWFvn37nrEm8o/BjEeIMY7Q/IJWdGnTvNzqkN+UaUykwmk8AovGI7BoPM7sxBmgM7EsIAUHB9O5c2dSU1NJTEwEwOv1kpqaypgxY0pcp3v37qSmpjJ+/HjfspSUFLp3716mbWdmZpKQkIDT6WTp0qW4XK4zruN0OnE6ncWWOxyOCnkTlqpfR00OBcVQuzCDI7s24ujYutzrkN9U1FjL2dF4BBaNR2DReJxaaV8XS0+xJSUlMWzYMLp06ULXrl2ZPXs22dnZvlltQ4cOpX79+kyfPh2AcePG0bt3b2bOnMnAgQNZuHAh3377LS+99JKvz8OHD7Nr1y727dsHwJYtW4Cio0+xsbFkZmbSr18/cnJyeOutt8jMzPSlyTp16mC328/nS3DOjoa3oPaRDAr2fQ/cbHU5IiIiVYKlAWnw4MEcPHiQyZMnk56eTnx8PMnJyb4LsXft2oXN9ttEux49erBgwQImTZrEgw8+SMuWLVmyZAnt27f3tVm6dKkvYAHceuutAEyZMoWpU6eyZs0avvrqKwBatGjhV8/27dtp0qRJRe1uhfDWaQNHvsD56xarSxEREakyLL9Ie8yYMac8pfbZZ58VWzZo0CAGDRp0yv6GDx/O8OHDT/n85ZdfjmmaZS0zYIU17ABboXbuNkzTxDAMq0sSERGp9Cy/1Yicm9rH78nW3NxN+tFca4sRERGpIhSQKrngmNZ4sFHDyGL7jm1WlyMiIlIlKCBVdg4Xhxz1ATiy4zuLixEREakaFJCqgGORRRebu/d/b3ElIiIiVYMCUhVg1m0DQOgRzWQTEREpDwpIVUBEo44A1M3bjtdbdWboiYiIWEUBqQqo3TQegObsYc/hbGuLERERqQIUkKqAoDotcBNEuJHHzu06zSYiInKuFJCqAruDA87GABzdud7iYkRERCo/BaQqIjuqaCabmfGDxZWIiIhUfgpIVYStblsAwo5stbgSERGRyk8BqYqIatIJgHoF2yn0eC2uRkREpHJTQKoiah2fydaMfew4mGltMSIiIpWcAlIVYavRmDycOA03e7fpG7VFRETOhQJSVWGzcSCkKQCZOzdYXIyIiEjlpoBUheRGtwLAOKiZbCIiIudCAakKCYotuidbRKZmsomIiJwLBaQqpEaTeADqu3eQX+ixthgREZFKTAGpCqnRpOimtU1IZ3v6YYurERERqbwUkKoQIzKOLCOcIMPL/p91obaIiMjZUkCqSgyDQ6HNAMjarYAkIiJythSQqpj8GkUz2WwHN1lciYiISOWlgFTFOOLaARCd9ZPFlYiIiFReCkhVTK3j92Rr6N5BTkGhxdWIiIhUTgpIVUxU46KZbI1sB/l5T4bF1YiIiFROCkhVTVhtjthqAJCxbb3FxYiIiFROCkhV0OGw5gDk7tlocSUiIiKVkwJSFVRQszUAQb9strgSERGRykkBqQpyNSiayVYj62eLKxEREamcFJCqoDrN4gFo4t3J0Vy3tcWIiIhUQgpIVVBY/fYAxBq/sm3XbourERERqXwUkKoiVySH7HUBOKSZbCIiImWmgFRFHQlvAUD+vu8trkRERKTyUUCqogprF81kc2gmm4iISJlZHpCee+45mjRpgsvlolu3bnz99denbb948WJat26Ny+WiQ4cOLFu2zO/5999/n379+lGrVi0Mw2DdunXF+sjLy+Puu++mVq1ahIeHc9NNN5GRUbW+dTq0QQcAauVoJpuIiEhZWRqQFi1aRFJSElOmTGHNmjV06tSJhIQEDhw4UGL7VatWMWTIEEaOHMnatWtJTEwkMTGRjRt/+0LE7OxsevXqxRNPPHHK7d577738+9//ZvHixfznP/9h37593HjjjeW+f1aq0zwegGbmLg4dy7O2GBERkUrG0oA0a9YsRo0axYgRI2jbti3z5s0jNDSU1157rcT2zzzzDP3792fChAm0adOGRx99lIsuuoi5c+f62txxxx1MnjyZPn36lNjH0aNHefXVV5k1axZXXnklnTt3Zv78+axatYovv/yyQvbTCiH12uDBRk0ji+07t1tdjoiISKUSZNWGCwoKWL16NRMnTvQts9ls9OnTh7S0tBLXSUtLIykpyW9ZQkICS5YsKfV2V69ejdvt9gtQrVu3plGjRqSlpXHJJZeUuF5+fj75+fm+x5mZmQC43W7c7vL7rqETfZ17n0H8ElSP2MK9HPp5Le5WLc69uGqq/MZEyoPGI7BoPAKLxuPMSvvaWBaQDh06hMfjISYmxm95TEwMmzeXfGFxenp6ie3T09NLvd309HSCg4OJjo4uUz/Tp0/nkUceKbZ8+fLlhIaGlnr7pZWSknLOfTS01SOWvRzc9AXLCCuHqqq38hgTKT8aj8Ci8QgsGo9Ty8nJKVU7ywJSZTNx4kS/o1eZmZk0bNiQfv36ERkZWW7bcbvdpKSk0LdvXxwOxzn19VPWF7D1W+pxkMsHDCinCquf8hwTOXcaj8Ci8QgsGo8zO3EG6EwsC0i1a9fGbrcXmz2WkZFBbGxsievExsaWqf2p+igoKODIkSN+R5HO1I/T6cTpdBZb7nA4KuRNWB79RjTuBFuhbt42goKCMAyjnKqrnipqrOXsaDwCi8YjsGg8Tq20r4tlF2kHBwfTuXNnUlNTfcu8Xi+pqal07969xHW6d+/u1x6KDiOeqn1JOnfujMPh8Otny5Yt7Nq1q0z9VAZ1j89ka27uJuOoZrKJiIiUlqWn2JKSkhg2bBhdunSha9euzJ49m+zsbEaMGAHA0KFDqV+/PtOnTwdg3Lhx9O7dm5kzZzJw4EAWLlzIt99+y0svveTr8/Dhw+zatYt9+/YBReEHio4cxcbGEhUVxciRI0lKSqJmzZpERkYyduxYunfvfsoLtCur4LoX4CaIcCOPjdu3EHthvNUliYiIVAqWBqTBgwdz8OBBJk+eTHp6OvHx8SQnJ/suxN61axc2228HuXr06MGCBQuYNGkSDz74IC1btmTJkiW0b9/e12bp0qW+gAVw6623AjBlyhSmTp0KwNNPP43NZuOmm24iPz+fhIQEnn/++fOwx+eZ3cGB4IbUL9jOke3rQQFJRESkVCy/SHvMmDGMGTOmxOc+++yzYssGDRrEoEGDTtnf8OHDGT58+Gm36XK5eO6553juuefKUmqllBXZEg5tx5Oue7KJiIiUluW3GpGKZcS0BSD06FaLKxEREak8FJCquIhGHQGIyduG12taXI2IiEjloIBUxdVpFg9Ac/ay55csa4sRERGpJBSQqrigWk3Jw4nTcLNnm65DEhERKQ0FpKrOZuOAqwkAR3d+Z20tIiIilYQCUjWQHdUSAPPADxZXIiIiUjkoIFUD9ph2AIRrJpuIiEipKCBVAzWaFM1kq1ewg0KP1+JqREREAp8CUjVQ6/hMtibsZ8eBX60tRkREpBJQQKoGbFH1yTLCcBge9v+8wepyREREAp4CUnVgGBwMaQZA1m4FJBERkTNRQKom8mpcAIBxYJPFlYiIiAQ+BaRqIii2aCZbxDHNZBMRETkTBaRqolbTTgA0cO8gv9BjcTUiIiKBTQGpmqjRpCggNeQg2/cdtLgaERGRwKaAVE0Y4XU4YovGZphk/Lze6nJEREQCmgJSNfJLaHMAsvdoJpuIiMjpKCBVI/k1WwFgP7jZ4kpEREQCmwJSNeKs1xaA6KyfLK5EREQksCkgVSO1j99ypJFnJzkFhdYWIyIiEsAUkKqRqMbHb1prHGbb7r0WVyMiIhK4FJCqE1cUh+x1ADjw8zpraxEREQlgCkjVzK9hLQDI3/e9xZWIiIgELgWkasZdqzUAQYc0k01ERORUFJCqmZAGRfdkq5n9s8WViIiIBC4FpGrmxEy2pt4dHM0psLYYERGRAKWAVM1E1G+HF4OaRhY7dm23uhwREZGApIBU3QSHciCoHgCHtn1ncTEiIiKBSQGpGjoaXjSTrWDfRosrERERCUwKSNWQp04bAJyHt1hciYiISGBSQKqGwhp0AKBWjmayiYiIlEQBqRqq2+JCAJqZu/nlWJ7F1YiIiAQeBaRqKCTmAgqxE2HksmP7j1aXIyIiEnAUkKqjoGAyHA0B+HX7eouLERERCTyWB6TnnnuOJk2a4HK56NatG19//fVp2y9evJjWrVvjcrno0KEDy5Yt83veNE0mT55MvXr1CAkJoU+fPmzdutWvzY8//sj1119P7dq1iYyMpFevXqxcubLc9y2QZUYWzWRzp/9gcSUiIiKBx9KAtGjRIpKSkpgyZQpr1qyhU6dOJCQkcODAgRLbr1q1iiFDhjBy5EjWrl1LYmIiiYmJbNz423T1J598kjlz5jBv3jy++uorwsLCSEhIIC/vt2ttrrnmGgoLC/n0009ZvXo1nTp14pprriE9Pb3C9zlg1G0LQMivmskmIiJyMksD0qxZsxg1ahQjRoygbdu2zJs3j9DQUF577bUS2z/zzDP079+fCRMm0KZNGx599FEuuugi5s6dCxQdPZo9ezaTJk3i+uuvp2PHjrz55pvs27ePJUuWAHDo0CG2bt3KAw88QMeOHWnZsiV///vfycnJ8QtaVV14w6KZbHXztmGapsXViIiIBJags1lp9+7dGIZBgwYNAPj6669ZsGABbdu2ZfTo0aXqo6CggNWrVzNx4kTfMpvNRp8+fUhLSytxnbS0NJKSkvyWJSQk+MLP9u3bSU9Pp0+fPr7no6Ki6NatG2lpadx6663UqlWLVq1a8eabb3LRRRfhdDp58cUXqVu3Lp07dz5lvfn5+eTn5/seZ2ZmAuB2u3G73aXa59I40Vd59lmSmk2KAlIzcw97Dh0lNjqsQrdXmZ2vMZHS0XgEFo1HYNF4nFlpX5uzCki33XYbo0eP5o477iA9PZ2+ffvSrl073n77bdLT05k8efIZ+zh06BAej4eYmBi/5TExMWzevLnEddLT00tsf+LU2Infp2tjGAYrVqwgMTGRiIgIbDYbdevWJTk5mRo1apyy3unTp/PII48UW758+XJCQ0PPsLdll5KSUu59+jG9JBCMyyhg2b/eIbZOzJnXqeYqfEykTDQegUXjEVg0HqeWk5NTqnZnFZA2btxI165dAXjnnXdo3749X3zxBcuXL+fOO+8sVUCyimma3H333dStW5f//ve/hISE8Morr3DttdfyzTffUK9evRLXmzhxot/Rq8zMTBo2bEi/fv2IjIwst/rcbjcpKSn07dsXh8NRbv2WZN8PjWlcsJUmUXDlgAEVuq3K7HyOiZyZxiOwaDwCi8bjzE6cATqTswpIbrcbp9MJwIoVK7juuusAaN26Nfv37y9VH7Vr18Zut5ORkeG3PCMjg9jY2BLXiY2NPW37E78zMjL8gk5GRgbx8fEAfPrpp3z44Yf8+uuvvmDz/PPPk5KSwhtvvMEDDzxQ4radTqdvn3/P4XBUyJuwovr9vezoC+DAVjiwWf8hlcL5GBMpPY1HYNF4BBaNx6mV9nU5q4u027Vrx7x58/jvf/9LSkoK/fv3B2Dfvn3UqlWrVH0EBwfTuXNnUlNTfcu8Xi+pqal07969xHW6d+/u1x6KDiOeaN+0aVNiY2P92mRmZvLVV1/52pw4tGaz+e+6zWbD6/WWqvaqwjg+ky306NYztBQREaleziogPfHEE7z44otcfvnlDBkyhE6dOgGwdOlS36m30khKSuLll1/mjTfeYNOmTdx1111kZ2czYsQIAIYOHep3Efe4ceNITk5m5syZbN68malTp/Ltt98yZswYoOj6ovHjx/PYY4+xdOlSNmzYwNChQ4mLiyMxMREoClk1atRg2LBhrF+/nh9//JEJEyawfft2Bg4ceDYvR6UV1bgjALH52/B6NZNNRETkhLM6xXb55Zdz6NAhMjMz/S5sHj16dJkuWB48eDAHDx5k8uTJpKenEx8fT3Jysu8i6127dvkd6enRowcLFixg0qRJPPjgg7Rs2ZIlS5bQvn17X5v777+f7OxsRo8ezZEjR+jVqxfJycm4XC6g6NRecnIyDz30EFdeeSVut5t27drxr3/9yxf0qou6zeMBaMJ+9v5ylIZ1oi2tR0REJFCcVUDKzc3FNE1fONq5cycffPABbdq0ISEhoUx9jRkzxncE6GSfffZZsWWDBg1i0KBBp+zPMAymTZvGtGnTTtmmS5cufPLJJ2WqsyoKqtGQbEIJM3LY89MGGta51OqSREREAsJZnWK7/vrrefPNNwE4cuQI3bp1Y+bMmSQmJvLCCy+Ua4FSgQyDAyFNATi26zuLixEREQkcZxWQ1qxZw6WXFh1tePfdd4mJiWHnzp28+eabzJkzp1wLlIqVE31B0T8O6J5sIiIiJ5xVQMrJySEiIgIo+qLEG2+8EZvNxiWXXMLOnTvLtUCpWEEx7QAIz/zJ4kpEREQCx1kFpBYtWrBkyRJ2797NJ598Qr9+/QA4cOBAuX5polS86CZFF6bHFeyg0FO9vuZARETkVM4qIE2ePJn77ruPJk2a0LVrV993DC1fvpwLL7ywXAuUilXn+Ey2RmSwK+MXa4sREREJEGc1i+3mm2+mV69e7N+/329q/FVXXcUNN9xQbsVJxbNF1OWIEUU0R9n/0zqaxfW1uiQRERHLnVVAgqLbesTGxrJnzx4AGjRoUKYviZTAcSi0GdHZa8navQFQQBIRETmrU2xer5dp06YRFRVF48aNady4MdHR0Tz66KPV7nYdVUF+jaKZbPaDmy2uREREJDCc1RGkhx56iFdffZW///3v9OzZE4D//e9/TJ06lby8PB5//PFyLVIqlqNee9iziMhjuiebiIgInGVAeuONN3jllVe47rrrfMs6duxI/fr1+fOf/6yAVMnUahoP30CDwp3kF3pwBtmtLklERMRSZ3WK7fDhw7Ru3brY8tatW3P48OFzLkrOr5pNOwAQZ/zCzr37La5GRETEemcVkDp16sTcuXOLLZ87dy4dO3Y856Lk/DJCanDIVhuA9J/WWVuMiIhIADirU2xPPvkkAwcOZMWKFb7vQEpLS2P37t0sW7asXAuU8+OXsObUPnaI3L0bgGusLkdERMRSZ3UEqXfv3vz444/ccMMNHDlyhCNHjnDjjTfy/fff849//KO8a5TzwF2zFQBBh7ZYXImIiIj1zvp7kOLi4opdjL1+/XpeffVVXnrppXMuTM4vZ/12sBNqZOmebCIiImd1BEmqnjrNim4R08izk5yCQourERERsZYCkgAQ3ag9XgxqG5ls37nD6nJEREQspYAkRYLDOGCPBeDQtvUWFyMiImKtMl2DdOONN572+SNHjpxLLWKxI+EtiD26n/y9G4GbrC5HRETEMmUKSFFRUWd8fujQoedUkFinsHZrOPpfHIc1k01ERKq3MgWk+fPnV1QdEgBC6neAn6FWtmayiYhI9aZrkMSnbot4AJp4d5GZW2BtMSIiIhZSQBKfiLg2FGIn0shlx/atVpcjIiJiGQUk+U1QMOmOBgAc3q6ZbCIiUn0pIImfzPAWABTs+97iSkRERKyjgCR+vHXaAOD8VTPZRESk+lJAEj9hDTsAUCfnZ4srERERsY4CkviJbVl0T7am5h5+ycyxuBoRERFrKCCJn5C6LcgnmBCjgF0/b7K6HBEREUsoIIk/m5304EYA/LrzO4uLERERsYYCkhRzLLIlAJ50zWQTEZHqSQFJijFi2gIQcuRHiysRERGxhgKSFBPRqCMAMXnbME3T4mpERETOP8sD0nPPPUeTJk1wuVx069aNr7/++rTtFy9eTOvWrXG5XHTo0IFly5b5PW+aJpMnT6ZevXqEhITQp08ftm4tftuMjz76iG7duhESEkKNGjVITEwsz92q1GJaFM1ka2Lu48CRYxZXIyIicv5ZGpAWLVpEUlISU6ZMYc2aNXTq1ImEhAQOHDhQYvtVq1YxZMgQRo4cydq1a0lMTCQxMZGNGzf62jz55JPMmTOHefPm8dVXXxEWFkZCQgJ5eXm+Nu+99x533HEHI0aMYP369XzxxRfcdtttFb6/lYWzZiOyCcFheNi9dYPV5YiIiJx3lgakWbNmMWrUKEaMGEHbtm2ZN28eoaGhvPbaayW2f+aZZ+jfvz8TJkygTZs2PProo1x00UXMnTsXKDp6NHv2bCZNmsT1119Px44defPNN9m3bx9LliwBoLCwkHHjxvHUU09x5513csEFF9C2bVtuueWW87Xbgc8wyHA1BSBzl2ayiYhI9WNZQCooKGD16tX06dPnt2JsNvr06UNaWlqJ66Slpfm1B0hISPC13759O+np6X5toqKi6Natm6/NmjVr2Lt3LzabjQsvvJB69epx9dVX+x2FEsiOKprJ5s3QdyGJiEj1E2TVhg8dOoTH4yEmJsZveUxMDJs3by5xnfT09BLbp6en+54/sexUbbZt2wbA1KlTmTVrFk2aNGHmzJlcfvnl/Pjjj9SsWbPEbefn55Ofn+97nJmZCYDb7cbtdpdqn0vjRF/l2edZqdMaMv5F2NEfra/FYgEzJgJoPAKNxiOwaDzOrLSvjWUBySperxeAhx56iJtuugmA+fPn06BBAxYvXsz//d//lbje9OnTeeSRR4otX758OaGhoeVeZ0pKSrn3WRb2o9ABiM3fzocfLcNmWFpOQLB6TMSfxiOwaDwCi8bj1HJySncbLcsCUu3atbHb7WRkZPgtz8jIIDY2tsR1YmNjT9v+xO+MjAzq1avn1yY+Ph7At7xt27a+551OJ82aNWPXrl2nrHfixIkkJSX5HmdmZtKwYUP69etHZGTkmXa31NxuNykpKfTt2xeHw1Fu/ZZVYeZF8OwTNCYDul5CgzolH1mrDgJlTKSIxiOwaDwCi8bjzE6cAToTywJScHAwnTt3JjU11TfF3uv1kpqaypgxY0pcp3v37qSmpjJ+/HjfspSUFLp37w5A06ZNiY2NJTU11ReIMjMz+eqrr7jrrrsA6Ny5M06nky1bttCrVy+g6A21Y8cOGjdufMp6nU4nTqez2HKHw1Ehb8KK6rfU26/ZgCNGJNFkcnD79zSNu9KyWgKF1WMi/jQegUXjEVg0HqdW2tfF0lNsSUlJDBs2jC5dutC1a1dmz55NdnY2I0aMAGDo0KHUr1+f6dOnAzBu3Dh69+7NzJkzGThwIAsXLuTbb7/lpZdeAsAwDMaPH89jjz1Gy5Ytadq0KQ8//DBxcXG+EBYZGcmdd97JlClTaNiwIY0bN+app54CYNCgQef/RQhUhsFBV1Oic9dzbPd3gAKSiIhUH5YGpMGDB3Pw4EEmT55Meno68fHxJCcn+y6y3rVrFzbbbxPtevTowYIFC5g0aRIPPvggLVu2ZMmSJbRv397X5v777yc7O5vRo0dz5MgRevXqRXJyMi6Xy9fmqaeeIigoiDvuuIPc3Fy6devGp59+So0aNc7fzlcCuTVaQe56OFDyRfMiIiJVleUXaY8ZM+aUp9Q+++yzYssGDRp02iM9hmEwbdo0pk2bdso2DoeDGTNmMGPGjDLXW504YtvBPojILP5N5CIiIlWZ5bcakcBVs2knAOq7d1Do8VpcjYiIyPmjgCSnVKdZPAD1jUPsSs84fWMREZEqRAFJTskWVoNDtloAHPhprcXViIiInD8KSHJah0KbA5C1W7diERGR6kMBSU6roMYFANgP6Z5sIiJSfSggyWkFx7UDIOrYzxZXIiIicv4oIMlp1T5+oXbDwh3kF3qsLUZEROQ8UUCS06rVpCMAdYyj7Np96nvViYiIVCUKSHJahjOcdPvxmwD/tM7aYkRERM4TBSQ5o8NhLQDI36uZbCIiUj0oIMkZuWu1AiDoF92TTUREqgcFJDmjkPpFNwOuka2ZbCIiUj0oIMkZ1WkeD0Bjz05y8wutLUZEROQ8UECSM6rRsB2F2IgyctixQ0eRRESk6lNAkjMLcpIe1ACAQ9t0TzYREan6FJCkVI6EF81kK9inmWwiIlL1KSBJqXhqtwYg+PAWiysRERGpeApIUiphDYpmstXO0TVIIiJS9SkgSanEtLgQgMbePWTm5ltcjYiISMVSQJJSiah3Afk4CDXy2fmzvjBSRESqNgUkKR17EOmORgD8un2dtbWIiIhUMAUkKbXMyJYAuPd/b3ElIiIiFUsBSUrNrNMGgJBfNZNNRESqNgUkKbXwhh0AqJO7zeJKREREKpYCkpRavZbHZ7KZezmcmW1xNSIiIhVHAUlKLaR2E3JwEWx42PWTvlFbRESqLgUkKT2bjf3OJgAc3bne2lpEREQqkAKSlEl25AUAeNJ/sLgSERGRiqOAJGVixLQFIPTIjxZXIiIiUnEUkKRMIht1BCA2fxumaVpcjYiISMVQQJIyiTk+k62hmc6Bw0esLUZERKSCKCBJmbii65FJBHbDZM9WXagtIiJVkwKSlI1hkO5qCsCxXRssLkZERKRiKCBJmeVEF81kMw9oJpuIiFRNARGQnnvuOZo0aYLL5aJbt258/fXXp22/ePFiWrdujcvlokOHDixbtszvedM0mTx5MvXq1SMkJIQ+ffqwdevWEvvKz88nPj4ewzBYt25dee1SlRYU2w6A8MySX1MREZHKzvKAtGjRIpKSkpgyZQpr1qyhU6dOJCQkcODAgRLbr1q1iiFDhjBy5EjWrl1LYmIiiYmJbNz42zc7P/nkk8yZM4d58+bx1VdfERYWRkJCAnl5ecX6u//++4mLi6uw/auKopsUzWSLy9+B16uZbCIiUvVYHpBmzZrFqFGjGDFiBG3btmXevHmEhoby2muvldj+mWeeoX///kyYMIE2bdrw6KOPctFFFzF37lyg6OjR7NmzmTRpEtdffz0dO3bkzTffZN++fSxZssSvr48//pjly5czY8aMit7NKiWmRdFMtvrGQfZllBxkRUREKjNLA1JBQQGrV6+mT58+vmU2m40+ffqQlpZW4jppaWl+7QESEhJ87bdv3056erpfm6ioKLp16+bXZ0ZGBqNGjeIf//gHoaGh5blbVZ4jvBaHjJoA7P9pnbXFiIiIVIAgKzd+6NAhPB4PMTExfstjYmLYvHlzieukp6eX2D49Pd33/Illp2pjmibDhw/nzjvvpEuXLuzYseOMtebn55Ofn+97nJmZCYDb7cbtdp9x/dI60Vd59lkRDoY0pXbOYY7tWo/bfaXV5VSoyjIm1YXGI7BoPAKLxuPMSvvaWBqQrPLss89y7NgxJk6cWOp1pk+fziOPPFJs+fLlyyvkCFRKSkq591mews06tAFydqwudpF8VRXoY1LdaDwCi8YjsGg8Ti0nJ6dU7SwNSLVr18Zut5ORkeG3PCMjg9jY2BLXiY2NPW37E78zMjKoV6+eX5v4+HgAPv30U9LS0nA6nX79dOnShdtvv5033nij2HYnTpxIUlKS73FmZiYNGzakX79+REZGlnKPz8ztdpOSkkLfvn1xOBzl1m9522zshDXJxJnpdBgwwOpyKlRlGZPqQuMRWDQegUXjcWYnzgCdiaUBKTg4mM6dO5OamkpiYiIAXq+X1NRUxowZU+I63bt3JzU1lfHjx/uWpaSk0L17dwCaNm1KbGwsqampvkCUmZnJV199xV133QXAnDlzeOyxx3zr79u3j4SEBBYtWkS3bt1K3K7T6SwWqAAcDkeFvAkrqt/yUqd5Z1gDDdw7sNmDsNsMq0uqcIE+JtWNxiOwaDwCi8bj1Er7ulh+ii0pKYlhw4bRpUsXunbtyuzZs8nOzmbEiBEADB06lPr16zN9+nQAxo0bR+/evZk5cyYDBw5k4cKFfPvtt7z00ksAGIbB+PHjeeyxx2jZsiVNmzbl4YcfJi4uzhfCGjVq5FdDeHg4AM2bN6dBgwbnac8rt7rNi6b61zWOsGPPbpqc9JqKiIhUZpYHpMGDB3Pw4EEmT55Meno68fHxJCcn+y6y3rVrFzbbb5PtevTowYIFC5g0aRIPPvggLVu2ZMmSJbRv397X5v777yc7O5vRo0dz5MgRevXqRXJyMi6X67zvX1Vlc0WQbosh1ptBxs9rFZBERKRKsTwgAYwZM+aUp9Q+++yzYssGDRrEoEGDTtmfYRhMmzaNadOmlWr7TZo0wTT1hYdldTi0ObFZGeTs3ghcb3U5IiIi5cbyL4qUyiu/VisA7L+U/JUMIiIilZUCkpw1Z1zRPdmis36yuBIREZHypYAkZ61O86JbjjQq3EmB22NxNSIiIuVHAUnOWu3G7Sg0bUQb2ezatc3qckRERMqNApKcNcMRQnpQfQAO/rzO2mJERETKkQKSnJNfw5sDkLd3g8WViIiIlB8FJDknnuMz2Ry/bLG4EhERkfKjgCTnJKR+BwBq5vxscSUiIiLlRwFJzkndFkUz2Rp7dpGb77a4GhERkfKhgCTnpEaD1hQQRJiRz85t+sJIERGpGhSQ5NzYg9gXVHQftsPb1llbi4iISDlRQJJzdjSiBQAF+7+3uBIREZHyoYAk58xbpw0AzsOaySYiIlWDApKcs7CG7QGonatv0xYRkapBAUnOWWzziwBo5N3DsZxci6sRERE5dwpIcs4iY5uRgwunUcjOrRutLkdEROScKSDJubPZ2B/cGIAjO9ZbXIyIiMi5U0CScnEssiUAhemaySYiIpWfApKUj7ptAQg98qPFhYiIiJw7BSQpFxGNOgJQVzPZRESkClBAknJRr0U8AA3MdA4fzbS2GBERkXOkgCTlIrRWAzIJJ8jwsmfrd1aXIyIick4UkKR8GAbpzqYAHN2pmWwiIlK5KSBJucmKvgAAb8YPFlciIiJybhSQpNzYY4pmsoUf3WpxJSIiIudGAUnKTVTjoplssfnbME3T4mpERETOngKSlJvYFhcCUJ+DHPrlsMXViIiInD0FJCk3rqg6/GLUAGDP1rUWVyMiInL2FJCkXB1wNQPg2C5N9RcRkcpLAUnKVe7xmWzGgU0WVyIiInL2FJCkXAXVawdARKZmsomISOWlgCTlqkbToplscQU78Ho1k01ERConBSQpV7HN4wGoa/zK/v17rS1GRETkLCkgSblyhEaRYdQFIOPnddYWIyIicpYCIiA999xzNGnSBJfLRbdu3fj6669P237x4sW0bt0al8tFhw4dWLZsmd/zpmkyefJk6tWrR0hICH369GHr1t+uidmxYwcjR46kadOmhISE0Lx5c6ZMmUJBQUGF7F91czC0aCZb1u6NFlciIiJydiwPSIsWLSIpKYkpU6awZs0aOnXqREJCAgcOHCix/apVqxgyZAgjR45k7dq1JCYmkpiYyMaNv30YP/nkk8yZM4d58+bx1VdfERYWRkJCAnl5eQBs3rwZr9fLiy++yPfff8/TTz/NvHnzePDBB8/LPld1+TVbAWA7pJlsIiJSOVkekGbNmsWoUaMYMWIEbdu2Zd68eYSGhvLaa6+V2P6ZZ56hf//+TJgwgTZt2vDoo49y0UUXMXfuXKDo6NHs2bOZNGkS119/PR07duTNN99k3759LFmyBID+/fszf/58+vXrR7Nmzbjuuuu47777eP/998/XbldpwcdnskUd+8niSkRERM5OkJUbLygoYPXq1UycONG3zGaz0adPH9LS0kpcJy0tjaSkJL9lCQkJvvCzfft20tPT6dOnj+/5qKgounXrRlpaGrfeemuJ/R49epSaNWuestb8/Hzy8/N9jzMzMwFwu9243e7T72gZnOirPPs836Iad4CvoYF7B3l5+djtlufwc1IVxqQq0XgEFo1HYNF4nFlpXxtLA9KhQ4fweDzExMT4LY+JiWHz5s0lrpOenl5i+/T0dN/zJ5adqs3JfvrpJ5599llmzJhxylqnT5/OI488Umz58uXLCQ0NPeV6ZyslJaXc+zxvPAXUNw1qGFm89e5CIiKira6oXFTqMamCNB6BReMRWDQep5aTk1OqdpYGpECwd+9e+vfvz6BBgxg1atQp202cONHvyFVmZiYNGzakX79+REZGlls9breblJQU+vbti8PhKLd+z7f9G6fSwLuH5nVcdLlygNXlnJOqMiZVhcYjsGg8AovG48xOnAE6E0sDUu3atbHb7WRkZPgtz8jIIDY2tsR1YmNjT9v+xO+MjAzq1avn1yY+Pt5vvX379nHFFVfQo0cPXnrppdPW6nQ6cTqdxZY7HI4KeRNWVL/ny5GIFjQ4uofgtNmsrd+Jru1bW13SOavsY1LVaDwCi8YjsGg8Tq20r4ulF4cEBwfTuXNnUlNTfcu8Xi+pqal07969xHW6d+/u1x6KDiWeaN+0aVNiY2P92mRmZvLVV1/59bl3714uv/xyOnfuzPz587HZKvd1MoGmfv8kcnHRmR9ouPhqPlj6Aaapb9YWEZHKwfJUkJSUxMsvv8wbb7zBpk2buOuuu8jOzmbEiBEADB061O8i7nHjxpGcnMzMmTPZvHkzU6dO5dtvv2XMmDEAGIbB+PHjeeyxx1i6dCkbNmxg6NChxMXFkZiYCPwWjho1asSMGTM4ePAg6enpp7xGScquRpve2EZ/yoHgRtQzDjNw9UgWPT+F7DxdOCgiIoHP8muQBg8ezMGDB5k8eTLp6enEx8eTnJzsu8h6165dfkd3evTowYIFC5g0aRIPPvggLVu2ZMmSJbRv397X5v777yc7O5vRo0dz5MgRevXqRXJyMi6XCyg64vTTTz/x008/0aBBA796dJSj/Djj2lEn6X/snD+Cxhmp3HrwGVbMXE+LP75Ck3p1rC5PRETklAxTieCsZGZmEhUVxdGjR8v9Iu1ly5YxYMCAqnP+2DTZ89ETxH77BEF42WI25tCAV+jZravVlZVKlRyTSkzjEVg0HoFF43Fmpf38tvwUm1QDhkGDax7g2KD3OGKLppWxkw7LrmfJwpfxepXPRUQk8CggyXlTo92VhI75gl1hHYg0ckjcfB8fzb6Lo1l5VpcmIiLiRwFJzqvgmg1odO+n/NT0DwBcm/lPts7qx9btO6wtTERE5HcUkOT8CwqmxbDn2H3Fs+TipIt3PWGvX8nnK5OtrkxERARQQBILNew9lILhKewPakCc8QvdPrudZa89RmGhx+rSRESkmlNAEktFNelE3b+sYkuNy3EahQzY9RRfzLyFQ7/+anVpIiJSjSkgieXsIVG0umcJm9vfh8c06J27gsNzLueH79dbXZqIiFRTCkgSGAyD1jc/TPr1C/nViOICcwf137mazz/8h9WViYhINaSAJAGl/kX9cdz1X352tiHKyOayb8fw6fP3kF9QYHVpIiJSjSggScAJr9uYpvf9h+/iBgFw5YE3+OGpBNLT91pcmYiIVBcKSBKQbA4nHUe/wg/dZ5BLMBe612DOu4wNX6+0ujQREakGFJAkoLVNGMWRIcnstdWjHoe44KOb+d+iGZher9WliYhIFaaAJAGvXqvO1ByfxnfhPXEahfTa9ChfPXM7udlZVpcmIiJVlAKSVAohkTXokPRvvm0xFo9pcMnRZeyddSl7t222ujQREamCFJCk0jBsdrr84TF+7PcmvxJJC882wt+8ku9WvmN1aSIiUsUoIEml06bndRT8aSVbgloRRTbtPxvN16/dh7ew0OrSRESkilBAkkoppkELmtz3GV/WugGbYdJ118t8P7M/mb8esLo0ERGpAhSQpNJyukK5ZOzrfNnpb+SawXTI/YbsOT3ZtXGV1aWJiEglp4Akld4lN9zNrhv+xR5iqWceIGbxdWxY+qzVZYmISCWmgCRVQqv4HoSM+S+rnZfgNNx0WDOJdc8NxVOQa3VpIiJSCSkgSZVRq3ZdOk34iJX1/w+vaRB/8F/sfOpSju77yerSRESkklFAkiolKCiIK0Y9yVe9XuZXM4Jm7q3wUm92fPkvq0sTEZFKRAFJqqTufQfxyx9S2GRrSRRZNPp4GN//8yHQLUpERKQUFJCkymrRsg1x937GyvBrsBkm7bbMZcvsgRQcO2x1aSIiEuAUkKRKi4oIp3fSW3zSYjJ5poNWmav4dXZ3Dv/0jdWliYhIAFNAkirPZjNI+MNfWJ+wmD1mXWI86YS+NYAdK160ujQREQlQCkhSbXTrcQWeUZ/xZVAXXBTQ5H/38+Mrf8R051ldmoiIBBgFJKlWGjeoT4f7Pubftf6I1zS4YM977J7Vm7xDO60uTUREAogCklQ7Ya5grhkzi+T4Z/nVDKdR7mbyn+vFwXXLrC5NREQChAKSVEuGYTDghjvYdsNH/EAzosxMai25jR0fPKKvAhAREQUkqd46x8cTNeZTkp0J2DBpsn4WO567HjP3V6tLExERCykgSbVXv3YNLr/vnyyKe4B800GTXz7n0Kzu5OxaZ3VpIiJikSCrCxAJBC6HnVtGPcDHy+PpsGosDd37yX+tLweuepIal/zB6vJEpJrzek3y3IXk5uaQl5tFQW4WBbk5uPOyj//k4C3Ixp2bRfbO7aQt3YPdEYxh2DHsQRi2IAx7EDabHcPuwLDZsNkd2I4/ZwuyY9gc2O3248vt2IKO/z6xPMiBPSgIuz0Ie5Cj6Lfdht1mYDcM7DYDwzCsfqnKjQKSyHGGYTAg4WrWNU1h9z9H0MNcS93U8ezalkatoOawvz6E1YDgcAgOK/qpQv8zEJGyM02TAo+XvNx88nKzyD/+U5CbTWF+NoV5Rb89+Tl4C3IwC3LwunPBnYPhzsUozMVWmIe9MBe7J48gbx6O4z/BZj7O4z8u8gmhgFDDLF1hGyp2v0/wmAYebLixk4cNDza82CnEhhcbHsNe9Bs75u8eew07Xux4DTumceKxDdMIwmvYMA07pmEnKmEiF8T3Oj87c5KACEjPPfccTz31FOnp6XTq1Ilnn32Wrl27nrL94sWLefjhh9mxYwctW7bkiSeeYMCAAb7nTdNkypQpvPzyyxw5coSePXvywgsv0LJlS1+bw4cPM3bsWP79739js9m46aabeOaZZwgPD6/QfZXAF39BUw7cu4x3XprAzcfeptH2RTQC2Po3v3ZeDPKNEArsIRTYQnEHheEJCsXjCMMbFIYZHA7OcGzOMAxnJPaQCBwh4ThCoggOjSQ4NILg0EgMZ8TxwBUONrsl+1yhPIUU5meRl5NFfs7xD4+8bNx5WRTmZePJz6YwPwdvfjZmQQ6mOwcKcjEKiz5AbJ5c7IW5BP3uw6NNocn27/9Ooc2F1x6Mx+bCG+TEtDvxBrnA7gSHC4Jc2BwhGA4XhiMEe7ALuyMEuzOEoOBQglwhOIJDCHaF4XCFEOwMxekKxRYcUjXHoppyF3rIycklN+coednHyM/JpCAnC3duJu68LDx5WXjzszDzszHdub7g4gsvnlyCvMfff558gs08v+DiooAow0NURe1ACX+HubGTj5N8w0mB4aTA5qLQ5sRtc5Fb4MEV7CiKK2YhhunFZnqKfuPBZh7/wev7bceDzTwRZY4/Pv7bzqlDmd0wseMBPKXbF/Ok32ewIfNg6RpWAMsD0qJFi0hKSmLevHl069aN2bNnk5CQwJYtW6hbt26x9qtWrWLIkCFMnz6da665hgULFpCYmMiaNWto3749AE8++SRz5szhjTfeoGnTpjz88MMkJCTwww8/4HK5ALj99tvZv38/KSkpuN1uRowYwejRo1mwYMF53X8JTHUjQ7nh3mdZsOAiLtj6ErU4SqiRTxi5hJOHzTCxYRJi5hBSmAP8AgXnvt08nOTZQsi3hZJvC6XQHuILXl5HOOaJIOUMx+aMwOYKJ8gVSVBoBMEhEThDowgOjyIkLBJHSCTYHafdnukpxJ2XXfSXb/Yx8vOyKcjNwp2Xjef3f/nmZ2O6i/76LfrLNwejMA9bYW7Rh8fxDxCHN59gb67vL18X+QRTSBAQfvyn3JTD6306btNOvhFMAcG4jWAKDCeFtmAKjWAKbU48tmA8dhdeuxOv3YVpD4YgF2ZQUTAzgl0YQb8FM1twKHaHyxfOHK4Q7A4XQY5g7A4HQUFOghzBBAU7cTic2IOCi0JaNTpKaZomuXl55GRlkpd9lPycLApyMynIOUZh7jEK87Lx5h87HqazoCAHmzsbW2FOUYguzMHhzSHYk4vTzMNl5hJi5hFCfsUEmBKGxmMa5BlO8vktuLiPBxePPQSP3YUnKATT7sJ0hIAjFMMRguEIweYMxRYcht0ZSpArDIczjCBXGMEhYThDwo//hGELDsVhd+Cg+H9TbrebZcuWMWDAAByO0//3X2qmCV4PmB7wFmJ6C/EUFlJYWIjX68ZT6MH0uCn0FGIWFuLxFOI9/uMpPN7eU4jp9eD1uPF6PJjHn8db9Nv0FuL1nOjfg+ktxPR4aNi8U/nsw1kwTNMsZY6rGN26dePiiy9m7ty5AHi9Xho2bMjYsWN54IEHirUfPHgw2dnZfPjhh75ll1xyCfHx8cybNw/TNImLi+Mvf/kL9913HwBHjx4lJiaG119/nVtvvZVNmzbRtm1bvvnmG7p06QJAcnIyAwYMYM+ePcTFxZ2x7szMTKKiojh69CiRkZHl8VIAFfTmlnNyKDOHf32cQtcel1LgNcjNLyQvNwt3Tibu3GMU5h3Dm3cMb/4xyM+CgixsBdnY3NnYC7NxeLIJKszB6c3B6c3FZeYSauYSZuQRRtGPwyjlX19llI+DXFzkGiHk21wEmYW/hRczH6fhrpDtlsRrGuQSTC4u31+9+YYLt82F2+4q+uA4/gHidYRAUAimI6zowyM4FFtwKDZnGAQ5+WnrjzRrGIfpKcB052G6c4u+Eb0wD6MwHzy52ArzMTz52D352Lz5BPl+CnCYRT/BZj4O042TApwUEFxB43C2vKZBIXYKDTtuHHiw4zaC8BCE5/e/j/94bUGYRhAem+O337YgTMOBaXdg2hxgCyoKc7YgsAeDzYFhL1pm2B3Y7A4ICsYIcmCzB2MEBWO3O7A5gouuTQkKxu4Ixh4UjN3hxDRsfJm2ik7tW+PJz6UwNxN3fjbe3N+CjFmQg82dhc2dg60wh6DCXF+QCfbm4jLzCDFzCSGvwscgDwe5hJBnuCiwhVBgC8FtD6EwKBSPPQRvUEhRcAkKgePvPSM4FHtwKHZnKHZnGEHOMBwhYQS7wnC6wgkODcMVGoHDGVb0R4mFoVafIWdW2s9vS48gFRQUsHr1aiZOnOhbZrPZ6NOnD2lpaSWuk5aWRlJSkt+yhIQElixZAsD27dtJT0+nT58+vuejoqLo1q0baWlp3HrrraSlpREdHe0LRwB9+vTBZrPx1VdfccMNNxTbbn5+Pvn5+b7HmZmZQNGb0e0uvw+ZE32VZ59ybkKDoLYLmtdy/e5/ONHn1Gehx0uu20NWgYcD+YXk5uZSkJuJO+cYBbmZePKy8OQdw8zPwsw/BgVZGO6iDxm7uyhwOTw5BHtyij5gvDmEUPTXcjh5vuDjxI0TN9HmseJHwH/3//AT4SXv+CH7fMNFgeH0hZfC43/9eoNcRR8gQSGYjtDjHyBh2IJDsAWHEuQMK/oAcYUSHBJOsCuM4ON/+Ya4Qgh22Ak+xw8Pt9tNRm4K8X36lusHQKHHy5ECN/n5ubjzcnHnZ1OQn0thfi6FBTl48nPxFOThcefhKcg5HszyTgpmedgK8zA8+dg8+di9RQEtyFuA3SzA4c33BbNgCnCYhRTFnEIcFGI/6foSm2ESTCHBFALH//9TxlMU50MTgO3n2MlJb4sCM4hcw0We4SLPOBFkXBTaQ4vCTFAoZlDo8aOqYRjB4didodhc4ThcEThCIoree6GROEMjcIVFEhwSht0WVP5HM49zm0BhYQX0XIYa9BlyRqV9bSwNSIcOHcLj8RATE+O3PCYmhs2bN5e4Tnp6eont09PTfc+fWHa6NiefvgsKCqJmzZq+NiebPn06jzzySLHly5cvJzQ09FS7eNZSUlLKvU85N+d/TCLAGQHOeqVewzTBY0JBYSFmYT6mOxcK8+D40RTTHgS2oqMCht2JLSgYI8iJ3e44PgPlHMr1ADlATiGQefyn4ljz34iz6MeIgmCKfsqJ1wSP14vpOXGKwQPm7/7tLSw6zeEtBNOD6fVimEXLDG8hhlnU3jA92I4/Nk489l2HUojN68FG4fHrUAqxm0WP7cevSwk6/m87hUXXn5xYhqcoxB1//Fuw81CInVycRWGGooB94iih2+byXR/jsTmLjhDajp+WDHIWXS8W5Cw6HRnkxHb8t2E/y4+nguM/mQXA4eM/1Y8+Q04tJyenVO0svwapspg4caLfkavMzEwaNmxIv379yv0UW0pKCn37lu9fx3L2NCaBReMRWNxuNyuPj0dNjYfl9N/HmZ04A3Qmlgak2rVrY7fbycjI8FuekZFBbGxsievExsaetv2J3xkZGdSrV8+vTXx8vK/NgQMH/PooLCzk8OHDp9yu0+nE6XQWW+5wOCrkTVhR/crZ05gEFo1HYNF4BBaNx6mV9nWx9Ju0g4OD6dy5M6mpqb5lXq+X1NRUunfvXuI63bt392sPRYcST7Rv2rQpsbGxfm0yMzP56quvfG26d+/OkSNHWL16ta/Np59+itfrpVu3buW2fyIiIlI5WX6KLSkpiWHDhtGlSxe6du3K7Nmzyc7OZsSIEQAMHTqU+vXrM336dADGjRtH7969mTlzJgMHDmThwoV8++23vPTSS0DRl/2NHz+exx57jJYtW/qm+cfFxZGYmAhAmzZt6N+/P6NGjWLevHm43W7GjBnDrbfeWqoZbCIiIlK1WR6QBg8ezMGDB5k8eTLp6enEx8eTnJzsu8h6165d2Gy/Hejq0aMHCxYsYNKkSTz44IO0bNmSJUuW+L4DCeD+++8nOzub0aNHc+TIEXr16kVycrLvO5AA3n77bcaMGcNVV13l+6LIOXPmnL8dFxERkYBleUACGDNmDGPGjCnxuc8++6zYskGDBjFo0KBT9mcYBtOmTWPatGmnbFOzZk19KaSIiIiUyNJrkEREREQCkQKSiIiIyEkUkEREREROooAkIiIichIFJBEREZGTKCCJiIiInEQBSUREROQkCkgiIiIiJ1FAEhERETlJQHyTdmVkmiZQdCPc8uR2u8nJySEzM1N3Yg4QGpPAovEILBqPwKLxOLMTn9snPsdPRQHpLB07dgyAhg0bWlyJiIiIlNWxY8eIioo65fOGeaYIJSXyer3s27ePiIgIDMMot34zMzNp2LAhu3fvJjIystz6lbOnMQksGo/AovEILBqPMzNNk2PHjhEXF4fNduorjXQE6SzZbDYaNGhQYf1HRkbqzR1gNCaBReMRWDQegUXjcXqnO3J0gi7SFhERETmJApKIiIjISRSQAozT6WTKlCk4nU6rS5HjNCaBReMRWDQegUXjUX50kbaIiIjISXQESUREROQkCkgiIiIiJ1FAEhERETmJApKIiIjISRSQAsxzzz1HkyZNcLlcdOvWja+//trqkqql6dOnc/HFFxMREUHdunVJTExky5YtVpclx/3973/HMAzGjx9vdSnV1t69e/nDH/5ArVq1CAkJoUOHDnz77bdWl1VteTweHn74YZo2bUpISAjNmzfn0UcfPeP9xuTUFJACyKJFi0hKSmLKlCmsWbOGTp06kZCQwIEDB6wurdr5z3/+w913382XX35JSkoKbrebfv36kZ2dbXVp1d4333zDiy++SMeOHa0updr69ddf6dmzJw6Hg48//pgffviBmTNnUqNGDatLq7aeeOIJXnjhBebOncumTZt44oknePLJJ3n22WetLq3S0jT/ANKtWzcuvvhi5s6dCxTd761hw4aMHTuWBx54wOLqqreDBw9St25d/vOf/3DZZZdZXU61lZWVxUUXXcTzzz/PY489Rnx8PLNnz7a6rGrngQce4IsvvuC///2v1aXIcddccw0xMTG8+uqrvmU33XQTISEhvPXWWxZWVnnpCFKAKCgoYPXq1fTp08e3zGaz0adPH9LS0iysTACOHj0KQM2aNS2upHq7++67GThwoN9/J3L+LV26lC5dujBo0CDq1q3LhRdeyMsvv2x1WdVajx49SE1N5ccffwRg/fr1/O9//+Pqq6+2uLLKSzerDRCHDh3C4/EQExPjtzwmJobNmzdbVJVA0ZG88ePH07NnT9q3b291OdXWwoULWbNmDd98843VpVR727Zt44UXXiApKYkHH3yQb775hnvuuYfg4GCGDRtmdXnV0gMPPEBmZiatW7fGbrfj8Xh4/PHHuf32260urdJSQBI5g7vvvpuNGzfyv//9z+pSqq3du3czbtw4UlJScLlcVpdT7Xm9Xrp06cLf/vY3AC688EI2btzIvHnzFJAs8s477/D222+zYMEC2rVrx7p16xg/fjxxcXEak7OkgBQgateujd1uJyMjw295RkYGsbGxFlUlY8aM4cMPP+Tzzz+nQYMGVpdTba1evZoDBw5w0UUX+ZZ5PB4+//xz5s6dS35+Pna73cIKq5d69erRtm1bv2Vt2rThvffes6gimTBhAg888AC33norAB06dGDnzp1Mnz5dAeks6RqkABEcHEznzp1JTU31LfN6vaSmptK9e3cLK6ueTNNkzJgxfPDBB3z66ac0bdrU6pKqtauuuooNGzawbt0630+XLl24/fbbWbduncLRedazZ89iX3vx448/0rhxY4sqkpycHGw2/490u92O1+u1qKLKT0eQAkhSUhLDhg2jS5cudO3aldmzZ5Odnc2IESOsLq3aufvuu1mwYAH/+te/iIiIID09HYCoqChCQkIsrq76iYiIKHb9V1hYGLVq1dJ1YRa499576dGjB3/729+45ZZb+Prrr3nppZd46aWXrC6t2rr22mt5/PHHadSoEe3atWPt2rXMmjWLP/7xj1aXVmlpmn+AmTt3Lk899RTp6enEx8czZ84cunXrZnVZ1Y5hGCUunz9/PsOHDz+/xUiJLr/8ck3zt9CHH37IxIkT2bp1K02bNiUpKYlRo0ZZXVa1dezYMR5++GE++OADDhw4QFxcHEOGDGHy5MkEBwdbXV6lpIAkIiIichJdgyQiIiJyEgUkERERkZMoIImIiIicRAFJRERE5CQKSCIiIiInUUASEREROYkCkoiIiMhJFJBERM6SYRgsWbLE6jJEpAIoIIlIpTR8+HAMwyj2079/f6tLE5EqQPdiE5FKq3///syfP99vmdPptKgaEalKdARJRCotp9NJbGys30+NGjWAotNfL7zwAldffTUhISE0a9aMd99912/9DRs2cOWVVxISEkKtWrUYPXo0WVlZfm1ee+012rVrh9PppF69eowZM8bv+UOHDnHDDTcQGhpKy5YtWbp0qe+5X3/9ldtvv506deoQEhJCy5YtiwU6EQlMCkgiUmU9/PDD3HTTTaxfv57bb7+dW2+9lU2bNgGQnZ1NQkICNWrU4JtvvmHx4sWsWLHCLwC98MIL3H333YwePZoNGzawdOlSWrRo4beNRx55hFtuuYXvvvuOAQMGcPvtt3P48GHf9n/44Qc+/vhjNm3axAsvvEDt2rXP3wsgImfPFBGphIYNG2ba7XYzLCzM7+fxxx83TdM0AfPOO+/0W6dbt27mXXfdZZqmab700ktmjRo1zKysLN/zH330kWmz2cz09HTTNE0zLi7OfOihh05ZA2BOmjTJ9zgrK8sEzI8//tg0TdO89tprzREjRpTPDovIeaVrkESk0rriiit44YUX/JbVrFnT9+/u3bv7Pde9e3fWrVsHwKZNm+jUqRNhYWG+53v27InX62XLli0YhsG+ffu46qqrTltDx44dff8OCwsjMjKSAwcOAHDXXXdx0003sWbNGvr160diYiI9evQ4q30VkfNLAUlEKq2wsLBip7zKS0hISKnaORwOv8eGYeD1egG4+uqr2blzJ8uWLSMlJYWrrrqKu+++mxkzZpR7vSJSvnQNkohUWV9++WWxx23atAGgTZs2rF+/nuzsbN/zX3zxBTabjVatWhEREUGTJk1ITU09pxrq1KnDsGHDeOutt5g9ezYvvfTSOfUnIueHjiCJSKWVn59Penq637KgoCDfhdCLFy+mS5cu9OrVi7fffpuvv/6aV199FYDbb7+dKVOmMGzYMKZOncrBgwcZO3Ysd9xxBzExMQBMnTqVO++8k7p163L11Vdz7NgxvvjiC8aOHVuq+iZPnkznzp1p164d+fn5fPjhh76AJiKBTQFJRCqt5ORk6tWr57esVatWbN68GSiaYbZw4UL+/Oc/U69ePf75z3/Stm1bAEJDQ/nkk08YN24cF198MaGhodx0003MmjXL19ewYcPIy8vj6aef5r777qN27drcfPPNpa4vODiYiRMnsmPHDkJCQrj00ktZuHBhOey5iFQ0wzRN0+oiRETKm2EYfPDBByQmJlpdiohUQroGSUREROQkCkgiIiIiJ9E1SCJSJenqARE5FzqCJCIiInISBSQRERGRkyggiYiIiJxEAUlERETkJApIIiIiIidRQBIRERE5iQKSiIiIyEkUkEREREROooAkIiIicpL/B0dTASBdQWdvAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "plt.plot(tr_loss_his_sc_en, label=\"Training loss\")\n", + "plt.plot(val_loss_his_sc_en, label=\"Validation loss\")\n", + "plt.ylabel(\"Loss\")\n", + "plt.xlabel(\"Epochs\")\n", + "plt.title(\"SCF training: MSE energy loss\")\n", + "plt.grid()\n", + "plt.legend()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "nan_check", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/intermediate_notebooks/periodic_systems_04.ipynb b/examples/intermediate_notebooks/periodic_systems_gamma_point_only_04.ipynb similarity index 98% rename from examples/intermediate_notebooks/periodic_systems_04.ipynb rename to examples/intermediate_notebooks/periodic_systems_gamma_point_only_04.ipynb index 1f47403..2a9cdf1 100644 --- a/examples/intermediate_notebooks/periodic_systems_04.ipynb +++ b/examples/intermediate_notebooks/periodic_systems_gamma_point_only_04.ipynb @@ -1,24 +1,41 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/XanaduAI/GradDFT/blob/main/examples/intermediate_notebooks/periodic_systems_gamma_point_only_04.ipynb)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# In colab run\n", + "# !pip install git+https://github.com/XanaduAI/GradDFT.git" + ] + }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Periodic systems in Grad DFT\n", "\n", - "In this tutorial, you will learn how to train a simple neural functional for solids. Presently, we only support periodic calculations where Brillouin Zone (BZ) samping is performed at the $\\Gamma$-point. This converges the electronic structure only at the large supercell limit. We won't work with large supercells in this tutorial, so for accurate results, please consider this.\n", + "In this tutorial, you will learn how to train a simple neural functional for solids. First Brillouin Zone (1BZ) samping is performed at the $\\Gamma$-point. This converges the electronic structure only at the large supercell limit. We won't work with large supercells in this tutorial, so for accurate results, please consider this.\n", "\n", - "Full BZ sampling will be coming soon.\n", + "Full 1BZ sampling is discussed in `~examples/intermediate_notebooks/periodic_systems_bz_sampling_05.ipynb`,\n", "\n", "## Perform solid-state calculations with PySCF\n", "\n", - "PySCF implements DFT and some wavefunction methods in periodic boundary conditions with integration over the first BZ. To begin, we need:\n", + "PySCF implements DFT and some wavefunction methods in periodic boundary conditions with integration over the 1BZ. To begin, we need:\n", "\n", "(1) A DFT starting point to prime Grad DFT. Let's use the LDA.\n", "\n", "(2) Accurate training and validation data. We'll use the periodic Coupled Cluster Singles and Doubles (CCSD) implemented in PySCF\n", "\n", - "Our calculations will be run using carbon in the diamond structure." + "Our calculations will be run using Carbon in the diamond structure." ] }, { @@ -495,13 +512,6 @@ "plt.grid()\n", "plt.legend()" ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "and that's all there is! Remember, full BZ sampling is coming soon!" - ] } ], "metadata": { diff --git a/grad_dft/__init__.py b/grad_dft/__init__.py index bdea5d0..d95b263 100644 --- a/grad_dft/__init__.py +++ b/grad_dft/__init__.py @@ -24,6 +24,9 @@ grad_density, coulomb_energy ) +from .solid import ( + Solid +) from .functional import ( DispersionFunctional, Functional, @@ -59,7 +62,8 @@ diff_scf_loop ) from .interface import ( - molecule_from_pyscf, + molecule_from_pyscf, + solid_from_pyscf, loader, saver ) diff --git a/grad_dft/evaluate.py b/grad_dft/evaluate.py index b075349..92967d1 100644 --- a/grad_dft/evaluate.py +++ b/grad_dft/evaluate.py @@ -23,16 +23,16 @@ import sys -from typing import Callable, Tuple, Optional +from typing import Callable, Tuple, Optional, Union from functools import partial, reduce import time from scipy.optimize import bisect from grad_dft import ( - Molecule, + Molecule, + Solid, abs_clip, make_rdm1, - orbital_grad, Functional, energy_predictor, ) @@ -91,7 +91,7 @@ def non_scf_predictor( **kwargs, ) -> Callable: r""" - Creates an non_scf_predictor function which when called non-self consistently + Creates an non_scf_predictor function, which when called, non-self consistently calculates the total energy at a fixed density. Main parameters @@ -103,25 +103,25 @@ def non_scf_predictor( Callable """ compute_energy = energy_predictor(functional, chunk_size=chunk_size, **kwargs) - def non_scf_predictor(params: PyTree, molecule: Molecule, *args) -> Molecule: + def non_scf_predictor(params: PyTree, atoms: Union[Molecule, Solid], *args) -> Union[Molecule, Solid]: r"""Calculates the total energy at a fixed density non-self consistently. Main parameters --------------- params: Pytree Parameters of the neural functional - molecule: Molecule - A Grad-DFT molecule object + atoms: Union[Molecule, Solid] + A Grad-DFT Molecule or Solid object Returns --------- Molecule - A Grad-DFT Molecule object with updated attributes + A Grad-DFT Molecule or Solid object with updated attributes """ - predicted_e, fock = compute_energy(params, molecule, *args) - molecule = molecule.replace(fock=fock) - molecule = molecule.replace(energy=predicted_e) - return molecule + predicted_e, fock = compute_energy(params, atoms, *args) + atoms = atoms.replace(fock=fock) + atoms = atoms.replace(energy=predicted_e) + return atoms return non_scf_predictor @@ -156,7 +156,7 @@ def simple_scf_loop( compute_energy = energy_predictor(functional, chunk_size=chunk_size, **kwargs) - def simple_scf_iterator(params: PyTree, molecule: Molecule, clip_cte = 1e-30, *args) -> Molecule: + def simple_scf_iterator(params: PyTree, atoms: Union[Molecule, Solid], clip_cte = 1e-30, *args) -> Union[Molecule, Solid]: r""" Implements a scf loop for a Molecule and a functional implicitly defined compute_energy with parameters params @@ -164,15 +164,15 @@ def simple_scf_iterator(params: PyTree, molecule: Molecule, clip_cte = 1e-30, *a Parameters ---------- params: PyTree - molecule: Molecule + atoms: Molecule or Solid class *args: Arguments to be passed to compute_energy function Returns ------- - Molecule + Molecule or solid class with updated attributes """ - nelectron = molecule.atom_index.sum() - molecule.charge + nelectron = atoms.atom_index.sum() - atoms.charge # predicted_e, fock = compute_energy(params, molecule, *args) # fock = abs_clip(fock, clip_cte) @@ -181,21 +181,20 @@ def simple_scf_iterator(params: PyTree, molecule: Molecule, clip_cte = 1e-30, *a for cycle in range(cycles): # Convergence criterion is energy difference (default 1) kcal/mol and norm of gradient of orbitals < g_conv start_time = time.time() - # old_e = molecule.energy if cycle == 0: - mo_energy = molecule.mo_energy - mo_coeff = molecule.mo_coeff - fock = molecule.fock + mo_energy = atoms.mo_energy + mo_coeff = atoms.mo_coeff + fock = atoms.fock else: # Diagonalize Fock matrix - overlap = abs_clip(molecule.s1e, clip_cte) + overlap = atoms.s1e mo_energy, mo_coeff = safe_fock_solver(fock, overlap) - molecule = molecule.replace(mo_coeff=mo_coeff) - molecule = molecule.replace(mo_energy=mo_energy) + atoms = atoms.replace(mo_coeff=mo_coeff) + atoms = atoms.replace(mo_energy=mo_energy) # Update the molecular occupation - mo_occ = molecule.get_occ() - molecule = molecule.replace(mo_occ=mo_occ) + mo_occ = atoms.get_occ() + atoms = atoms.replace(mo_occ=mo_occ) if verbose > 2: print( f"Cycle {cycle} took {time.time() - start_time:.1e} seconds to compute and diagonalize Fock matrix" @@ -203,26 +202,23 @@ def simple_scf_iterator(params: PyTree, molecule: Molecule, clip_cte = 1e-30, *a # Update the density matrix if cycle == 0: - old_rdm1 = molecule.make_rdm1() + old_rdm1 = atoms.make_rdm1() else: - rdm1 = (1 - mixing_factor)*old_rdm1 + mixing_factor*abs_clip(molecule.make_rdm1(), clip_cte) - rdm1 = abs_clip(rdm1, clip_cte) - molecule = molecule.replace(rdm1=rdm1) + rdm1 = (1 - mixing_factor)*old_rdm1 + mixing_factor*atoms.make_rdm1() + atoms = atoms.replace(rdm1=rdm1) old_rdm1 = rdm1 - - computed_charge = jnp.einsum( - "r,ra,rb,sab->", molecule.grid.weights, molecule.ao, molecule.ao, molecule.rdm1 - ) + computed_charge = jnp.einsum("r,rs->", atoms.grid.weights, atoms.density()) + # This assertion was removed because the forward pass number of electrons is correct, but in backward pass, this assertion will fail. + # This doesn't mean there is an error though. Just because of batching in backwrd pass. assert jnp.isclose( nelectron, computed_charge, atol=1e-3 - ), "Total charge is not conserved" + ), "Total charge is not conserved. given electrons: %.3f, computed electrons: %.3f" % (nelectron, computed_charge) exc_start_time = time.time() - predicted_e, fock = compute_energy(params, molecule, *args) - fock = abs_clip(fock, clip_cte) - + predicted_e, fock = compute_energy(params, atoms, *args) + exc_time = time.time() if verbose > 2: @@ -231,7 +227,7 @@ def simple_scf_iterator(params: PyTree, molecule: Molecule, clip_cte = 1e-30, *a ) # Compute the norm of the gradient - norm_gorb = jnp.linalg.norm(orbital_grad(mo_coeff, mo_occ, fock)) + norm_gorb = jnp.linalg.norm(atoms.get_mo_grads()) if verbose > 1: print( @@ -247,10 +243,10 @@ def simple_scf_iterator(params: PyTree, molecule: Molecule, clip_cte = 1e-30, *a print( f"cycle: {cycle}, predicted energy: {predicted_e:.7e}, energy difference: {abs(predicted_e - old_e):.4e}, norm_gradient_orbitals: {norm_gorb:.2e}" ) - # Ensure molecule is fully updated - molecule = molecule.replace(fock=fock) - molecule = molecule.replace(energy=predicted_e) - return molecule + # Ensure atoms are fully updated + atoms = atoms.replace(fock=fock) + atoms = atoms.replace(energy=predicted_e) + return atoms return simple_scf_iterator @@ -276,25 +272,25 @@ def diff_simple_scf_loop(functional: Functional, cycles: int = 25, mixing_factor @jit def simple_scf_jitted_iterator( params: PyTree, - molecule: Molecule, + atoms: Union[Molecule, Solid], *args - ) -> Molecule: + ) -> Union[Molecule, Solid]: r""" Implements a scf loop intented for use in a jax.jit compiled function (training loop). If you are looking for a more flexible but not differentiable scf loop, see evaluate.py scf_loop. - It asks for a Molecule and a functional implicitly defined compute_energy with + It asks for a Molecule or Solid and a functional implicitly defined compute_energy with parameters params Parameters ---------- params: PyTree - molecule: Molecule + atoms: Molecule or Solid *args: Arguments to be passed to compute_energy function Returns ------- - molecule: Molecule + atoms: Molecule or Solid with updated attributes Notes: ------ @@ -305,50 +301,50 @@ def simple_scf_jitted_iterator( old_e = jnp.inf norm_gorb = jnp.inf - predicted_e, fock = compute_energy(params, molecule, *args) - molecule = molecule.replace(fock=fock) - molecule = molecule.replace(energy=predicted_e) + predicted_e, fock = compute_energy(params, atoms, *args) + atoms = atoms.replace(fock=fock) + atoms = atoms.replace(energy=predicted_e) - state = (molecule, predicted_e, old_e, norm_gorb) + state = (atoms, predicted_e, old_e, norm_gorb) def loop_body(cycle, state): old_state = state - molecule, predicted_e, old_e, norm_gorb = old_state + atoms, predicted_e, old_e, norm_gorb = old_state old_e = predicted_e - old_rdm1 = molecule.rdm1 - fock = molecule.fock + old_rdm1 = atoms.rdm1 + fock = atoms.fock # Diagonalize Fock matrix - mo_energy, mo_coeff = safe_fock_solver(fock, molecule.s1e) - molecule = molecule.replace(mo_coeff=mo_coeff) - molecule = molecule.replace(mo_energy=mo_energy) + mo_energy, mo_coeff = safe_fock_solver(fock, atoms.s1e) + atoms = atoms.replace(mo_coeff=mo_coeff) + atoms = atoms.replace(mo_energy=mo_energy) # Update the molecular occupation - mo_occ = molecule.get_occ() - molecule = molecule.replace(mo_occ=mo_occ) + mo_occ = atoms.get_occ() + atoms = atoms.replace(mo_occ=mo_occ) # Update the density matrix with linear mixing - unmixed_new_rdm1 = molecule.make_rdm1() + unmixed_new_rdm1 = atoms.make_rdm1() rdm1 = (1 - mixing_factor)*old_rdm1 + mixing_factor*unmixed_new_rdm1 - molecule = molecule.replace(rdm1=rdm1) + atoms = atoms.replace(rdm1=rdm1) # Compute the new energy and Fock matrix - predicted_e, fock = compute_energy(params, molecule, *args) - molecule = molecule.replace(fock=fock) + predicted_e, fock = compute_energy(params, atoms, *args) + atoms = atoms.replace(fock=fock) # Compute the norm of the gradient - norm_gorb = jnp.linalg.norm(orbital_grad(mo_coeff, mo_occ, fock)) + norm_gorb = jnp.linalg.norm(atoms.get_mo_grads()) - state = (molecule, predicted_e, old_e, norm_gorb) + state = (atoms, predicted_e, old_e, norm_gorb) return state # Compute the scf loop final_state = fori_loop(0, cycles, body_fun=loop_body, init_val=state) - molecule, predicted_e, old_e, norm_gorb = final_state - molecule = molecule.replace(energy=predicted_e) - return molecule + atoms, predicted_e, old_e, norm_gorb = final_state + atoms = atoms.replace(energy=predicted_e) + return atoms return simple_scf_jitted_iterator @@ -400,7 +396,8 @@ def scf_iterator(params: PyTree, molecule: Molecule, *args) -> Molecule: ------- Molecule """ - + if isinstance(molecule, Solid): + raise NotImplementedError("Solids with full BZ zampling not yet supported. Use simple_scf_loop or diff_simple_scf_loop instead.") # Needed to be able to update the chi tensor mol = mol_from_Molecule(molecule) _, mf = process_mol( @@ -535,7 +532,7 @@ def nelec_cost_fn(m, mo_es, sigma, _nelectron): ) # Compute the norm of the gradient - norm_gorb = jnp.linalg.norm(orbital_grad(mo_coeff, mo_occ, fock)) + norm_gorb = jnp.linalg.norm(molecule.get_mo_grads()) if verbose > 1: print( @@ -578,7 +575,7 @@ def nelec_cost_fn(m, mo_es, sigma, _nelectron): predicted_e, fock = compute_energy(params, molecule, *args) # Compute the norm of the gradient - norm_gorb = jnp.linalg.norm(orbital_grad(mo_coeff, mo_occ, fock)) + norm_gorb = jnp.linalg.norm(molecule.get_mo_grads()) if verbose > 1: print( @@ -716,7 +713,8 @@ def neural_iterator( ------- molecule: Molecule """ - + if isinstance(molecule, Solid): + raise NotImplementedError("Solids with full BZ zampling not yet supported. Use simple_scf_loop instead.") old_e = jnp.inf cycle = 0 @@ -931,6 +929,8 @@ def diff_scf_loop(functional: Functional, cycles: int = 25, **kwargs) -> Callabl compute_energy = energy_predictor(functional, chunk_size=None, **kwargs) + @jaxtyped + @typechecked @jit def scf_jitted_iterator( params: PyTree, @@ -1006,7 +1006,7 @@ def loop_body(cycle, state): molecule = molecule.replace(fock=fock) # Compute the norm of the gradient - norm_gorb = jnp.linalg.norm(orbital_grad(mo_coeff, mo_occ, fock)) + norm_gorb = jnp.linalg.norm(molecule.get_mo_grads()) state = (molecule, predicted_e, old_e, norm_gorb, diis_data) diff --git a/grad_dft/functional.py b/grad_dft/functional.py index 37ffb1f..6e5ffef 100644 --- a/grad_dft/functional.py +++ b/grad_dft/functional.py @@ -39,7 +39,8 @@ from grad_dft import ( abs_clip, Grid, - Molecule + Molecule, + Solid ) from grad_dft.utils.types import DType, default_dtype @@ -68,13 +69,13 @@ class Functional(nn.Module): A function that computes and returns the energy densities e_\theta that can be autodifferentiated with respect to the reduced density matrix. - densities(molecule: Molecule, *args, **kwargs) -> Array + densities(atoms: Union[Molecule, Solid], *args, **kwargs) -> Array nograd_densities : Callable, optional - A function that calculates the molecule energy densities e_\theta where gradient with respect to the + A function that calculates the energy densities e_\theta where gradient with respect to the reduced density matrix is computed via in densitygrads. - nograd_densities(molecule: Molecule, *args, **kwargs) -> Array + nograd_densities(atoms: Union[Molecule, Solid], *args, **kwargs) -> Array featuregrads: Callable, optional A function to compute contributions to the Fock matrix for energy densities @@ -82,7 +83,7 @@ class Functional(nn.Module): If given has signature - featuregrads(functional: nn.Module, params: PyTree, molecule: Molecule, + featuregrads(functional: nn.Module, params: PyTree, atoms: Union[Molecule, Solid], nograd_densities: Array, coefficient_inputs: Array, grad_densities, *args) - > Fock matrix: Array of shape (2, nao, nao) combine_densities : Callable, optional @@ -93,13 +94,13 @@ class Functional(nn.Module): A function that computes the inputs to the coefficients function, that can be autodifferentiated with respect to the reduced density matrix. - coefficient_inputs(molecule: Molecule, *args, **kwargs) -> Array + coefficient_inputs(atoms: Union[Molecule, Solid], *args, **kwargs) -> Array nograd_coefficient_inputs : Callable, optional A function that computes the inputs to the coefficients function, where gradient with respect to the reduced density matrix is computed via in coefficient_input_grads. - nograd_coefficient_inputs(molecule: Molecule, *args, **kwargs) -> Array + nograd_coefficient_inputs(atoms: Union[Molecule, Solid], *args, **kwargs) -> Array coefficient_inputs_grads: Callable, optional A function to compute contributions to the Fock matrix for coefficient inputs @@ -107,7 +108,7 @@ class Functional(nn.Module): If given has signature - coefficient_inputs_grads(functional: nn.Module, params: PyTree, molecule: Molecule, + coefficient_inputs_grads(functional: nn.Module, params: PyTree, atoms: Union[Molecule, Solid], nograd_coefficient_inputs: Array, grad_coefficient_inputs: Array, densities, *args) - > Fock matrix: Array of shape (2, nao, nao) combine_coefficient_inputs : Callable, optional @@ -156,14 +157,14 @@ def __call__(self, coefficient_inputs) -> Scalar: return self.coefficients(self, coefficient_inputs) - def compute_densities(self, molecule: Molecule, clip_cte: float = 1e-30, *args, **kwargs): + def compute_densities(self, atoms: Union[Molecule, Solid], clip_cte: float = 1e-30, *args, **kwargs): r""" Computes the densities for the functional, both with and without autodifferentiation. Parameters ---------- - molecule: Molecule - The molecule to compute the densities for + atoms: Union[Molecule, Solid] + The atoms to compute the densities for Returns ------- @@ -171,26 +172,26 @@ def compute_densities(self, molecule: Molecule, clip_cte: float = 1e-30, *args, """ if self.nograd_densities and self.energy_densities: - densities = self.energy_densities(molecule, *args, **kwargs) - nograd_densities = stop_gradient(self.nograd_densities(molecule, *args, **kwargs)) + densities = self.energy_densities(atoms, *args, **kwargs) + nograd_densities = stop_gradient(self.nograd_densities(atoms, *args, **kwargs)) densities = self.combine_densities(densities, nograd_densities) elif self.energy_densities: - densities = self.energy_densities(molecule, *args, **kwargs) + densities = self.energy_densities(atoms, *args, **kwargs) elif self.nograd_densities: - densities = stop_gradient(self.nograd_densities(molecule, *args, **kwargs)) + densities = stop_gradient(self.nograd_densities(atoms, *args, **kwargs)) densities = abs_clip(densities, clip_cte) #todo: investigate if we can lower this return densities - def compute_coefficient_inputs(self, molecule: Molecule, *args, **kwargs): + def compute_coefficient_inputs(self, atoms: Union[Molecule, Solid], *args, **kwargs): r""" Computes the inputs to the coefficients method in the functional Parameters ---------- - molecule: Molecule - The molecule to compute the inputs for the coefficients + atoms: Union[Molecule, Solid] + The atoms to compute the inputs for the coefficients Returns ------- @@ -198,17 +199,17 @@ def compute_coefficient_inputs(self, molecule: Molecule, *args, **kwargs): """ if self.nograd_coefficient_inputs and self.coefficient_inputs: - cinputs = self.coefficient_inputs(molecule, *args, **kwargs) + cinputs = self.coefficient_inputs(atoms, *args, **kwargs) nograd_cinputs = stop_gradient( - self.nograd_coefficient_inputs(molecule, *args, **kwargs) + self.nograd_coefficient_inputs(atoms, *args, **kwargs) ) cinputs = self.combine_inputs(cinputs, nograd_cinputs) elif self.coefficient_inputs: - cinputs = self.coefficient_inputs(molecule, *args, **kwargs) + cinputs = self.coefficient_inputs(atoms, *args, **kwargs) elif self.nograd_coefficient_inputs: - cinputs = stop_gradient(self.nograd_coefficient_inputs(molecule, *args, **kwargs)) + cinputs = stop_gradient(self.nograd_coefficient_inputs(atoms, *args, **kwargs)) else: cinputs = None @@ -251,7 +252,7 @@ def xc_energy( xc_energy_density = abs_clip(xc_energy_density, clip_cte) return self._integrate(xc_energy_density, grid.weights) - def energy(self, params: PyTree, molecule: Molecule, *args, **kwargs) -> Scalar: + def energy(self, params: PyTree, atoms: Union[Molecule, Solid], *args, **kwargs) -> Scalar: r""" Total energy of local functional @@ -259,7 +260,7 @@ def energy(self, params: PyTree, molecule: Molecule, *args, **kwargs) -> Scalar: --------- params: PyTree params of the neural network if there is one in self.f - molecule: Molecule + atoms: Union[Molecule, Solid] *args: other arguments to compute_densities or compute_coefficient_inputs **kwargs: other key word arguments to densities and self.xc_energy @@ -272,19 +273,45 @@ def energy(self, params: PyTree, molecule: Molecule, *args, **kwargs) -> Scalar: ------- Integrates the energy over the grid. If the function is_xc, it will add the rest of the energy components - computed with function molecule.nonXC() + computed with function atoms.nonXC() """ - densities = self.compute_densities(molecule, *args, **kwargs) - # sys.exit() - cinputs = self.compute_coefficient_inputs(molecule, *args) + densities = self.compute_densities(atoms, *args, **kwargs) + + cinputs = self.compute_coefficient_inputs(atoms, *args) - energy = self.xc_energy(params, molecule.grid, cinputs, densities, **kwargs) + energy = self.xc_energy(params, atoms.grid, cinputs, densities, **kwargs) if self.is_xc: - energy += molecule.nonXC() + energy += atoms.nonXC() return energy + + def energy_xc_only(self, params: PyTree, atoms: Union[Molecule, Solid], *args, **kwargs) -> Scalar: + r""" + Compute the XC only using the same function signature as functional.energy + + Parameters + --------- + params: PyTree + params of the neural network if there is one in self.f + atoms: Union[Molecule, Solid] + + *args: other arguments to compute_densities or compute_coefficient_inputs + **kwargs: other key word arguments to densities and self.xc_energy + + Returns + ------- + Scalar + """ + + densities = self.compute_densities(atoms, *args, **kwargs) + + cinputs = self.compute_coefficient_inputs(atoms, *args) + + Exc = self.xc_energy(params, atoms.grid, cinputs, densities, **kwargs) + + return Exc def _integrate( self, @@ -474,14 +501,13 @@ def load_checkpoint( ######################## DM21 ######################## -def dm21_coefficient_inputs(molecule: Molecule, clip_cte: Optional[float] = 1e-30, *_, **__): +def dm21_coefficient_inputs(atoms: Union[Molecule, Solid], clip_cte: Optional[float] = 1e-30, *_, **__): r""" Computes the electronic density and derivatives Parameters ---------- - molecule: - class Molecule + atoms: Union[Molecule, Solid] clip_cte: Optional[float] Needed to make sure it default 1e-30 (chosen carefully, take care if decrease) @@ -491,11 +517,11 @@ class Molecule Array: shape (n_grid, 7) where 7 is the number of features """ - rho = molecule.density() + rho = atoms.density() # We need to clip rho away from 0 to obtain good gradients. rho = jnp.maximum(abs(rho), clip_cte) * jnp.sign(rho) - grad_rho = molecule.grad_density() - tau = molecule.kinetic_density() + grad_rho = atoms.grad_density() + tau = atoms.kinetic_density() grad_rho_norm = jnp.sum(grad_rho**2, axis=-1) grad_rho_norm_sumspin = jnp.sum(grad_rho.sum(axis=1, keepdims=True) ** 2, axis=-1) @@ -506,7 +532,7 @@ class Molecule def dm21_densities( - molecule: Molecule, + atoms: Union[Molecule, Solid], functional_type: Optional[Union[str, Dict[str, int]]] = "LDA", clip_cte: float = 1e-30, *_, @@ -517,8 +543,7 @@ def dm21_densities( Parameters: ---------- - molecule: - class Molecule + atoms: Union[Molecule, Solid] functional_type: Either one of 'LDA', 'GGA', 'MGGA' or Dictionary @@ -559,10 +584,10 @@ class Molecule f"Functional type {functional_type} not recognized, must be one of LDA, GGA, MGGA." ) - # Molecule preprocessing data - rho = molecule.density() - grad_rho = molecule.grad_density() - tau = molecule.kinetic_density() + # Atoms preprocessing data + rho = atoms.density() + grad_rho = atoms.grad_density() + tau = atoms.kinetic_density() grad_rho_norm_sq = jnp.sum(grad_rho**2, axis=-1) # LDA preprocessing data @@ -654,7 +679,7 @@ def dm21_combine_densities( def dm21_hfgrads_densities( functional: nn.Module, params: PyTree, - molecule: Molecule, + atoms: Union[Molecule, Solid], ehf: Float[Array, "omega spin grid"], coefficient_inputs: Float[Array, "grid cinputs"], densities_wout_hf: Float[Array, "grid densities_whf"], @@ -670,8 +695,8 @@ def dm21_hfgrads_densities( The functional to calculate the Hartree-Fock matrix contribution for. params: PyTree The parameters of the functional. - molecule: Molecule - The molecule to calculate the Hartree-Fock matrix contribution for. + atoms: Union[Molecule, Solid] + The atoms to calculate the Hartree-Fock matrix contribution for. ehf: Float[Array, "omega spin grid"] The Hartree-Fock energy density. coefficient_inputs: Float[Array, "grid cinputs"] @@ -686,7 +711,7 @@ def dm21_hfgrads_densities( ---------- Float[Array, "spin orbitals orbitals"] """ - vxc_hf = molecule.HF_density_grad_2_Fock( + vxc_hf = atoms.HF_density_grad_2_Fock( functional, params, omegas, ehf, coefficient_inputs, densities_wout_hf ) return vxc_hf.sum(axis=0) # Sum over omega @@ -696,7 +721,7 @@ def dm21_hfgrads_densities( def dm21_hfgrads_cinputs( functional: nn.Module, params: PyTree, - molecule: Molecule, + atoms: Union[Molecule, Solid], ehf: Float[Array, "omega spin grid"], cinputs_wout_hf: Float[Array, "grid cinputs_whf"], densities: Float[Array, "grid densities"], @@ -712,8 +737,8 @@ def dm21_hfgrads_cinputs( The functional to calculate the Hartree-Fock matrix contribution for. params: PyTree The parameters of the functional. - molecule: Molecule - The molecule to calculate the Hartree-Fock matrix contribution for. + atoms: Union[Molecule, Solid] + The atoms to calculate the Hartree-Fock matrix contribution for. ehf: Float[Array, "omega spin grid"] The Hartree-Fock energy density. cinputs_wout_hf: Float[Array, "grid cinputs_whf"] @@ -727,7 +752,7 @@ def dm21_hfgrads_cinputs( ---------- Float[Array, "spin orbitals orbitals"] """ - vxc_hf = molecule.HF_coefficient_input_grad_2_Fock( + vxc_hf = atoms.HF_coefficient_input_grad_2_Fock( functional, params, omegas, ehf, cinputs_wout_hf, densities ) return vxc_hf.sum(axis=0) # Sum over omega @@ -743,20 +768,20 @@ class DM21(NeuralFunctional): coefficients: Callable = lambda self, inputs: self.default_nn(inputs) energy_densities: Callable = dm21_densities - nograd_densities: staticmethod = lambda molecule, *_, **__: molecule.HF_energy_density( + nograd_densities: staticmethod = lambda atoms, *_, **__: atoms.HF_energy_density( jnp.array([0.0, 0.4]) ) - densitygrads: staticmethod = lambda self, params, molecule, nograd_densities, cinputs, grad_densities, *_, **__: dm21_hfgrads_densities( - self, params, molecule, nograd_densities, cinputs, grad_densities, jnp.array([0.0, 0.4]) + densitygrads: staticmethod = lambda self, params, atoms, nograd_densities, cinputs, grad_densities, *_, **__: dm21_hfgrads_densities( + self, params, atoms, nograd_densities, cinputs, grad_densities, jnp.array([0.0, 0.4]) ) combine_densities: staticmethod = dm21_combine_densities coefficient_inputs: staticmethod = dm21_coefficient_inputs - nograd_coefficient_inputs: staticmethod = lambda molecule, *_, **__: molecule.HF_energy_density( + nograd_coefficient_inputs: staticmethod = lambda atoms, *_, **__: atoms.HF_energy_density( jnp.array([0.0, 0.4]) ) - coefficient_input_grads: staticmethod = lambda self, params, molecule, nograd_cinputs, grad_cinputs, densities, *_, **__: dm21_hfgrads_cinputs( - self, params, molecule, nograd_cinputs, grad_cinputs, densities, jnp.array([0.0, 0.4]) + coefficient_input_grads: staticmethod = lambda self, params, atoms, nograd_cinputs, grad_cinputs, densities, *_, **__: dm21_hfgrads_cinputs( + self, params, atoms, nograd_cinputs, grad_cinputs, densities, jnp.array([0.0, 0.4]) ) combine_inputs: staticmethod = dm21_combine_cinputs @@ -1021,7 +1046,7 @@ def fzeta(z): def densities( - molecule: Molecule, + atoms: Union[Molecule, Solid], functional_type: Optional[Union[str, Dict[str, int]]] = "LDA", clip_cte: float = 1e-30, *_, @@ -1032,8 +1057,7 @@ def densities( Parameters: ---------- - molecule: - class Molecule + atoms: Union[Molecule, Solid] functional_type: Either one of 'LDA', 'GGA', 'MGGA' or Dictionary @@ -1074,10 +1098,10 @@ class Molecule f"Functional type {functional_type} not recognized, must be one of LDA, GGA, MGGA." ) - # Molecule preprocessing data - rho = molecule.density() - grad_rho = molecule.grad_density() - tau = molecule.kinetic_density() + # Atoms preprocessing data + rho = atoms.density() + grad_rho = atoms.grad_density() + tau = atoms.kinetic_density() grad_rho_norm_sq = jnp.sum(grad_rho**2, axis=-1) # LDA preprocessing data @@ -1234,11 +1258,14 @@ def head(self, x: Array, local_features, sigmoid_scale_factor): return jnp.squeeze(out) # Eliminating unnecessary dimensions - def energy(self, params: PyTree, molecule: Molecule): + def energy(self, params: PyTree, atoms: Union[Molecule, Solid]): r""" Calculates the energy of the functional. """ - R_AB, ai = calculate_distances(molecule.nuclear_pos, molecule.atom_index) + if isinstance(atoms, Solid): + raise NotImplementedError("Dispersion functionals are not presently implemented for solids") + + R_AB, ai = calculate_distances(atoms.nuclear_pos, atoms.atom_index) result = 0 for n in range(3, 6): @@ -1250,7 +1277,7 @@ def energy(self, params: PyTree, molecule: Molecule): def calculate_distances(positions, atoms): r""" - Calculates the distances between all atoms in the molecule. + Calculates the distances between all atoms. """ pairwise_distances = jnp.linalg.norm(positions[:, None] - positions, axis=-1) atom_pairs = jnp.array( diff --git a/grad_dft/interface/__init__.py b/grad_dft/interface/__init__.py index 29cebee..177738d 100644 --- a/grad_dft/interface/__init__.py +++ b/grad_dft/interface/__init__.py @@ -14,6 +14,7 @@ from .pyscf import ( molecule_from_pyscf, + solid_from_pyscf, grid_from_pyscf, mol_from_Molecule, saver, diff --git a/grad_dft/interface/pyscf.py b/grad_dft/interface/pyscf.py index 6abd096..387e683 100644 --- a/grad_dft/interface/pyscf.py +++ b/grad_dft/interface/pyscf.py @@ -14,7 +14,7 @@ from random import shuffle from typing import List, Optional, Tuple, Union, Sequence, Dict -from itertools import chain, combinations_with_replacement +from itertools import chain, combinations_with_replacement, product import os import numpy as np @@ -24,12 +24,19 @@ from pyscf import scf # type: ignore from pyscf.dft import Grids, numint # type: ignore +from pyscf.pbc.dft import numint as pbc_numint from pyscf.gto import Mole import pyscf.data.elements as elements from pyscf.pbc.gto.cell import Cell +from pyscf.pbc.lib.kpts import KPoints +from pyscf.pbc.df.fft import FFTDF +from pyscf.pbc.df.mdf import MDF +from pyscf.pbc.df.df import GDF +from pyscf.ao2mo import restore # from qdft.reaction import Reaction, make_reaction, get_grad from grad_dft.molecule import Grid, Molecule, Reaction, make_reaction +from grad_dft.solid import Solid, KPointInfo from grad_dft.utils import DType, default_dtype, DensityFunctional, HartreeFock from jax.tree_util import tree_map from grad_dft.external import NeuralNumInt @@ -40,7 +47,7 @@ import h5py from pyscf import cc, dft, scf -from jaxtyping import Array, Scalar, Int +from jaxtyping import Array, Scalar, Int, Bool from grad_dft.external import _nu_chunk @@ -52,6 +59,48 @@ def grid_from_pyscf(grids: Grids, dtype: Optional[DType] = None) -> Grid: return Grid(coords, weights) +def kpt_info_from_pyscf(kmf: DensityFunctional): + kpts = kmf.kpts + if isinstance(kpts, KPoints): + msg = """PySCF KPoint object detected. Symmetry adapted calculations are not yet possible. Please ensure + that the supplied k-points to the PySCF Molecule object have space_group_symmetry=False and time_reversal_symmetry=False. + """ + raise NotImplementedError(msg) + # 1BZ single k-points: kinetic + external terms + kpts_abs = kpts.kpts + kpts_scaled = kpts.kpts_scaled + weights = kpts.weights_ibz + bz2ibz_map = kpts.bz2ibz + ibz2bz_map = kpts.ibz2bz + kpts_ir_abs = kpts.kpts_ibz + kpts_ir_scaled = kpts.kpts_scaled_ibz + else: + # No symmetries used + + # Equal weights + + # bz2ibz_map = None + # ibz2bz_map = None + # kpts_ir_abs = None + # kpts_ir_scaled = None + kpts_abs, kpts_scaled, weights = \ + to_device_arrays( + kpts, + kmf.cell.get_scaled_kpts(kpts), + np.ones(shape=(kpts.shape[0],))/kpts.shape[0], + dtype=None + ) + return KPointInfo( + kpts_abs, + kpts_scaled, + weights, + # bz2ibz_map, + # ibz2bz_map, + # kpts_ir_abs, + # kpts_ir_scaled, + ) + + def molecule_from_pyscf( mf: DensityFunctional, @@ -62,7 +111,10 @@ def molecule_from_pyscf( scf_iteration: Scalar = jnp.int32(50), chunk_size: Optional[Scalar] = jnp.int32(1024), grad_order: Optional[Scalar] = jnp.int32(2), -) -> Molecule: +) -> Molecule: + if hasattr(mf, "kpts"): + if not np.array_equal(mf.kpts, np.array([[0.0, 0.0, 0.0]])): + raise RuntimeError("Input was periodic with BZ sampling beyond gamma-point only. Use solid_from_pyscf instead.") # mf, grids = _maybe_run_kernel(mf, grids) grid = grid_from_pyscf(mf.grids, dtype=dtype) @@ -81,11 +133,12 @@ def molecule_from_pyscf( s1e, fock, rep_tensor, + kpt_info, ) = to_device_arrays(*_package_outputs(mf, mf.grids, scf_iteration, grad_order), dtype=dtype) atom_index, nuclear_pos = to_device_arrays( [elements.ELEMENTS.index(e) for e in mf.mol.elements], - mf.mol.atom_coords(unit="angstrom"), + mf.mol.atom_coords(unit="bohr"), dtype=dtype, ) @@ -146,6 +199,108 @@ def molecule_from_pyscf( scf_iteration, fock, ) + +def solid_from_pyscf( + kmf: DensityFunctional, + dtype: Optional[DType] = None, + omegas: Optional[Array] = None, + energy: Optional[Scalar] = None, + name: Optional[Array] = None, + scf_iteration: Scalar = jnp.int32(50), + chunk_size: Optional[Scalar] = jnp.int32(1024), + grad_order: Optional[Scalar] = jnp.int32(2), +) -> Solid: + if np.array_equal(kmf.kpts, np.array([[0.0, 0.0, 0.0]])): + raise RuntimeError("Use molecule_from_pyscf for Gamma point only calculations") + elif not hasattr(kmf, "cell"): + raise RuntimeError("Input was an isolated system. Use molecule_from_pyscf instead.") + + + grid = grid_from_pyscf(kmf.grids, dtype=dtype) + pyscf_dat = _package_outputs(kmf, kmf.grids, scf_iteration, grad_order) + kpt_info = pyscf_dat[-1] + ( + ao, + grad_ao, + grad_n_ao, + rdm1, + energy_nuc, + h1e, + vj, + mo_coeff, + mo_energy, + mo_occ, + mf_e_tot, + s1e, + fock, + rep_tensor, + ) = to_device_arrays(*pyscf_dat[0:-1], dtype=dtype) + + atom_index, nuclear_pos = to_device_arrays( + [elements.ELEMENTS.index(e) for e in kmf.mol.elements], + kmf.mol.atom_coords(unit="bohr"), + dtype=dtype, + ) + + basis = jnp.array( + [ord(char) for char in kmf.mol.basis] + ) # jax doesn't support strings, so we convert it to integers + unit_Angstrom = True + if name: + name = jnp.array([ord(char) for char in name]) + + if omegas is not None: + chi = generate_chi_tensor( + rdm1=rdm1, + ao=ao, + grid_coords=grid.coords, + mol=kmf.mol, + omegas=omegas, + chunk_size=chunk_size, + ) + # chi = to_device_arrays(chi, dtype=dtype) + # omegas = to_device_arrays(omegas, dtype=dtype) + else: + chi = None + + spin = jnp.int32(kmf.mol.spin) + charge = jnp.int32(kmf.mol.charge) + if isinstance(kmf.grids, Grids): # check if it's the open boundary grid. Otherwise we have a uniform grid with no level + grid_level = jnp.int32(kmf.grids.level) + else: + grid_level = None + lattice_vectors = kmf.cell.lattice_vectors() + return Solid( + grid, + kpt_info, + atom_index, + lattice_vectors, + nuclear_pos, + ao, + grad_ao, + grad_n_ao, + rdm1, + energy_nuc, + h1e, + vj, + mo_coeff, + mo_occ, + mo_energy, + mf_e_tot, + s1e, + omegas, + chi, + rep_tensor, + energy, + basis, + name, + spin, + charge, + unit_Angstrom, + grid_level, + scf_iteration, + fock, + ) def mol_from_Molecule(molecule: Molecule): @@ -525,10 +680,6 @@ def ao_grads(mol: Mole, coords: Array, order=2) -> Dict: .. math:: \nabla^n \psi - Parameters - ---------- - mf: PySCF Density Functional object. - Outputs ---------- Dict @@ -553,17 +704,113 @@ def ao_grads(mol: Mole, coords: Array, order=2) -> Dict: i += 1 return result +def pbc_ao_grads(cell: Cell, coords: Array, order=2, kpts=None) -> Dict: + r"""Function to compute nth order crystal atomic orbital grads, for n > 1. + + .. math:: + \nabla^n \psi + + Outputs + ---------- + Dict + For each order n > 1, result[n] is an array of shape + (n_kpt, n_grid, n_ao, 3) where the fourth coordinate indicates + .. math:: + \frac{\partial^n \psi}{\partial x_i^n} + + for :math:`x_i` is one of the usual cartesian coordinates x, y or z. + """ + if kpts is None: + # Default is Gamma only + ao_ = pbc_numint.eval_ao_kpts(cell, coords, kpts=np.zeros(3), deriv=order) + ao_ = np.asarray(ao_) + aos = ao_[:, 0, :, :] + res_shape = (1, aos.shape[1], aos.shape[2], 0) + else: + ao_ = pbc_numint.eval_ao_kpts(cell, coords, kpts=kpts, deriv=order) + ao_ = np.asarray(ao_) + aos = ao_[:, 0, :, :] + res_shape = (kpts.shape[0], aos.shape[1], aos.shape[2], 0) + if order == 0: + return ao_ + result = {} + i = 4 + for n in range(2, order + 1): + result[n] = jnp.empty(res_shape) + for c in combinations_with_replacement("xyz", r=n): + if len(set(c)) == 1: + result[n] = jnp.concatenate((result[n], jnp.expand_dims(ao_[:, i, :, :], axis=3)), axis=3) + i += 1 + return result + +def calc_eri_with_pyscf(mf, kpts=np.zeros(3)) -> np.ndarray: + r"""Calculate the ERIs using the method detected from the PySCF mean field object. + + Inputs + ---------- + + mf: + PySCF mean field object + kpts: + Array of k-points (absolute, not fractional). + + Outputs + ---------- + np.ndarray + + The ERIs. Output shape is (nao, nao, nao, nao) for isolated molecules and gamma-point only + periodic calculations. For full BZ calculations, the output shape is (nkpt, nkpt, nao, nao, nao, nao). + """ + # Solid or Isolated molecule? + if hasattr(mf, "cell"): # Periodic system + + # Check for the three density fitting methods. DF is always used for periodic calculations + if isinstance(mf.with_df, FFTDF): + density_fitter = FFTDF(mf.cell, kpts=kpts) + elif isinstance(mf.with_df, MDF): # Check for MDF before GDF becuase MDF inherits from GDF + density_fitter = MDF(mf.cell, kpts=kpts) + elif isinstance(mf.with_df, GDF): + density_fitter = GDF(mf.cell, kpts=kpts) + + # Calculate the Periodic ERI's. + if np.array_equal(kpts, np.zeros(3)): + # Assume Gamma point only + eri_compressed = density_fitter.get_eri(kpts=np.zeros(3)) + eri = restore(1, eri_compressed, mf.cell.nao_nr()) + else: + # Loop over all k-pairs. This will be a fall back in the future. We will encourage users + # to save ERIs to disk after a PySCF calculation. + nkpt = kpts.shape[0] + nao = mf.cell.nao_nr() + # Empty array for all k points in uncompressed format. + eri = np.empty(shape=(nkpt, nkpt, nao, nao, nao, nao), dtype=np.complex128) + for ikpt, jkpt in product(range(nkpt), range(nkpt)): + k_quartet = np.array([kpts[ikpt], kpts[ikpt], kpts[jkpt], kpts[jkpt]]) + eri_kquartet =\ + density_fitter.get_eri(compact=False, kpts=k_quartet).reshape(nao, nao, nao, nao) + eri[ikpt, jkpt, :, :, :, :] = eri_kquartet + + else: # Isolated system + try: + _ = mf.with_df + except(AttributeError): + eri = mf.mol.intor("int2e") + return eri + # Use default DF method when DF is used on molecules + density_fitter = df.DF(mf.mol) + eri_compressed = density_fitter.get_eri() + eri = restore(1, eri_compressed, mf.mol.nao_nr()) + return eri + + + def _package_outputs( mf: DensityFunctional, grids: Optional[Grids] = None, scf_iteration: Scalar = jnp.int32(50), grad_order: Scalar = jnp.int32(2), -): - ao_ = numint.eval_ao(mf.mol, grids.coords, deriv=1) # , non0tab=grids.non0tab) - ao = ao_[0] - nao = ao.shape[1] - +): if scf_iteration != 0: rdm1 = mf.make_rdm1(mf.mo_coeff, mf.mo_occ) else: @@ -574,6 +821,10 @@ def _package_outputs( # Restricted (non-spin polarized), open boundary conditions if rdm1.ndim == 2 and not hasattr(mf, "cell"): + ao_and_1deriv = numint.eval_ao(mf.mol, grids.coords, deriv=1) # , non0tab=grids.non0tab) + ao = ao_and_1deriv[0] + grad_ao = ao_and_1deriv[1:4].transpose(1, 2, 0) + grad_n_ao = ao_grads(mf.mol, jnp.array(mf.grids.coords), order=grad_order) s1e = mf.get_ovlp(mf.mol) h1e = mf.get_hcore(mf.mol) half_dm = rdm1 / 2 @@ -590,10 +841,15 @@ def _package_outputs( ) # The 2 is to compensate for the /2 in the definition of the density matrix dm = mf.make_rdm1(mf.mo_coeff, mf.mo_occ) fock = np.stack([h1e, h1e], axis=0) + mf.get_veff(mf.mol, dm) - rep_tensor = mf.mol.intor("int2e") + rep_tensor = calc_eri_with_pyscf(mf) + kpt_info = None # Unrestricted (spin polarized), open boundary conditions elif rdm1.ndim == 3 and not hasattr(mf, "cell"): + ao_and_1deriv = numint.eval_ao(mf.mol, grids.coords, deriv=1) # , non0tab=grids.non0tab) + ao = ao_and_1deriv[0] + grad_ao = ao_and_1deriv[1:4].transpose(1, 2, 0) + grad_n_ao = ao_grads(mf.mol, jnp.array(mf.grids.coords), order=grad_order) s1e = mf.get_ovlp(mf.mol) h1e = mf.get_hcore(mf.mol) mo_coeff = np.stack(mf.mo_coeff, axis=0) @@ -604,11 +860,17 @@ def _package_outputs( ) # The 2 is to compensate for the /2 in the definition of the density matrix dm = mf.make_rdm1(mf.mo_coeff, mf.mo_occ) fock = np.stack([h1e, h1e], axis=0) + mf.get_veff(mf.mol, dm) - rep_tensor = mf.mol.intor("int2e") + rep_tensor = calc_eri_with_pyscf(mf) + kpt_info = None # Restricted (non-spin polarized), periodic boundary conditions, full BZ sampling elif rdm1.ndim == 3 and hasattr(mf, "cell") and rdm1.shape[0] != 1: - print(rdm1.shape) + ao_and_1deriv = pbc_numint.eval_ao_kpts(mf.cell, grids.coords, kpts=mf.kpts, deriv=1) + ao_and_1deriv = np.asarray(ao_and_1deriv) + ao = ao_and_1deriv[:, 0, :, :] + grad_ao = ao_and_1deriv[:, 1:4, :, :].transpose(0, 2, 3, 1) + grad_n_ao = pbc_ao_grads(mf.cell, jnp.array(mf.grids.coords), order=grad_order, kpts=mf.kpts) + # grad_n_ao = ao_grads(mf.mol, jnp.array(mf.grids.coords), order=grad_order) s1e = mf.get_ovlp(mf.mol) h1e = mf.get_hcore(mf.mol) @@ -627,11 +889,20 @@ def _package_outputs( ) # The 2 is to compensate for the /2 in the definition of the density matrix dm = mf.make_rdm1(mf.mo_coeff, mf.mo_occ) fock = np.stack([h1e, h1e], axis=0) + mf.get_veff(mf.mol, dm) - rep_tensor = df.DF(mf.cell).get_eri(compact=False).reshape(nao, nao, nao, nao) + + kpt_info = kpt_info_from_pyscf(mf) + # Compute ERIs for all pairs of k-points. Needed for Coulomb energy calculation + rep_tensor = calc_eri_with_pyscf(mf, kpts=mf.kpts) # Unrestricted (spin polarized), periodic boundary conditions, full BZ sampling elif rdm1.ndim == 4 and hasattr(mf, "cell") and rdm1.shape[1] != 1: + ao_and_1deriv = pbc_numint.eval_ao_kpts(mf.cell, grids.coords, kpts=mf.kpts, deriv=1) + ao_and_1deriv = np.asarray(ao_and_1deriv) + ao = ao_and_1deriv[:, 0, :, :] + grad_ao = ao_and_1deriv[:, 1:4, :, :].transpose(0, 2, 3, 1) + grad_n_ao = pbc_ao_grads(mf.cell, jnp.array(mf.grids.coords), order=grad_order, kpts=mf.kpts) + s1e = mf.get_ovlp(mf.mol) h1e = mf.get_hcore(mf.mol) mo_coeff = np.stack(mf.mo_coeff, axis=0) @@ -643,13 +914,25 @@ def _package_outputs( ) dm = mf.make_rdm1(mf.mo_coeff, mf.mo_occ) - print("right") fock = np.stack([h1e, h1e], axis=0) + mf.get_veff(mf.mol, dm) - rep_tensor = df.DF(mf.cell).get_eri(compact=False).reshape(nao, nao, nao, nao) + + kpt_info = kpt_info_from_pyscf(mf) + # Compute ERIs for all pairs of k-points. Needed for Coulomb energy calculation + rep_tensor = calc_eri_with_pyscf(mf, kpts=mf.kpts) # Restricted (non-spin polarized), periodic boundary conditions, gamma point only elif rdm1.ndim == 3 and hasattr(mf, "cell") and rdm1.shape[0] == 1: + ao_and_1deriv = pbc_numint.eval_ao_kpts(mf.cell, grids.coords, kpts=mf.kpts, deriv=1) + ao_and_1deriv = np.asarray(ao_and_1deriv) + ao = ao_and_1deriv[:, 0, :, :] + grad_ao = ao_and_1deriv[:, 1:4, :, :].transpose(0, 2, 3, 1) + grad_n_ao = pbc_ao_grads(mf.cell, jnp.array(mf.grids.coords), order=grad_order) # Collapse the redundant extra dimension from k-points: gamma only + ao = np.squeeze(ao, axis=0) + grad_ao = np.squeeze(grad_ao, axis=0) + for key in grad_n_ao.keys(): + grad_n_ao[key] = np.squeeze(grad_n_ao[key], axis=0) + s1e = mf.get_ovlp(mf.mol) s1e = np.squeeze(s1e, axis=0) h1e = mf.get_hcore(mf.mol) @@ -679,10 +962,22 @@ def _package_outputs( fock = np.squeeze(fock, axis=1) vj = np.squeeze(vj, axis=1) h1e = np.squeeze(h1e, axis=0) - rep_tensor = df.DF(mf.cell).get_eri(compact=False).reshape(nao, nao, nao, nao) + rep_tensor = calc_eri_with_pyscf(mf) + kpt_info = None # Unrestricted (spin polarized), periodic boundary conditions, gamma point only elif rdm1.ndim == 4 and hasattr(mf, "cell") and rdm1.shape[1] == 1: + ao_and_1deriv = pbc_numint.eval_ao_kpts(mf.cell, grids.coords, kpts=mf.kpts, deriv=1) + ao_and_1deriv = np.asarray(ao_and_1deriv) + ao = ao_and_1deriv[:, 0, :, :] + grad_ao = ao_and_1deriv[:, 1:4, :, :].transpose(0, 2, 3, 1) + grad_n_ao = pbc_ao_grads(mf.cell, jnp.array(mf.grids.coords), order=grad_order) + + # Collapse the redundant extra dimension from k-points: gamma only + for key in grad_n_ao.keys(): + grad_n_ao[key] = np.squeeze(grad_n_ao[key], axis=0) + ao = np.squeeze(ao, axis=0) + grad_ao = np.squeeze(grad_ao, axis=0) s1e = mf.get_ovlp(mf.mol) s1e = np.squeeze(s1e, axis=0) h1e = mf.get_hcore(mf.mol) @@ -706,15 +1001,13 @@ def _package_outputs( fock = np.squeeze(fock, axis=1) vj = np.squeeze(vj, axis=1) h1e = np.squeeze(h1e, axis=0) - rep_tensor = df.DF(mf.cell).get_ao_eri(compact=False).reshape(nao, nao, nao, nao) + rep_tensor = calc_eri_with_pyscf(mf) + kpt_info = None else: raise RuntimeError( f"Invalid density matrix shape. Got {rdm1.shape} for AO shape {ao.shape}" ) - - grad_ao = ao_[1:4].transpose(1, 2, 0) - grad_n_ao = ao_grads(mf.mol, jnp.array(mf.grids.coords), order=grad_order) mf_e_tot = mf.e_tot energy_nuc = mf.energy_nuc() @@ -733,6 +1026,7 @@ def _package_outputs( s1e, fock, rep_tensor, + kpt_info ) diff --git a/grad_dft/molecule.py b/grad_dft/molecule.py index f8532bd..580430b 100644 --- a/grad_dft/molecule.py +++ b/grad_dft/molecule.py @@ -10,7 +10,7 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. +# limitations under the License. from typing import List, Optional, Union, Sequence, Tuple, NamedTuple from dataclasses import fields @@ -37,9 +37,6 @@ class Grid: coords: Array weights: Array - # def __repr__(self): - # return f"{self.__class__.__name__}(size={len(self)})" - def __len__(self): return self.weights.shape[0] @@ -107,6 +104,15 @@ class Molecule: @property def grid_size(self): return len(self.grid) + + def get_coulomb_potential(self, *args, **kwargs) -> Float[Array, "orbitals orbitals"]: + r"""Compute the Coulomb potential matrix. + + Returns + ------- + Float[Array, "spin orbitals orbitals"] + """ + return coulomb_potential(self.rdm1.sum(axis=0), self.rep_tensor, *args, **kwargs) def density(self, *args, **kwargs) -> Array: r""" Computes the electronic density of a molecule at each grid point. @@ -312,6 +318,16 @@ def get_occ(self) -> Array: nelecs = jnp.array([self.mo_occ[i].sum() for i in range(2)], dtype=jnp.int64) naos = self.mo_occ.shape[1] return get_occ(self.mo_energy, nelecs, naos) + + def get_mo_grads(self, *args, **kwargs): + r"""Compute the gradient of the electronic energy with respect + to the molecular orbital coefficients. + + Returns: + ------- + Float[Array, "orbitals orbitals"] + """ + return orbital_grad(self.mo_coeff, self.mo_occ, self.fock, *args, **kwargs) def to_dict(self) -> dict: r""" Returns a dictionary with the attributes of the molecule.""" @@ -331,7 +347,8 @@ def orbital_grad( F: Float[Array, "spin orbitals orbitals"], precision: Precision = Precision.HIGHEST ) -> Float[Array, "orbitals orbitals"]: - r""" Computes the restricted Hartree Fock orbital gradients + r"""Compute the gradient of the electronic energy with respect + to the molecular orbital coefficients. Parameters: ---------- @@ -350,7 +367,7 @@ def orbital_grad( Notes: ----- - # Similar to pyscf/scf/hf.py: + # Performs same task as pyscf/scf/hf.py: occidx = mo_occ > 0 viridx = ~occidx g = reduce(jnp.dot, (mo_coeff[:,viridx].conj().T, fock_ao, diff --git a/grad_dft/solid.py b/grad_dft/solid.py new file mode 100644 index 0000000..f6a3ea2 --- /dev/null +++ b/grad_dft/solid.py @@ -0,0 +1,674 @@ +# Copyright 2023 Xanadu Quantum Technologies Inc. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import jax.numpy as jnp +from jax.lax import Precision +from typing import List, Optional +import jax + +from typeguard import typechecked +from grad_dft.utils import vmap_chunked +from functools import partial +from jax import jit, vmap +from jax.lax import fori_loop, cond + +from dataclasses import fields + +from flax import struct +from flax import linen as nn +from jaxtyping import Array, PyTree, Scalar, Float, Int, Complex, jaxtyped + + +@struct.dataclass +class Grid: + r""" Base class for the grid coordinates and integration grids.""" + coords: Array + weights: Array + + def __len__(self): + return self.weights.shape[0] + + def to_dict(self) -> dict: + return {"coords": self.coords, "weights": self.weights} + + def integrate(self, vals: Array, axis: int = 0) -> Array: + r""" + A function that performs grid quadrature (integration) in a differentiable way (using jax.numpy). + + This function is glorified tensor contraction but it sometimes helps + with readability and expresing intent in the rest of the code. + + Parameters + ---------- + vals : Array + Local features/ function values to weigh. + Expected shape: (..., n_lattice, ...) + axis : int, optional + Axis to contract. vals.shape[axis] == n_lattice + has to hold. + + Returns + ------- + Array + Integrals of the same as `vals` but with `axis` contracted out. + If vals.ndim==(1,), then the output is squeezed to a scalar. + """ + + return jnp.tensordot(self.weights, vals, axes=(0, axis)) + +@struct.dataclass +class KPointInfo: + r"""Contains the neccesary information about BZ sampling needed for total energy calculations. + Most simply, we need the array of k-points in absolute and fractional forms with equal weights. + To properly take advantage of space-group and time-reversal symmetry, informations about mappings + between the BZ -> IBZ and vice versa is needed as well as weights which are not neccesarily equal. + + n_kpts_or_n_ikpts in weights could be the total number of points in the full BZ or the number of + points in the IBZ, context dependent. I.e, if the next variables are set to None, + the first case applies. If they are not None, the second does. + """ + + kpts_abs: Float[Array, "n_kpts 3"] + kpts_scaled: Float[Array, "n_kpts 3"] + weights: Float[Array, "n_kpts_or_n_ir_kpts"] + # Coming Soon: take advantage of Space Group symmetry for efficient simulation + # bz2ibz_map: Optional[Float[Array, "n_kpts"]] + # ibz2bz_map: Optional[Float[Array, "n_kpts_ir"]] + # kpts_ir_abs: Optional[Float[Array, "n_kpts_ir 3"]] + # kpts_ir_scaled: Optional[Float[Array, "n_kpts_ir 3"]] + + def to_dict(self) -> dict: + info = { + "kpts_abs": self.kpts_abs, + "kpts_scaled": self.kpts_scaled, + "kpt_weights": self.weights + } + return info + +@struct.dataclass +class Solid: + r"""Base class for storing data pertaining to DFT calculations with solids. + This shares many simlarities ~/grad_dft/molecule.py's `Molecule` class, but many arrays + must have an extra dimension to house the number of k-points. + + Typically, for those arrays which need a k-point index, if a spin index is required, + dimension 1 will be dimension n_kpt. If spin is not required, dimension 0 will be + n_kpt. + """ + + grid: Grid + kpt_info: KPointInfo + atom_index: Int[Array, "n_atom"] + lattice_vectors: Float[Array, "3 3"] + nuclear_pos: Float[Array, "n_atom 3"] + ao: Complex[Array, "n_kpt n_flat_grid n_orbitals"] # ao = Crystal Atomic Orbitals in PBC case + grad_ao: Complex[Array, "nkpt n_flat_grid n_orbitals 3"] + grad_n_ao: PyTree + rdm1: Complex[Array, "n_spin n_kpt n_orbitals n_orbitals"] + nuclear_repulsion: Scalar + h1e: Complex[Array, "n_kpt n_orbitals n_orbitals"] + vj: Complex[Array, "n_spin n_kpt n_orbitals n_orbitals"] + mo_coeff: Complex[Array, "n_spin n_kpt n_orbitals n_orbitals"] + mo_occ: Float[Array, "n_spin n_kpt n_orbitals"] + mo_energy: Float[Array, "n_spin n_kpt n_orbitals"] + mf_energy: Optional[Scalar] = None + s1e: Optional[Complex[Array, "n_kpt n_orbitals n_orbitals"]] = None + omegas: Optional[Float[Array, "omega"]] = None + chi: Optional[Float[Array, "grid omega spin orbitals"]] = None # Come back to this to figure out correct dims for k-points + rep_tensor: Optional[Complex[Array, "n_k4pt n_orbitals n_orbitals n_orbitals n_orbitals"]] = None + energy: Optional[Scalar] = None + basis: Optional[Int[Array, '...']] = None # The name is saved as a list of integers, JAX does not accept str + name: Optional[Int[Array, '...']] = None # The name is saved as a list of integers, JAX does not accept str + spin: Optional[Scalar] = 0 + charge: Optional[Scalar] = 0 + unit_Angstrom: Optional[bool] = True + grid_level: Optional[Scalar] = 2 + scf_iteration: Optional[Scalar] = 50 + fock: Optional[Complex[Array, "n_spin n_kpt n_orbitals n_orbitals"]] = None + + def density(self, *args, **kwargs) -> Array: + r"""Compute the electronic density at each grid point. + + Returns + ------- + Float[Array, "grid spin"] + """ + return density(self.rdm1, self.ao, self.kpt_info.weights, *args, **kwargs) + + def nonXC(self, *args, **kwargs) -> Scalar: + r"""Compute all terms in the KS total energy with the exception of the XC component + + Returns + ------- + Scalar + """ + return non_xc(self.rdm1.sum(axis=0), self.h1e, self.rep_tensor, self.nuclear_repulsion, self.kpt_info.weights, *args, **kwargs) + + def make_rdm1(self, *args, **kwargs) -> Complex[Array, "n_spin n_kpt n_orbitals n_orbitals"]: + r"""Compute the 1-body reduced density matrix for each k-point. + + Returns + ------- + Complex[Array, "n_spin n_kpt n_orbitals n_orbitals"] + """ + return make_rdm1(self.mo_coeff, self.mo_occ, *args, **kwargs) + + def get_occ(self) -> Array: + r"""Compute the occupations of the molecular orbitals for each spin and k-point. + + Returns + ------- + Float[Array, "n_spin n_kpt n_orbitals"] + """ + # each k-channel has same total number of electrons, so just use index 0 in nelec calculation + # when indexing self.mo_occ + nelecs = jnp.array([self.mo_occ[i, 0].sum() for i in range(2)], dtype=jnp.int64) + return get_occ(self.mo_energy, nelecs) + + def grad_density(self, *args, **kwargs) -> Array: + r"""Compute the gradient of the electronic density at each grid point. + + Returns + ------- + Float[Array, "n_flat_grid n_spin 3"] + """ + return grad_density(self.rdm1, self.ao, self.grad_ao, self.kpt_info.weights, *args, **kwargs) + + def lapl_density(self, *args, **kwargs) -> Array: + r"""Compute the laplacian of the electronic density at each grid point. + + Returns + ------- + Float[Array, "n_flat_grid n_spin"] + """ + return lapl_density(self.rdm1, self.ao, self.grad_ao, self.grad_n_ao[2], self.kpt_info.weights, *args, **kwargs) + + def kinetic_density(self, *args, **kwargs) -> Array: + r"""Compute the kinetic energy density at each grid point. + + Returns + ------- + Float[Array, "n_flat_grid n_spin"] + """ + return kinetic_density(self.rdm1, self.grad_ao, self.kpt_info.weights, *args, **kwargs) + + def to_dict(self) -> dict: + r"""Return a dictionary with the attributes of the solid. + + Returns + ------- + Dict + """ + grid_dict = self.grid.to_dict() + kpt_dict = self.kpt_info.to_dict() + rest = {field.name: getattr(self, field.name) for field in fields(self)[2:]} + return dict(**grid_dict, **kpt_dict, **rest) + + def get_coulomb_potential(self, *args, **kwargs) -> Complex[Array, "n_kpts_or_n_ir_kpts n_orbitals n_orbitals"]: + r""" + Computes the Coulomb potential matrix for all k-points. + + Returns + ------- + Complex[Array, "n_kpts_or_n_ir_kpts n_orbitals n_orbitals"] + """ + return coulomb_potential(self.rdm1.sum(axis=0), self.rep_tensor, self.kpt_info.weights, *args, **kwargs) + + def select_HF_omegas(self, omegas: Float[Array, "omega"]) -> Array: + raise NotImplementedError("Hartree-Fock methods (for computation of Hybrid functionals) will come in a later release.") + + def HF_energy_density(self, omegas: Float[Array, "omega"], *args, **kwargs) -> Array: + raise NotImplementedError("Hartree-Fock methods (for computation of Hybrid functionals) will come in a later release.") + + def HF_density_grad_2_Fock( + self, + functional: nn.Module, + params: PyTree, + omegas: Float[Array, "omega"], + ehf: Float[Array, "omega spin grid"], + coefficient_inputs: Float[Array, "grid cinputs"], + densities_wout_hf: Float[Array, "grid densities_w"], + **kwargs, + ) -> Float[Array, "omega spin orbitals orbitals"]: + raise NotImplementedError("Hartree-Fock methods (for computation of Hybrid functionals) will come in a later release.") + + def HF_coefficient_input_grad_2_Fock( + self, + functional: nn.Module, + params: PyTree, + omegas: Float[Array, "omega"], + ehf: Float[Array, "omega spin grid"], + cinputs_wout_hf: Float[Array, "grid cinputs_w"], + densities: Float[Array, "grid densities"], + **kwargs, + ) -> Float[Array, "omega spin orbitals orbitals"]: + raise NotImplementedError("Hartree-Fock methods (for computation of Hybrid functionals) will come in a later release.") + + def get_mo_grads(self, *args, **kwargs): + r"""Compute the gradient of the electronic energy with respect + to the molecular orbital coefficients. + + Returns: + ------- + Float[Array, "orbitals orbitals"] + """ + return orbital_grad(self.mo_coeff, self.mo_occ, self.fock, *args, **kwargs) + + +@jaxtyped +@typechecked +@partial(jit, static_argnames=["precision"]) +def one_body_energy( + rdm1: Complex[Array, "n_kpt n_orbitals n_orbitals"], + h1e: Complex[Array, "n_kpt n_orbitals n_orbitals"], + weights: Float[Array, "n_kpts_or_n_ir_kpts"], + precision=Precision.HIGHEST, +) -> Scalar: + r"""Compute the one-body (kinetic + external) component of the KS total energy. + + Parameters + ---------- + rdm1 : Complex[Array, "n_kpt n_orbitals n_orbitals"] + The 1-body reduced density matrix for each k-point. Spin has been summed over before input. + h1e : Complex[Array, "n_kpt orbitals orbitals"] + The 1-electron Hamiltonian for each k-point. + weights : Float[Array, "n_kpts_or_n_ir_kpts"] + The weights for each k-point which together sum to 1. If we are working + in the full 1BZ, weights are equal. If we are working in the + irreducible 1BZ, weights may not be equal if symmetry can be + exploited. + + Returns + ------- + Scalar + """ + h1e_energy = jnp.einsum("k,kij,kji->", weights, rdm1, h1e, precision=precision) + return h1e_energy.real + +@jaxtyped +@typechecked +@partial(jit, static_argnames=["precision"]) +def coulomb_potential( + rdm1: Complex[Array, "n_kpt n_orbitals n_orbitals"], + rep_tensor: Complex[Array, "n_kpt n_kpt n_orbitals n_orbitals n_orbitals n_orbitals"], + weights: Float[Array, "n_kpts_or_n_ir_kpts"], + precision=Precision.HIGHEST +) -> Complex[Array, "n_kpts_or_n_ir_kpts n_orbitals n_orbitals"]: + r""" + Computes the Coulomb potential matrix for all k-points. + + Parameters + ---------- + rdm1 : Complex[Array, "n_kpt n_orbitals n_orbitals"] + The 1-body reduced density matrix. Spin has been summed over before input. + rep_tensor : Complex[Array, "n_kpt n_kpt n_orbitals n_orbitals n_orbitals n_orbitals"] + The repulsion tensor computed on a grid of nkpt x nkpt + precision : Precision, optional + The precision to use for the computation, by default Precision.HIGHEST + + Returns + ------- + Complex[Array, "n_kpts_or_n_ir_kpts n_orbitals n_orbitals"] + """ + + # k and q are k-point indices while i, j, l and m are orbital indices + v_k = jnp.einsum("k,kqijlm,qml->kij", weights, rep_tensor, rdm1, precision=precision) + return v_k + + +@jaxtyped +@typechecked +@partial(jit, static_argnames=["precision"]) +def coulomb_energy( + rdm1: Complex[Array, "n_kpt n_orbitals n_orbitals"], + rep_tensor: Complex[Array, "n_kpt n_kpt n_orbitals n_orbitals n_orbitals n_orbitals"], + weights: Float[Array, "n_kpts_or_n_ir_kpts"], + precision=Precision.HIGHEST +) -> Scalar: + """ + Compute the Coulomb energy + + Parameters + ---------- + rdm1 : Complex[Array, "n_kpt n_orbitals n_orbitals"] + The 1-body reduced density matrix. Spin has been summed over before input. + rep_tensor : Complex[Array, "n_kpt n_kpt n_orbitals n_orbitals n_orbitals n_orbitals"] + The repulsion tensor computed on a grid of nkpt x nkpt + weights : Float[Array, "n_kpts_or_n_ir_kpts"] + The weights for each k-point which together sum to 1. If we are working + in the full 1BZ, weights are equal. If we are working in the + irreducible 1BZ, weights may not be equal if symmetry can be + exploited. + precision : Precision, optional + The precision to use for the computation, by default Precision.HIGHEST + + Returns + ------- + Scalar + """ + v_k = coulomb_potential(rdm1, rep_tensor, weights, precision) + coulomb_energy = jnp.einsum("k,kij,kji->", weights, rdm1, v_k)/2.0 + return coulomb_energy.real + +@jaxtyped +@typechecked +@partial(jit, static_argnames=["precision"]) +def non_xc( + rdm1: Complex[Array, "n_kpt n_orbitals n_orbitals"], + h1e: Complex[Array, "n_kpt n_orbitals n_orbitals"], + rep_tensor: Complex[Array, "n_kpt n_kpt n_orbitals n_orbitals n_orbitals n_orbitals"], + nuclear_repulsion: Scalar, + weights: Float[Array, "n_kpts_or_n_ir_kpts"], + precision=Precision.HIGHEST, +) -> Scalar: + r"""Compute all terms in the KS total energy with the exception of the XC component + + Parameters + ---------- + rdm1 : Complex[Array, "n_kpt n_orbitals n_orbitals"] + The 1-body reduced density matrix. Spin has been summed over before input. + h1e : Complex[Array, "n_kpt orbitals orbitals"] + The 1-electron Hamiltonian for each k-point. + Equivalent to mf.get_hcore(mf.mol) in pyscf. + rep_tensor : Complex[Array, "n_kpt n_kpt n_orbitals n_orbitals n_orbitals n_orbitals"] + The repulsion tensor computed on a grid of nkpt x nkpt + nuclear_repulsion : Scalar + The nuclear repulsion energy. + Equivalent to mf.mol.energy_nuc() in pyscf. + weights : Float[Array, "n_kpts_or_n_ir_kpts"] + The weights for each k-point which together sum to 1. If we are working + in the full 1BZ, weights are equal. If we are working in the + irreducible 1BZ, weights may not be equal if symmetry can be + exploited. + precision : Precision, optional + The precision to use for the computation, by default Precision.HIGHEST + + Returns + ------- + Scalar + """ + kinetic_and_external = one_body_energy(rdm1, h1e, weights, precision) + # jax.debug.print("h1e_energy is {x}", x=h1e_energy) + coulomb = coulomb_energy(rdm1, rep_tensor, weights, precision) + # jax.debug.print("coulomb2e_energy is {x}", x=coulomb2e_energy) + # jax.debug.print("nuclear_repulsion is {x}", x=nuclear_repulsion) + + return nuclear_repulsion + kinetic_and_external + coulomb + + +@jaxtyped +@typechecked +@partial(jit, static_argnames=["precision"]) +def make_rdm1( + mo_coeff: Complex[Array, "n_spin n_kpt n_orbitals n_orbitals"], + mo_occ: Float[Array, "n_spin n_kpt n_orbitals"], + precision=Precision.HIGHEST +) -> Complex[Array, "n_spin n_kpt n_orbitals n_orbitals"]: + r""" + One-body reduced density matrix for each k-point in AO representation + + Parameters: + ---------- + mo_coeff : Complex[Array, "n_spin n_kpt n_orbitals n_orbitals"] + Spin-orbital coefficients for each k-point. + mo_occ : Float[Array, "n_spin n_kpt n_orbitals"] + Spin-orbital occupancies for each k-point. + + Returns: + ------- + Complex[Array, "n_spin n_kpt n_orbitals n_orbitals"] + """ + + return jnp.einsum("skij,skj,sklj -> skil", mo_coeff, mo_occ, mo_coeff.conj(), precision=precision) + + +@jaxtyped +@typechecked +def get_occ( + mo_energies: Float[Array, "n_spin n_kpt n_orbitals"], + nelecs: Int[Array, "spin"], +) -> Float[Array, "n_spin n_kpt n_orbitals"]: + r"""Compute the occupations of the molecular orbitals for each + spin and k-point. + + Parameters + ---------- + mo_energies : Float[Array, "n_spin n_kpt n_orbitals"] + The molecular orbital energies. + nelecs : Int[Array, "n_spin"] + The number of electrons in each spin channel. + + Returns + ------- + Int[Array, "n_spin n_kpt n_orbitals"] + """ + nkpt = mo_energies.shape[1] + nmo = mo_energies.shape[2] + def get_occ_spin_k_pair(mo_energy_spin_k, nelec_spin, nmo): + sorted_indices = jnp.argsort(mo_energy_spin_k) + + mo_occ = jnp.zeros_like(mo_energy_spin_k) + + def assign_values(i, mo_occ): + value = cond(jnp.less(i, nelec_spin), lambda _: 1, lambda _: 0, operand=None) + idx = sorted_indices[i] + mo_occ = mo_occ.at[idx].set(value) + return mo_occ + + mo_occ = fori_loop(0, nmo, assign_values, mo_occ) + + return mo_occ + + + mo_occ = jnp.stack( + jnp.asarray([[get_occ_spin_k_pair(mo_energies[s, k], jnp.int64(nelecs[s]), nmo) for k in range(nkpt)] for s in range(2)]), axis=0 + ) + + return mo_occ + + +""" +Note: while the below functions related to the density and it's gradients take a k-point weights parameter, +modification is needed before they support unequal weights as they would appear in a symmetry adapted code. I.e, +the whole 1BZ need to be considered which would involve use of rotation matrices to map 1RDM's in the IBZ to the full +1BZ. +""" + +@jaxtyped +@typechecked +@partial(jit, static_argnames="precision") +def density(rdm1: Complex[Array, "n_spin n_kpt n_orbitals n_orbitals"], + ao: Complex[Array, "n_kpt n_flat_grid n_orbitals"], + weights: Float[Array, "n_kpts_or_n_ir_kpts"], + precision: Precision = Precision.HIGHEST +) -> Float[Array, "n_flat_grid n_spin"]: + r""" Calculates electronic density from atomic orbitals. + + Parameters + ---------- + rdm1 : Complex[Array, "n_spin n_kpt n_orbitals n_orbitals"] + The 1-body reduced density matrix. + ao : Complex[Array, "n_kpt n_flat_grid n_orbitals"] + Crystal atomic orbitals. + weights : Float[Array, "n_kpts_or_n_ir_kpts"] + The weights for each k-point which together sum to 1. If we are working + in the full 1BZ, weights are equal. If we are working in the + irreducible 1BZ, weights may not be equal if symmetry can be + exploited. + precision : jax.lax.Precision, optional + Jax `Precision` enum member, indicating desired numerical precision. + By default jax.lax.Precision.HIGHEST. + + Returns + ------- + Float[Array, "n_flat_grid n_spin"] + """ + den = jnp.einsum("k,skab,kra,krb->rs", weights, rdm1, ao, ao, precision=precision).real + return den + +@jaxtyped +@typechecked +@partial(jit, static_argnames="precision") +def grad_density( + rdm1: Complex[Array, "n_spin n_kpt n_orbitals n_orbitals"], + ao: Complex[Array, "n_kpt n_flat_grid n_orbitals"], + grad_ao: Complex[Array, "n_kpt n_flat_grid n_orbitals 3"], + weights: Float[Array, "n_kpts_or_n_ir_kpts"], + precision: Precision = Precision.HIGHEST +) -> Float[Array, "n_flat_grid n_spin 3"]: + r"""Compute the electronic density gradient using crystal atomic orbitals. + + Parameters + ---------- + rdm1 : Complex[Array, "n_spin n_kpt n_orbitals n_orbitals"] + The 1-body reduced density matrix. + ao : Complex[Array, "n_kpt n_flat_grid n_orbitals"] + Crystal atomic orbitals. + grad_ao : Complex[Array, "n_kpt n_flat_grid n_orbitals 3"] + Gradients of crystal atomic orbitals. + weights : Float[Array, "n_kpts_or_n_ir_kpts"] + The weights for each k-point which together sum to 1. If we are working + in the full 1BZ, weights are equal. If we are working in the + irreducible 1BZ, weights may not be equal if symmetry can be + exploited. + precision : jax.lax.Precision, optional + Jax `Precision` enum member, indicating desired numerical precision. + By default jax.lax.Precision.HIGHEST. + + Returns + ------- + Array + The density gradient: Float[Array, "n_flat_grid n_spin 3"] + """ + + return 2 * jnp.einsum("k,...kab,kra,krbj->r...j", weights, rdm1, ao, grad_ao, precision=precision).real + +@jaxtyped +@typechecked +@partial(jit, static_argnames="precision") +def lapl_density( + rdm1: Complex[Array, "n_spin n_kpt n_orbitals n_orbitals"], + ao: Complex[Array, "n_kpt n_flat_grid n_orbitals"], + grad_ao: Complex[Array, "n_kpt n_flat_grid n_orbitals 3"], + grad_2_ao: Complex[Array, "n_kpt n_flat_grid n_orbitals 3"], + weights: Float[Array, "n_kpts_or_n_ir_kpts"], + precision: Precision = Precision.HIGHEST, +) -> Float[Array, "n_flat_grid n_spin"]: + r"""Compute the laplacian of the electronic density. + + Parameters + ---------- + rdm1 : Complex[Array, "n_spin n_kpt n_orbitals n_orbitals"] + The 1-body reduced density matrix. + ao : Complex[Array, "b_flat_grid n_orbitals"] + Crystal atomic orbitals. + grad_ao : Complex[Array, "n_flat_grid n_orbitals 3"] + Gradients of crystal atomic orbitals. + grad_2_ao : Complex[Array, "n_flat_grid n_orbitals 3"] + Vector of second derivatives of crystal atomic orbitals. + weights : Float[Array, "n_kpts_or_n_ir_kpts"] + The weights for each k-point which together sum to 1. If we are working + in the full 1BZ, weights are equal. If we are working in the + irreducible 1BZ, weights may not be equal if symmetry can be + exploited. + precision : jax.lax.Precision, optional + Jax `Precision` enum member, indicating desired numerical precision. + By default jax.lax.Precision.HIGHEST. + + Returns + ------- + Float[Array, "n_flat_grid n_spin"] + """ + return (2 * jnp.einsum( + "k,...kab,kraj,krbj->r...", weights, rdm1, grad_ao, grad_ao, precision=precision + ) + 2 * jnp.einsum("k,...kab,kra,krbi->r...", weights, rdm1, ao, grad_2_ao, precision=precision)).real + +@jaxtyped +@typechecked +@partial(jit, static_argnames="precision") +def kinetic_density( + rdm1 : Complex[Array, "n_spin n_kpt n_orbitals n_orbitals"], + grad_ao: Complex[Array, "n_kpt n_flat_grid n_orbitals 3"], + weights: Float[Array, "n_kpts_or_n_ir_kpts"], + precision: Precision = Precision.HIGHEST +) -> Float[Array, "n_flat_grid n_spin"]: + r""" Compute the kinetic energy density using crystal atomic orbitals. + + Parameters + ---------- + rdm1 : Complex[Array, "n_spin n_kpt n_orbitals n_orbitals"] + The 1-body reduced density matrix. + grad_ao : Complex[Array, "n_kpt n_flat_grid n_orbitals 3"] + Gradients of crystal atomic orbitals. + weights : Float[Array, "n_kpts_or_n_ir_kpts"] + The weights for each k-point which together sum to 1. If we are working + in the full 1BZ, weights are equal. If we are working in the + irreducible 1BZ, weights may not be equal if symmetry can be + exploited. + precision : jax.lax.Precision, optional + Jax `Precision` enum member, indicating desired numerical precision. + By default jaxx.lax.Precision.HIGHEST. + + Returns + ------- + Array + The kinetic energy density: Float[Array, "n_flat_grid n_spin"] + """ + + return 0.5 * jnp.einsum("k,...kab,kraj,krbj->r...", weights, rdm1, grad_ao, grad_ao, precision=precision).real + +@jaxtyped +@typechecked +@partial(jit, static_argnames="precision") +def orbital_grad( + mo_coeff: Complex[Array, "n_spin n_kpt n_orbitals n_orbitals"], + mo_occ: Float[Array, "n_spin n_kpt n_orbitals"], + fock: Complex[Array, "n_spin n_kpt n_orbitals n_orbitals"], + precision: Precision = Precision.HIGHEST + ) -> Float[Array, "n_kpt n_orbitals n_orbitals"]: + r"""Compute the gradient of the electronic energy with respect + to the molecular orbital coefficients. + + Parameters: + ---------- + mo_coeff: Complex[Array, "n_spin n_kpt n_orbitals n_orbitals"] + Orbital coefficients + mo_occ: Float[Array, "n_spin n_kpt n_orbitals"] + Orbital occupancy + fock: Complex[Array, "n_spin n_kpt n_orbitals n_orbitals"] + Fock matrix in AO representation + precision: jax.lax.Precision, optional + + Returns: + ------- + Float[Array, "n_kpt n_orbitals n_orbitals"] + + + Notes: + ----- + # Performs same task as pyscf/scf/hf.py but we have k-point sampling: + occidx = mo_occ > 0 + viridx = ~occidx + g = reduce(jnp.dot, (mo_coeff[:,viridx].conj().T, fock_ao, + mo_coeff[:,occidx])) * 2 + return g.ravel() + """ + + C_occ = vmap(jnp.where, in_axes=(None, 2, None), out_axes=2)(mo_occ > 0, mo_coeff, 0) + C_vir = vmap(jnp.where, in_axes=(None, 2, None), out_axes=2)(mo_occ == 0, mo_coeff, 0) + + return jnp.einsum("skab,skac,skcd->kbd", C_vir.conj(), fock, C_occ, precision = precision).real + + + diff --git a/grad_dft/train.py b/grad_dft/train.py index bd2e930..e81680f 100644 --- a/grad_dft/train.py +++ b/grad_dft/train.py @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Callable, Optional, Tuple +from typing import Callable, Optional, Tuple, Union from functools import partial -from jaxtyping import Array, PRNGKeyArray, PyTree, Scalar, Float +from jaxtyping import Array, PRNGKeyArray, PyTree, Scalar, Float, Complex from jax import numpy as jnp, vmap -from jax import value_and_grad +from jax import value_and_grad, grad from jax.profiler import annotate_function from jax.lax import stop_gradient from optax import OptState, GradientTransformation, apply_updates @@ -26,7 +26,8 @@ coulomb_energy, DispersionFunctional, Functional, - Molecule, + Molecule, + Solid, abs_clip, ) @@ -37,7 +38,7 @@ def energy_predictor( **kwargs, ) -> Callable: r"""Generate a function that predicts the energy - energy of a `Molecule` and a corresponding Fock matrix + energy of a `Molecule` or `Solid` and a corresponding Fock matrix Parameters ---------- @@ -46,10 +47,10 @@ def energy_predictor( exchange-correlation energy given some parameters. A callable must have the following signature: - fxc.energy(params: Array, molecule: Molecule, **functional_kwargs) -> Scalar + fxc.energy(params: Array, atoms: Union[Molecule, Solid], **functional_kwargs) -> Scalar - where `params` is any parameter pytree, and `molecule` - is a Molecule class instance. + where `params` is any parameter pytree, and `atoms` + is a `Molecule` or `Solid` class instance. Returns ------- @@ -58,7 +59,7 @@ def energy_predictor( the predicted energy with the corresponding Fock matrix. Signature: - (params: PyTree, molecule: Molecule, *args) -> Tuple[Scalar, Array] + (params: PyTree, atoms: Union[Molecule, Solid], *args) -> Tuple[Scalar, Array] Notes ----- @@ -81,17 +82,19 @@ def energy_predictor( >>> fock.shape == molecule.density_matrix.shape True """ - + @partial(value_and_grad, argnums=1) - def energy_and_grads( - params: PyTree, - rdm1: Float[Array, "spin orbitals orbitals"], - molecule: Molecule, - *args, - **functional_kwargs, + def xc_energy_and_grads( + params: PyTree, + rdm1: Union[Float[Array, "spin orbitals orbitals"], + Complex[Array, "spin kpt orbitals orbitals"] + ], + atoms: Union[Molecule, Solid], + *args, + **functional_kwargs ) -> Scalar: r""" - Computes the energy and gradients with respect to the density matrix + Computes the xc energy and gradients with respect to the density matrix. Parameters ---------- @@ -99,26 +102,26 @@ def energy_and_grads( Functional parameters rdm1: Float[Array, "spin orbitals orbitals"] The reduced density matrix. - molecule: Molecule - the molecule + atoms: Union[Molecule, Solid] + The collection of atoms. + *args + **kwargs Returns ----------- - Scalar - The energy of the molecule when the state of the system is given by rdm1. + Tuple[Scalar, Float[Array, "spin orbitals orbitals"]] """ - - molecule = molecule.replace(rdm1=rdm1) - - e = functional.energy(params, molecule, *args, **functional_kwargs) + atoms = atoms.replace(rdm1=rdm1) + densities = functional.compute_densities(atoms, *args, **functional_kwargs) + cinputs = functional.compute_coefficient_inputs(atoms, *args) if nlc_functional: e = e + nlc_functional.energy( - {"params": params["dispersion"]}, molecule, **functional_kwargs + {"params": params["dispersion"]}, atoms, **functional_kwargs ) - return e + return functional.xc_energy(params, atoms.grid, cinputs, densities, **functional_kwargs) @partial(annotate_function, name="predict") - def predict(params: PyTree, molecule: Molecule, *args) -> Tuple[Scalar, Array]: + def predict(params: PyTree, atoms: Union[Molecule, Solid], *args) -> Tuple[Scalar, Array]: r"""A DFT functional wrapper, returning the predicted exchange-correlation energy as well as the corresponding Fock matrix. This function does **not** require that the provided `feature_fn` returns derivatives (Jacobian matrix) of provided @@ -128,53 +131,67 @@ def predict(params: PyTree, molecule: Molecule, *args) -> Tuple[Scalar, Array]: ---------- params : PyTree The functional parameters. - molecule : Molecule - The `Molecule` object to predict properties of. + atoms: Union[Molecule, Solid] + The collection of atoms. *args Returns ------- Tuple[Scalar, Array] A tuple of the predicted exchange-correlation energy and the corresponding - Fock matrix of the same shape as `molecule.density_matrix`: - (*batch_size, n_spin, n_orbitals, n_orbitals). + Fock matrix of the same shape as `atoms.rdm1`: + (*batch_size, n_spin, n_orbitals, n_orbitals) for a `Molecule` or + (*batch_size, n_spin, n_kpt, n_orbitals, n_orbitals) for a `Solid`. """ - - energy, fock = energy_and_grads(params, molecule.rdm1, molecule, *args) + + Exc, fock_xc = xc_energy_and_grads(params, atoms.rdm1, atoms, *args) + fock_noxc = atoms.h1e + atoms.get_coulomb_potential() + + energy = Exc + atoms.nonXC() + + if isinstance(atoms, Molecule): + transpose_dims = (0, 2, 1) + fock = fock_noxc + fock_xc + elif isinstance(atoms, Solid): + transpose_dims = (0, 1, 3, 2) + # auto-diffed xc gradient is divided by n_k=number of k-points. Undo this. + fock = fock_noxc + (fock_xc*atoms.rdm1.shape[1]) + + # Improve stability by clipping and symmetrizing fock = abs_clip(fock, clip_cte) - fock = 1 / 2 * (fock + fock.transpose(0, 2, 1)) + fock = 1 / 2 * (fock + fock.transpose(transpose_dims).conj()) fock = abs_clip(fock, clip_cte) # Compute the features that should be autodifferentiated if functional.energy_densities and functional.densitygrads: - grad_densities = functional.energy_densities(molecule, *args, **kwargs) - nograd_densities = stop_gradient(functional.nograd_densities(molecule, *args, **kwargs)) + grad_densities = functional.energy_densities(atoms, *args, **kwargs) + nograd_densities = stop_gradient(functional.nograd_densities(atoms, *args, **kwargs)) densities = functional.combine_densities(grad_densities, nograd_densities) elif functional.energy_densities: - grad_densities = functional.energy_densities(molecule, *args, **kwargs) + grad_densities = functional.energy_densities(atoms, *args, **kwargs) nograd_densities = None densities = grad_densities elif functional.densitygrads: grad_densities = None - nograd_densities = stop_gradient(functional.nograd_densities(molecule, *args, **kwargs)) + nograd_densities = stop_gradient(functional.nograd_densities(atoms, *args, **kwargs)) densities = nograd_densities else: densities, grad_densities, nograd_densities = None, None, None if functional.coefficient_input_grads and functional.coefficient_inputs: - grad_cinputs = functional.coefficient_inputs(molecule, *args, **kwargs) + grad_cinputs = functional.coefficient_inputs(atoms, *args, **kwargs) nograd_cinputs = stop_gradient( - functional.nograd_coefficient_inputs(molecule, *args, **kwargs) + functional.nograd_coefficient_inputs(atoms, *args, **kwargs) ) cinputs = functional.combine_inputs(grad_cinputs, nograd_cinputs) elif functional.coefficient_inputs: - grad_cinputs = functional.coefficient_inputs(molecule, *args, **kwargs) + grad_cinputs = functional.coefficient_inputs(atoms, *args, **kwargs) nograd_cinputs = None cinputs = grad_cinputs elif functional.coefficient_input_grads: grad_cinputs = None nograd_cinputs = stop_gradient( - functional.nograd_coefficient_inputs(molecule, *args, **kwargs) + functional.nograd_coefficient_inputs(atoms, *args, **kwargs) ) cinputs = nograd_cinputs else: @@ -183,20 +200,19 @@ def predict(params: PyTree, molecule: Molecule, *args) -> Tuple[Scalar, Array]: # Compute the derivatives with respect to nograd_densities if functional.densitygrads: vxc_expl = functional.densitygrads( - functional, params, molecule, nograd_densities, cinputs, grad_densities + functional, params, atoms, nograd_densities, cinputs, grad_densities ) - fock += vxc_expl + vxc_expl.transpose(0, 2, 1) # Sum over omega + fock += vxc_expl + vxc_expl.transpose(transpose_dims) # Sum over omega fock = abs_clip(fock, clip_cte) if functional.coefficient_input_grads: vxc_expl = functional.coefficient_input_grads( - functional, params, molecule, nograd_cinputs, grad_cinputs, densities + functional, params, atoms, nograd_cinputs, grad_cinputs, densities ) - fock += vxc_expl + vxc_expl.transpose(0, 2, 1) # Sum over omega + fock += vxc_expl + vxc_expl.transpose(transpose_dims) # Sum over omega fock = abs_clip(fock, clip_cte) fock = abs_clip(fock, clip_cte) - return energy, fock return predict @@ -226,7 +242,7 @@ def Harris_energy_predictor( def xc_energy_and_grads( params: PyTree, rdm1: Float[Array, "spin orbitals orbitals"], - molecule: Molecule, + atoms: Union[Molecule, Solid], *args, **kwargs ) -> Scalar: @@ -239,8 +255,8 @@ def xc_energy_and_grads( Functional parameters rdm1: Float[Array, "spin orbitals orbitals"] The reduced density matrix. - molecule: Molecule - the molecule + atoms: Union[Molecule, Solid] + The collection of atoms. *args **kwargs @@ -248,12 +264,13 @@ def xc_energy_and_grads( ----------- Tuple[Scalar, Float[Array, "spin orbitals orbitals"]] """ - molecule = molecule.replace(rdm1 = rdm1) - densities = functional.compute_densities(molecule, *args, **kwargs) - cinputs = functional.compute_coefficient_inputs(molecule, *args) - return functional.xc_energy(params, molecule.grid, cinputs, densities, **kwargs) - + atoms = atoms.replace(rdm1=rdm1) + densities = functional.compute_densities(atoms, *args, **kwargs) + cinputs = functional.compute_coefficient_inputs(atoms, *args) + return functional.xc_energy(params, atoms.grid, cinputs, densities, **kwargs) + + # Works for Molecules only for now def Harris_energy( params: PyTree, molecule: Molecule, @@ -300,7 +317,7 @@ def train_kernel(tx: GradientTransformation, loss: Callable) -> Callable: tx : GradientTransformation An optax gradient transformation. loss : Callable - A loss function that takes in the parameters, a `Molecule` object, and the ground truth energy + A loss function that takes in the parameters, a `Molecule` or `Solid` object, and the ground truth energy and returns a tuple of the loss value and the gradients. Returns @@ -309,7 +326,7 @@ def train_kernel(tx: GradientTransformation, loss: Callable) -> Callable: """ def kernel( - params: PyTree, opt_state: OptState, molecule: Molecule, ground_truth_energy: float, *args + params: PyTree, opt_state: OptState, atoms: Union[Molecule, Solid], ground_truth_energy: float, *args ) -> Tuple[PyTree, OptState, Scalar, Scalar]: r"""" The training kernel updating the parameters according to the loss @@ -321,8 +338,8 @@ def kernel( The parameters of the functional. opt_state : OptState The optimizer state. - molecule : Molecule - The molecule. + atoms: Union[Molecule, Solid] + The collection of atoms ground_truth_energy : float The ground truth energy. *args @@ -332,7 +349,7 @@ def kernel( Tuple[PyTree, OptState, Scalar, Scalar] The updated parameters, optimizer state, loss value, and predicted energy. """ - (cost_value, predictedenergy), grads = loss(params, molecule, ground_truth_energy) + (cost_value, predictedenergy), grads = loss(params, atoms, ground_truth_energy) updates, opt_state = tx.update(grads, opt_state, params) params = apply_updates(params, updates) @@ -344,6 +361,7 @@ def kernel( ##################### Regularization ##################### +# Regularization terms only support `Molecule` object for now def fock_grad_regularization(molecule: Molecule, F: Float[Array, "spin ao ao"]) -> Scalar: """Calculates the Fock alternative regularization term for a `Molecule` given a Fock matrix. @@ -460,15 +478,20 @@ def get_grad( def mse_energy_loss( params: PyTree, compute_energy: Callable, - molecules: list[Molecule], + atoms_list: Union[list[Molecule], + list[Solid], + list[Union[Molecule,Solid]], + Molecule, + Solid + ], truth_energies: Float[Array, "energy"], elec_num_norm: Scalar = True, ) -> Scalar: r""" Computes the mean-squared error between predicted and truth energies. - This loss function does not yet support parallel execution for the loss contributions - and instead implemented a simple for loop. + This loss function does not yet support parallel execution for the loss contributions. + We instead use a simple serial for loop. Parameters ---------- @@ -476,8 +499,9 @@ def mse_energy_loss( functional parameters (weights) compute_energy: Callable(molecule, params) -> molecule. any non SCF or SCF method in evaluate.py. The output molecule contains the predicted energy. - molecule: Molecule - a Grad-DFT Molecule object + atoms_list: Union[list[Molecule], list[Solid], list[Union[Molecule,Solid]], Molecule, Solid] + A list of `Molecule` or `Solid` objects or a combination of both. Passing + a single `Molecule` or `Solid` wraps it in a list internally. truth_energies: Float[Array, "energy"] the truth values of the energy to measure the predictions against elec_num_norm: Scalar @@ -487,25 +511,28 @@ def mse_energy_loss( ---------- Scalar: the mean-squared error between predicted and truth energies """ - if isinstance(molecules, Molecule): molecules = [molecules] + # Catch the case where a list of atoms was not passed. I.e, dealing with a single + # instance. + if isinstance(atoms_list, Molecule) or isinstance(atoms_list, Solid): + atoms_list = [atoms_list] sum = 0 - for i, molecule in enumerate(molecules): - molecule_out = compute_energy(params, molecule) - E_predict = molecule_out.energy + for i, atoms in enumerate(atoms_list): + atoms_out = compute_energy(params, atoms) + E_predict = atoms_out.energy diff = E_predict - truth_energies[i] # Not jittable because of if. - num_elec = jnp.sum(molecule.atom_index) - molecule.charge + num_elec = jnp.sum(atoms.atom_index) - atoms.charge if elec_num_norm: diff = diff / num_elec sum += (diff) ** 2 - cost_value = sum / len(molecules) + cost_value = sum / len(atoms_list) return cost_value @partial(value_and_grad, has_aux=True) def simple_energy_loss(params: PyTree, compute_energy: Callable, - molecule: Molecule, + atoms: Union[Molecule, Solid], truth_energy: Float, ): r""" @@ -517,9 +544,13 @@ def simple_energy_loss(params: PyTree, functional parameters (weights) compute_energy: Callable. any non SCF or SCF method in evaluate.py + atoms: Union[Molecule, Solid] + The collcection of atoms. + truth_energy: Float + The energy value we are training against """ - molecule_out = compute_energy(params, molecule) - E_predict = molecule_out.energy + atoms_out = compute_energy(params, atoms) + E_predict = atoms_out.energy diff = E_predict - truth_energy return diff**2, E_predict @@ -527,7 +558,7 @@ def simple_energy_loss(params: PyTree, def sq_electron_err_int( pred_density: Float[Array, "ngrid nspin"], truth_density: Float[Array, "ngrid nspin"], - molecule: Molecule, + atoms: Union[Molecule, Solid], clip_cte=1e-30 ) -> Scalar: r""" @@ -541,8 +572,8 @@ def sq_electron_err_int( Density predicted by a neural functional truth_density: Float[Array, "ngrid nspin"] A accurate density used as a truth value in training - molecule: Molecule - A Grad-DFT Molecule + atoms: Union[Molecule, Solid] + The collection of atoms. Returns Scalar: the value epsilon described above @@ -552,14 +583,19 @@ def sq_electron_err_int( truth_density = jnp.clip(truth_density, a_min=clip_cte) diff_up = jnp.clip(jnp.clip(pred_density[:, 0] - truth_density[:, 0], a_min=clip_cte) ** 2, a_min=clip_cte) diff_dn = jnp.clip(jnp.clip(pred_density[:, 1] - truth_density[:, 1], a_min=clip_cte) ** 2, a_min=clip_cte) - err_int = jnp.sum(diff_up * molecule.grid.weights) + jnp.sum(diff_dn * molecule.grid.weights) + err_int = jnp.sum(diff_up * atoms.grid.weights) + jnp.sum(diff_dn * atoms.grid.weights) return err_int def mse_density_loss( params: PyTree, compute_energy: Callable, - molecules: list[Molecule], + atoms_list: Union[list[Molecule], + list[Solid], + list[Union[Molecule,Solid]], + Molecule, + Solid + ], truth_rhos: list[Float[Array, "ngrid nspin"]], elec_num_norm: Scalar = True, ) -> Scalar: @@ -575,8 +611,9 @@ def mse_density_loss( functional parameters (weights) compute_energy: Callable. any non SCF or SCF method in evaluate.py - molecule: Molecule - a Grad-DFT Molecule object + atoms_list: Union[list[Molecule], list[Solid], list[Union[Molecule,Solid]], Molecule, Solid] + A list of `Molecule` or `Solid` objects or a combination of both. Passing + a single `Molecule` or `Solid` wraps it in a list internally. truth_densities: list[Float[Array, "ngrid nspin"]] the truth values of the density to measure the predictions against elec_num_norm: Scalar @@ -586,17 +623,21 @@ def mse_density_loss( ---------- Scalar: the mean-squared error between predicted and truth densities """ + # Catch the case where a list of atoms was not passed. I.e, dealing with a single + # instance. + if isinstance(atoms_list, Molecule) or isinstance(atoms_list, Solid): + atoms_list = [atoms_list] sum = 0 - for i, molecule in enumerate(molecules): - molecule_out = compute_energy(params, molecule) - rho_predict = molecule_out.density() - diff = sq_electron_err_int(rho_predict, truth_rhos[i], molecule) + for i, atoms in enumerate(atoms_list): + atoms_out = compute_energy(params, atoms) + rho_predict = atoms_out.density() + diff = sq_electron_err_int(rho_predict, truth_rhos[i], atoms) # Not jittable because of if. - num_elec = jnp.sum(molecule.atom_index) - molecule.charge + num_elec = jnp.sum(atoms.atom_index) - atoms.charge if elec_num_norm: diff = diff / num_elec**2 sum += diff - cost_value = sum / len(molecules) + cost_value = sum / len(atoms_list) return cost_value @@ -604,7 +645,12 @@ def mse_density_loss( def mse_energy_and_density_loss( params: PyTree, compute_energy: Callable, - molecules: list[Molecule], + atoms_list: Union[list[Molecule], + list[Solid], + list[Union[Molecule,Solid]], + Molecule, + Solid + ], truth_densities: list[Float[Array, "ngrid nspin"]], truth_energies: Float[Array, "energy"], rho_factor: Scalar = 1.0, @@ -623,8 +669,9 @@ def mse_energy_and_density_loss( functional parameters (weights) compute_energy: Callable. any non SCF or SCF method in evaluate.py - molecule: Molecule - a Grad-DFT Molecule object + atoms_list: Union[list[Molecule], list[Solid], list[Union[Molecule,Solid]], Molecule, Solid] + A list of `Molecule` or `Solid` objects or a combination of both. Passing + a single `Molecule` or `Solid` wraps it in a list internally. truth_densities: list[Float[Array, "ngrid, nspin"]] the truth values of the density to measure the predictions against truth_energies: Float[Array, "energy"] @@ -634,28 +681,32 @@ def mse_energy_and_density_loss( density_factor: Scalar A weighting factor for the density portion of the loss. Default = 1.0 elec_num_norm: Scalar - True to normalize the loss function by the number of electrons in a Molecule. + True to normalize the loss function by the number of electrons in an atoms instance. Returns ---------- Scalar: the mean-squared error of both energies and densities each with it's own weight. """ + # Catch the case where a list of atoms was not passed. I.e, dealing with a single + # instance. + if isinstance(atoms_list, Molecule) or isinstance(atoms_list, Solid): + atoms_list = [atoms_list] sum_energy = 0 sum_rho = 0 - for i, molecule in enumerate(molecules): - molecule_out = compute_energy(params, molecule) - rho_predict = molecule_out.density() - energy_predict = molecule_out.energy - diff_rho = sq_electron_err_int(rho_predict, truth_densities[i], molecule) + for i, atoms in enumerate(atoms_list): + atoms_out = compute_energy(params, atoms) + rho_predict = atoms_out.density() + energy_predict = atoms_out.energy + diff_rho = sq_electron_err_int(rho_predict, truth_densities[i], atoms) diff_energy = energy_predict - truth_energies[i] # Not jittable because of if. - num_elec = jnp.sum(molecule.atom_index) - molecule.charge + num_elec = jnp.sum(atoms.atom_index) - atoms.charge if elec_num_norm: diff_rho = diff_rho / num_elec**2 diff_energy = diff_energy / num_elec sum_rho += diff_rho sum_energy += diff_energy**2 - energy_contrib = energy_factor * sum_energy / len(molecules) - rho_contrib = rho_factor * sum_rho / len(molecules) + energy_contrib = energy_factor * sum_energy / len(atoms_list) + rho_contrib = rho_factor * sum_rho / len(atoms_list) return energy_contrib + rho_contrib diff --git a/grad_dft/utils/eigenproblem.py b/grad_dft/utils/eigenproblem.py index b04f128..2afcfc8 100644 --- a/grad_dft/utils/eigenproblem.py +++ b/grad_dft/utils/eigenproblem.py @@ -34,8 +34,8 @@ def safe_eigh(A: Array) -> tuple[Array, Array]: Returns: tuple[Array, Array]: the eigenvalues and eigenvectors of the input real symmetric matrix. """ - evecs, evals = jnp.linalg.eigh(A) - return evecs, evals + evals, evecs = jnp.linalg.eigh(A) + return evals, evecs def safe_eigh_fwd(A: Array) -> tuple[tuple[Array, Array], tuple[tuple[Array, Array], Array]]: @@ -104,6 +104,7 @@ def safe_eigh_rev(res: tuple[tuple[Array, Array], Array], g: Array) -> tuple[Arr safe_eigh.defvjp(safe_eigh_fwd, safe_eigh_rev) +safe_eigh_vec = jnp.vectorize(safe_eigh, signature="(m,m)->(n),(n,n)") def safe_general_eigh(A: Array, B: Array) -> tuple[Array, Array]: @@ -122,9 +123,9 @@ def safe_general_eigh(A: Array, B: Array) -> tuple[Array, Array]: """ L = jnp.linalg.cholesky(B) L_inv = jnp.linalg.inv(L) - C = L_inv @ A @ L_inv.T - eigenvalues, eigenvectors_transformed = safe_eigh(C) - eigenvectors_original = L_inv.T @ eigenvectors_transformed + C = L_inv @ A @ jnp.moveaxis(L_inv, -1, -2) + eigenvalues, eigenvectors_transformed = safe_eigh_vec(C) + eigenvectors_original = jnp.moveaxis(L_inv, -1, -2) @ eigenvectors_transformed return eigenvalues, eigenvectors_original diff --git a/requirements.txt b/requirements.txt index 1914608..a236acd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,8 +3,10 @@ jaxlib>=0.4.14 pyscf>=2.3.0 attrs>=23.1.0 flax>=0.7.2 -tensorflow>=2.13.0 +tensorflow>=2.13.0,<=2.14.0 tensorflow-hub>=0.14.0 typeguard==2.13.3 +typing_extensions>=4.8.0 jaxtyping -pytest +pytest>=7.4.3 + diff --git a/tests/integration/test_Harris.py b/tests/integration/molecules/test_Harris.py similarity index 100% rename from tests/integration/test_Harris.py rename to tests/integration/molecules/test_Harris.py diff --git a/tests/integration/test_functional_implementations.py b/tests/integration/molecules/test_functional_implementations.py similarity index 100% rename from tests/integration/test_functional_implementations.py rename to tests/integration/molecules/test_functional_implementations.py diff --git a/tests/integration/test_non_xc_energy.py b/tests/integration/molecules/test_non_xc_energy.py similarity index 100% rename from tests/integration/test_non_xc_energy.py rename to tests/integration/molecules/test_non_xc_energy.py diff --git a/tests/integration/test_predict_B3LYP.py b/tests/integration/molecules/test_predict_B3LYP.py similarity index 91% rename from tests/integration/test_predict_B3LYP.py rename to tests/integration/molecules/test_predict_B3LYP.py index 8dc8af8..fdab7d3 100644 --- a/tests/integration/test_predict_B3LYP.py +++ b/tests/integration/molecules/test_predict_B3LYP.py @@ -48,9 +48,14 @@ # This test will only pass if you set B3LYP_WITH_VWN5 = True in pyscf_conf.py. # See pyscf_conf.py in .github/workflows + # This test differs slightly due to the use of the original LYP functional definition # in C. Lee, W. Yang, and R. G. Parr., Phys. Rev. B 37, 785 (1988) (doi: 10.1103/PhysRevB.37.785) # instead of the one in libxc: B. Miehlich, A. Savin, H. Stoll, and H. Preuss., Chem. Phys. Lett. 157, 200 (1989) (doi: 10.1016/0009-2614(89)87234-3) + +# This test is now NOT included in the CI because of implementation differences between B3LYP in Grad DFT +# versus PySCF. See above. + @pytest.mark.parametrize("mol_and_name", [(MOL_WATER, "water"), (MOL_LI, "Li")]) def test_predict(mol_and_name: tuple[gto.Mole, str]) -> None: r"""Compare the total energy predicted by Grad-DFT for the B3LYP functional versus PySCF. @@ -74,4 +79,4 @@ def test_predict(mol_and_name: tuple[gto.Mole, str]) -> None: molecule_out = iterator(PARAMS, molecule) e_XND = molecule_out.energy kcalmoldiff = (e_XND - e_DM) * Hartree2kcalmol - assert jnp.allclose(kcalmoldiff, 0, atol=1), f"Energy difference with PySCF for B3LYP on {name} exceeds the threshold." \ No newline at end of file + assert jnp.allclose(kcalmoldiff, 0, atol=10), f"Energy difference with PySCF for B3LYP on {name} exceeds the threshold." \ No newline at end of file diff --git a/tests/integration/test_predict_B88.py b/tests/integration/molecules/test_predict_B88.py similarity index 100% rename from tests/integration/test_predict_B88.py rename to tests/integration/molecules/test_predict_B88.py diff --git a/tests/integration/test_predict_DM21.py b/tests/integration/molecules/test_predict_DM21.py similarity index 100% rename from tests/integration/test_predict_DM21.py rename to tests/integration/molecules/test_predict_DM21.py diff --git a/tests/integration/test_training.py b/tests/integration/molecules/test_training.py similarity index 100% rename from tests/integration/test_training.py rename to tests/integration/molecules/test_training.py diff --git a/tests/integration/solids/test_functional_implementations.py b/tests/integration/solids/test_functional_implementations.py new file mode 100644 index 0000000..8d26662 --- /dev/null +++ b/tests/integration/solids/test_functional_implementations.py @@ -0,0 +1,146 @@ +# Copyright 2023 Xanadu Quantum Technologies Inc. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from flax.core import freeze +from jax import numpy as jnp +import pytest +from grad_dft import ( + solid_from_pyscf, + energy_predictor, # A class, needs to be instanciated! + B88, LSDA, VWN, PW92 +) +from grad_dft.utils.types import Hartree2kcalmol +import numpy as np + + +# This file aims to test, given some electronic density, whether our +# implementation of popular functionals closely matches libxc (pyscf default). + +# These tests are specifically for solids. We test only LDAs and GGGas here. + +# again, this only works on startup! +from jax import config +config.update("jax_enable_x64", True) + +# First we define a molecule: +from pyscf.pbc import gto, dft + +PARAMS = freeze({"params": {}}) +DIFF_TOL = 1e-3 # in KCal/Mol so is quite small +KPTS = [2, 1, 1] + +# Look at ficticious solid Hydrogen and Lithium + +# Bond lengths in Angstroms. Taken from https://cccbdb.nist.gov/diatomicexpbondx.asp. +# This is for the molecule obviously, but we will use it as the solid lattice constant. +H2_EXP_BOND_LENGTH = 1.3984 +LI2_EXP_BOND_LENGTH = 5.0512 + +LAT_VEC_H = 2 * np.array( + [ + [H2_EXP_BOND_LENGTH, 0.0, 0.0], + [0.0, H2_EXP_BOND_LENGTH, 0.0], + [0.0, 0.0, H2_EXP_BOND_LENGTH] + ] +) + +LAT_VEC_LI = 2 * np.array( + [ + [LI2_EXP_BOND_LENGTH, 0.0, 0.0], + [0.0, LI2_EXP_BOND_LENGTH, 0.0], + [0.0, 0.0, LI2_EXP_BOND_LENGTH] + ] +) + +GEOM_H = "H 0.0 0.0 0.0; H %.5f 0.0 0.0" % (H2_EXP_BOND_LENGTH) +GEOM_LI = "H 0.0 0.0 0.0; H %.5f 0.0 0.0" % (H2_EXP_BOND_LENGTH) + +sols = [ + gto.M( + a = LAT_VEC_H, + atom=GEOM_H, + basis="sto-3g", + ), + gto.M( + a = LAT_VEC_LI, + atom=GEOM_LI, + basis="sto-3g", + ) +] + +#### LSDA #### +@pytest.mark.parametrize("sol", sols) +def test_lda(sol): + kmf = dft.KRKS(sol, kpts=sol.make_kpts(KPTS)) + kmf.xc = "LDA" # LDA is the same as LDA_X. + ground_truth_energy = kmf.kernel() + + gd_sol = solid_from_pyscf(kmf) + compute_energy = energy_predictor(LSDA) + predicted_e, fock = compute_energy(PARAMS, gd_sol) + + lsdadiff = (ground_truth_energy - predicted_e) * Hartree2kcalmol + + assert not jnp.isnan(fock).any() + assert jnp.allclose(lsdadiff, 0, atol=DIFF_TOL) + +##### B88 #### +@pytest.mark.parametrize("sol", sols) +def test_b88(sol): + kmf = dft.KRKS(sol, kpts=sol.make_kpts(KPTS)) + kmf.xc = "B88" + ground_truth_energy = kmf.kernel() + + gd_sol = solid_from_pyscf(kmf) + compute_energy = energy_predictor(B88) + predicted_e, fock = compute_energy(PARAMS, gd_sol) + + b88diff = (ground_truth_energy - predicted_e) * Hartree2kcalmol + + assert not jnp.isnan(fock).any() + assert jnp.allclose(b88diff, 0, atol=DIFF_TOL) + + +##### VWN #### +@pytest.mark.parametrize("sol", sols) +def test_vwn(sol): + kmf = dft.KRKS(sol, kpts=sol.make_kpts(KPTS)) + kmf.xc = "LDA_C_VWN" + ground_truth_energy = kmf.kernel() + + gd_sol = solid_from_pyscf(kmf) + compute_energy = energy_predictor(VWN) + predicted_e, fock = compute_energy(PARAMS, gd_sol) + + vwndiff = (ground_truth_energy - predicted_e) * Hartree2kcalmol + + assert not jnp.isnan(fock).any() + assert jnp.allclose(vwndiff, 0, atol=DIFF_TOL) + + +#### PW92 #### +@pytest.mark.parametrize("sol", sols) +def test_pw92(sol): + kmf = dft.KRKS(sol, kpts=sol.make_kpts(KPTS)) + kmf.xc = "LDA_C_PW" + ground_truth_energy = kmf.kernel() + + gd_sol = solid_from_pyscf(kmf) + compute_energy = energy_predictor(PW92) + predicted_e, fock = compute_energy(PARAMS, gd_sol) + + pw92diff = (ground_truth_energy - predicted_e) * Hartree2kcalmol + + assert not jnp.isnan(fock).any() + assert jnp.allclose(pw92diff, 0, atol=DIFF_TOL) diff --git a/tests/integration/solids/test_non_xc_energy.py b/tests/integration/solids/test_non_xc_energy.py new file mode 100644 index 0000000..4dff83e --- /dev/null +++ b/tests/integration/solids/test_non_xc_energy.py @@ -0,0 +1,111 @@ +# Copyright 2023 Xanadu Quantum Technologies Inc. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The goal of this module is to test that the implementation of the total +energy is correct in Grad-DFT (for solids) ignoring the exchange and correlation functional. +We do so by comparing the energy volume curve of [ficticious] solid Hydrogren to the energy +calculated by PySCF. +""" + +from grad_dft import solid_from_pyscf + +import numpy as np + +from pyscf.pbc import gto, dft + +from jax import config + +import pytest + +config.update("jax_enable_x64", True) + +# Bond lengths in Angstroms. Taken from https://cccbdb.nist.gov/diatomicexpbondx.asp. +# This is for the molecule obviously, but we will use it as the solid lattice constant. +H2_EXP_BOND_LENGTH = 1.3984 + +SCF_ITERS = 200 +NUM_POINTS_CURVE = 10 +LAT_PARAM_FRAC_CHANGE = 0.1 +ERR_TOL = 1e-8 +KPTS = [2, 1, 1] + +H2_LAT_VECS = [ + np.array( + [ + [p, 0.0, 0.0], + [0.0, p, 0.0], + [0.0, 0.0, p] + ] + ) for p in np.linspace( + 2 * (1 - LAT_PARAM_FRAC_CHANGE) * H2_EXP_BOND_LENGTH, + 2 * (1 + LAT_PARAM_FRAC_CHANGE) * H2_EXP_BOND_LENGTH, + NUM_POINTS_CURVE, + ) +] + +H2_GEOMS = [ + """ + H 0.0 0.0 0.0 + H %.5f 0.0 0.0 + """ + % (bl) + for bl in np.linspace( + (1 - LAT_PARAM_FRAC_CHANGE) * H2_EXP_BOND_LENGTH, + (1 + LAT_PARAM_FRAC_CHANGE) * H2_EXP_BOND_LENGTH, + NUM_POINTS_CURVE, + ) +] + +H2_TRAJ = [ + gto.M( + a = lat_vec, + atom=geom, + basis="sto-3g", + ) + for geom, lat_vec in zip(H2_GEOMS, H2_LAT_VECS) +] + + + +def solid_and_energies(geom) -> tuple[float, float]: + r"""Calculate the total energy of crystal geometry with PySCF and Grad-DFT with no XC component in the electronic energy + + Args: + geom (gto.M): The periodicic gaussian orbital object from PySCF. Contains atomic positions, basis set and lattice vectors + Returns: + tuple[float, float]: the energy predicts by PySCF and Grad-DFT + """ + kmf = dft.KRKS(geom, kpts=geom.make_kpts(KPTS)) + kmf.xc = "0.00*LDA" # quick way of having no XC energy in PySCF + E_pyscf = kmf.kernel(max_cycle=SCF_ITERS) + sol = solid_from_pyscf(kmf) + E_gdft = sol.nonXC() + return E_pyscf, E_gdft + + +@pytest.mark.parametrize( + "geom", + H2_TRAJ, +) +def test_diatomic_molecule_energy(geom) -> None: + """Compare the total energies as a function of solid lattice parameter predicted by PySCF and Grad-DFT with no XC component in the electronic energy + + Args: + geom (gto.M): The periodicic gaussian orbital object from PySCF. Contains atomic positions, basis set and lattice vectors + """ + E_pyscf, E_gdft = solid_and_energies(geom) + tot_energy_error = np.abs(E_pyscf - E_gdft) + assert ( + tot_energy_error < ERR_TOL + ), f"Total energy difference exceeds threshold: {tot_energy_error}" diff --git a/tests/integration/solids/test_training.py b/tests/integration/solids/test_training.py new file mode 100644 index 0000000..ace766a --- /dev/null +++ b/tests/integration/solids/test_training.py @@ -0,0 +1,234 @@ +# Copyright 2023 Xanadu Quantum Technologies Inc. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The goal of this module is to test that the loss functions in ~/grad_dft/train.py are +trainable and minimizable given a simple neural functional and only two H Solid objects in +the training set. These tests do the same as in ~/tests/molecules/test_training.py but we are testing for Solids +instead. + +For SCF and non-SCF training, we should have: + + +(1) Loss function gradients free of NaN's + +(2) Decreased loss after 5 iterations of an optimizer + +""" + +from jax.random import PRNGKey +import jax.numpy as jnp +import numpy as np +from jax import grad +import pytest +from pyscf.pbc import gto, scf, cc, ci + +from grad_dft import ( + solid_from_pyscf, + mse_energy_loss, + mse_density_loss, + mse_energy_and_density_loss, + diff_simple_scf_loop, + simple_scf_loop, + non_scf_predictor, + Solid, + NeuralFunctional +) + +from jax.nn import sigmoid, gelu +from flax import linen as nn +from optax import adam, apply_updates +from jax import config, value_and_grad +config.update("jax_enable_x64", True) + +# Two H solid geometries. Small basis set +LAT_VEC = np.array( + [[3.6, 0.0, 0.0], + [0.0, 3.6, 0.0], + [0.0, 0.0, 3.6]] +) + +PYSCF_SOLS = [ + gto.M( + a = LAT_VEC, + atom = """H 0.0 0.0 0.0 + H 1.4 0.0 0.0""", + basis = 'sto-3g', + space_group_symmetry=False, + symmorphic=False, + ), + + gto.M( + a = LAT_VEC, + atom = """H 0.0 0.0 0. + H 1.4 0.0 0.0""", + basis = 'sto-3g', + space_group_symmetry=False, + symmorphic=False, + ), + +] + +SCF_ITERS = 5 + +# Truth values are decided to be from MP2 calculations. +TRUTH_ENERGIES = [] +TRUTH_DENSITIES = [] +SOLIDS = [] +KPTS = [2, 1, 1] + +for sol in PYSCF_SOLS: + kmf = scf.KRKS(sol, kpts=sol.make_kpts(KPTS)) + kmf.xc = "LDA" + E_pyscf = kmf.kernel(max_cycle=SCF_ITERS) + solid = solid_from_pyscf(kmf) + SOLIDS.append(solid) + + khf = scf.KRHF(sol, kpts=sol.make_kpts(KPTS)) + khf = khf.run() + mp2 = khf.MP2().run() + mp2_rdm1 = np.asarray(mp2.make_rdm1()) + E_tr = mp2.e_tot + + # DFT calculations used for their grids to calculate MP2 densities + kmf_dft_dummy = scf.KRKS(sol, kpts=sol.make_kpts(KPTS)) + kmf_dft_dummy.kernel(max_cycle=1) + grad_dft_sol_dummy = solid_from_pyscf(kmf_dft_dummy) + dft_kccsd_rdm1 = grad_dft_sol_dummy.replace(rdm1=jnp.asarray(mp2_rdm1)) + # Works because we use the same AOs for DFT and MP2 + den_tr = grad_dft_sol_dummy.density() + + TRUTH_ENERGIES.append(E_tr) + TRUTH_DENSITIES.append(den_tr) + +# Define a simple neural functional and its initial parameters + +def coefficient_inputs(solid: Solid, clip_cte: float = 1e-30, *_, **__): + rho = jnp.clip(solid.density(), a_min = clip_cte) + return jnp.concatenate((rho, ), axis = 1) + +def energy_densities(solid: Solid, clip_cte: float = 1e-30, *_, **__): + r"""Auxiliary function to generate the features of LSDA.""" + rho = solid.density() + # To avoid numerical issues in JAX we limit too small numbers. + rho = jnp.clip(rho, a_min = clip_cte) + # Now we can implement the LSDA exchange energy density + lda_e = -3/2 * (3/(4*jnp.pi)) ** (1/3) * (rho**(4/3)).sum(axis = 1, keepdims = True) + return lda_e + +out_features = 1 +def coefficients(instance, rhoinputs): + r""" + Instance is an instance of the class Functional or NeuralFunctional. + rhoinputs is the input to the neural network, in the form of an array. + localfeatures represents the potentials e_\theta(r). + + The output of this function is the energy density of the system. + """ + + x = nn.Dense(features=out_features)(rhoinputs) + x = nn.LayerNorm()(x) + x = gelu(x) + return sigmoid(x) + +NF = NeuralFunctional(coefficients, energy_densities, coefficient_inputs) +KEY = PRNGKey(42) +CINPUTS = coefficient_inputs(SOLIDS[0]) +PARAMS = NF.init(KEY, CINPUTS) + + +# Only linear mixing SCF and non SCF training implemented for Solid objects at present +TRAIN_RECIPES = [ + # Non-SCF training on the energy only + (mse_energy_loss, [PARAMS, non_scf_predictor(NF), SOLIDS, TRUTH_ENERGIES, True]), + + # Linear mixing SCF training on the energy only + (mse_energy_loss, [PARAMS, simple_scf_loop(NF, cycles=SCF_ITERS), SOLIDS, TRUTH_ENERGIES, True]), + # Linear mixing SCF training on the density only + (mse_density_loss, [PARAMS, simple_scf_loop(NF, cycles=SCF_ITERS), SOLIDS, TRUTH_DENSITIES, True]), + # Linear SCF training on energy and density + (mse_energy_and_density_loss, [PARAMS, simple_scf_loop(NF, cycles=SCF_ITERS), SOLIDS, TRUTH_DENSITIES, TRUTH_ENERGIES, 1.0, 1.0, True]), + + # Jitted Linear mixing SCF training on the energy only + (mse_energy_loss, [PARAMS, diff_simple_scf_loop(NF, cycles=SCF_ITERS), SOLIDS, TRUTH_ENERGIES, True]), + # Jitted Linear mixing SCF training on the density only + (mse_density_loss, [PARAMS, diff_simple_scf_loop(NF, cycles=SCF_ITERS), SOLIDS, TRUTH_DENSITIES, True]), + # Jitted Linear SCF training on energy and density + (mse_energy_and_density_loss, [PARAMS, diff_simple_scf_loop(NF, cycles=SCF_ITERS), SOLIDS, TRUTH_DENSITIES, TRUTH_ENERGIES, 1.0, 1.0, True]), +] + + +@pytest.mark.parametrize("train_recipe", TRAIN_RECIPES) +def test_loss_functions(train_recipe: tuple) -> None: + r"""Same objectives as the unit test: test_loss.py but the predictors are now real DFT calculations + with Neural functionals. + + Args: + train_recipe (tuple): information regarding the loss, its arguments and the predictor. See TRAIN_RECIPES variable above. + """ + loss_func, loss_args = train_recipe + predictor_name = loss_args[1].__name__ + loss = loss_func(*loss_args) + # Pure loss test + assert not jnp.isnan( + loss + ).any(), f"Loss for loss function {loss_func.__name__} contains a NaN. It should not." + + assert ( + loss >= 0 + ), f"Loss for loss function {loss_func.__name__} is less than 0 which shouldn't be possible" + + # Gradient tests + grad_fn = grad(loss_func) + gradient = grad_fn(*loss_args) + assert not jnp.isnan( + gradient["params"]["Dense_0"]["bias"] + ).any(), f"Bias loss gradients for loss function {loss_func.__name__} and predictor {predictor_name} contains a NaN. It should not." + assert not jnp.isnan( + gradient["params"]["Dense_0"]["kernel"] + ).any(), f"Kernel loss gradients for loss function {loss_func.__name__} and predictor {predictor_name} contains a NaN. It should not." + +LR = 0.001 +MOMENTUM = 0.9 + +# and implement the optimization loop +N_EPOCHS = 5 + +@pytest.mark.parametrize("train_recipe", TRAIN_RECIPES) +def test_minimize(train_recipe: tuple) -> None: + r"""Check that the loss functions with different predictords are minimizable in 5 iterations. + + Args: + train_recipe (tuple):train_recipe (tuple): information regarding the loss, its arguments and the predictor. See TRAIN_RECIPES variable above. + """ + + loss_func, loss_args = train_recipe + predictor_name = loss_args[1].__name__ + + tr_params = NF.init(KEY, CINPUTS) + loss_args[0] = tr_params + + tx = adam(learning_rate=LR, b1=MOMENTUM) + opt_state = tx.init(PARAMS) + loss_and_grad = value_and_grad(loss_func) + cost_history = [] + for i in range(N_EPOCHS): + cost_value, grads = loss_and_grad(*loss_args) + # print(grads) + cost_history.append(cost_value) + updates, opt_state = tx.update(grads, opt_state, tr_params) + tr_params = apply_updates(tr_params, updates) + loss_args[0] = tr_params + assert ( + cost_history[-1] <= cost_history[0] + ), f"Training recipe for loss function {loss_func.__name__} and {predictor_name} did not reduce the cost in 5 iterations" \ No newline at end of file diff --git a/tests/unit/test_loss.py b/tests/unit/test_loss.py index 805f786..501b3a5 100644 --- a/tests/unit/test_loss.py +++ b/tests/unit/test_loss.py @@ -51,23 +51,23 @@ @struct.dataclass -class dummy_grid: +class Grid: r"""A dummy Grid object used only to access the weights attribute used in the density loss functions """ weights: Array -GRID = dummy_grid(GRID_WEIGHTS) +GRID = Grid(GRID_WEIGHTS) @struct.dataclass -class dummy_molecule: +class Molecule: r"""A dummy Molecule object used only to access the atom_index and charge attributes used in loss functions """ atom_index: Int[Array, "atoms"] - grid: dummy_grid + grid: Grid charge: Scalar = 0 energy: Optional[Scalar] = 0 rdm1: Optional[Float[Array, "spin orbitals orbitals"]] = 0 @@ -77,11 +77,11 @@ def density(self): return self.rho -MOLECULES = [dummy_molecule(jnp.array([1, 1]), GRID), - dummy_molecule(jnp.array([1, 8, 1]), GRID)] +MOLECULES = [Molecule(jnp.array([1, 1]), GRID), + Molecule(jnp.array([1, 8, 1]), GRID)] -def dummy_predictor(params: PyTree, molecule: dummy_molecule) -> dummy_molecule: +def dummy_predictor(params: PyTree, molecule: Molecule) -> Molecule: r"""A dummy function matching the signature of the predictor functions in Grad-DFT Args: