From 6ff165ac7e0f703ae9cee21b42f619b14ac5d868 Mon Sep 17 00:00:00 2001 From: alex-rakowski Date: Sun, 17 Dec 2023 14:55:11 -0800 Subject: [PATCH 01/19] changing install requirements --- setup.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 631f23f9a..c199930f0 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ author_email="ben.savitzky@gmail.com", license="GNU GPLv3", keywords="STEM 4DSTEM", - python_requires=">=3.9,<=3.12", + python_requires=">=3.9,<3.13", install_requires=[ "numpy >= 1.19", "scipy >= 1.5.2", @@ -47,12 +47,18 @@ "ipyparallel": ["ipyparallel >= 6.2.4", "dill >= 0.3.3"], "cuda": ["cupy >= 10.0.0"], "acom": ["pymatgen >= 2022", "mp-api == 0.24.1"], - "aiml": ["tensorflow == 2.4.1", "tensorflow-addons <= 0.14.0", "crystal4D"], + "aiml": [ + "tensorflow == 2.8.0", + "tensorflow-addons <= 0.16.1", + "crystal4D", + "typeguard == 2.7", + ], "aiml-cuda": [ - "tensorflow == 2.4.1", - "tensorflow-addons <= 0.14.0", + "tensorflow == 2.8.0", + "tensorflow-addons <= 0.16.1", "crystal4D", "cupy >= 10.0.0", + "typeguard == 2.7", ], "numba": ["numba >= 0.49.1"], }, From 374f4506d5891a3bec3c2740f3086f89fb56b73d Mon Sep 17 00:00:00 2001 From: alex-rakowski Date: Sun, 17 Dec 2023 14:55:44 -0800 Subject: [PATCH 02/19] adding initial fixes --- py4DSTEM/braggvectors/diskdetection_aiml.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/py4DSTEM/braggvectors/diskdetection_aiml.py b/py4DSTEM/braggvectors/diskdetection_aiml.py index 4d23ebf6c..e5d0b7b73 100644 --- a/py4DSTEM/braggvectors/diskdetection_aiml.py +++ b/py4DSTEM/braggvectors/diskdetection_aiml.py @@ -884,7 +884,7 @@ def _get_latest_model(model_path=None): + "https://www.tensorflow.org/install" + "for more information" ) - from py4DSTEM.io.google_drive_downloader import download_file_from_google_drive + from py4DSTEM.io.google_drive_downloader import gdrive_download tf.keras.backend.clear_session() @@ -894,7 +894,7 @@ def _get_latest_model(model_path=None): except: pass # download the json file with the meta data - download_file_from_google_drive("FCU-Net", "./tmp/model_metadata.json") + gdrive_download("1-KX0saEYfhZ9IJAOwabH38PCVtfXidJi", destination="./tmp/", filename="model_metadata.json") with open("./tmp/model_metadata.json") as f: metadata = json.load(f) file_id = metadata["file_id"] @@ -918,7 +918,11 @@ def _get_latest_model(model_path=None): else: print("Checking the latest model on the cloud... \n") filename = file_path + file_type - download_file_from_google_drive(file_id, filename) + print(f"{file_path = }") + print(f"{filename = }") + from pathlib import Path + filename = Path(filename) + gdrive_download(file_id, destination='./tmp', filename=filename.name) try: shutil.unpack_archive(filename, "./tmp", format="zip") except: From 790eff63df0c8960bf59dadc2d831ec7a2fbe56d Mon Sep 17 00:00:00 2001 From: alex-rakowski Date: Sun, 17 Dec 2023 16:46:13 -0800 Subject: [PATCH 03/19] changing how peaks are accessed --- py4DSTEM/braggvectors/diskdetection_aiml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py4DSTEM/braggvectors/diskdetection_aiml.py b/py4DSTEM/braggvectors/diskdetection_aiml.py index e5d0b7b73..9c05e960d 100644 --- a/py4DSTEM/braggvectors/diskdetection_aiml.py +++ b/py4DSTEM/braggvectors/diskdetection_aiml.py @@ -518,7 +518,7 @@ def find_Bragg_disks_aiml_serial( subpixel=subpixel, upsample_factor=upsample_factor, filter_function=filter_function, - peaks=peaks.vectors_uncal.get_pointlist(Rx, Ry), + peaks=peaks.get_pointlist(Rx, Ry), model_path=model_path, ) t2 = time() - t0 From 193e65b70bd666ee9134eba9cb5e3fdd1b4f7419 Mon Sep 17 00:00:00 2001 From: alex-rakowski Date: Sun, 17 Dec 2023 16:57:24 -0800 Subject: [PATCH 04/19] changing to PLA from BraggVectors --- py4DSTEM/braggvectors/diskdetection_aiml.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/py4DSTEM/braggvectors/diskdetection_aiml.py b/py4DSTEM/braggvectors/diskdetection_aiml.py index 9c05e960d..4767b5d26 100644 --- a/py4DSTEM/braggvectors/diskdetection_aiml.py +++ b/py4DSTEM/braggvectors/diskdetection_aiml.py @@ -437,9 +437,9 @@ def find_Bragg_disks_aiml_serial( raise ImportError("Import Error: Please install crystal4D before proceeding") # Make the peaks PointListArray - # dtype = [('qx',float),('qy',float),('intensity',float)] - peaks = BraggVectors(datacube.Rshape, datacube.Qshape) - + dtype = [('qx',float),('qy',float),('intensity',float)] + # peaks = BraggVectors(datacube.Rshape, datacube.Qshape) + peaks = PointListArray(dtype=dtype, shape=(datacube.R_Nx, datacube.R_Ny)) # check that the filtered DP is the right size for the probe kernel: if filter_function: assert callable(filter_function), "filter_function must be callable" From b214edc702b0735b5d9fce91104c2c83d80ac09a Mon Sep 17 00:00:00 2001 From: alex-rakowski Date: Sun, 17 Dec 2023 17:20:21 -0800 Subject: [PATCH 05/19] changing to PLA from BraggVectors --- py4DSTEM/braggvectors/diskdetection_aiml_cuda.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/py4DSTEM/braggvectors/diskdetection_aiml_cuda.py b/py4DSTEM/braggvectors/diskdetection_aiml_cuda.py index ffa6e891b..10dc8356a 100644 --- a/py4DSTEM/braggvectors/diskdetection_aiml_cuda.py +++ b/py4DSTEM/braggvectors/diskdetection_aiml_cuda.py @@ -124,8 +124,9 @@ def find_Bragg_disks_aiml_CUDA( """ # Make the peaks PointListArray - # dtype = [('qx',float),('qy',float),('intensity',float)] - peaks = BraggVectors(datacube.Rshape, datacube.Qshape) + dtype = [('qx',float),('qy',float),('intensity',float)] + # peaks = BraggVectors(datacube.Rshape, datacube.Qshape) + peaks = PointListArray(dtype=dtype, shape=(datacube.R_Nx, datacube.R_Ny)) # check that the filtered DP is the right size for the probe kernel: if filter_function: @@ -221,7 +222,7 @@ def find_Bragg_disks_aiml_CUDA( subpixel=subpixel, upsample_factor=upsample_factor, filter_function=filter_function, - peaks=peaks.vectors_uncal.get_pointlist(Rx, Ry), + peaks=peaks.get_pointlist(Rx, Ry), get_maximal_points=get_maximal_points, blocks=blocks, threads=threads, From d14e29451a1f007be3e3857abfbc57c8187f0898 Mon Sep 17 00:00:00 2001 From: alex-rakowski Date: Sun, 17 Dec 2023 17:21:46 -0800 Subject: [PATCH 06/19] moving import to top of file --- py4DSTEM/braggvectors/diskdetection_aiml.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/py4DSTEM/braggvectors/diskdetection_aiml.py b/py4DSTEM/braggvectors/diskdetection_aiml.py index 4767b5d26..73a2e7186 100644 --- a/py4DSTEM/braggvectors/diskdetection_aiml.py +++ b/py4DSTEM/braggvectors/diskdetection_aiml.py @@ -8,6 +8,8 @@ import json import shutil import numpy as np +from pathlib import Path + from scipy.ndimage import gaussian_filter from time import time @@ -437,7 +439,7 @@ def find_Bragg_disks_aiml_serial( raise ImportError("Import Error: Please install crystal4D before proceeding") # Make the peaks PointListArray - dtype = [('qx',float),('qy',float),('intensity',float)] + dtype = [("qx", float), ("qy", float), ("intensity", float)] # peaks = BraggVectors(datacube.Rshape, datacube.Qshape) peaks = PointListArray(dtype=dtype, shape=(datacube.R_Nx, datacube.R_Ny)) # check that the filtered DP is the right size for the probe kernel: @@ -894,7 +896,11 @@ def _get_latest_model(model_path=None): except: pass # download the json file with the meta data - gdrive_download("1-KX0saEYfhZ9IJAOwabH38PCVtfXidJi", destination="./tmp/", filename="model_metadata.json") + gdrive_download( + "1-KX0saEYfhZ9IJAOwabH38PCVtfXidJi", + destination="./tmp/", + filename="model_metadata.json", + ) with open("./tmp/model_metadata.json") as f: metadata = json.load(f) file_id = metadata["file_id"] @@ -920,9 +926,8 @@ def _get_latest_model(model_path=None): filename = file_path + file_type print(f"{file_path = }") print(f"{filename = }") - from pathlib import Path filename = Path(filename) - gdrive_download(file_id, destination='./tmp', filename=filename.name) + gdrive_download(file_id, destination="./tmp", filename=filename.name) try: shutil.unpack_archive(filename, "./tmp", format="zip") except: From 61550fe58dfd4cc06dba231aa0dd8309ce6d0e57 Mon Sep 17 00:00:00 2001 From: alex-rakowski Date: Sun, 17 Dec 2023 17:22:16 -0800 Subject: [PATCH 07/19] black --- py4DSTEM/braggvectors/diskdetection_aiml_cuda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py4DSTEM/braggvectors/diskdetection_aiml_cuda.py b/py4DSTEM/braggvectors/diskdetection_aiml_cuda.py index 10dc8356a..bd2736719 100644 --- a/py4DSTEM/braggvectors/diskdetection_aiml_cuda.py +++ b/py4DSTEM/braggvectors/diskdetection_aiml_cuda.py @@ -124,7 +124,7 @@ def find_Bragg_disks_aiml_CUDA( """ # Make the peaks PointListArray - dtype = [('qx',float),('qy',float),('intensity',float)] + dtype = [("qx", float), ("qy", float), ("intensity", float)] # peaks = BraggVectors(datacube.Rshape, datacube.Qshape) peaks = PointListArray(dtype=dtype, shape=(datacube.R_Nx, datacube.R_Ny)) From c5d23d43b3cb6608357e209a83073c8764357aab Mon Sep 17 00:00:00 2001 From: alex-rakowski Date: Sun, 17 Dec 2023 18:11:09 -0800 Subject: [PATCH 08/19] removing hardcoded link to sample collections --- py4DSTEM/braggvectors/diskdetection_aiml.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/py4DSTEM/braggvectors/diskdetection_aiml.py b/py4DSTEM/braggvectors/diskdetection_aiml.py index 73a2e7186..4ae97ad88 100644 --- a/py4DSTEM/braggvectors/diskdetection_aiml.py +++ b/py4DSTEM/braggvectors/diskdetection_aiml.py @@ -897,7 +897,8 @@ def _get_latest_model(model_path=None): pass # download the json file with the meta data gdrive_download( - "1-KX0saEYfhZ9IJAOwabH38PCVtfXidJi", + # "1-KX0saEYfhZ9IJAOwabH38PCVtfXidJi", + "FCU-Net", destination="./tmp/", filename="model_metadata.json", ) From a199e07529fbaadf23528a01b9912baa5639634f Mon Sep 17 00:00:00 2001 From: alex-rakowski Date: Sun, 17 Dec 2023 19:10:19 -0800 Subject: [PATCH 09/19] adding overwrite --- py4DSTEM/braggvectors/diskdetection_aiml.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/py4DSTEM/braggvectors/diskdetection_aiml.py b/py4DSTEM/braggvectors/diskdetection_aiml.py index 4ae97ad88..260c04235 100644 --- a/py4DSTEM/braggvectors/diskdetection_aiml.py +++ b/py4DSTEM/braggvectors/diskdetection_aiml.py @@ -897,10 +897,11 @@ def _get_latest_model(model_path=None): pass # download the json file with the meta data gdrive_download( - # "1-KX0saEYfhZ9IJAOwabH38PCVtfXidJi", + "1-KX0saEYfhZ9IJAOwabH38PCVtfXidJi", "FCU-Net", destination="./tmp/", filename="model_metadata.json", + overwrite=True, ) with open("./tmp/model_metadata.json") as f: metadata = json.load(f) From 535982d04ce75087c210c749373c40d907ebf209 Mon Sep 17 00:00:00 2001 From: alex-rakowski Date: Sun, 17 Dec 2023 19:14:16 -0800 Subject: [PATCH 10/19] typo --- Trying_to_get_FCU_net_working.ipynb | 422 ++++++++++++++++++++++++++++ 1 file changed, 422 insertions(+) create mode 100644 Trying_to_get_FCU_net_working.ipynb diff --git a/Trying_to_get_FCU_net_working.ipynb b/Trying_to_get_FCU_net_working.ipynb new file mode 100644 index 000000000..c69244a4b --- /dev/null +++ b/Trying_to_get_FCU_net_working.ipynb @@ -0,0 +1,422 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "aa07ba8a-7250-4004-a0eb-2c58654c765f", + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "51569eaf-b2aa-46a3-9b90-5d7d162a4783", + "metadata": {}, + "outputs": [], + "source": [ + "!pip install tensorflow==2.8.0\n", + "!pip install typeguard==2.7\n", + "!pip install tensorflow-addons==0.16.1\n", + "!pip install py4dstem==0.13.17\n", + "!pip install ivy\n", + "!pip install crystal4D" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "c59f0639-f5d4-47be-addc-3b3d7cbf708d", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import py4DSTEM\n", + "import crystal4D" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "8812318a-f8f1-4e46-a94f-5a84cd5b3328", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "crystal4D.utils.lrScheduler(128)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "d57d2349-6db6-4f0c-8a90-4b3845c8f59d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'0.14.9'" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "py4DSTEM.__version__" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "id": "b1960fb4-f8e5-4c8a-a9ae-9e0d6e216692", + "metadata": {}, + "outputs": [], + "source": [ + "import tensorflow_addons as tfa" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6925d62f-76a9-49ae-ba60-0c76c8672aa8", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 43, + "id": "325f83a2-38f2-4d21-8494-792418f58a85", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: At this time, the v2.11+ optimizer `tf.keras.optimizers.LAMB` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.LAMB`.\n" + ] + }, + { + "ename": "AttributeError", + "evalue": "'LAMB' object has no attribute '_set_hyper'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[43], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mtfa\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moptimizers\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mLAMB\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m128\u001b[39;49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/miniconda3/envs/py4dstem-fcunet/lib/python3.10/site-packages/typeguard/__init__.py:810\u001b[0m, in \u001b[0;36mtypechecked..wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 808\u001b[0m memo \u001b[38;5;241m=\u001b[39m _CallMemo(python_func, _localns, args\u001b[38;5;241m=\u001b[39margs, kwargs\u001b[38;5;241m=\u001b[39mkwargs)\n\u001b[1;32m 809\u001b[0m check_argument_types(memo)\n\u001b[0;32m--> 810\u001b[0m retval \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 811\u001b[0m check_return_type(retval, memo)\n\u001b[1;32m 813\u001b[0m \u001b[38;5;66;03m# If a generator is returned, wrap it if its yield/send/return types can be checked\u001b[39;00m\n", + "File \u001b[0;32m~/miniconda3/envs/py4dstem-fcunet/lib/python3.10/site-packages/tensorflow_addons/optimizers/lamb.py:97\u001b[0m, in \u001b[0;36mLAMB.__init__\u001b[0;34m(self, learning_rate, beta_1, beta_2, epsilon, weight_decay, exclude_from_weight_decay, exclude_from_layer_adaptation, name, **kwargs)\u001b[0m\n\u001b[1;32m 89\u001b[0m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m(name, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 91\u001b[0m \u001b[38;5;66;03m# Just adding the square of the weights to the loss function is *not*\u001b[39;00m\n\u001b[1;32m 92\u001b[0m \u001b[38;5;66;03m# the correct way of using L2 regularization/weight decay with Adam,\u001b[39;00m\n\u001b[1;32m 93\u001b[0m \u001b[38;5;66;03m# since that will interact with the m and v parameters in strange ways.\u001b[39;00m\n\u001b[1;32m 94\u001b[0m \u001b[38;5;66;03m#\u001b[39;00m\n\u001b[1;32m 95\u001b[0m \u001b[38;5;66;03m# Instead we want to decay the weights in a manner that doesn't interact\u001b[39;00m\n\u001b[1;32m 96\u001b[0m \u001b[38;5;66;03m# with the m/v parameters.\u001b[39;00m\n\u001b[0;32m---> 97\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_set_hyper\u001b[49m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mweight_decay\u001b[39m\u001b[38;5;124m\"\u001b[39m, weight_decay)\n\u001b[1;32m 98\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_set_hyper(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlearning_rate\u001b[39m\u001b[38;5;124m\"\u001b[39m, kwargs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlr\u001b[39m\u001b[38;5;124m\"\u001b[39m, learning_rate))\n\u001b[1;32m 100\u001b[0m \u001b[38;5;66;03m# This is learning rate decay for using keras learning rate schedule.\u001b[39;00m\n", + "\u001b[0;31mAttributeError\u001b[0m: 'LAMB' object has no attribute '_set_hyper'" + ] + } + ], + "source": [ + "tfa.optimizers.LAMB(128)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "f01faf07-a47e-4e66-b8bb-d094f3deb08b", + "metadata": {}, + "outputs": [], + "source": [ + "import tensorflow as tf" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "799d18bf-3053-42ce-8e2d-4684c5c6bcf7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "keras.src.optimizers.schedules.learning_rate_schedule.LearningRateSchedule" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tf.keras.optimizers.schedules.LearningRateSchedule" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "id": "5a34efff-d239-4fd6-b941-525dbd4b4dd9", + "metadata": {}, + "outputs": [ + { + "ename": "AttributeError", + "evalue": "module 'keras.api._v2.keras.optimizers.legacy' has no attribute 'LAMB'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[38], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mtf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkeras\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moptimizers\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlegacy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mLAMB\u001b[49m\n", + "\u001b[0;31mAttributeError\u001b[0m: module 'keras.api._v2.keras.optimizers.legacy' has no attribute 'LAMB'" + ] + } + ], + "source": [ + "tf.keras.optimizers.legacy.LAMB" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "2afcfd08-565b-4afa-a526-e535c0265e44", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'2.13.0'" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tf.__version__" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "e021f1bd-70a6-45a3-847d-4385917a87a2", + "metadata": {}, + "outputs": [ + { + "ename": "AttributeError", + "evalue": "module 'keras.api._v2.keras.optimizers.legacy' has no attribute 'Lamb'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[36], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mtf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkeras\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moptimizers\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlegacy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mLamb\u001b[49m\n", + "\u001b[0;31mAttributeError\u001b[0m: module 'keras.api._v2.keras.optimizers.legacy' has no attribute 'Lamb'" + ] + } + ], + "source": [ + "tf.keras.optimizers.legacy.Lamb" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2d894dc8-ce0a-4858-8616-3085635012d0", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "A file already exists at ./tmp/model_metadata.json, skipping...\n", + "Checking the latest model on the cloud... \n", + "\n", + "file_path = './tmp/fcunet_strain_mapping_weights_latest'\n", + "filename = './tmp/fcunet_strain_mapping_weights_latest.zip'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Downloading...\n", + "From (uriginal): https://drive.google.com/uc?id=1qzh69ZWRedpihTyLVsMWZl8RqqRZIubO\n", + "From (redirected): https://drive.google.com/uc?id=1qzh69ZWRedpihTyLVsMWZl8RqqRZIubO&confirm=t&uuid=3115ed68-b121-48cf-8eb8-8f9822ac3c4a\n", + "To: /Users/arakowski/Documents/git_repos/py4dstem-alex/tmp/fcunet_strain_mapping_weights_latest.zip\n", + "100%|████████████████████████████████████████████████████████████████████████████████| 239M/239M [00:18<00:00, 13.0MB/s]\n", + "WARNING: SavedModel saved prior to TF 2.5 detected when loading Keras model. Please ensure that you are saving the model with model.save() or tf.keras.models.save_model(), *NOT* tf.saved_model.save(). To confirm, there should be a file named \"keras_metadata.pb\" in the SavedModel directory.\n", + "WARNING: Mixed precision compatibility check (mixed_float16): WARNING\n", + "The dtype policy mixed_float16 may run slowly because this machine does not have a GPU. Only Nvidia GPUs with compute capability of at least 7.0 run quickly with mixed_float16.\n", + "If you will use compatible GPU(s) not attached to this host, e.g. by running a multi-worker model, you can ignore this warning. This message will only be logged once\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loading the model... \n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: At this time, the v2.11+ optimizer `tf.keras.optimizers.LAMB` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.LAMB`.\n" + ] + }, + { + "ename": "ValueError", + "evalue": "decay is deprecated in the new Keras optimizer, please check the docstring for valid arguments, or use the legacy optimizer, e.g., tf.keras.optimizers.legacy.LAMB.", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[28], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m m \u001b[38;5;241m=\u001b[39m \u001b[43mpy4DSTEM\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbraggvectors\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdiskdetection_aiml\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_latest_model\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Documents/git_repos/py4dstem-alex/py4DSTEM/braggvectors/diskdetection_aiml.py:934\u001b[0m, in \u001b[0;36m_get_latest_model\u001b[0;34m(model_path)\u001b[0m\n\u001b[1;32m 931\u001b[0m os\u001b[38;5;241m.\u001b[39mrename(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m./tmp/model_metadata.json\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m./tmp/model_metadata_old.json\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 932\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mLoading the model... \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 934\u001b[0m model \u001b[38;5;241m=\u001b[39m \u001b[43mtf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkeras\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodels\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload_model\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 935\u001b[0m \u001b[43m \u001b[49m\u001b[43mmodel_path\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 936\u001b[0m \u001b[43m \u001b[49m\u001b[43mcustom_objects\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mlrScheduler\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mcrystal4D\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mutils\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mutils\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlrScheduler\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m128\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 937\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 938\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 939\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mLoading the user provided model... \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n", + "File \u001b[0;32m~/miniconda3/envs/py4dstem-fcunet/lib/python3.10/site-packages/keras/src/saving/saving_api.py:238\u001b[0m, in \u001b[0;36mload_model\u001b[0;34m(filepath, custom_objects, compile, safe_mode, **kwargs)\u001b[0m\n\u001b[1;32m 230\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m saving_lib\u001b[38;5;241m.\u001b[39mload_model(\n\u001b[1;32m 231\u001b[0m filepath,\n\u001b[1;32m 232\u001b[0m custom_objects\u001b[38;5;241m=\u001b[39mcustom_objects,\n\u001b[1;32m 233\u001b[0m \u001b[38;5;28mcompile\u001b[39m\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mcompile\u001b[39m,\n\u001b[1;32m 234\u001b[0m safe_mode\u001b[38;5;241m=\u001b[39msafe_mode,\n\u001b[1;32m 235\u001b[0m )\n\u001b[1;32m 237\u001b[0m \u001b[38;5;66;03m# Legacy case.\u001b[39;00m\n\u001b[0;32m--> 238\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mlegacy_sm_saving_lib\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload_model\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 239\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilepath\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcustom_objects\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcustom_objects\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mcompile\u001b[39;49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mcompile\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\n\u001b[1;32m 240\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/miniconda3/envs/py4dstem-fcunet/lib/python3.10/site-packages/keras/src/utils/traceback_utils.py:70\u001b[0m, in \u001b[0;36mfilter_traceback..error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 67\u001b[0m filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n\u001b[1;32m 68\u001b[0m \u001b[38;5;66;03m# To get the full stack trace, call:\u001b[39;00m\n\u001b[1;32m 69\u001b[0m \u001b[38;5;66;03m# `tf.debugging.disable_traceback_filtering()`\u001b[39;00m\n\u001b[0;32m---> 70\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m e\u001b[38;5;241m.\u001b[39mwith_traceback(filtered_tb) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 71\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 72\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m filtered_tb\n", + "File \u001b[0;32m~/miniconda3/envs/py4dstem-fcunet/lib/python3.10/site-packages/typeguard/__init__.py:810\u001b[0m, in \u001b[0;36mtypechecked..wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 808\u001b[0m memo \u001b[38;5;241m=\u001b[39m _CallMemo(python_func, _localns, args\u001b[38;5;241m=\u001b[39margs, kwargs\u001b[38;5;241m=\u001b[39mkwargs)\n\u001b[1;32m 809\u001b[0m check_argument_types(memo)\n\u001b[0;32m--> 810\u001b[0m retval \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 811\u001b[0m check_return_type(retval, memo)\n\u001b[1;32m 813\u001b[0m \u001b[38;5;66;03m# If a generator is returned, wrap it if its yield/send/return types can be checked\u001b[39;00m\n", + "File \u001b[0;32m~/miniconda3/envs/py4dstem-fcunet/lib/python3.10/site-packages/tensorflow_addons/optimizers/lamb.py:89\u001b[0m, in \u001b[0;36mLAMB.__init__\u001b[0;34m(self, learning_rate, beta_1, beta_2, epsilon, weight_decay, exclude_from_weight_decay, exclude_from_layer_adaptation, name, **kwargs)\u001b[0m\n\u001b[1;32m 86\u001b[0m weight_decay \u001b[38;5;241m=\u001b[39m kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mweight_decay_rate\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m 87\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mweight_decay_rate\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[0;32m---> 89\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 91\u001b[0m \u001b[38;5;66;03m# Just adding the square of the weights to the loss function is *not*\u001b[39;00m\n\u001b[1;32m 92\u001b[0m \u001b[38;5;66;03m# the correct way of using L2 regularization/weight decay with Adam,\u001b[39;00m\n\u001b[1;32m 93\u001b[0m \u001b[38;5;66;03m# since that will interact with the m and v parameters in strange ways.\u001b[39;00m\n\u001b[1;32m 94\u001b[0m \u001b[38;5;66;03m#\u001b[39;00m\n\u001b[1;32m 95\u001b[0m \u001b[38;5;66;03m# Instead we want to decay the weights in a manner that doesn't interact\u001b[39;00m\n\u001b[1;32m 96\u001b[0m \u001b[38;5;66;03m# with the m/v parameters.\u001b[39;00m\n\u001b[1;32m 97\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_set_hyper(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mweight_decay\u001b[39m\u001b[38;5;124m\"\u001b[39m, weight_decay)\n", + "\u001b[0;31mValueError\u001b[0m: decay is deprecated in the new Keras optimizer, please check the docstring for valid arguments, or use the legacy optimizer, e.g., tf.keras.optimizers.legacy.LAMB." + ] + } + ], + "source": [ + "m = py4DSTEM.braggvectors.diskdetection_aiml._get_latest_model()" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "7982b31a-ab73-417a-b057-059fb45d4cdd", + "metadata": {}, + "outputs": [], + "source": [ + "filename = './tmp/fcunet_strain_mapping_weights_latest.zip'" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "eb6c99f8-8f30-40f3-b6b0-9b9948ceb3c0", + "metadata": {}, + "outputs": [], + "source": [ + "p = Path(filename)" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "cf99dda6-a0a7-4612-a0c9-0838d3520e49", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "('tmp', 'fcunet_strain_mapping_weights_latest.zip')" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "p.parts" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "d09c93f4-be60-4507-863a-ee9d54c8e911", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "609dd77e-5335-44bb-ad32-b22673f31801", + "metadata": {}, + "outputs": [ + { + "ename": "FileExistsError", + "evalue": "[Errno 17] File exists: './tmp'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mFileExistsError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[7], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmkdir\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m./tmp\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n", + "\u001b[0;31mFileExistsError\u001b[0m: [Errno 17] File exists: './tmp'" + ] + } + ], + "source": [ + "os.mkdir(\"./tmp\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "ab915b49-6b1f-42df-b4c3-a93022ec3a62", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Downloading...\n", + "From: https://drive.google.com/uc?id=1-KX0saEYfhZ9IJAOwabH38PCVtfXidJi\n", + "To: /Users/arakowski/Documents/git_repos/py4dstem-alex/tmp/model_metadata.json\n", + "100%|███████████████████████████████████████████████████████████████████████████████████| 200/200 [00:00<00:00, 226kB/s]\n" + ] + } + ], + "source": [ + "py4DSTEM.io.gdrive_download(id_=\"1-KX0saEYfhZ9IJAOwabH38PCVtfXidJi\", destination=\"./tmp/\", filename=\"model_metadata.json\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5c004167-5629-4b76-a01a-31a4ed5aa0a4", + "metadata": {}, + "outputs": [], + "source": [ + "py4DSTEM.io.gdrive_download" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 17e13f25771cf71b2643dc3dbb9d788788d662e0 Mon Sep 17 00:00:00 2001 From: alex-rakowski Date: Sun, 17 Dec 2023 19:14:42 -0800 Subject: [PATCH 11/19] typo --- py4DSTEM/braggvectors/diskdetection_aiml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py4DSTEM/braggvectors/diskdetection_aiml.py b/py4DSTEM/braggvectors/diskdetection_aiml.py index 260c04235..a099540e9 100644 --- a/py4DSTEM/braggvectors/diskdetection_aiml.py +++ b/py4DSTEM/braggvectors/diskdetection_aiml.py @@ -897,7 +897,7 @@ def _get_latest_model(model_path=None): pass # download the json file with the meta data gdrive_download( - "1-KX0saEYfhZ9IJAOwabH38PCVtfXidJi", + # "1-KX0saEYfhZ9IJAOwabH38PCVtfXidJi", "FCU-Net", destination="./tmp/", filename="model_metadata.json", From ebe8933ddc18ae2561c4c83a1828af6d7a3ac85d Mon Sep 17 00:00:00 2001 From: alex-rakowski Date: Sun, 17 Dec 2023 19:17:01 -0800 Subject: [PATCH 12/19] removing accidental noteobok --- Trying_to_get_FCU_net_working.ipynb | 422 ---------------------------- 1 file changed, 422 deletions(-) delete mode 100644 Trying_to_get_FCU_net_working.ipynb diff --git a/Trying_to_get_FCU_net_working.ipynb b/Trying_to_get_FCU_net_working.ipynb deleted file mode 100644 index c69244a4b..000000000 --- a/Trying_to_get_FCU_net_working.ipynb +++ /dev/null @@ -1,422 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "aa07ba8a-7250-4004-a0eb-2c58654c765f", - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "51569eaf-b2aa-46a3-9b90-5d7d162a4783", - "metadata": {}, - "outputs": [], - "source": [ - "!pip install tensorflow==2.8.0\n", - "!pip install typeguard==2.7\n", - "!pip install tensorflow-addons==0.16.1\n", - "!pip install py4dstem==0.13.17\n", - "!pip install ivy\n", - "!pip install crystal4D" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "id": "c59f0639-f5d4-47be-addc-3b3d7cbf708d", - "metadata": {}, - "outputs": [], - "source": [ - "from pathlib import Path\n", - "import py4DSTEM\n", - "import crystal4D" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "id": "8812318a-f8f1-4e46-a94f-5a84cd5b3328", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 34, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "crystal4D.utils.lrScheduler(128)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "d57d2349-6db6-4f0c-8a90-4b3845c8f59d", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'0.14.9'" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "py4DSTEM.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "id": "b1960fb4-f8e5-4c8a-a9ae-9e0d6e216692", - "metadata": {}, - "outputs": [], - "source": [ - "import tensorflow_addons as tfa" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6925d62f-76a9-49ae-ba60-0c76c8672aa8", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 43, - "id": "325f83a2-38f2-4d21-8494-792418f58a85", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING: At this time, the v2.11+ optimizer `tf.keras.optimizers.LAMB` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.LAMB`.\n" - ] - }, - { - "ename": "AttributeError", - "evalue": "'LAMB' object has no attribute '_set_hyper'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[43], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mtfa\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moptimizers\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mLAMB\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m128\u001b[39;49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/miniconda3/envs/py4dstem-fcunet/lib/python3.10/site-packages/typeguard/__init__.py:810\u001b[0m, in \u001b[0;36mtypechecked..wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 808\u001b[0m memo \u001b[38;5;241m=\u001b[39m _CallMemo(python_func, _localns, args\u001b[38;5;241m=\u001b[39margs, kwargs\u001b[38;5;241m=\u001b[39mkwargs)\n\u001b[1;32m 809\u001b[0m check_argument_types(memo)\n\u001b[0;32m--> 810\u001b[0m retval \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 811\u001b[0m check_return_type(retval, memo)\n\u001b[1;32m 813\u001b[0m \u001b[38;5;66;03m# If a generator is returned, wrap it if its yield/send/return types can be checked\u001b[39;00m\n", - "File \u001b[0;32m~/miniconda3/envs/py4dstem-fcunet/lib/python3.10/site-packages/tensorflow_addons/optimizers/lamb.py:97\u001b[0m, in \u001b[0;36mLAMB.__init__\u001b[0;34m(self, learning_rate, beta_1, beta_2, epsilon, weight_decay, exclude_from_weight_decay, exclude_from_layer_adaptation, name, **kwargs)\u001b[0m\n\u001b[1;32m 89\u001b[0m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m(name, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 91\u001b[0m \u001b[38;5;66;03m# Just adding the square of the weights to the loss function is *not*\u001b[39;00m\n\u001b[1;32m 92\u001b[0m \u001b[38;5;66;03m# the correct way of using L2 regularization/weight decay with Adam,\u001b[39;00m\n\u001b[1;32m 93\u001b[0m \u001b[38;5;66;03m# since that will interact with the m and v parameters in strange ways.\u001b[39;00m\n\u001b[1;32m 94\u001b[0m \u001b[38;5;66;03m#\u001b[39;00m\n\u001b[1;32m 95\u001b[0m \u001b[38;5;66;03m# Instead we want to decay the weights in a manner that doesn't interact\u001b[39;00m\n\u001b[1;32m 96\u001b[0m \u001b[38;5;66;03m# with the m/v parameters.\u001b[39;00m\n\u001b[0;32m---> 97\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_set_hyper\u001b[49m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mweight_decay\u001b[39m\u001b[38;5;124m\"\u001b[39m, weight_decay)\n\u001b[1;32m 98\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_set_hyper(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlearning_rate\u001b[39m\u001b[38;5;124m\"\u001b[39m, kwargs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlr\u001b[39m\u001b[38;5;124m\"\u001b[39m, learning_rate))\n\u001b[1;32m 100\u001b[0m \u001b[38;5;66;03m# This is learning rate decay for using keras learning rate schedule.\u001b[39;00m\n", - "\u001b[0;31mAttributeError\u001b[0m: 'LAMB' object has no attribute '_set_hyper'" - ] - } - ], - "source": [ - "tfa.optimizers.LAMB(128)" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "id": "f01faf07-a47e-4e66-b8bb-d094f3deb08b", - "metadata": {}, - "outputs": [], - "source": [ - "import tensorflow as tf" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "id": "799d18bf-3053-42ce-8e2d-4684c5c6bcf7", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "keras.src.optimizers.schedules.learning_rate_schedule.LearningRateSchedule" - ] - }, - "execution_count": 31, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "tf.keras.optimizers.schedules.LearningRateSchedule" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "id": "5a34efff-d239-4fd6-b941-525dbd4b4dd9", - "metadata": {}, - "outputs": [ - { - "ename": "AttributeError", - "evalue": "module 'keras.api._v2.keras.optimizers.legacy' has no attribute 'LAMB'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[38], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mtf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkeras\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moptimizers\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlegacy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mLAMB\u001b[49m\n", - "\u001b[0;31mAttributeError\u001b[0m: module 'keras.api._v2.keras.optimizers.legacy' has no attribute 'LAMB'" - ] - } - ], - "source": [ - "tf.keras.optimizers.legacy.LAMB" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "id": "2afcfd08-565b-4afa-a526-e535c0265e44", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'2.13.0'" - ] - }, - "execution_count": 30, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "tf.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "id": "e021f1bd-70a6-45a3-847d-4385917a87a2", - "metadata": {}, - "outputs": [ - { - "ename": "AttributeError", - "evalue": "module 'keras.api._v2.keras.optimizers.legacy' has no attribute 'Lamb'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[36], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mtf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkeras\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moptimizers\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlegacy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mLamb\u001b[49m\n", - "\u001b[0;31mAttributeError\u001b[0m: module 'keras.api._v2.keras.optimizers.legacy' has no attribute 'Lamb'" - ] - } - ], - "source": [ - "tf.keras.optimizers.legacy.Lamb" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "id": "2d894dc8-ce0a-4858-8616-3085635012d0", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "A file already exists at ./tmp/model_metadata.json, skipping...\n", - "Checking the latest model on the cloud... \n", - "\n", - "file_path = './tmp/fcunet_strain_mapping_weights_latest'\n", - "filename = './tmp/fcunet_strain_mapping_weights_latest.zip'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Downloading...\n", - "From (uriginal): https://drive.google.com/uc?id=1qzh69ZWRedpihTyLVsMWZl8RqqRZIubO\n", - "From (redirected): https://drive.google.com/uc?id=1qzh69ZWRedpihTyLVsMWZl8RqqRZIubO&confirm=t&uuid=3115ed68-b121-48cf-8eb8-8f9822ac3c4a\n", - "To: /Users/arakowski/Documents/git_repos/py4dstem-alex/tmp/fcunet_strain_mapping_weights_latest.zip\n", - "100%|████████████████████████████████████████████████████████████████████████████████| 239M/239M [00:18<00:00, 13.0MB/s]\n", - "WARNING: SavedModel saved prior to TF 2.5 detected when loading Keras model. Please ensure that you are saving the model with model.save() or tf.keras.models.save_model(), *NOT* tf.saved_model.save(). To confirm, there should be a file named \"keras_metadata.pb\" in the SavedModel directory.\n", - "WARNING: Mixed precision compatibility check (mixed_float16): WARNING\n", - "The dtype policy mixed_float16 may run slowly because this machine does not have a GPU. Only Nvidia GPUs with compute capability of at least 7.0 run quickly with mixed_float16.\n", - "If you will use compatible GPU(s) not attached to this host, e.g. by running a multi-worker model, you can ignore this warning. This message will only be logged once\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loading the model... \n", - "\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING: At this time, the v2.11+ optimizer `tf.keras.optimizers.LAMB` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.LAMB`.\n" - ] - }, - { - "ename": "ValueError", - "evalue": "decay is deprecated in the new Keras optimizer, please check the docstring for valid arguments, or use the legacy optimizer, e.g., tf.keras.optimizers.legacy.LAMB.", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[28], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m m \u001b[38;5;241m=\u001b[39m \u001b[43mpy4DSTEM\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbraggvectors\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdiskdetection_aiml\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_latest_model\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Documents/git_repos/py4dstem-alex/py4DSTEM/braggvectors/diskdetection_aiml.py:934\u001b[0m, in \u001b[0;36m_get_latest_model\u001b[0;34m(model_path)\u001b[0m\n\u001b[1;32m 931\u001b[0m os\u001b[38;5;241m.\u001b[39mrename(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m./tmp/model_metadata.json\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m./tmp/model_metadata_old.json\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 932\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mLoading the model... \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 934\u001b[0m model \u001b[38;5;241m=\u001b[39m \u001b[43mtf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkeras\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodels\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload_model\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 935\u001b[0m \u001b[43m \u001b[49m\u001b[43mmodel_path\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 936\u001b[0m \u001b[43m \u001b[49m\u001b[43mcustom_objects\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mlrScheduler\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mcrystal4D\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mutils\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mutils\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlrScheduler\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m128\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 937\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 938\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 939\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mLoading the user provided model... \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n", - "File \u001b[0;32m~/miniconda3/envs/py4dstem-fcunet/lib/python3.10/site-packages/keras/src/saving/saving_api.py:238\u001b[0m, in \u001b[0;36mload_model\u001b[0;34m(filepath, custom_objects, compile, safe_mode, **kwargs)\u001b[0m\n\u001b[1;32m 230\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m saving_lib\u001b[38;5;241m.\u001b[39mload_model(\n\u001b[1;32m 231\u001b[0m filepath,\n\u001b[1;32m 232\u001b[0m custom_objects\u001b[38;5;241m=\u001b[39mcustom_objects,\n\u001b[1;32m 233\u001b[0m \u001b[38;5;28mcompile\u001b[39m\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mcompile\u001b[39m,\n\u001b[1;32m 234\u001b[0m safe_mode\u001b[38;5;241m=\u001b[39msafe_mode,\n\u001b[1;32m 235\u001b[0m )\n\u001b[1;32m 237\u001b[0m \u001b[38;5;66;03m# Legacy case.\u001b[39;00m\n\u001b[0;32m--> 238\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mlegacy_sm_saving_lib\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload_model\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 239\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilepath\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcustom_objects\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcustom_objects\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mcompile\u001b[39;49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mcompile\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\n\u001b[1;32m 240\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/miniconda3/envs/py4dstem-fcunet/lib/python3.10/site-packages/keras/src/utils/traceback_utils.py:70\u001b[0m, in \u001b[0;36mfilter_traceback..error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 67\u001b[0m filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n\u001b[1;32m 68\u001b[0m \u001b[38;5;66;03m# To get the full stack trace, call:\u001b[39;00m\n\u001b[1;32m 69\u001b[0m \u001b[38;5;66;03m# `tf.debugging.disable_traceback_filtering()`\u001b[39;00m\n\u001b[0;32m---> 70\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m e\u001b[38;5;241m.\u001b[39mwith_traceback(filtered_tb) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 71\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 72\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m filtered_tb\n", - "File \u001b[0;32m~/miniconda3/envs/py4dstem-fcunet/lib/python3.10/site-packages/typeguard/__init__.py:810\u001b[0m, in \u001b[0;36mtypechecked..wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 808\u001b[0m memo \u001b[38;5;241m=\u001b[39m _CallMemo(python_func, _localns, args\u001b[38;5;241m=\u001b[39margs, kwargs\u001b[38;5;241m=\u001b[39mkwargs)\n\u001b[1;32m 809\u001b[0m check_argument_types(memo)\n\u001b[0;32m--> 810\u001b[0m retval \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 811\u001b[0m check_return_type(retval, memo)\n\u001b[1;32m 813\u001b[0m \u001b[38;5;66;03m# If a generator is returned, wrap it if its yield/send/return types can be checked\u001b[39;00m\n", - "File \u001b[0;32m~/miniconda3/envs/py4dstem-fcunet/lib/python3.10/site-packages/tensorflow_addons/optimizers/lamb.py:89\u001b[0m, in \u001b[0;36mLAMB.__init__\u001b[0;34m(self, learning_rate, beta_1, beta_2, epsilon, weight_decay, exclude_from_weight_decay, exclude_from_layer_adaptation, name, **kwargs)\u001b[0m\n\u001b[1;32m 86\u001b[0m weight_decay \u001b[38;5;241m=\u001b[39m kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mweight_decay_rate\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m 87\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mweight_decay_rate\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[0;32m---> 89\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 91\u001b[0m \u001b[38;5;66;03m# Just adding the square of the weights to the loss function is *not*\u001b[39;00m\n\u001b[1;32m 92\u001b[0m \u001b[38;5;66;03m# the correct way of using L2 regularization/weight decay with Adam,\u001b[39;00m\n\u001b[1;32m 93\u001b[0m \u001b[38;5;66;03m# since that will interact with the m and v parameters in strange ways.\u001b[39;00m\n\u001b[1;32m 94\u001b[0m \u001b[38;5;66;03m#\u001b[39;00m\n\u001b[1;32m 95\u001b[0m \u001b[38;5;66;03m# Instead we want to decay the weights in a manner that doesn't interact\u001b[39;00m\n\u001b[1;32m 96\u001b[0m \u001b[38;5;66;03m# with the m/v parameters.\u001b[39;00m\n\u001b[1;32m 97\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_set_hyper(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mweight_decay\u001b[39m\u001b[38;5;124m\"\u001b[39m, weight_decay)\n", - "\u001b[0;31mValueError\u001b[0m: decay is deprecated in the new Keras optimizer, please check the docstring for valid arguments, or use the legacy optimizer, e.g., tf.keras.optimizers.legacy.LAMB." - ] - } - ], - "source": [ - "m = py4DSTEM.braggvectors.diskdetection_aiml._get_latest_model()" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "7982b31a-ab73-417a-b057-059fb45d4cdd", - "metadata": {}, - "outputs": [], - "source": [ - "filename = './tmp/fcunet_strain_mapping_weights_latest.zip'" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "eb6c99f8-8f30-40f3-b6b0-9b9948ceb3c0", - "metadata": {}, - "outputs": [], - "source": [ - "p = Path(filename)" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "id": "cf99dda6-a0a7-4612-a0c9-0838d3520e49", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "('tmp', 'fcunet_strain_mapping_weights_latest.zip')" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "p.parts" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "d09c93f4-be60-4507-863a-ee9d54c8e911", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "609dd77e-5335-44bb-ad32-b22673f31801", - "metadata": {}, - "outputs": [ - { - "ename": "FileExistsError", - "evalue": "[Errno 17] File exists: './tmp'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mFileExistsError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[7], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmkdir\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m./tmp\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n", - "\u001b[0;31mFileExistsError\u001b[0m: [Errno 17] File exists: './tmp'" - ] - } - ], - "source": [ - "os.mkdir(\"./tmp\")" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "ab915b49-6b1f-42df-b4c3-a93022ec3a62", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Downloading...\n", - "From: https://drive.google.com/uc?id=1-KX0saEYfhZ9IJAOwabH38PCVtfXidJi\n", - "To: /Users/arakowski/Documents/git_repos/py4dstem-alex/tmp/model_metadata.json\n", - "100%|███████████████████████████████████████████████████████████████████████████████████| 200/200 [00:00<00:00, 226kB/s]\n" - ] - } - ], - "source": [ - "py4DSTEM.io.gdrive_download(id_=\"1-KX0saEYfhZ9IJAOwabH38PCVtfXidJi\", destination=\"./tmp/\", filename=\"model_metadata.json\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5c004167-5629-4b76-a01a-31a4ed5aa0a4", - "metadata": {}, - "outputs": [], - "source": [ - "py4DSTEM.io.gdrive_download" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} From 812638d961bcc8b6d3706ce851acc5e075c1d53a Mon Sep 17 00:00:00 2001 From: alex-rakowski Date: Sun, 17 Dec 2023 19:17:57 -0800 Subject: [PATCH 13/19] back to hardcoded id --- py4DSTEM/braggvectors/diskdetection_aiml.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/py4DSTEM/braggvectors/diskdetection_aiml.py b/py4DSTEM/braggvectors/diskdetection_aiml.py index a099540e9..ba70a2110 100644 --- a/py4DSTEM/braggvectors/diskdetection_aiml.py +++ b/py4DSTEM/braggvectors/diskdetection_aiml.py @@ -897,8 +897,8 @@ def _get_latest_model(model_path=None): pass # download the json file with the meta data gdrive_download( - # "1-KX0saEYfhZ9IJAOwabH38PCVtfXidJi", - "FCU-Net", + "1-KX0saEYfhZ9IJAOwabH38PCVtfXidJi", + # "FCU-Net", destination="./tmp/", filename="model_metadata.json", overwrite=True, From 0496da1d39c9271505873fa22a945289552ea0c8 Mon Sep 17 00:00:00 2001 From: alex-rakowski Date: Sun, 17 Dec 2023 19:22:51 -0800 Subject: [PATCH 14/19] changing filename for FCU-Net file --- py4DSTEM/io/google_drive_downloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py4DSTEM/io/google_drive_downloader.py b/py4DSTEM/io/google_drive_downloader.py index 5b53f19ae..ff3907ba5 100644 --- a/py4DSTEM/io/google_drive_downloader.py +++ b/py4DSTEM/io/google_drive_downloader.py @@ -57,7 +57,7 @@ ), "small_dm3_3Dstack": ("small_dm3_3Dstack.dm3", "1B-xX3F65JcWzAg0v7f1aVwnawPIfb5_o"), "FCU-Net": ( - "filename.name", + "model_metadata.json", "1-KX0saEYfhZ9IJAOwabH38PCVtfXidJi", ), "small_datacube": ( From 7f6779a9f164750c8fff31f27f6722fd581ddb4c Mon Sep 17 00:00:00 2001 From: alex-rakowski Date: Sun, 17 Dec 2023 19:27:09 -0800 Subject: [PATCH 15/19] Adding ability to force filename for collection Items --- py4DSTEM/io/google_drive_downloader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/py4DSTEM/io/google_drive_downloader.py b/py4DSTEM/io/google_drive_downloader.py index ff3907ba5..fe4a4e8fd 100644 --- a/py4DSTEM/io/google_drive_downloader.py +++ b/py4DSTEM/io/google_drive_downloader.py @@ -221,7 +221,8 @@ def gdrive_download( kwargs = {"fuzzy": True} if id_ in file_ids: f = file_ids[id_] - filename = f[0] + # Use the name in the collection filename passed + filename = filename if filename is not None else f[0] kwargs["id"] = f[1] # if its not in the list of files we expect From 57f867f0461add880d21c9d5a4c181a96ee2f2eb Mon Sep 17 00:00:00 2001 From: alex-rakowski Date: Sun, 17 Dec 2023 19:32:39 -0800 Subject: [PATCH 16/19] back to non hardcoded now gdrive function fixed --- py4DSTEM/braggvectors/diskdetection_aiml.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/py4DSTEM/braggvectors/diskdetection_aiml.py b/py4DSTEM/braggvectors/diskdetection_aiml.py index ba70a2110..a099540e9 100644 --- a/py4DSTEM/braggvectors/diskdetection_aiml.py +++ b/py4DSTEM/braggvectors/diskdetection_aiml.py @@ -897,8 +897,8 @@ def _get_latest_model(model_path=None): pass # download the json file with the meta data gdrive_download( - "1-KX0saEYfhZ9IJAOwabH38PCVtfXidJi", - # "FCU-Net", + # "1-KX0saEYfhZ9IJAOwabH38PCVtfXidJi", + "FCU-Net", destination="./tmp/", filename="model_metadata.json", overwrite=True, From 11b64a925b6d068d234542d19e01d938773cadcf Mon Sep 17 00:00:00 2001 From: alex-rakowski Date: Mon, 18 Dec 2023 01:19:27 -0800 Subject: [PATCH 17/19] removing print statements --- py4DSTEM/braggvectors/diskdetection_aiml.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/py4DSTEM/braggvectors/diskdetection_aiml.py b/py4DSTEM/braggvectors/diskdetection_aiml.py index a099540e9..8e3607edb 100644 --- a/py4DSTEM/braggvectors/diskdetection_aiml.py +++ b/py4DSTEM/braggvectors/diskdetection_aiml.py @@ -897,7 +897,6 @@ def _get_latest_model(model_path=None): pass # download the json file with the meta data gdrive_download( - # "1-KX0saEYfhZ9IJAOwabH38PCVtfXidJi", "FCU-Net", destination="./tmp/", filename="model_metadata.json", @@ -926,8 +925,6 @@ def _get_latest_model(model_path=None): else: print("Checking the latest model on the cloud... \n") filename = file_path + file_type - print(f"{file_path = }") - print(f"{filename = }") filename = Path(filename) gdrive_download(file_id, destination="./tmp", filename=filename.name) try: From 870eb00072cc40b1cf3f08154c9822dda8723739 Mon Sep 17 00:00:00 2001 From: alex-rakowski Date: Mon, 18 Dec 2023 01:26:51 -0800 Subject: [PATCH 18/19] bumping allowed version of tensorflow --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index c199930f0..58f61dc4a 100644 --- a/setup.py +++ b/setup.py @@ -48,13 +48,13 @@ "cuda": ["cupy >= 10.0.0"], "acom": ["pymatgen >= 2022", "mp-api == 0.24.1"], "aiml": [ - "tensorflow == 2.8.0", + "tensorflow <= 2.10.0", "tensorflow-addons <= 0.16.1", "crystal4D", "typeguard == 2.7", ], "aiml-cuda": [ - "tensorflow == 2.8.0", + "tensorflow <= 2.10.0", "tensorflow-addons <= 0.16.1", "crystal4D", "cupy >= 10.0.0", From 300aaa3967039a2177ae82200a40e28c3b4022e7 Mon Sep 17 00:00:00 2001 From: alex-rakowski Date: Mon, 18 Dec 2023 01:33:12 -0800 Subject: [PATCH 19/19] black --- py4DSTEM/io/google_drive_downloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py4DSTEM/io/google_drive_downloader.py b/py4DSTEM/io/google_drive_downloader.py index fe4a4e8fd..a50fb2c09 100644 --- a/py4DSTEM/io/google_drive_downloader.py +++ b/py4DSTEM/io/google_drive_downloader.py @@ -221,7 +221,7 @@ def gdrive_download( kwargs = {"fuzzy": True} if id_ in file_ids: f = file_ids[id_] - # Use the name in the collection filename passed + # Use the name in the collection filename passed filename = filename if filename is not None else f[0] kwargs["id"] = f[1]