From ba82714d0d7fb1f44d845a6584fe337e9552ede0 Mon Sep 17 00:00:00 2001
From: Ruge Li <rugeli0605@gmail.com>
Date: Wed, 5 Jul 2023 13:50:33 -0700
Subject: [PATCH 01/16] starting code

---
 .../upy/simularium/simularium_helper.py         | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/cellpack/autopack/upy/simularium/simularium_helper.py b/cellpack/autopack/upy/simularium/simularium_helper.py
index ca8d068b3..a0c6f9bf9 100644
--- a/cellpack/autopack/upy/simularium/simularium_helper.py
+++ b/cellpack/autopack/upy/simularium/simularium_helper.py
@@ -21,6 +21,9 @@
 from cellpack.autopack.upy import hostHelper
 import collada
 
+import boto3
+import logging
+
 
 class Instance:
     def __init__(self, name, instance_id, unique_id, radius, viz_type, mesh=None):
@@ -1331,6 +1334,7 @@ def writeToFile(self, file_name, bb, recipe_name, version):
             time_units=UnitData("ns"),  # nanoseconds
             spatial_units=UnitData("nm"),  # nanometers
         )
+        print("simularium helper---", vars(converted_data))
         TrajectoryConverter(converted_data).save(file_name, False)
 
     def raycast(self, **kw):
@@ -1345,3 +1349,16 @@ def raycast(self, **kw):
 
     def raycast_test(self, obj, start, end, length, **kw):
         return
+    
+    # def store_results_in_s3(bucket_name, file_name, object_name=None):
+    #     if object_name is None:
+    #         object_name = file_name
+
+    #     s3_client = boto3.client("3s")
+    #     try:
+    #         response = s3_client.upload_file(file_name, bucket_name, object_name)
+    #     except ClientError as e:
+    #         logging.error(e)
+    #         return False
+    #     return True
+

From 32bf257a24bf26d583eadaba149789f0af59af5c Mon Sep 17 00:00:00 2001
From: Ruge Li <rugeli0605@gmail.com>
Date: Thu, 6 Jul 2023 14:12:09 -0700
Subject: [PATCH 02/16] store results to S3 and firebase

---
 cellpack/autopack/AWSHandler.py               | 42 +++++++++++++++++
 cellpack/autopack/FirebaseHandler.py          | 12 +++++
 .../upy/simularium/simularium_helper.py       | 45 ++++++++++++-------
 3 files changed, 84 insertions(+), 15 deletions(-)
 create mode 100644 cellpack/autopack/AWSHandler.py

diff --git a/cellpack/autopack/AWSHandler.py b/cellpack/autopack/AWSHandler.py
new file mode 100644
index 000000000..2ea313c48
--- /dev/null
+++ b/cellpack/autopack/AWSHandler.py
@@ -0,0 +1,42 @@
+import logging
+import boto3
+from botocore.exceptions import ClientError
+import os
+
+class AWSHandler(object):
+    """
+    Handles all the AWS S3 operations
+    """
+
+    def __init__(self, bucket_name, aws_access_key_id=None, aws_secret_access_key=None, region_name=None):
+        self.bucket_name = bucket_name
+        self.s3_client = boto3.client(
+            's3',
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key, # TODO: check how do we want to handle credentials?
+            region_name=region_name
+        )
+
+
+    def upload_file(self, file_name, folder_name=None, object_name=None):
+        """Upload a file to an S3 bucket
+
+        :param file_name: File to upload
+        :param bucket: Bucket to upload to
+        :param object_name: S3 object name. If not specified then file_name is used
+        :return: True if file was uploaded, else False
+        """
+
+        # If S3 object_name was not specified, use file_name
+        if object_name is None:
+            object_name = folder_name + os.path.basename(file_name)
+        else:
+            object_name = folder_name + object_name
+
+        # Upload the file
+        try:
+            response = self.s3_client.upload_file(file_name, self.bucket_name, object_name)
+        except ClientError as e:
+            logging.error(e)
+            return False
+        return True
\ No newline at end of file
diff --git a/cellpack/autopack/FirebaseHandler.py b/cellpack/autopack/FirebaseHandler.py
index e42955024..e7d60cfbc 100644
--- a/cellpack/autopack/FirebaseHandler.py
+++ b/cellpack/autopack/FirebaseHandler.py
@@ -42,6 +42,14 @@ def get_creds():
         if creds is None or "firebase" not in creds:
             creds = FirebaseHandler.write_creds_path()
         return creds["firebase"]
+    
+    @staticmethod
+    def get_username():
+        creds = read_json_file("./.creds")
+        try:
+            return creds["username"]
+        except KeyError:
+            raise ValueError("No username found in .creds file")
 
     def db_name(self):
         return self.name
@@ -53,6 +61,10 @@ def doc_id(doc):
     @staticmethod
     def create_path(collection, doc_id):
         return f"firebase:{collection}/{doc_id}"
+    
+    @staticmethod
+    def create_timestamp():
+        return firestore.SERVER_TIMESTAMP
 
     @staticmethod
     def get_path_from_ref(doc):
diff --git a/cellpack/autopack/upy/simularium/simularium_helper.py b/cellpack/autopack/upy/simularium/simularium_helper.py
index a0c6f9bf9..7c44d0fbc 100644
--- a/cellpack/autopack/upy/simularium/simularium_helper.py
+++ b/cellpack/autopack/upy/simularium/simularium_helper.py
@@ -17,13 +17,12 @@
 )
 from simulariumio.cellpack import CellpackConverter, HAND_TYPE
 from simulariumio.constants import DISPLAY_TYPE, VIZ_TYPE
+from cellpack.autopack.FirebaseHandler import FirebaseHandler
 
 from cellpack.autopack.upy import hostHelper
+from cellpack.autopack.AWSHandler import AWSHandler
 import collada
 
-import boto3
-import logging
-
 
 class Instance:
     def __init__(self, name, instance_id, unique_id, radius, viz_type, mesh=None):
@@ -1334,8 +1333,11 @@ def writeToFile(self, file_name, bb, recipe_name, version):
             time_units=UnitData("ns"),  # nanoseconds
             spatial_units=UnitData("nm"),  # nanometers
         )
-        print("simularium helper---", vars(converted_data))
         TrajectoryConverter(converted_data).save(file_name, False)
+        print("file--", file_name)
+        simularium_file = file_name + ".simularium"
+        print("sim_file", simularium_file)
+        # simulariumHelper.store_results_to_s3(simularium_file,folder_name="simularium/")
 
     def raycast(self, **kw):
         intersect = False
@@ -1350,15 +1352,28 @@ def raycast(self, **kw):
     def raycast_test(self, obj, start, end, length, **kw):
         return
     
-    # def store_results_in_s3(bucket_name, file_name, object_name=None):
-    #     if object_name is None:
-    #         object_name = file_name
-
-    #     s3_client = boto3.client("3s")
-    #     try:
-    #         response = s3_client.upload_file(file_name, bucket_name, object_name)
-    #     except ClientError as e:
-    #         logging.error(e)
-    #         return False
-    #     return True
+    @staticmethod
+    def store_results_to_s3(file_name, folder_name=None, object_name=None):
+        handler = AWSHandler(
+            bucket_name="cellpack-results",
+            aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID"),
+            aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY"),
+            region_name="us-west-2",
+        )
+        print(handler.upload_file(file_name,folder_name,object_name))
+
+    def upload_metadata_to_firebase(result_name, aws_url):
+        db = FirebaseHandler()
+        username = db.get_username()
+        timestamp = db.create_timestamp()
+        db.set_doc(
+            "results", 
+            result_name,
+            {
+            "user": username,
+            "created": timestamp,
+            "aws_url": aws_url
+            }
+        )
+    upload_metadata_to_firebase("testing result2", "testing_url2")
 

From 286dfbffca088ea4acdfe84fe0fb00d0696c401a Mon Sep 17 00:00:00 2001
From: Ruge Li <rugeli0605@gmail.com>
Date: Tue, 11 Jul 2023 17:16:41 -0700
Subject: [PATCH 03/16] draft - get aws url and open in simularium

---
 cellpack/autopack/AWSHandler.py               | 45 ++++++++++++++++---
 .../upy/simularium/simularium_helper.py       | 28 ++++++++++--
 2 files changed, 62 insertions(+), 11 deletions(-)

diff --git a/cellpack/autopack/AWSHandler.py b/cellpack/autopack/AWSHandler.py
index 2ea313c48..2d025caeb 100644
--- a/cellpack/autopack/AWSHandler.py
+++ b/cellpack/autopack/AWSHandler.py
@@ -3,21 +3,27 @@
 from botocore.exceptions import ClientError
 import os
 
+
 class AWSHandler(object):
     """
     Handles all the AWS S3 operations
     """
 
-    def __init__(self, bucket_name, aws_access_key_id=None, aws_secret_access_key=None, region_name=None):
+    def __init__(
+        self,
+        bucket_name,
+        aws_access_key_id=None,
+        aws_secret_access_key=None,
+        region_name=None,
+    ):
         self.bucket_name = bucket_name
         self.s3_client = boto3.client(
-            's3',
+            "s3",
             aws_access_key_id=aws_access_key_id,
-            aws_secret_access_key=aws_secret_access_key, # TODO: check how do we want to handle credentials?
-            region_name=region_name
+            aws_secret_access_key=aws_secret_access_key,  # TODO: check how do we want to handle credentials?
+            region_name=region_name,
         )
 
-
     def upload_file(self, file_name, folder_name=None, object_name=None):
         """Upload a file to an S3 bucket
 
@@ -35,8 +41,33 @@ def upload_file(self, file_name, folder_name=None, object_name=None):
 
         # Upload the file
         try:
-            response = self.s3_client.upload_file(file_name, self.bucket_name, object_name)
+            response = self.s3_client.upload_file(
+                file_name, self.bucket_name, object_name
+            )
+        # TODO: check what is response, return object name if successful
         except ClientError as e:
             logging.error(e)
             return False
-        return True
\ No newline at end of file
+        return True
+
+    def create_presigned_url(self, object_name, expiration=3600):
+        """Generate a presigned URL to share an S3 object
+
+        :param object_name: string
+        :param expiration: Time in seconds for the presigned URL to remain valid
+        :return: Presigned URL as string. If error, returns None.
+        """
+
+        # Generate a presigned URL for the S3 object
+        try:
+            response = self.s3_client.generate_presigned_url(
+                "get_object",
+                Params={"Bucket": self.bucket_name, "Key": object_name},
+                ExpiresIn=expiration,
+            )
+        except ClientError as e:
+            logging.error(e)
+            return None
+
+        # The response contains the presigned URL
+        return response
diff --git a/cellpack/autopack/upy/simularium/simularium_helper.py b/cellpack/autopack/upy/simularium/simularium_helper.py
index 7c44d0fbc..a9f00f173 100644
--- a/cellpack/autopack/upy/simularium/simularium_helper.py
+++ b/cellpack/autopack/upy/simularium/simularium_helper.py
@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 # standardmodule
 import os
+import webbrowser
 import matplotlib
 import numpy as np
 import trimesh
@@ -1334,10 +1335,20 @@ def writeToFile(self, file_name, bb, recipe_name, version):
             spatial_units=UnitData("nm"),  # nanometers
         )
         TrajectoryConverter(converted_data).save(file_name, False)
-        print("file--", file_name)
+        # check if the file exists, if so, store it to S3 and upload the metadata in Firebase
         simularium_file = file_name + ".simularium"
-        print("sim_file", simularium_file)
-        # simulariumHelper.store_results_to_s3(simularium_file,folder_name="simularium/")
+        if simulariumHelper.check_file_exists(simularium_file):
+            try:
+                simulariumHelper.store_results_to_s3(simularium_file,folder_name="simularium/")
+            except Exception as e:
+                print(f"An error occurred while storing the file {simularium_file} to S3:", e)
+        else:
+            print(f"File {simularium_file} does not exist.")
+        
+
+    @staticmethod
+    def check_file_exists(file_name):
+        return os.path.isfile(file_name)
 
     def raycast(self, **kw):
         intersect = False
@@ -1360,7 +1371,9 @@ def store_results_to_s3(file_name, folder_name=None, object_name=None):
             aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY"),
             region_name="us-west-2",
         )
-        print(handler.upload_file(file_name,folder_name,object_name))
+        # TODO: get object_name
+        handler.upload_file(file_name,folder_name,object_name)
+        return handler.create_presigned_url(object_name)
 
     def upload_metadata_to_firebase(result_name, aws_url):
         db = FirebaseHandler()
@@ -1375,5 +1388,12 @@ def upload_metadata_to_firebase(result_name, aws_url):
             "aws_url": aws_url
             }
         )
+    # TODO: where to call this func?
     upload_metadata_to_firebase("testing result2", "testing_url2")
 
+    @staticmethod
+    def open_in_simularium(aws_url):
+        # TODO: save the endpoint,routes and options somewhere? 
+        webbrowser.open_new_tab(f"https://simularium.allencell.org/viewer?trajUrl={aws_url}")
+    open_in_simularium()
+

From bc52d38e02cc342669f1749f5d57e5686f029a8a Mon Sep 17 00:00:00 2001
From: meganrm <meganrm@gmail.com>
Date: Thu, 13 Jul 2023 14:40:21 -0700
Subject: [PATCH 04/16] add boto3

---
 setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup.py b/setup.py
index baa49d464..f67571595 100644
--- a/setup.py
+++ b/setup.py
@@ -45,6 +45,7 @@
 ]
 
 requirements = [
+    "boto3>=1.28.3",
     "fire>=0.4.0",
     "firebase_admin>=6.0.1",
     "matplotlib>=3.3.4",

From 5840c2ebfa9026fa3c4787bf7bb27507afee55ca Mon Sep 17 00:00:00 2001
From: meganrm <meganrm@gmail.com>
Date: Thu, 13 Jul 2023 18:57:20 -0700
Subject: [PATCH 05/16] upload file to s3 and open it

---
 cellpack/autopack/AWSHandler.py               | 48 +++++++-----
 cellpack/autopack/FirebaseHandler.py          |  6 ++
 .../upy/simularium/simularium_helper.py       | 77 +++++++++++--------
 cellpack/autopack/writers/__init__.py         |  5 +-
 4 files changed, 82 insertions(+), 54 deletions(-)

diff --git a/cellpack/autopack/AWSHandler.py b/cellpack/autopack/AWSHandler.py
index 2d025caeb..05bc285ba 100644
--- a/cellpack/autopack/AWSHandler.py
+++ b/cellpack/autopack/AWSHandler.py
@@ -1,7 +1,7 @@
 import logging
 import boto3
 from botocore.exceptions import ClientError
-import os
+from pathlib import Path
 
 
 class AWSHandler(object):
@@ -12,43 +12,53 @@ class AWSHandler(object):
     def __init__(
         self,
         bucket_name,
+        sub_folder_name=None, 
         aws_access_key_id=None,
         aws_secret_access_key=None,
         region_name=None,
     ):
         self.bucket_name = bucket_name
-        self.s3_client = boto3.client(
+        self.folder_name = sub_folder_name
+        session = boto3.Session()
+        self.s3_client = session.client(
             "s3",
+            endpoint_url=f"https://{bucket_name}.s3.{region_name}.amazonaws.com",
             aws_access_key_id=aws_access_key_id,
-            aws_secret_access_key=aws_secret_access_key,  # TODO: check how do we want to handle credentials?
+            aws_secret_access_key=aws_secret_access_key,
             region_name=region_name,
         )
 
-    def upload_file(self, file_name, folder_name=None, object_name=None):
+    def get_aws_object_key(self, object_name):
+        if self.folder_name is not None: 
+            object_name = self.folder_name + object_name
+        else:
+            object_name = object_name
+        return object_name
+
+
+    def upload_file(self, file_path):
         """Upload a file to an S3 bucket
 
-        :param file_name: File to upload
+        :param file_path: File to upload
         :param bucket: Bucket to upload to
-        :param object_name: S3 object name. If not specified then file_name is used
+        :param object_name: S3 object name. If not specified then file_path is used
         :return: True if file was uploaded, else False
         """
 
-        # If S3 object_name was not specified, use file_name
-        if object_name is None:
-            object_name = folder_name + os.path.basename(file_name)
-        else:
-            object_name = folder_name + object_name
+        file_name = Path(file_path).stem
 
+        object_name = self.get_aws_object_key(file_name)
         # Upload the file
         try:
-            response = self.s3_client.upload_file(
-                file_name, self.bucket_name, object_name
+            self.s3_client.upload_file(
+                file_path, self.bucket_name, object_name
             )
-        # TODO: check what is response, return object name if successful
+            self.s3_client.put_object_acl( ACL='public-read', Bucket=self.bucket_name, Key=object_name )
+        
         except ClientError as e:
             logging.error(e)
             return False
-        return True
+        return file_name
 
     def create_presigned_url(self, object_name, expiration=3600):
         """Generate a presigned URL to share an S3 object
@@ -57,10 +67,10 @@ def create_presigned_url(self, object_name, expiration=3600):
         :param expiration: Time in seconds for the presigned URL to remain valid
         :return: Presigned URL as string. If error, returns None.
         """
-
+        object_name = self.get_aws_object_key(object_name)
         # Generate a presigned URL for the S3 object
         try:
-            response = self.s3_client.generate_presigned_url(
+            url = self.s3_client.generate_presigned_url(
                 "get_object",
                 Params={"Bucket": self.bucket_name, "Key": object_name},
                 ExpiresIn=expiration,
@@ -68,6 +78,6 @@ def create_presigned_url(self, object_name, expiration=3600):
         except ClientError as e:
             logging.error(e)
             return None
-
         # The response contains the presigned URL
-        return response
+        # https://{self.bucket_name}.s3.{region}.amazonaws.com/{object_key}
+        return url
diff --git a/cellpack/autopack/FirebaseHandler.py b/cellpack/autopack/FirebaseHandler.py
index e7d60cfbc..320e06843 100644
--- a/cellpack/autopack/FirebaseHandler.py
+++ b/cellpack/autopack/FirebaseHandler.py
@@ -78,6 +78,12 @@ def get_collection_id_from_path(path):
         id = components[1]
         return collection, id
 
+    def update_doc(self, collection, id, data):
+        doc_ref = self.db.collection(collection).document(id)
+        doc_ref.update(data)
+        print(f"successfully uploaded to path: {doc_ref.path}")
+        return doc_ref
+    
     @staticmethod
     def update_reference_on_doc(doc_ref, index, new_item_ref):
         doc_ref.update({index: new_item_ref})
diff --git a/cellpack/autopack/upy/simularium/simularium_helper.py b/cellpack/autopack/upy/simularium/simularium_helper.py
index a9f00f173..2e395b75a 100644
--- a/cellpack/autopack/upy/simularium/simularium_helper.py
+++ b/cellpack/autopack/upy/simularium/simularium_helper.py
@@ -5,6 +5,8 @@
 import matplotlib
 import numpy as np
 import trimesh
+from pathlib import Path
+from botocore.exceptions import NoCredentialsError
 
 from simulariumio import (
     TrajectoryConverter,
@@ -1335,20 +1337,28 @@ def writeToFile(self, file_name, bb, recipe_name, version):
             spatial_units=UnitData("nm"),  # nanometers
         )
         TrajectoryConverter(converted_data).save(file_name, False)
-        # check if the file exists, if so, store it to S3 and upload the metadata in Firebase
-        simularium_file = file_name + ".simularium"
-        if simulariumHelper.check_file_exists(simularium_file):
-            try:
-                simulariumHelper.store_results_to_s3(simularium_file,folder_name="simularium/")
-            except Exception as e:
-                print(f"An error occurred while storing the file {simularium_file} to S3:", e)
-        else:
-            print(f"File {simularium_file} does not exist.")
-        
+        return file_name
 
-    @staticmethod
-    def check_file_exists(file_name):
-        return os.path.isfile(file_name)
+
+    def post_and_open_file(self, file_name):
+        simularium_file = Path(f"{file_name}.simularium")
+        url = None
+        try:
+            url = simulariumHelper.store_results_to_s3(
+                simularium_file
+            )
+        except Exception as e:
+            if isinstance(e, NoCredentialsError):
+                # TODO: add info on AWS installation and setup to our documetation
+                # TODO: link to documentation section on setting up AWS CLI and boto3 authentation
+                print(f"need to configure your aws account (link to readme here)")
+            else:
+                print(
+                    f"An error occurred while storing the file {simularium_file} to S3:",
+                    e,
+                )
+        if url is not None:
+            simulariumHelper.open_in_simularium(url)
 
     def raycast(self, **kw):
         intersect = False
@@ -1362,38 +1372,39 @@ def raycast(self, **kw):
 
     def raycast_test(self, obj, start, end, length, **kw):
         return
-    
+
     @staticmethod
-    def store_results_to_s3(file_name, folder_name=None, object_name=None):
+    def store_results_to_s3(file_path):
         handler = AWSHandler(
             bucket_name="cellpack-results",
             aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID"),
             aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY"),
             region_name="us-west-2",
+            sub_folder_name="simularium/"
         )
-        # TODO: get object_name
-        handler.upload_file(file_name,folder_name,object_name)
-        return handler.create_presigned_url(object_name)
+        file_name = handler.upload_file(file_path)
+        url = handler.create_presigned_url(file_name)
+        # simulariumHelper.upload_metadata_to_firebase(file_name, url)
+        return url
 
-    def upload_metadata_to_firebase(result_name, aws_url):
+    @staticmethod
+    def upload_metadata_to_firebase(result_file_name, aws_url):
+        # TODO: use db handler (rename to not have recipe in it)
+        # write a update_doc function that in the firebase handler
+        # class update doc instead of set doc
+        # check to make sure it writes a doc if non exists
         db = FirebaseHandler()
         username = db.get_username()
         timestamp = db.create_timestamp()
-        db.set_doc(
-            "results", 
-            result_name,
-            {
-            "user": username,
-            "created": timestamp,
-            "aws_url": aws_url
-            }
+        db.update_doc(
+            "results",
+            result_file_name,
+            {"user": username, "created": timestamp, "aws_url": aws_url},
         )
-    # TODO: where to call this func?
-    upload_metadata_to_firebase("testing result2", "testing_url2")
 
     @staticmethod
     def open_in_simularium(aws_url):
-        # TODO: save the endpoint,routes and options somewhere? 
-        webbrowser.open_new_tab(f"https://simularium.allencell.org/viewer?trajUrl={aws_url}")
-    open_in_simularium()
-
+        # TODO: save the endpoint,routes and options somewhere?
+        webbrowser.open_new_tab(
+            f"https://simularium.allencell.org/viewer?trajUrl={aws_url}"
+        )
diff --git a/cellpack/autopack/writers/__init__.py b/cellpack/autopack/writers/__init__.py
index f4e8b3d1d..1d2e167d2 100644
--- a/cellpack/autopack/writers/__init__.py
+++ b/cellpack/autopack/writers/__init__.py
@@ -101,10 +101,11 @@ def save_as_simularium(
                 autopack.helper.add_grid_data_to_scene(
                     f"{gradient.name}-distances", grid_positions, values
                 )
-
-        autopack.helper.writeToFile(
+        
+        file_name = autopack.helper.writeToFile(
             f"{result_file_path}_results", env.boundingBox, env.name, env.version
         )
+        autopack.helper.post_and_open_file(file_name)
 
     def save_Mixed_asJson(
         self,

From 05e811ab7cde1a58c74c48107d8de5b73107fbff Mon Sep 17 00:00:00 2001
From: Ruge Li <rugeli0605@gmail.com>
Date: Fri, 14 Jul 2023 14:19:32 -0700
Subject: [PATCH 06/16] *rename DBHandler and update/upload metadata to DB

---
 .../{DBRecipeHandler.py => DBHandler.py}      | 31 +++++++++++++------
 cellpack/autopack/FirebaseHandler.py          |  2 +-
 cellpack/autopack/__init__.py                 |  8 ++---
 .../upy/simularium/simularium_helper.py       | 16 +++++-----
 cellpack/bin/upload.py                        |  4 +--
 cellpack/tests/test_comp_doc.py               |  2 +-
 cellpack/tests/test_db_recipe_handler.py      | 26 ++++++++--------
 cellpack/tests/test_gradient_doc.py           |  2 +-
 cellpack/tests/test_object_doc.py             |  2 +-
 9 files changed, 52 insertions(+), 41 deletions(-)
 rename cellpack/autopack/{DBRecipeHandler.py => DBHandler.py} (96%)

diff --git a/cellpack/autopack/DBRecipeHandler.py b/cellpack/autopack/DBHandler.py
similarity index 96%
rename from cellpack/autopack/DBRecipeHandler.py
rename to cellpack/autopack/DBHandler.py
index 94e03717d..b643bdbc7 100644
--- a/cellpack/autopack/DBRecipeHandler.py
+++ b/cellpack/autopack/DBHandler.py
@@ -4,6 +4,8 @@
 
 from cellpack.autopack.utils import deep_merge
 
+from google.cloud.exceptions import NotFound
+
 
 class DataDoc(object):
     def __init__(
@@ -124,7 +126,7 @@ def resolve_local_regions(self, local_data, recipe_data, db):
         Recursively resolves the regions of a composition from local data.
         Restructure the local data to match the db data.
         """
-        unpack_recipe_data = DBRecipeHandler.prep_data_for_db(recipe_data)
+        unpack_recipe_data = DBHandler.prep_data_for_db(recipe_data)
         prep_recipe_data = ObjectDoc.convert_representation(unpack_recipe_data, db)
         # `gradients` is a list, convert it to dict for easy access and replace
         CompositionDoc.gradient_list_to_dict(prep_recipe_data)
@@ -326,7 +328,7 @@ def should_write(self, db):
                 # if there is repr in the obj doc from db
                 full_doc_data = ObjectDoc.convert_representation(doc, db)
                 # unpack objects to dicts in local data for comparison
-                local_data = DBRecipeHandler.prep_data_for_db(self.as_dict())
+                local_data = DBHandler.prep_data_for_db(self.as_dict())
                 difference = DeepDiff(full_doc_data, local_data, ignore_order=True)
                 if not difference:
                     return doc, db.doc_id(doc)
@@ -342,7 +344,7 @@ def should_write(self, db, grad_name):
         docs = db.get_doc_by_name("gradients", grad_name)
         if docs and len(docs) >= 1:
             for doc in docs:
-                local_data = DBRecipeHandler.prep_data_for_db(db.doc_to_dict(doc))
+                local_data = DBHandler.prep_data_for_db(db.doc_to_dict(doc))
                 db_data = db.doc_to_dict(doc)
                 difference = DeepDiff(db_data, local_data, ignore_order=True)
                 if not difference:
@@ -350,7 +352,7 @@ def should_write(self, db, grad_name):
         return None, None
 
 
-class DBRecipeHandler(object):
+class DBHandler(object):
     def __init__(self, db_handler):
         self.db = db_handler
         self.objects_to_path_map = {}
@@ -381,20 +383,20 @@ def prep_data_for_db(data):
         modified_data = {}
         for key, value in data.items():
             # convert 2d array to dict
-            if DBRecipeHandler.is_nested_list(value):
+            if DBHandler.is_nested_list(value):
                 flatten_dict = dict(zip([str(i) for i in range(len(value))], value))
-                modified_data[key] = DBRecipeHandler.prep_data_for_db(flatten_dict)
+                modified_data[key] = DBHandler.prep_data_for_db(flatten_dict)
             # If the value is an object, we want to convert it to dict
             elif isinstance(value, object) and "__dict__" in dir(value):
                 unpacked_value = vars(value)
                 modified_data[key] = unpacked_value
                 if isinstance(unpacked_value, dict):
-                    modified_data[key] = DBRecipeHandler.prep_data_for_db(
+                    modified_data[key] = DBHandler.prep_data_for_db(
                         unpacked_value
                     )
             # If the value is a dictionary, recursively convert its nested lists to dictionaries
             elif isinstance(value, dict):
-                modified_data[key] = DBRecipeHandler.prep_data_for_db(value)
+                modified_data[key] = DBHandler.prep_data_for_db(value)
             else:
                 modified_data[key] = value
         return modified_data
@@ -404,7 +406,7 @@ def upload_data(self, collection, data, id=None):
         If should_write is true, upload the data to the database
         """
         # check if we need to convert part of the data(2d arrays and objs to dict)
-        modified_data = DBRecipeHandler.prep_data_for_db(data)
+        modified_data = DBHandler.prep_data_for_db(data)
         if id is None:
             name = modified_data["name"]
             doc = self.db.upload_doc(collection, modified_data)
@@ -546,6 +548,17 @@ def upload_recipe(self, recipe_meta_data, recipe_data):
         key = self.get_recipe_id(recipe_to_save)
         self.upload_data("recipes", recipe_to_save, key)
 
+    def update_or_create_metadata(self, collection, id, data):
+        """
+        If the input id exists, update the metadata. If not, create a new file.
+        """
+        try:
+            self.db.update_doc(collection, id, data)
+        except NotFound:
+            self.db.set_doc(collection, id, data)
+
+
+
     def prep_db_doc_for_download(self, db_doc):
         """
         convert data from db and resolve references.
diff --git a/cellpack/autopack/FirebaseHandler.py b/cellpack/autopack/FirebaseHandler.py
index 320e06843..a0b067605 100644
--- a/cellpack/autopack/FirebaseHandler.py
+++ b/cellpack/autopack/FirebaseHandler.py
@@ -81,7 +81,7 @@ def get_collection_id_from_path(path):
     def update_doc(self, collection, id, data):
         doc_ref = self.db.collection(collection).document(id)
         doc_ref.update(data)
-        print(f"successfully uploaded to path: {doc_ref.path}")
+        print(f"successfully updated to path: {doc_ref.path}")
         return doc_ref
     
     @staticmethod
diff --git a/cellpack/autopack/__init__.py b/cellpack/autopack/__init__.py
index f2b12ab5a..76e92790c 100755
--- a/cellpack/autopack/__init__.py
+++ b/cellpack/autopack/__init__.py
@@ -50,7 +50,7 @@
 
 from cellpack.autopack.interface_objects.meta_enum import MetaEnum
 from cellpack.autopack.FirebaseHandler import FirebaseHandler
-from cellpack.autopack.DBRecipeHandler import DBRecipeHandler
+from cellpack.autopack.DBHandler import DBHandler
 from cellpack.autopack.loaders.utils import read_json_file, write_json_file
 
 
@@ -392,9 +392,9 @@ def load_file(filename, destination="", cache="geometries", force=None):
         # command example: `pack -r firebase:recipes/peroxisomes_surface_gradient_v-linear -c examples/packing-configs/peroxisome_packing_config.json`
         if database_name == "firebase":
             recipe_id = file_path.split("/")[-1]
-            db = FirebaseHandler()
-            db_doc, _ = db.get_doc_by_id(collection="recipes", id=recipe_id)
-            db_handler = DBRecipeHandler(db)
+            # TODO: do we need to include the case where the recipe is downloaded from firebase?
+            db_handler = DBHandler(FirebaseHandler())
+            db_doc, _ = db_handler.db.get_doc_by_id(collection="recipes", id=recipe_id)
             downloaded_recipe_data = db_handler.prep_db_doc_for_download(db_doc)
             return downloaded_recipe_data, database_name
         else:
diff --git a/cellpack/autopack/upy/simularium/simularium_helper.py b/cellpack/autopack/upy/simularium/simularium_helper.py
index 2e395b75a..59510e580 100644
--- a/cellpack/autopack/upy/simularium/simularium_helper.py
+++ b/cellpack/autopack/upy/simularium/simularium_helper.py
@@ -20,6 +20,7 @@
 )
 from simulariumio.cellpack import CellpackConverter, HAND_TYPE
 from simulariumio.constants import DISPLAY_TYPE, VIZ_TYPE
+from cellpack.autopack.DBHandler import DBHandler
 from cellpack.autopack.FirebaseHandler import FirebaseHandler
 
 from cellpack.autopack.upy import hostHelper
@@ -1352,6 +1353,7 @@ def post_and_open_file(self, file_name):
                 # TODO: add info on AWS installation and setup to our documetation
                 # TODO: link to documentation section on setting up AWS CLI and boto3 authentation
                 print(f"need to configure your aws account (link to readme here)")
+            # TODO: an ValueError is raised (in autopack/__init__ L396) when firebase app is already initialized, i.e. storing the recipe that downloaded from firebase
             else:
                 print(
                     f"An error occurred while storing the file {simularium_file} to S3:",
@@ -1384,19 +1386,15 @@ def store_results_to_s3(file_path):
         )
         file_name = handler.upload_file(file_path)
         url = handler.create_presigned_url(file_name)
-        # simulariumHelper.upload_metadata_to_firebase(file_name, url)
+        simulariumHelper.upload_metadata_to_firebase(file_name, url)
         return url
 
     @staticmethod
     def upload_metadata_to_firebase(result_file_name, aws_url):
-        # TODO: use db handler (rename to not have recipe in it)
-        # write a update_doc function that in the firebase handler
-        # class update doc instead of set doc
-        # check to make sure it writes a doc if non exists
-        db = FirebaseHandler()
-        username = db.get_username()
-        timestamp = db.create_timestamp()
-        db.update_doc(
+        db_handler = DBHandler(FirebaseHandler())
+        username = db_handler.db.get_username()
+        timestamp = db_handler.db.create_timestamp()
+        db_handler.update_or_create_metadata(
             "results",
             result_file_name,
             {"user": username, "created": timestamp, "aws_url": aws_url},
diff --git a/cellpack/bin/upload.py b/cellpack/bin/upload.py
index 0a823f28a..fde2bff09 100644
--- a/cellpack/bin/upload.py
+++ b/cellpack/bin/upload.py
@@ -1,7 +1,7 @@
 from enum import Enum
 import fire
 from cellpack.autopack.FirebaseHandler import FirebaseHandler
-from cellpack.autopack.DBRecipeHandler import DBRecipeHandler
+from cellpack.autopack.DBHandler import DBHandler
 
 from cellpack.autopack.loaders.recipe_loader import RecipeLoader
 
@@ -26,7 +26,7 @@ def upload(
         recipe_loader = RecipeLoader(recipe_path)
         recipe_full_data = recipe_loader.recipe_data
         recipe_meta_data = recipe_loader.get_only_recipe_metadata()
-        recipe_db_handler = DBRecipeHandler(db_handler)
+        recipe_db_handler = DBHandler(db_handler)
         recipe_db_handler.upload_recipe(recipe_meta_data, recipe_full_data)
 
 
diff --git a/cellpack/tests/test_comp_doc.py b/cellpack/tests/test_comp_doc.py
index 29e429f24..38b5862bc 100644
--- a/cellpack/tests/test_comp_doc.py
+++ b/cellpack/tests/test_comp_doc.py
@@ -1,4 +1,4 @@
-from cellpack.autopack.DBRecipeHandler import CompositionDoc
+from cellpack.autopack.DBHandler import CompositionDoc
 from cellpack.tests.mocks.mock_db import MockDB
 
 mock_db = MockDB({})
diff --git a/cellpack/tests/test_db_recipe_handler.py b/cellpack/tests/test_db_recipe_handler.py
index 3a6a11621..6230fc8d3 100644
--- a/cellpack/tests/test_db_recipe_handler.py
+++ b/cellpack/tests/test_db_recipe_handler.py
@@ -1,13 +1,13 @@
-from cellpack.autopack.DBRecipeHandler import DBRecipeHandler
+from cellpack.autopack.DBHandler import DBHandler
 from cellpack.tests.mocks.mock_db import MockDB
 
 mock_db = MockDB({})
 
 
 def test_is_nested_list():
-    assert DBRecipeHandler.is_nested_list([]) is False
-    assert DBRecipeHandler.is_nested_list([[], []]) is True
-    assert DBRecipeHandler.is_nested_list([[1, 2], [3, 4]]) is True
+    assert DBHandler.is_nested_list([]) is False
+    assert DBHandler.is_nested_list([[], []]) is True
+    assert DBHandler.is_nested_list([[1, 2], [3, 4]]) is True
 
 
 def test_prep_data_for_db():
@@ -24,7 +24,7 @@ def test_prep_data_for_db():
         },
         "max_jitter": [1, 1, 0],
     }
-    new_data = DBRecipeHandler.prep_data_for_db(input_data)
+    new_data = DBHandler.prep_data_for_db(input_data)
     assert new_data == converted_data
 
 
@@ -37,7 +37,7 @@ def test_upload_data_with_recipe_and_id():
         "composition": {"test": {"inherit": "firebase:test_collection/test_id"}},
     }
     id = "test_id"
-    recipe_doc = DBRecipeHandler(mock_db)
+    recipe_doc = DBHandler(mock_db)
     expected_result = recipe_doc.upload_data(collection, data, id)
 
     assert expected_result[0] == "test_id"
@@ -50,7 +50,7 @@ def test_upload_data_with_object():
         "name": "test",
         "test_key": "test_value",
     }
-    object_doc = DBRecipeHandler(mock_db)
+    object_doc = DBHandler(mock_db)
     expected_result = object_doc.upload_data(collection, data)
 
     assert expected_result[0] == "test_id"
@@ -59,7 +59,7 @@ def test_upload_data_with_object():
 
 def test_upload_objects():
     data = {"test": {"test_key": "test_value"}}
-    object_doc = DBRecipeHandler(mock_db)
+    object_doc = DBHandler(mock_db)
     object_doc.upload_objects(data)
 
     assert object_doc.objects_to_path_map == {"test": "firebase:objects/test_id"}
@@ -83,7 +83,7 @@ def test_upload_compositions():
         },
     }
 
-    composition_doc = DBRecipeHandler(mock_db)
+    composition_doc = DBHandler(mock_db)
     references_to_update = composition_doc.upload_compositions(
         composition, recipe_to_save, recipe_data
     )
@@ -97,7 +97,7 @@ def test_upload_compositions():
 
 def test_upload_gradients():
     data = [{"name": "test_grad_name", "test_key": "test_value"}]
-    gradient_doc = DBRecipeHandler(mock_db)
+    gradient_doc = DBHandler(mock_db)
     gradient_doc.upload_gradients(data)
 
 
@@ -108,7 +108,7 @@ def test_get_recipe_id():
         "objects": None,
         "composition": {},
     }
-    recipe_doc = DBRecipeHandler(mock_db)
+    recipe_doc = DBHandler(mock_db)
     assert recipe_doc.get_recipe_id(recipe_data) == "test_v-1.0.0"
 
 
@@ -132,7 +132,7 @@ def test_upload_collections():
         },
     }
 
-    recipe_doc = DBRecipeHandler(mock_db)
+    recipe_doc = DBHandler(mock_db)
     expected_result = {
         "name": "one_sphere",
         "version": "1.0.0",
@@ -166,7 +166,7 @@ def test_upload_recipe():
         },
     }
 
-    recipe_doc = DBRecipeHandler(mock_db)
+    recipe_doc = DBHandler(mock_db)
     recipe_doc.upload_recipe(recipe_meta_data, recipe_data)
     assert recipe_doc.comp_to_path_map == {
         "space": {"path": "firebase:composition/test_id", "id": "test_id"},
diff --git a/cellpack/tests/test_gradient_doc.py b/cellpack/tests/test_gradient_doc.py
index 34b093a95..430474add 100644
--- a/cellpack/tests/test_gradient_doc.py
+++ b/cellpack/tests/test_gradient_doc.py
@@ -1,4 +1,4 @@
-from cellpack.autopack.DBRecipeHandler import GradientDoc
+from cellpack.autopack.DBHandler import GradientDoc
 from cellpack.tests.mocks.mock_db import MockDB
 
 mock_db = MockDB({})
diff --git a/cellpack/tests/test_object_doc.py b/cellpack/tests/test_object_doc.py
index 6b82d2eac..94d1a60b8 100644
--- a/cellpack/tests/test_object_doc.py
+++ b/cellpack/tests/test_object_doc.py
@@ -1,4 +1,4 @@
-from cellpack.autopack.DBRecipeHandler import ObjectDoc
+from cellpack.autopack.DBHandler import ObjectDoc
 from cellpack.tests.mocks.mock_db import MockDB
 
 mock_db = MockDB({})

From 898561a313672f56c364c56cfad4a816ca4b918f Mon Sep 17 00:00:00 2001
From: Ruge Li <rugeli0605@gmail.com>
Date: Fri, 14 Jul 2023 15:24:07 -0700
Subject: [PATCH 07/16] removed credentials and used AWS CLI instead

---
 cellpack/autopack/AWSHandler.py               | 19 +++++++------------
 .../upy/simularium/simularium_helper.py       | 13 ++++++-------
 2 files changed, 13 insertions(+), 19 deletions(-)

diff --git a/cellpack/autopack/AWSHandler.py b/cellpack/autopack/AWSHandler.py
index 05bc285ba..0ec3c929d 100644
--- a/cellpack/autopack/AWSHandler.py
+++ b/cellpack/autopack/AWSHandler.py
@@ -12,9 +12,7 @@ class AWSHandler(object):
     def __init__(
         self,
         bucket_name,
-        sub_folder_name=None, 
-        aws_access_key_id=None,
-        aws_secret_access_key=None,
+        sub_folder_name=None,
         region_name=None,
     ):
         self.bucket_name = bucket_name
@@ -22,20 +20,17 @@ def __init__(
         session = boto3.Session()
         self.s3_client = session.client(
             "s3",
-            endpoint_url=f"https://{bucket_name}.s3.{region_name}.amazonaws.com",
-            aws_access_key_id=aws_access_key_id,
-            aws_secret_access_key=aws_secret_access_key,
+            endpoint_url=f"https://s3.{region_name}.amazonaws.com",
             region_name=region_name,
         )
 
     def get_aws_object_key(self, object_name):
-        if self.folder_name is not None: 
+        if self.folder_name is not None:
             object_name = self.folder_name + object_name
         else:
             object_name = object_name
         return object_name
 
-
     def upload_file(self, file_path):
         """Upload a file to an S3 bucket
 
@@ -50,11 +45,11 @@ def upload_file(self, file_path):
         object_name = self.get_aws_object_key(file_name)
         # Upload the file
         try:
-            self.s3_client.upload_file(
-                file_path, self.bucket_name, object_name
+            self.s3_client.upload_file(file_path, self.bucket_name, object_name)
+            self.s3_client.put_object_acl(
+                ACL="public-read", Bucket=self.bucket_name, Key=object_name
             )
-            self.s3_client.put_object_acl( ACL='public-read', Bucket=self.bucket_name, Key=object_name )
-        
+
         except ClientError as e:
             logging.error(e)
             return False
diff --git a/cellpack/autopack/upy/simularium/simularium_helper.py b/cellpack/autopack/upy/simularium/simularium_helper.py
index 59510e580..571a4a9ad 100644
--- a/cellpack/autopack/upy/simularium/simularium_helper.py
+++ b/cellpack/autopack/upy/simularium/simularium_helper.py
@@ -1340,14 +1340,11 @@ def writeToFile(self, file_name, bb, recipe_name, version):
         TrajectoryConverter(converted_data).save(file_name, False)
         return file_name
 
-
     def post_and_open_file(self, file_name):
         simularium_file = Path(f"{file_name}.simularium")
         url = None
         try:
-            url = simulariumHelper.store_results_to_s3(
-                simularium_file
-            )
+            url = simulariumHelper.store_results_to_s3(simularium_file)
         except Exception as e:
             if isinstance(e, NoCredentialsError):
                 # TODO: add info on AWS installation and setup to our documetation
@@ -1379,10 +1376,8 @@ def raycast_test(self, obj, start, end, length, **kw):
     def store_results_to_s3(file_path):
         handler = AWSHandler(
             bucket_name="cellpack-results",
-            aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID"),
-            aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY"),
+            sub_folder_name="simularium/",
             region_name="us-west-2",
-            sub_folder_name="simularium/"
         )
         file_name = handler.upload_file(file_path)
         url = handler.create_presigned_url(file_name)
@@ -1399,6 +1394,10 @@ def upload_metadata_to_firebase(result_file_name, aws_url):
             result_file_name,
             {"user": username, "created": timestamp, "aws_url": aws_url},
         )
+        # result_file_name examples:
+        # github:autopack_recipe.json
+        # firebase:recipes/peroxisomes_surface_gradient_v-linear
+        # results_seed_0_test_spheres_test_spheres_config_1.0.0_results
 
     @staticmethod
     def open_in_simularium(aws_url):

From ad56993ee2ad88a7fefe7387bf7332299cde7f90 Mon Sep 17 00:00:00 2001
From: Ruge Li <rugeli0605@gmail.com>
Date: Fri, 14 Jul 2023 15:24:34 -0700
Subject: [PATCH 08/16] formatting

---
 cellpack/autopack/DBHandler.py        | 6 +-----
 cellpack/autopack/FirebaseHandler.py  | 6 +++---
 cellpack/autopack/writers/__init__.py | 2 +-
 3 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/cellpack/autopack/DBHandler.py b/cellpack/autopack/DBHandler.py
index b643bdbc7..64341ce57 100644
--- a/cellpack/autopack/DBHandler.py
+++ b/cellpack/autopack/DBHandler.py
@@ -391,9 +391,7 @@ def prep_data_for_db(data):
                 unpacked_value = vars(value)
                 modified_data[key] = unpacked_value
                 if isinstance(unpacked_value, dict):
-                    modified_data[key] = DBHandler.prep_data_for_db(
-                        unpacked_value
-                    )
+                    modified_data[key] = DBHandler.prep_data_for_db(unpacked_value)
             # If the value is a dictionary, recursively convert its nested lists to dictionaries
             elif isinstance(value, dict):
                 modified_data[key] = DBHandler.prep_data_for_db(value)
@@ -557,8 +555,6 @@ def update_or_create_metadata(self, collection, id, data):
         except NotFound:
             self.db.set_doc(collection, id, data)
 
-
-
     def prep_db_doc_for_download(self, db_doc):
         """
         convert data from db and resolve references.
diff --git a/cellpack/autopack/FirebaseHandler.py b/cellpack/autopack/FirebaseHandler.py
index a0b067605..a88c2528b 100644
--- a/cellpack/autopack/FirebaseHandler.py
+++ b/cellpack/autopack/FirebaseHandler.py
@@ -42,7 +42,7 @@ def get_creds():
         if creds is None or "firebase" not in creds:
             creds = FirebaseHandler.write_creds_path()
         return creds["firebase"]
-    
+
     @staticmethod
     def get_username():
         creds = read_json_file("./.creds")
@@ -61,7 +61,7 @@ def doc_id(doc):
     @staticmethod
     def create_path(collection, doc_id):
         return f"firebase:{collection}/{doc_id}"
-    
+
     @staticmethod
     def create_timestamp():
         return firestore.SERVER_TIMESTAMP
@@ -83,7 +83,7 @@ def update_doc(self, collection, id, data):
         doc_ref.update(data)
         print(f"successfully updated to path: {doc_ref.path}")
         return doc_ref
-    
+
     @staticmethod
     def update_reference_on_doc(doc_ref, index, new_item_ref):
         doc_ref.update({index: new_item_ref})
diff --git a/cellpack/autopack/writers/__init__.py b/cellpack/autopack/writers/__init__.py
index 1d2e167d2..04aeb3d06 100644
--- a/cellpack/autopack/writers/__init__.py
+++ b/cellpack/autopack/writers/__init__.py
@@ -101,7 +101,7 @@ def save_as_simularium(
                 autopack.helper.add_grid_data_to_scene(
                     f"{gradient.name}-distances", grid_positions, values
                 )
-        
+
         file_name = autopack.helper.writeToFile(
             f"{result_file_path}_results", env.boundingBox, env.name, env.version
         )

From 8636b8ca2ce6cae2143bc08b2f25f97982145616 Mon Sep 17 00:00:00 2001
From: Ruge Li <91452427+rugeli@users.noreply.github.com>
Date: Tue, 18 Jul 2023 22:29:09 -0700
Subject: [PATCH 09/16] add documentation for remote DBs

---
 README.md | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/README.md b/README.md
index 9d4dd1ebe..3c9eaf632 100644
--- a/README.md
+++ b/README.md
@@ -71,6 +71,21 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for information related to developing the
    each set of changes to `main` atomic and as a side effect naturally encourages small
    well defined PR's.
 
+## Introduction to Remote Databases 
+### AWS S3
+1. Pre-requisites
+   * Obtain an AWS account for AICS. Please contact the IT team or the code owner. 
+   * Generate an `aws_access_key_id` and `aws_secret_access_key` in your AWS account.
+
+2. Step-by-step Guide
+   * Download and install the [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)
+   * Configure AWS CLI by running `aws configure`, then enter your credentials as prompted
+   * Install Boto3, the AWS SDK for Python. Run `conda install -c conda-forge boto3` or `pip install boto3`.  
+
+### Firebase Firestore
+1. Step-by-step Guide
+   * Create a Firebase project in test mode with your google account, select `firebase_admin` as the SDK. [Firebase Firestore tutorial](https://firebase.google.com/docs/firestore))
+   * Generate a new private key by navigating to "Project settings">"Service account" in the project's dashboard 
 
 **MIT license**
 

From 676374478085cb3c9dd2717be22d143b25c87f50 Mon Sep 17 00:00:00 2001
From: Ruge Li <rugeli0605@gmail.com>
Date: Tue, 18 Jul 2023 22:33:20 -0700
Subject: [PATCH 10/16] add readme link

---
 cellpack/autopack/upy/simularium/simularium_helper.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/cellpack/autopack/upy/simularium/simularium_helper.py b/cellpack/autopack/upy/simularium/simularium_helper.py
index 571a4a9ad..4d3ae36ef 100644
--- a/cellpack/autopack/upy/simularium/simularium_helper.py
+++ b/cellpack/autopack/upy/simularium/simularium_helper.py
@@ -1347,9 +1347,7 @@ def post_and_open_file(self, file_name):
             url = simulariumHelper.store_results_to_s3(simularium_file)
         except Exception as e:
             if isinstance(e, NoCredentialsError):
-                # TODO: add info on AWS installation and setup to our documetation
-                # TODO: link to documentation section on setting up AWS CLI and boto3 authentation
-                print(f"need to configure your aws account (link to readme here)")
+                print(f"need to configure your aws account, find instructions here: https://github.com/mesoscope/cellpack/blob/feature/main/README.md#aws-s3")
             # TODO: an ValueError is raised (in autopack/__init__ L396) when firebase app is already initialized, i.e. storing the recipe that downloaded from firebase
             else:
                 print(

From edaabe782ad7f839338319a868473d547c339c42 Mon Sep 17 00:00:00 2001
From: Ruge Li <91452427+rugeli@users.noreply.github.com>
Date: Wed, 19 Jul 2023 11:47:59 -0700
Subject: [PATCH 11/16] fix typo

---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 3c9eaf632..02822c0a1 100644
--- a/README.md
+++ b/README.md
@@ -79,13 +79,13 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for information related to developing the
 
 2. Step-by-step Guide
    * Download and install the [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)
-   * Configure AWS CLI by running `aws configure`, then enter your credentials as prompted
+   * Configure AWS CLI by running `aws configure`, then enter your credentials as prompted.
    * Install Boto3, the AWS SDK for Python. Run `conda install -c conda-forge boto3` or `pip install boto3`.  
 
 ### Firebase Firestore
 1. Step-by-step Guide
-   * Create a Firebase project in test mode with your google account, select `firebase_admin` as the SDK. [Firebase Firestore tutorial](https://firebase.google.com/docs/firestore))
-   * Generate a new private key by navigating to "Project settings">"Service account" in the project's dashboard 
+   * Create a Firebase project in test mode with your google account, select `firebase_admin` as the SDK. [Firebase Firestore tutorial](https://firebase.google.com/docs/firestore)
+   * Generate a new private key by navigating to "Project settings">"Service account" in the project's dashboard.
 
 **MIT license**
 

From 0b9b4a208308bffaa4d7dd2c2842d4a10b1de12f Mon Sep 17 00:00:00 2001
From: Ruge Li <91452427+rugeli@users.noreply.github.com>
Date: Thu, 20 Jul 2023 14:26:34 -0700
Subject: [PATCH 12/16] Update README.md boto3

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 02822c0a1..a356e0f02 100644
--- a/README.md
+++ b/README.md
@@ -80,7 +80,7 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for information related to developing the
 2. Step-by-step Guide
    * Download and install the [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)
    * Configure AWS CLI by running `aws configure`, then enter your credentials as prompted.
-   * Install Boto3, the AWS SDK for Python. Run `conda install -c conda-forge boto3` or `pip install boto3`.  
+   * Ensure that Boto3, the AWS SDK for Python is installed and included in the requirements section of `setup.py`. 
 
 ### Firebase Firestore
 1. Step-by-step Guide

From 97bf31bb0733cf6d9116ff90a62d0cfc182edf9e Mon Sep 17 00:00:00 2001
From: Ruge Li <rugeli0605@gmail.com>
Date: Thu, 20 Jul 2023 17:16:17 -0700
Subject: [PATCH 13/16] enable saving remote recipes results and metadata

---
 cellpack/autopack/DBHandler.py                   |  4 ++++
 cellpack/autopack/__init__.py                    |  1 -
 .../autopack/upy/simularium/simularium_helper.py | 16 ++++++----------
 3 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/cellpack/autopack/DBHandler.py b/cellpack/autopack/DBHandler.py
index 64341ce57..2db8bd42a 100644
--- a/cellpack/autopack/DBHandler.py
+++ b/cellpack/autopack/DBHandler.py
@@ -6,6 +6,8 @@
 
 from google.cloud.exceptions import NotFound
 
+from cellpack.autopack import FirebaseHandler
+
 
 class DataDoc(object):
     def __init__(
@@ -355,6 +357,8 @@ def should_write(self, db, grad_name):
 class DBHandler(object):
     def __init__(self, db_handler):
         self.db = db_handler
+        if isinstance(db_handler, FirebaseHandler):
+            DBHandler.firebase_handler = db_handler
         self.objects_to_path_map = {}
         self.comp_to_path_map = {}
         self.grad_to_path_map = {}
diff --git a/cellpack/autopack/__init__.py b/cellpack/autopack/__init__.py
index 76e92790c..e300309fa 100755
--- a/cellpack/autopack/__init__.py
+++ b/cellpack/autopack/__init__.py
@@ -392,7 +392,6 @@ def load_file(filename, destination="", cache="geometries", force=None):
         # command example: `pack -r firebase:recipes/peroxisomes_surface_gradient_v-linear -c examples/packing-configs/peroxisome_packing_config.json`
         if database_name == "firebase":
             recipe_id = file_path.split("/")[-1]
-            # TODO: do we need to include the case where the recipe is downloaded from firebase?
             db_handler = DBHandler(FirebaseHandler())
             db_doc, _ = db_handler.db.get_doc_by_id(collection="recipes", id=recipe_id)
             downloaded_recipe_data = db_handler.prep_db_doc_for_download(db_doc)
diff --git a/cellpack/autopack/upy/simularium/simularium_helper.py b/cellpack/autopack/upy/simularium/simularium_helper.py
index 4d3ae36ef..5828702a9 100644
--- a/cellpack/autopack/upy/simularium/simularium_helper.py
+++ b/cellpack/autopack/upy/simularium/simularium_helper.py
@@ -1348,11 +1348,9 @@ def post_and_open_file(self, file_name):
         except Exception as e:
             if isinstance(e, NoCredentialsError):
                 print(f"need to configure your aws account, find instructions here: https://github.com/mesoscope/cellpack/blob/feature/main/README.md#aws-s3")
-            # TODO: an ValueError is raised (in autopack/__init__ L396) when firebase app is already initialized, i.e. storing the recipe that downloaded from firebase
             else:
                 print(
-                    f"An error occurred while storing the file {simularium_file} to S3:",
-                    e,
+                    f"An error occurred while storing the file {simularium_file} to S3: {e}"
                 )
         if url is not None:
             simulariumHelper.open_in_simularium(url)
@@ -1384,22 +1382,20 @@ def store_results_to_s3(file_path):
 
     @staticmethod
     def upload_metadata_to_firebase(result_file_name, aws_url):
-        db_handler = DBHandler(FirebaseHandler())
+        if DBHandler.firebase_handler is None:
+            db_handler = DBHandler(FirebaseHandler())
+        else:
+            db_handler = DBHandler(DBHandler.firebase_handler)
         username = db_handler.db.get_username()
         timestamp = db_handler.db.create_timestamp()
         db_handler.update_or_create_metadata(
             "results",
             result_file_name,
-            {"user": username, "created": timestamp, "aws_url": aws_url},
+            {"user": username, "created": timestamp, "aws_url": aws_url.split("?")[0]},
         )
-        # result_file_name examples:
-        # github:autopack_recipe.json
-        # firebase:recipes/peroxisomes_surface_gradient_v-linear
-        # results_seed_0_test_spheres_test_spheres_config_1.0.0_results
 
     @staticmethod
     def open_in_simularium(aws_url):
-        # TODO: save the endpoint,routes and options somewhere?
         webbrowser.open_new_tab(
             f"https://simularium.allencell.org/viewer?trajUrl={aws_url}"
         )

From a0c61aa74fd82ab36162c4da59b9d26af0c3be25 Mon Sep 17 00:00:00 2001
From: Ruge Li <rugeli0605@gmail.com>
Date: Thu, 20 Jul 2023 19:28:55 -0700
Subject: [PATCH 14/16] fix attribute error

---
 cellpack/autopack/DBHandler.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cellpack/autopack/DBHandler.py b/cellpack/autopack/DBHandler.py
index 2db8bd42a..86a8679a3 100644
--- a/cellpack/autopack/DBHandler.py
+++ b/cellpack/autopack/DBHandler.py
@@ -355,6 +355,7 @@ def should_write(self, db, grad_name):
 
 
 class DBHandler(object):
+    firebase_handler = None
     def __init__(self, db_handler):
         self.db = db_handler
         if isinstance(db_handler, FirebaseHandler):

From af2a480abfbd58835c56d72012fd8149338d8022 Mon Sep 17 00:00:00 2001
From: Ruge Li <rugeli0605@gmail.com>
Date: Mon, 24 Jul 2023 10:39:46 -0700
Subject: [PATCH 15/16] formatting

---
 cellpack/autopack/DBHandler.py                        | 1 +
 cellpack/autopack/upy/simularium/simularium_helper.py | 5 ++++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/cellpack/autopack/DBHandler.py b/cellpack/autopack/DBHandler.py
index 86a8679a3..53f4b5da5 100644
--- a/cellpack/autopack/DBHandler.py
+++ b/cellpack/autopack/DBHandler.py
@@ -356,6 +356,7 @@ def should_write(self, db, grad_name):
 
 class DBHandler(object):
     firebase_handler = None
+
     def __init__(self, db_handler):
         self.db = db_handler
         if isinstance(db_handler, FirebaseHandler):
diff --git a/cellpack/autopack/upy/simularium/simularium_helper.py b/cellpack/autopack/upy/simularium/simularium_helper.py
index 5828702a9..bda530a8a 100644
--- a/cellpack/autopack/upy/simularium/simularium_helper.py
+++ b/cellpack/autopack/upy/simularium/simularium_helper.py
@@ -1346,8 +1346,11 @@ def post_and_open_file(self, file_name):
         try:
             url = simulariumHelper.store_results_to_s3(simularium_file)
         except Exception as e:
+            aws_readme_url = "https://github.com/mesoscope/cellpack/blob/feature/main/README.md#aws-s3"
             if isinstance(e, NoCredentialsError):
-                print(f"need to configure your aws account, find instructions here: https://github.com/mesoscope/cellpack/blob/feature/main/README.md#aws-s3")
+                print(
+                    f"need to configure your aws account, find instructions here: {aws_readme_url}"
+                )
             else:
                 print(
                     f"An error occurred while storing the file {simularium_file} to S3: {e}"

From f6533c98a2b8c3a660d0054b1da74b241811080f Mon Sep 17 00:00:00 2001
From: Ruge Li <rugeli0605@gmail.com>
Date: Mon, 24 Jul 2023 11:50:29 -0700
Subject: [PATCH 16/16] get user with windows compatible solution

---
 cellpack/autopack/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cellpack/autopack/__init__.py b/cellpack/autopack/__init__.py
index e300309fa..9db5df00b 100755
--- a/cellpack/autopack/__init__.py
+++ b/cellpack/autopack/__init__.py
@@ -41,7 +41,7 @@
 import re
 import shutil
 from os import path, environ
-import pwd
+import getpass
 from pathlib import Path
 import urllib.request as urllib
 from collections import OrderedDict
@@ -541,7 +541,7 @@ def clearCaches(*args):
 
 
 def write_username_to_creds():
-    username = pwd.getpwuid(os.getuid())[0]
+    username = getpass.getuser()
     creds = read_json_file("./.creds")
     if creds is None or "username" not in creds:
         creds = {}