From f14bb6e0580c69d77f20ab67ee1a7e7f04852063 Mon Sep 17 00:00:00 2001 From: Ruge Li Date: Tue, 29 Aug 2023 15:02:12 -0700 Subject: [PATCH 1/6] upload local recipes to S3 --- cellpack/autopack/AWSHandler.py | 76 +++++++++++++++++++ .../upy/simularium/simularium_helper.py | 41 ++++++++++ cellpack/autopack/writers/__init__.py | 3 +- 3 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 cellpack/autopack/AWSHandler.py diff --git a/cellpack/autopack/AWSHandler.py b/cellpack/autopack/AWSHandler.py new file mode 100644 index 000000000..dd9b9fc83 --- /dev/null +++ b/cellpack/autopack/AWSHandler.py @@ -0,0 +1,76 @@ +import logging +import boto3 +from botocore.exceptions import ClientError +from pathlib import Path + + +class AWSHandler(object): + """ + Handles all the AWS S3 operations + """ + + def __init__( + self, + bucket_name, + sub_folder_name=None, + region_name=None, + ): + self.bucket_name = bucket_name + self.folder_name = sub_folder_name + session = boto3.Session() + self.s3_client = session.client( + "s3", + endpoint_url=f"https://s3.{region_name}.amazonaws.com", + region_name=region_name, + ) + + def get_aws_object_key(self, object_name): + if self.folder_name is not None: + object_name = self.folder_name + object_name + else: + object_name = object_name + return object_name + + def upload_file(self, file_path): + """Upload a file to an S3 bucket + :param file_path: File to upload + :param bucket: Bucket to upload to + :param object_name: S3 object name. If not specified then file_path is used + :return: True if file was uploaded, else False + """ + + file_name = Path(file_path).name + + object_name = self.get_aws_object_key(file_name) + # Upload the file + try: + self.s3_client.upload_file(file_path, self.bucket_name, object_name) + self.s3_client.put_object_acl( + ACL="public-read", Bucket=self.bucket_name, Key=object_name + ) + + except ClientError as e: + logging.error(e) + return False + return file_name + + def create_presigned_url(self, object_name, expiration=3600): + """Generate a presigned URL to share an S3 object + :param object_name: string + :param expiration: Time in seconds for the presigned URL to remain valid + :return: Presigned URL as string. If error, returns None. + """ + object_name = self.get_aws_object_key(object_name) + # Generate a presigned URL for the S3 object + try: + url = self.s3_client.generate_presigned_url( + "get_object", + Params={"Bucket": self.bucket_name, "Key": object_name}, + ExpiresIn=expiration, + ) + except ClientError as e: + logging.error(e) + return None + # The response contains the presigned URL + # https://{self.bucket_name}.s3.{region}.amazonaws.com/{object_key} + return url \ No newline at end of file diff --git a/cellpack/autopack/upy/simularium/simularium_helper.py b/cellpack/autopack/upy/simularium/simularium_helper.py index ac0db6bc3..7708f2ee5 100644 --- a/cellpack/autopack/upy/simularium/simularium_helper.py +++ b/cellpack/autopack/upy/simularium/simularium_helper.py @@ -1,9 +1,12 @@ # -*- coding: utf-8 -*- # standardmodule import os +import webbrowser import matplotlib import numpy as np import trimesh +from pathlib import Path +from botocore.exceptions import NoCredentialsError from simulariumio import ( TrajectoryConverter, @@ -19,6 +22,7 @@ from simulariumio.constants import DISPLAY_TYPE, VIZ_TYPE from cellpack.autopack.upy import hostHelper +from cellpack.autopack.AWSHandler import AWSHandler import collada @@ -1335,6 +1339,8 @@ def writeToFile(self, file_name, bb, recipe_name, version): spatial_units=UnitData("nm"), # nanometers ) TrajectoryConverter(converted_data).save(file_name, False) + return file_name + def raycast(self, **kw): intersect = False @@ -1348,3 +1354,38 @@ def raycast(self, **kw): def raycast_test(self, obj, start, end, length, **kw): return + + def post_and_open_file(self, file_name): + simularium_file = Path(f"{file_name}.simularium") + url = None + try: + url = simulariumHelper.store_results_to_s3(simularium_file) + except Exception as e: + aws_readme_url = "https://github.com/mesoscope/cellpack/blob/feature/main/README.md#aws-s3" + if isinstance(e, NoCredentialsError): + print( + f"need to configure your aws account, find instructions here: {aws_readme_url}" + ) + else: + print( + f"An error occurred while storing the file {simularium_file} to S3: {e}" + ) + if url is not None: + simulariumHelper.open_in_simularium(url) + + @staticmethod + def store_results_to_s3(file_path): + handler = AWSHandler( + bucket_name="cellpack-results", + sub_folder_name="simularium/", + region_name="us-west-2", + ) + file_name = handler.upload_file(file_path) + url = handler.create_presigned_url(file_name) + return url + + @staticmethod + def open_in_simularium(aws_url): + webbrowser.open_new_tab( + f"https://simularium.allencell.org/viewer?trajUrl={aws_url}" + ) \ No newline at end of file diff --git a/cellpack/autopack/writers/__init__.py b/cellpack/autopack/writers/__init__.py index 64736a0a9..744568f7a 100644 --- a/cellpack/autopack/writers/__init__.py +++ b/cellpack/autopack/writers/__init__.py @@ -172,7 +172,8 @@ def save_as_simularium(self, env, all_ingr_as_array, compartments): env.helper.add_grid_data_to_scene( f"{gradient.name}-weights", grid_positions, gradient.weight ) - env.helper.writeToFile(env.result_file, env.boundingBox, env.name, env.version) + file_name = env.helper.writeToFile(env.result_file, env.boundingBox, env.name, env.version) + autopack.helper.post_and_open_file(file_name) def save_Mixed_asJson( self, From 59b80b4b2fadcf340d32f4fe85f8a3a663ae6be9 Mon Sep 17 00:00:00 2001 From: Ruge Li Date: Tue, 29 Aug 2023 15:03:11 -0700 Subject: [PATCH 2/6] update README and add boto3 into requirements --- README.md | 15 +++++++++++++++ setup.py | 1 + 2 files changed, 16 insertions(+) diff --git a/README.md b/README.md index 9d4dd1ebe..a356e0f02 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,21 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for information related to developing the each set of changes to `main` atomic and as a side effect naturally encourages small well defined PR's. +## Introduction to Remote Databases +### AWS S3 +1. Pre-requisites + * Obtain an AWS account for AICS. Please contact the IT team or the code owner. + * Generate an `aws_access_key_id` and `aws_secret_access_key` in your AWS account. + +2. Step-by-step Guide + * Download and install the [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) + * Configure AWS CLI by running `aws configure`, then enter your credentials as prompted. + * Ensure that Boto3, the AWS SDK for Python is installed and included in the requirements section of `setup.py`. + +### Firebase Firestore +1. Step-by-step Guide + * Create a Firebase project in test mode with your google account, select `firebase_admin` as the SDK. [Firebase Firestore tutorial](https://firebase.google.com/docs/firestore) + * Generate a new private key by navigating to "Project settings">"Service account" in the project's dashboard. **MIT license** diff --git a/setup.py b/setup.py index 577e7ab34..ec4a11fb5 100644 --- a/setup.py +++ b/setup.py @@ -46,6 +46,7 @@ ] requirements = [ + "boto3>=1.28.3", "fire>=0.4.0", "firebase_admin>=6.0.1", "matplotlib>=3.3.4", From 565487ce8ad6b64f49564ccdb0935fd6a0a633e8 Mon Sep 17 00:00:00 2001 From: Ruge Li Date: Tue, 29 Aug 2023 15:03:59 -0700 Subject: [PATCH 3/6] lint --- cellpack/autopack/AWSHandler.py | 2 +- cellpack/autopack/upy/simularium/simularium_helper.py | 5 ++--- cellpack/autopack/writers/__init__.py | 4 +++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cellpack/autopack/AWSHandler.py b/cellpack/autopack/AWSHandler.py index dd9b9fc83..c29eec7a4 100644 --- a/cellpack/autopack/AWSHandler.py +++ b/cellpack/autopack/AWSHandler.py @@ -73,4 +73,4 @@ def create_presigned_url(self, object_name, expiration=3600): return None # The response contains the presigned URL # https://{self.bucket_name}.s3.{region}.amazonaws.com/{object_key} - return url \ No newline at end of file + return url diff --git a/cellpack/autopack/upy/simularium/simularium_helper.py b/cellpack/autopack/upy/simularium/simularium_helper.py index 7708f2ee5..ee2789f77 100644 --- a/cellpack/autopack/upy/simularium/simularium_helper.py +++ b/cellpack/autopack/upy/simularium/simularium_helper.py @@ -1340,7 +1340,6 @@ def writeToFile(self, file_name, bb, recipe_name, version): ) TrajectoryConverter(converted_data).save(file_name, False) return file_name - def raycast(self, **kw): intersect = False @@ -1383,9 +1382,9 @@ def store_results_to_s3(file_path): file_name = handler.upload_file(file_path) url = handler.create_presigned_url(file_name) return url - + @staticmethod def open_in_simularium(aws_url): webbrowser.open_new_tab( f"https://simularium.allencell.org/viewer?trajUrl={aws_url}" - ) \ No newline at end of file + ) diff --git a/cellpack/autopack/writers/__init__.py b/cellpack/autopack/writers/__init__.py index 744568f7a..74fb711db 100644 --- a/cellpack/autopack/writers/__init__.py +++ b/cellpack/autopack/writers/__init__.py @@ -172,7 +172,9 @@ def save_as_simularium(self, env, all_ingr_as_array, compartments): env.helper.add_grid_data_to_scene( f"{gradient.name}-weights", grid_positions, gradient.weight ) - file_name = env.helper.writeToFile(env.result_file, env.boundingBox, env.name, env.version) + file_name = env.helper.writeToFile( + env.result_file, env.boundingBox, env.name, env.version + ) autopack.helper.post_and_open_file(file_name) def save_Mixed_asJson( From c97d5a81523a44bd121235acc695351e41cd8799 Mon Sep 17 00:00:00 2001 From: Ruge Li Date: Tue, 29 Aug 2023 15:14:58 -0700 Subject: [PATCH 4/6] rerange import order --- cellpack/autopack/AWSHandler.py | 3 ++- cellpack/autopack/upy/simularium/simularium_helper.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cellpack/autopack/AWSHandler.py b/cellpack/autopack/AWSHandler.py index c29eec7a4..3f400ee2d 100644 --- a/cellpack/autopack/AWSHandler.py +++ b/cellpack/autopack/AWSHandler.py @@ -1,7 +1,8 @@ import logging +from pathlib import Path + import boto3 from botocore.exceptions import ClientError -from pathlib import Path class AWSHandler(object): diff --git a/cellpack/autopack/upy/simularium/simularium_helper.py b/cellpack/autopack/upy/simularium/simularium_helper.py index ee2789f77..3b26dbe97 100644 --- a/cellpack/autopack/upy/simularium/simularium_helper.py +++ b/cellpack/autopack/upy/simularium/simularium_helper.py @@ -2,10 +2,11 @@ # standardmodule import os import webbrowser +from pathlib import Path + import matplotlib import numpy as np import trimesh -from pathlib import Path from botocore.exceptions import NoCredentialsError from simulariumio import ( From d09b64afcb57432769a9b7f5b53555fd8aba2fc9 Mon Sep 17 00:00:00 2001 From: Ruge Li Date: Wed, 6 Sep 2023 15:58:38 -0700 Subject: [PATCH 5/6] add save_file for S3 uploads and url generation --- cellpack/autopack/AWSHandler.py | 8 ++++++++ cellpack/autopack/upy/simularium/simularium_helper.py | 5 +++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/cellpack/autopack/AWSHandler.py b/cellpack/autopack/AWSHandler.py index 3f400ee2d..878638bd0 100644 --- a/cellpack/autopack/AWSHandler.py +++ b/cellpack/autopack/AWSHandler.py @@ -75,3 +75,11 @@ def create_presigned_url(self, object_name, expiration=3600): # The response contains the presigned URL # https://{self.bucket_name}.s3.{region}.amazonaws.com/{object_key} return url + + def save_file(self, file_path): + """ + Uploads a file to S3 and returns the presigned url + """ + file_name = self.upload_file(file_path) + if file_name: + return self.create_presigned_url(file_name) diff --git a/cellpack/autopack/upy/simularium/simularium_helper.py b/cellpack/autopack/upy/simularium/simularium_helper.py index 3b26dbe97..d94e70426 100644 --- a/cellpack/autopack/upy/simularium/simularium_helper.py +++ b/cellpack/autopack/upy/simularium/simularium_helper.py @@ -1380,8 +1380,9 @@ def store_results_to_s3(file_path): sub_folder_name="simularium/", region_name="us-west-2", ) - file_name = handler.upload_file(file_path) - url = handler.create_presigned_url(file_name) + url = handler.save_file(file_path) + if url is None: + raise Exception("Unable to store file to S3") return url @staticmethod From 2b22d689e57eed9079e33769b33732116fd975d4 Mon Sep 17 00:00:00 2001 From: Ruge Li Date: Wed, 6 Sep 2023 16:12:24 -0700 Subject: [PATCH 6/6] delete redundant error msg --- cellpack/autopack/upy/simularium/simularium_helper.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cellpack/autopack/upy/simularium/simularium_helper.py b/cellpack/autopack/upy/simularium/simularium_helper.py index d94e70426..5118a5d6f 100644 --- a/cellpack/autopack/upy/simularium/simularium_helper.py +++ b/cellpack/autopack/upy/simularium/simularium_helper.py @@ -1381,8 +1381,6 @@ def store_results_to_s3(file_path): region_name="us-west-2", ) url = handler.save_file(file_path) - if url is None: - raise Exception("Unable to store file to S3") return url @staticmethod