From b6fc051c258bd700c47872b6903bb0c95620ba2c Mon Sep 17 00:00:00 2001 From: Ruge Li <91452427+rugeli@users.noreply.github.com> Date: Fri, 22 Sep 2023 14:18:39 -0700 Subject: [PATCH] Feature/upload local recipes to s3 (#189) * upload local recipes to S3 * update README and add boto3 into requirements * lint * rerange import order * add save_file for S3 uploads and url generation * delete redundant error msg --- README.md | 15 ++++ cellpack/autopack/AWSHandler.py | 85 +++++++++++++++++++ .../upy/simularium/simularium_helper.py | 40 +++++++++ cellpack/autopack/writers/__init__.py | 5 +- setup.py | 1 + 5 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 cellpack/autopack/AWSHandler.py diff --git a/README.md b/README.md index 9d4dd1ebe..a356e0f02 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,21 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for information related to developing the each set of changes to `main` atomic and as a side effect naturally encourages small well defined PR's. +## Introduction to Remote Databases +### AWS S3 +1. Pre-requisites + * Obtain an AWS account for AICS. Please contact the IT team or the code owner. + * Generate an `aws_access_key_id` and `aws_secret_access_key` in your AWS account. + +2. Step-by-step Guide + * Download and install the [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) + * Configure AWS CLI by running `aws configure`, then enter your credentials as prompted. + * Ensure that Boto3, the AWS SDK for Python is installed and included in the requirements section of `setup.py`. + +### Firebase Firestore +1. Step-by-step Guide + * Create a Firebase project in test mode with your google account, select `firebase_admin` as the SDK. [Firebase Firestore tutorial](https://firebase.google.com/docs/firestore) + * Generate a new private key by navigating to "Project settings">"Service account" in the project's dashboard. **MIT license** diff --git a/cellpack/autopack/AWSHandler.py b/cellpack/autopack/AWSHandler.py new file mode 100644 index 000000000..878638bd0 --- /dev/null +++ b/cellpack/autopack/AWSHandler.py @@ -0,0 +1,85 @@ +import logging +from pathlib import Path + +import boto3 +from botocore.exceptions import ClientError + + +class AWSHandler(object): + """ + Handles all the AWS S3 operations + """ + + def __init__( + self, + bucket_name, + sub_folder_name=None, + region_name=None, + ): + self.bucket_name = bucket_name + self.folder_name = sub_folder_name + session = boto3.Session() + self.s3_client = session.client( + "s3", + endpoint_url=f"https://s3.{region_name}.amazonaws.com", + region_name=region_name, + ) + + def get_aws_object_key(self, object_name): + if self.folder_name is not None: + object_name = self.folder_name + object_name + else: + object_name = object_name + return object_name + + def upload_file(self, file_path): + """Upload a file to an S3 bucket + :param file_path: File to upload + :param bucket: Bucket to upload to + :param object_name: S3 object name. If not specified then file_path is used + :return: True if file was uploaded, else False + """ + + file_name = Path(file_path).name + + object_name = self.get_aws_object_key(file_name) + # Upload the file + try: + self.s3_client.upload_file(file_path, self.bucket_name, object_name) + self.s3_client.put_object_acl( + ACL="public-read", Bucket=self.bucket_name, Key=object_name + ) + + except ClientError as e: + logging.error(e) + return False + return file_name + + def create_presigned_url(self, object_name, expiration=3600): + """Generate a presigned URL to share an S3 object + :param object_name: string + :param expiration: Time in seconds for the presigned URL to remain valid + :return: Presigned URL as string. If error, returns None. + """ + object_name = self.get_aws_object_key(object_name) + # Generate a presigned URL for the S3 object + try: + url = self.s3_client.generate_presigned_url( + "get_object", + Params={"Bucket": self.bucket_name, "Key": object_name}, + ExpiresIn=expiration, + ) + except ClientError as e: + logging.error(e) + return None + # The response contains the presigned URL + # https://{self.bucket_name}.s3.{region}.amazonaws.com/{object_key} + return url + + def save_file(self, file_path): + """ + Uploads a file to S3 and returns the presigned url + """ + file_name = self.upload_file(file_path) + if file_name: + return self.create_presigned_url(file_name) diff --git a/cellpack/autopack/upy/simularium/simularium_helper.py b/cellpack/autopack/upy/simularium/simularium_helper.py index ac0db6bc3..5118a5d6f 100644 --- a/cellpack/autopack/upy/simularium/simularium_helper.py +++ b/cellpack/autopack/upy/simularium/simularium_helper.py @@ -1,9 +1,13 @@ # -*- coding: utf-8 -*- # standardmodule import os +import webbrowser +from pathlib import Path + import matplotlib import numpy as np import trimesh +from botocore.exceptions import NoCredentialsError from simulariumio import ( TrajectoryConverter, @@ -19,6 +23,7 @@ from simulariumio.constants import DISPLAY_TYPE, VIZ_TYPE from cellpack.autopack.upy import hostHelper +from cellpack.autopack.AWSHandler import AWSHandler import collada @@ -1335,6 +1340,7 @@ def writeToFile(self, file_name, bb, recipe_name, version): spatial_units=UnitData("nm"), # nanometers ) TrajectoryConverter(converted_data).save(file_name, False) + return file_name def raycast(self, **kw): intersect = False @@ -1348,3 +1354,37 @@ def raycast(self, **kw): def raycast_test(self, obj, start, end, length, **kw): return + + def post_and_open_file(self, file_name): + simularium_file = Path(f"{file_name}.simularium") + url = None + try: + url = simulariumHelper.store_results_to_s3(simularium_file) + except Exception as e: + aws_readme_url = "https://github.com/mesoscope/cellpack/blob/feature/main/README.md#aws-s3" + if isinstance(e, NoCredentialsError): + print( + f"need to configure your aws account, find instructions here: {aws_readme_url}" + ) + else: + print( + f"An error occurred while storing the file {simularium_file} to S3: {e}" + ) + if url is not None: + simulariumHelper.open_in_simularium(url) + + @staticmethod + def store_results_to_s3(file_path): + handler = AWSHandler( + bucket_name="cellpack-results", + sub_folder_name="simularium/", + region_name="us-west-2", + ) + url = handler.save_file(file_path) + return url + + @staticmethod + def open_in_simularium(aws_url): + webbrowser.open_new_tab( + f"https://simularium.allencell.org/viewer?trajUrl={aws_url}" + ) diff --git a/cellpack/autopack/writers/__init__.py b/cellpack/autopack/writers/__init__.py index 64736a0a9..74fb711db 100644 --- a/cellpack/autopack/writers/__init__.py +++ b/cellpack/autopack/writers/__init__.py @@ -172,7 +172,10 @@ def save_as_simularium(self, env, all_ingr_as_array, compartments): env.helper.add_grid_data_to_scene( f"{gradient.name}-weights", grid_positions, gradient.weight ) - env.helper.writeToFile(env.result_file, env.boundingBox, env.name, env.version) + file_name = env.helper.writeToFile( + env.result_file, env.boundingBox, env.name, env.version + ) + autopack.helper.post_and_open_file(file_name) def save_Mixed_asJson( self, diff --git a/setup.py b/setup.py index 577e7ab34..ec4a11fb5 100644 --- a/setup.py +++ b/setup.py @@ -46,6 +46,7 @@ ] requirements = [ + "boto3>=1.28.3", "fire>=0.4.0", "firebase_admin>=6.0.1", "matplotlib>=3.3.4",