Skip to content

Commit

Permalink
Made boto3/botocore optional installs and moved boto3/botocore import…
Browse files Browse the repository at this point in the history
…s into gempyor.utils.download_file_from_s3, GH-238
  • Loading branch information
TimothyWillard committed Jun 25, 2024
1 parent 0b32f2e commit 7a83279
Show file tree
Hide file tree
Showing 2 changed files with 11 additions and 5 deletions.
5 changes: 3 additions & 2 deletions flepimop/gempyor_pkg/setup.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,16 @@ install_requires =
dask
scipy
graphviz
boto3

# see https://stackoverflow.com/questions/58826164/dependencies-requirements-for-setting-up-testing-and-installing-a-python-lib
# installed for pip install -e ".[test]"
[options.extras_require]
test =
pytest
mock

aws =
boto3
botocore

[options.entry_points]
console_scripts =
Expand Down
11 changes: 8 additions & 3 deletions flepimop/gempyor_pkg/src/gempyor/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,8 @@
import subprocess
import shutil
import logging
import boto3
from gempyor import file_paths
from typing import List, Dict
from botocore.exceptions import ClientError

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -509,12 +507,19 @@ def download_file_from_s3(name_map: Dict[str, str]) -> None:
>>> download_file_from_s3(name_map)
# This will raise a ValueError indicating the invalid S3 URI format.
"""
try:
import boto3
from botocore.exceptions import ClientError
except:
raise ModuleNotFoundError((
"No module named 'boto3', which is required for "
"gempyor.utils.download_file_from_s3. Please install the aws target."
))
s3 = boto3.client("s3")
first_output_filename = next(iter(name_map.values()))
output_dir = os.path.dirname(first_output_filename)
if not os.path.exists(output_dir):
os.makedirs(output_dir)

for s3_uri in name_map:
try:
if s3_uri.startswith("s3://"):
Expand Down

0 comments on commit 7a83279

Please sign in to comment.