Skip to content

Commit

Permalink
Wire --object_storage_storage_class through s3.py
Browse files Browse the repository at this point in the history
PiperOrigin-RevId: 705616161
  • Loading branch information
pmkc authored and copybara-github committed Dec 12, 2024
1 parent 4c639ac commit d35fae7
Show file tree
Hide file tree
Showing 3 changed files with 17 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@
'Storage multiregion for GCS in object storage benchmark.',
)

flags.DEFINE_string(
OBJECT_STORAGE_STORAGE_CLASS = flags.DEFINE_string(
'object_storage_storage_class',
None,
'Storage class to use in object storage benchmark.',
Expand Down Expand Up @@ -1940,6 +1940,9 @@ def Run(benchmark_spec):

metadata.update(service.Metadata(vms[0]))

if OBJECT_STORAGE_STORAGE_CLASS.value:
metadata['object_storage_class'] = OBJECT_STORAGE_STORAGE_CLASS.value

results = []
test_script_path = '/tmp/run/%s' % API_TEST_SCRIPT
try:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,9 @@
"""The generic superclass for object storage API providers."""

import abc
import six


class ObjectStorageServiceBase(six.with_metaclass(abc.ABCMeta, object)):
class ObjectStorageServiceBase(metaclass=abc.ABCMeta):
"""Our interface to an object storage service."""

@abc.abstractmethod
Expand Down Expand Up @@ -88,7 +87,7 @@ def BulkDeleteObjects(self, bucket, objects_to_delete, delay_time):
pass

@abc.abstractmethod
def WriteObjectFromBuffer(self, bucket, object, stream, size):
def WriteObjectFromBuffer(self, bucket, object_name, stream, size):
"""Write an object to a bucket.
Exceptions are propagated to the caller, which can decide whether
Expand All @@ -97,7 +96,7 @@ def WriteObjectFromBuffer(self, bucket, object, stream, size):
Args:
bucket: the name of the bucket to write to.
object: the name of the object.
object_name: the name of the object.
stream: a read()-able and seek()-able stream to transfer.
size: the number of bytes to transfer.
Expand All @@ -108,15 +107,15 @@ def WriteObjectFromBuffer(self, bucket, object, stream, size):
pass

@abc.abstractmethod
def ReadObject(self, bucket, object):
def ReadObject(self, bucket, object_name):
"""Read an object.
Exceptions are propagated to the caller, which can decide whether
to tolerate them or not.
Args:
bucket: the name of the bucket.
object: the name of the object.
object_name: the name of the object.
Returns:
A tuple of (start_time, latency)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,14 @@ def WriteObjectFromBuffer(self, bucket, object_name, stream, size):
start_time = time.time()
stream.seek(0)
obj = six.BytesIO(stream.read(size))
self.client.put_object(Body=obj, Bucket=bucket, Key=object_name)
# boto requires additional parameters to be not passed or valid.
# Passing StorageClass=None causes a type error. So use kwargs.
extra_kwargs = {}
if FLAGS.object_storage_class:
extra_kwargs['StorageClass'] = FLAGS.object_storage_class
self.client.put_object(
Body=obj, Bucket=bucket, Key=object_name, **extra_kwargs
)
latency = time.time() - start_time
return start_time, latency

Expand Down

0 comments on commit d35fae7

Please sign in to comment.