diff --git a/CHANGES.next.md b/CHANGES.next.md index 28eded85cf..776af8c224 100644 --- a/CHANGES.next.md +++ b/CHANGES.next.md @@ -404,3 +404,4 @@ - Moved flags from `pkb.py` to `flags.py` to help avoid circular dependencies. - Updated tracer dstat to use pcp dstat. - Removed Windows 2012 after loss of support on all clouds. +- Formatted entire directory with https://github.com/google/pyink. diff --git a/perfkitbenchmarker/app_service.py b/perfkitbenchmarker/app_service.py index 547f9ab1ea..c78513ebd0 100644 --- a/perfkitbenchmarker/app_service.py +++ b/perfkitbenchmarker/app_service.py @@ -1,4 +1,5 @@ """Module containing class for BaseAppService and BaseAppServiceSpec.""" + import logging import threading import time diff --git a/perfkitbenchmarker/archive.py b/perfkitbenchmarker/archive.py index be62d05890..3315a345f9 100644 --- a/perfkitbenchmarker/archive.py +++ b/perfkitbenchmarker/archive.py @@ -24,10 +24,13 @@ from perfkitbenchmarker.providers.aws.util import AWS_PATH -def ArchiveRun(run_temp_directory, target_bucket, - prefix='', - gsutil_path='gsutil', - aws_path=AWS_PATH): +def ArchiveRun( + run_temp_directory, + target_bucket, + prefix='', + gsutil_path='gsutil', + aws_path=AWS_PATH, +): """Archive a run directory to GCS or S3. Args: @@ -45,24 +48,25 @@ def ArchiveRun(run_temp_directory, target_bucket, raise ValueError('{0} is not a directory.'.format(run_temp_directory)) tar_file_name = '{}{}.tar.gz'.format( - prefix, datetime.datetime.now().strftime('%Y%m%d%H%M%S')) + prefix, datetime.datetime.now().strftime('%Y%m%d%H%M%S') + ) prefix_len = 5 - prefixes = { - 's3://': [aws_path, 's3', 'cp'], - 'gs://': [gsutil_path, 'cp'] - } + prefixes = {'s3://': [aws_path, 's3', 'cp'], 'gs://': [gsutil_path, 'cp']} assert all(len(key) == prefix_len for key in prefixes), prefixes try: - cmd = (prefixes[target_bucket[:prefix_len]] + - ['-', posixpath.join(target_bucket, tar_file_name)]) + cmd = prefixes[target_bucket[:prefix_len]] + [ + '-', + posixpath.join(target_bucket, tar_file_name), + ] except KeyError: raise ValueError('Unsupported bucket name: {0}'.format(target_bucket)) - logging.info('Streaming %s to %s\n%s', run_temp_directory, tar_file_name, - ' '.join(cmd)) + logging.info( + 'Streaming %s to %s\n%s', run_temp_directory, tar_file_name, ' '.join(cmd) + ) p = subprocess.Popen(cmd, stdin=subprocess.PIPE) with p.stdin: diff --git a/perfkitbenchmarker/background_tasks.py b/perfkitbenchmarker/background_tasks.py index 37aaa8733f..8bae3e0736 100644 --- a/perfkitbenchmarker/background_tasks.py +++ b/perfkitbenchmarker/background_tasks.py @@ -77,7 +77,7 @@ # For situations where an interruptable wait is necessary, a loop of waits with # long timeouts is used instead. This is because some of Python's built-in wait # methods are non-interruptable without a timeout. -_LONG_TIMEOUT = 1000. +_LONG_TIMEOUT = 1000.0 # Constants used for polling waits. See _WaitForCondition. _WAIT_MIN_RECHECK_DELAY = 0.001 # 1 ms @@ -95,8 +95,10 @@ # important for the cluster_boot benchmark where we want to launch all of the # VMs in parallel. flags.DEFINE_integer( - 'max_concurrent_threads', None, 'Maximum number of concurrent threads to ' - 'use when running a benchmark.') + 'max_concurrent_threads', + None, + 'Maximum number of concurrent threads to use when running a benchmark.', +) FLAGS = flags.FLAGS @@ -111,8 +113,9 @@ def _GetCallString(target_arg_tuple): target = target.func arg_strings = [str(a) for a in args] arg_strings.extend(['{0}={1}'.format(k, v) for k, v in six.iteritems(kwargs)]) - return '{0}({1})'.format(getattr(target, '__name__', target), - ', '.join(arg_strings)) + return '{0}({1})'.format( + getattr(target, '__name__', target), ', '.join(arg_strings) + ) def _WaitForCondition(condition_callback, timeout=None): @@ -124,10 +127,10 @@ def _WaitForCondition(condition_callback, timeout=None): Args: condition_callback: Callable that returns a value that evaluates True to end - the wait or evaluates False to continue the wait. + the wait or evaluates False to continue the wait. timeout: Optional float. Number of seconds to wait before giving up. If - provided, the condition is still checked at least once before giving up. - If not provided, the wait does not time out. + provided, the condition is still checked at least once before giving up. + If not provided, the wait does not time out. Returns: True if condition_callback returned a value that evaluated True. False if @@ -139,8 +142,9 @@ def _WaitForCondition(condition_callback, timeout=None): while True: if condition_callback(): return True - remaining_time = (_WAIT_MAX_RECHECK_DELAY if deadline is None - else deadline - time.time()) + remaining_time = ( + _WAIT_MAX_RECHECK_DELAY if deadline is None else deadline - time.time() + ) if remaining_time <= 0: return False time.sleep(delay) @@ -230,11 +234,11 @@ class _BackgroundTask(object): args: Series of unnamed arguments to be passed to the target. kwargs: dict. Keyword arguments to be passed to the target. context: _BackgroundTaskThreadContext. Thread-specific state to be inherited - from parent to child thread. + from parent to child thread. return_value: Return value if the call was executed successfully, or None - otherwise. + otherwise. traceback: The traceback string if the call raised an exception, or None - otherwise. + otherwise. """ def __init__(self, target, args, kwargs, thread_context): @@ -259,7 +263,7 @@ class _BackgroundTaskManager(six.with_metaclass(abc.ABCMeta, object)): Attributes: tasks: list of _BackgroundTask instances. Contains one _BackgroundTask per - started task, in the order that they were started. + started task, in the order that they were started. """ def __init__(self, max_concurrency): @@ -283,7 +287,7 @@ def StartTask(self, target, args, kwargs, thread_context): args: Series of unnamed arguments to be passed to the target. kwargs: dict. Keyword arguments to be passed to the target. thread_context: _BackgroundTaskThreadContext. Thread-specific state to be - inherited from parent to child thread. + inherited from parent to child thread. """ raise NotImplementedError() @@ -313,17 +317,17 @@ def _ExecuteBackgroundThreadTasks(worker_id, task_queue, response_queue): Args: worker_id: int. Identifier for the child thread relative to other child - threads. + threads. task_queue: _NonPollingSingleReaderQueue. Queue from which input is read. - Each value in the queue can be one of three types of values. If it is a - (task_id, _BackgroundTask) pair, the task is executed on this thread. - If it is _THREAD_STOP_PROCESSING, the thread stops executing. If it is - _THREAD_WAIT_FOR_KEYBOARD_INTERRUPT, the thread waits for a - KeyboardInterrupt. + Each value in the queue can be one of three types of values. If it is a + (task_id, _BackgroundTask) pair, the task is executed on this thread. If + it is _THREAD_STOP_PROCESSING, the thread stops executing. If it is + _THREAD_WAIT_FOR_KEYBOARD_INTERRUPT, the thread waits for a + KeyboardInterrupt. response_queue: _SingleReaderQueue. Queue to which output is written. It - receives worker_id when this thread's bootstrap code has completed and - receives a (worker_id, task_id) pair for each task completed on this - thread. + receives worker_id when this thread's bootstrap code has completed and + receives a (worker_id, task_id) pair for each task completed on this + thread. """ try: response_queue.Put(worker_id) @@ -341,8 +345,11 @@ def _ExecuteBackgroundThreadTasks(worker_id, task_queue, response_queue): # TODO(user): Detect when the log would be unhelpful (e.g. if the # current thread was spinning in the _THREAD_WAIT_FOR_KEYBOARD_INTERRUPT # sub-loop). Only log in helpful cases, like when the task is interrupted. - logging.debug('Child thread %s received a KeyboardInterrupt from its ' - 'parent.', worker_id, exc_info=True) + logging.debug( + 'Child thread %s received a KeyboardInterrupt from its parent.', + worker_id, + exc_info=True, + ) class _BackgroundThreadTaskManager(_BackgroundTaskManager): @@ -360,7 +367,8 @@ def __init__(self, *args, **kwargs): self._task_queues.append(task_queue) thread = threading.Thread( target=_ExecuteBackgroundThreadTasks, - args=(worker_id, task_queue, self._response_queue)) + args=(worker_id, task_queue, self._response_queue), + ) thread.daemon = True self._threads.append(thread) thread.start() @@ -380,8 +388,9 @@ def __exit__(self, *unused_args, **unused_kwargs): _WaitForCondition(lambda: not thread.is_alive()) def StartTask(self, target, args, kwargs, thread_context): - assert self._available_worker_ids, ('StartTask called when no threads were ' - 'available') + assert ( + self._available_worker_ids + ), 'StartTask called when no threads were available' task = _BackgroundTask(target, args, kwargs, thread_context) task_id = len(self.tasks) self.tasks.append(task) @@ -397,7 +406,8 @@ def HandleKeyboardInterrupt(self): # Raise a KeyboardInterrupt in each child thread. for thread in self._threads: ctypes.pythonapi.PyThreadState_SetAsyncExc( - ctypes.c_long(thread.ident), ctypes.py_object(KeyboardInterrupt)) + ctypes.c_long(thread.ident), ctypes.py_object(KeyboardInterrupt) + ) # Wake threads up from possible non-interruptable wait states so they can # actually see the KeyboardInterrupt. for task_queue, thread in zip(self._task_queues, self._threads): @@ -425,12 +435,14 @@ def _ExecuteProcessTask(task): element is the exception traceback string, or None if the function succeeded. """ + def handle_sigint(signum, frame): # Ignore any new SIGINTs since we are already tearing down. signal.signal(signal.SIGINT, signal.SIG_IGN) # Execute the default SIGINT handler which throws a KeyboardInterrupt # in the main thread of the process. signal.default_int_handler(signum, frame) + signal.signal(signal.SIGINT, handle_sigint) task.Run() return task.return_value, task.traceback @@ -470,8 +482,10 @@ def AwaitAnyTask(self): completed_tasks = None while not completed_tasks: completed_tasks, _ = futures.wait( - self._active_futures, timeout=_LONG_TIMEOUT, - return_when=futures.FIRST_COMPLETED) + self._active_futures, + timeout=_LONG_TIMEOUT, + return_when=futures.FIRST_COMPLETED, + ) future = completed_tasks.pop() task_id = self._active_futures.pop(future) task = self.tasks[task_id] @@ -486,19 +500,24 @@ def HandleKeyboardInterrupt(self): self._executor.shutdown(wait=True) -def _RunParallelTasks(target_arg_tuples, max_concurrency, get_task_manager, - parallel_exception_class, post_task_delay=0): +def _RunParallelTasks( + target_arg_tuples, + max_concurrency, + get_task_manager, + parallel_exception_class, + post_task_delay=0, +): """Executes function calls concurrently in separate threads or processes. Args: target_arg_tuples: list of (target, args, kwargs) tuples. Each tuple - contains the function to call and the arguments to pass it. - max_concurrency: int or None. The maximum number of concurrent new - threads or processes. + contains the function to call and the arguments to pass it. + max_concurrency: int or None. The maximum number of concurrent new threads + or processes. get_task_manager: Callable that accepts an int max_concurrency arg and - returns a _TaskManager. + returns a _TaskManager. parallel_exception_class: Type of exception to raise upon an exception in - one of the called functions. + one of the called functions. post_task_delay: Delay in seconds between parallel task invocations. Returns: @@ -517,8 +536,10 @@ def _RunParallelTasks(target_arg_tuples, max_concurrency, get_task_manager, with get_task_manager(max_concurrency) as task_manager: try: while started_task_count < len(target_arg_tuples) or active_task_count: - if (started_task_count < len(target_arg_tuples) and - active_task_count < max_concurrency): + if ( + started_task_count < len(target_arg_tuples) + and active_task_count < max_concurrency + ): # Start a new task. target, args, kwargs = target_arg_tuples[started_task_count] task_manager.StartTask(target, args, kwargs, thread_context) @@ -536,16 +557,18 @@ def _RunParallelTasks(target_arg_tuples, max_concurrency, get_task_manager, # for other tasks. stacktrace = task_manager.tasks[task_id].traceback if stacktrace: - msg = ('Exception occurred while calling {0}:{1}{2}'.format( - _GetCallString(target_arg_tuples[task_id]), os.linesep, - stacktrace)) + msg = 'Exception occurred while calling {0}:{1}{2}'.format( + _GetCallString(target_arg_tuples[task_id]), os.linesep, stacktrace + ) logging.error(msg) error_strings.append(msg) except KeyboardInterrupt: logging.error( 'Received KeyboardInterrupt while executing parallel tasks. Waiting ' - 'for %s tasks to clean up.', active_task_count) + 'for %s tasks to clean up.', + active_task_count, + ) task_manager.HandleKeyboardInterrupt() raise @@ -555,7 +578,8 @@ def _RunParallelTasks(target_arg_tuples, max_concurrency, get_task_manager, # type. raise parallel_exception_class( 'The following exceptions occurred during parallel execution:' - '{0}{1}'.format(os.linesep, os.linesep.join(error_strings))) + '{0}{1}'.format(os.linesep, os.linesep.join(error_strings)) + ) results = [task.return_value for task in task_manager.tasks] assert len(target_arg_tuples) == len(results), (target_arg_tuples, results) return results @@ -566,9 +590,8 @@ def RunParallelThreads(target_arg_tuples, max_concurrency, post_task_delay=0): Args: target_arg_tuples: list of (target, args, kwargs) tuples. Each tuple - contains the function to call and the arguments to pass it. - max_concurrency: int or None. The maximum number of concurrent new - threads. + contains the function to call and the arguments to pass it. + max_concurrency: int or None. The maximum number of concurrent new threads. post_task_delay: Delay in seconds between parallel task invocations. Returns: @@ -580,23 +603,26 @@ def RunParallelThreads(target_arg_tuples, max_concurrency, post_task_delay=0): called functions. """ return _RunParallelTasks( - target_arg_tuples, max_concurrency, _BackgroundThreadTaskManager, - errors.VmUtil.ThreadException, post_task_delay) + target_arg_tuples, + max_concurrency, + _BackgroundThreadTaskManager, + errors.VmUtil.ThreadException, + post_task_delay, + ) -def RunThreaded(target, - thread_params, - max_concurrent_threads=None, - post_task_delay=0): +def RunThreaded( + target, thread_params, max_concurrent_threads=None, post_task_delay=0 +): """Runs the target method in parallel threads. The method starts up threads with one arg from thread_params as the first arg. Args: target: The method to invoke in the thread. - thread_params: A thread is launched for each value in the list. The items - in the list can either be a singleton or a (args, kwargs) tuple/list. - Usually this is a list of VMs. + thread_params: A thread is launched for each value in the list. The items in + the list can either be a singleton or a (args, kwargs) tuple/list. Usually + this is a list of VMs. max_concurrent_threads: The maximum number of concurrent threads to allow. post_task_delay: Delay in seconds between commands. @@ -626,7 +652,8 @@ def RunThreaded(target, """ if max_concurrent_threads is None: max_concurrent_threads = ( - FLAGS.max_concurrent_threads or MAX_CONCURRENT_THREADS) + FLAGS.max_concurrent_threads or MAX_CONCURRENT_THREADS + ) if not isinstance(thread_params, list): raise ValueError('Param "thread_params" must be a list') @@ -637,28 +664,33 @@ def RunThreaded(target, if not isinstance(thread_params[0], tuple): target_arg_tuples = [(target, (arg,), {}) for arg in thread_params] - elif (not isinstance(thread_params[0][0], tuple) or - not isinstance(thread_params[0][1], dict)): + elif not isinstance(thread_params[0][0], tuple) or not isinstance( + thread_params[0][1], dict + ): raise ValueError('If Param is a tuple, the tuple must be (tuple, dict)') else: - target_arg_tuples = [(target, args, kwargs) - for args, kwargs in thread_params] + target_arg_tuples = [ + (target, args, kwargs) for args, kwargs in thread_params + ] - return RunParallelThreads(target_arg_tuples, - max_concurrency=max_concurrent_threads, - post_task_delay=post_task_delay) + return RunParallelThreads( + target_arg_tuples, + max_concurrency=max_concurrent_threads, + post_task_delay=post_task_delay, + ) -def RunParallelProcesses(target_arg_tuples, max_concurrency, - post_process_delay=0): +def RunParallelProcesses( + target_arg_tuples, max_concurrency, post_process_delay=0 +): """Executes function calls concurrently in separate processes. Args: target_arg_tuples: list of (target, args, kwargs) tuples. Each tuple - contains the function to call and the arguments to pass it. + contains the function to call and the arguments to pass it. max_concurrency: int or None. The maximum number of concurrent new - processes. If None, it will default to the number of processors on the - machine. + processes. If None, it will default to the number of processors on the + machine. post_process_delay: Delay in seconds between parallel process invocations. Returns: @@ -669,18 +701,25 @@ def RunParallelProcesses(target_arg_tuples, max_concurrency, errors.VmUtil.CalledProcessException: When an exception occurred in any of the called functions. """ + def handle_sigint(signum, frame): # Ignore any SIGINTS in the parent process, but let users know # that the child processes are getting cleaned up. - logging.error('Got SIGINT while executing parallel tasks. ' - 'Waiting for tasks to clean up.') + logging.error( + 'Got SIGINT while executing parallel tasks. ' + 'Waiting for tasks to clean up.' + ) + old_handler = None try: old_handler = signal.signal(signal.SIGINT, handle_sigint) ret_val = _RunParallelTasks( - target_arg_tuples, max_concurrency, _BackgroundProcessTaskManager, + target_arg_tuples, + max_concurrency, + _BackgroundProcessTaskManager, errors.VmUtil.CalledProcessException, - post_task_delay=post_process_delay) + post_task_delay=post_process_delay, + ) finally: if old_handler: signal.signal(signal.SIGINT, old_handler) diff --git a/perfkitbenchmarker/background_workload.py b/perfkitbenchmarker/background_workload.py index 95eb803463..bd3ca5c96c 100644 --- a/perfkitbenchmarker/background_workload.py +++ b/perfkitbenchmarker/background_workload.py @@ -34,7 +34,8 @@ def __init__(cls, name, bases, dct): class BaseBackgroundWorkload( - six.with_metaclass(AutoRegisterBackgroundWorkloadMeta, object)): + six.with_metaclass(AutoRegisterBackgroundWorkloadMeta, object) +): """Baseclass for background workloads.""" EXCLUDED_OS_TYPES = [] @@ -81,7 +82,9 @@ def Start(vm): """Starts the background workload on this VM.""" vm.RemoteCommand( 'nohup sysbench --num-threads=%s --test=cpu --cpu-max-prime=10000000 ' - 'run 1> /dev/null 2> /dev/null &' % vm.background_cpu_threads) + 'run 1> /dev/null 2> /dev/null &' + % vm.background_cpu_threads + ) @staticmethod def Stop(vm): @@ -108,8 +111,9 @@ def Prepare(vm): def Start(vm): """Starts the background workload on this VM.""" vm.AllowPort(BACKGROUND_IPERF_PORT) - vm.RemoteCommand('nohup iperf --server --port %s &> /dev/null &' % - BACKGROUND_IPERF_PORT) + vm.RemoteCommand( + 'nohup iperf --server --port %s &> /dev/null &' % BACKGROUND_IPERF_PORT + ) stdout, _ = vm.RemoteCommand('pgrep iperf -n') vm.server_pid = stdout.strip() @@ -117,10 +121,15 @@ def Start(vm): ip_address = vm.ip_address else: ip_address = vm.internal_ip - iperf_cmd = ('nohup iperf --client %s --port %s --time %s -u -b %sM ' - '&> /dev/null &' % (ip_address, BACKGROUND_IPERF_PORT, - BACKGROUND_IPERF_SECONDS, - vm.background_network_mbits_per_sec)) + iperf_cmd = ( + 'nohup iperf --client %s --port %s --time %s -u -b %sM &> /dev/null &' + % ( + ip_address, + BACKGROUND_IPERF_PORT, + BACKGROUND_IPERF_SECONDS, + vm.background_network_mbits_per_sec, + ) + ) vm.RemoteCommand(iperf_cmd) stdout, _ = vm.RemoteCommand('pgrep iperf -n') diff --git a/perfkitbenchmarker/beam_benchmark_helper.py b/perfkitbenchmarker/beam_benchmark_helper.py index 6836ebb3c2..f4ec438023 100644 --- a/perfkitbenchmarker/beam_benchmark_helper.py +++ b/perfkitbenchmarker/beam_benchmark_helper.py @@ -29,42 +29,75 @@ BEAM_JAVA_SDK = 'java' BEAM_PYTHON_SDK = 'python' -flags.DEFINE_string('gradle_binary', None, - 'Set to use a different gradle binary than gradle wrapper ' - 'from the repository') -flags.DEFINE_string('beam_location', None, - 'Location of already checked out Beam codebase.') -flags.DEFINE_string('beam_it_module', None, - 'Gradle module containing integration test. Use full ' - 'module starting and separated by colon, like :sdk:python') -flags.DEFINE_boolean('beam_prebuilt', False, - 'Set this to indicate that the repo in beam_location ' - 'does not need to be rebuilt before being used') +flags.DEFINE_string( + 'gradle_binary', + None, + 'Set to use a different gradle binary than gradle wrapper ' + 'from the repository', +) +flags.DEFINE_string( + 'beam_location', None, 'Location of already checked out Beam codebase.' +) +flags.DEFINE_string( + 'beam_it_module', + None, + 'Gradle module containing integration test. Use full ' + 'module starting and separated by colon, like :sdk:python', +) +flags.DEFINE_boolean( + 'beam_prebuilt', + False, + 'Set this to indicate that the repo in beam_location ' + 'does not need to be rebuilt before being used', +) flags.DEFINE_integer('beam_it_timeout', 600, 'Integration Test Timeout.') flags.DEFINE_string('git_binary', 'git', 'Path to git binary.') -flags.DEFINE_string('beam_version', None, - 'Version of Beam to download. Use tag from Github ' - 'as value. If not specified, will use HEAD.') -flags.DEFINE_enum('beam_sdk', None, [BEAM_JAVA_SDK, BEAM_PYTHON_SDK], - 'Which BEAM SDK is used to build the benchmark pipeline.') -flags.DEFINE_string('beam_python_attr', 'IT', - 'Test decorator that is used in Beam Python to filter a ' - 'specific category.') -flags.DEFINE_string('beam_python_sdk_location', None, - 'Python SDK tar ball location. It is a required option to ' - 'run Python pipeline.') - -flags.DEFINE_string('beam_extra_properties', None, - 'Allows to specify list of key-value pairs that will be ' - 'forwarded to target mvn command as system properties') - -flags.DEFINE_string('beam_runner', 'dataflow', 'Defines runner which will be used in tests') -flags.DEFINE_string('beam_runner_option', None, - 'Overrides any pipeline options to specify the runner.') - -flags.DEFINE_string('beam_filesystem', None, - 'Defines filesystem which will be used in tests. ' - 'If not specified it will use runner\'s local filesystem.') +flags.DEFINE_string( + 'beam_version', + None, + 'Version of Beam to download. Use tag from Github ' + 'as value. If not specified, will use HEAD.', +) +flags.DEFINE_enum( + 'beam_sdk', + None, + [BEAM_JAVA_SDK, BEAM_PYTHON_SDK], + 'Which BEAM SDK is used to build the benchmark pipeline.', +) +flags.DEFINE_string( + 'beam_python_attr', + 'IT', + 'Test decorator that is used in Beam Python to filter a specific category.', +) +flags.DEFINE_string( + 'beam_python_sdk_location', + None, + 'Python SDK tar ball location. It is a required option to ' + 'run Python pipeline.', +) + +flags.DEFINE_string( + 'beam_extra_properties', + None, + 'Allows to specify list of key-value pairs that will be ' + 'forwarded to target mvn command as system properties', +) + +flags.DEFINE_string( + 'beam_runner', 'dataflow', 'Defines runner which will be used in tests' +) +flags.DEFINE_string( + 'beam_runner_option', + None, + 'Overrides any pipeline options to specify the runner.', +) + +flags.DEFINE_string( + 'beam_filesystem', + None, + 'Defines filesystem which will be used in tests. ' + "If not specified it will use runner's local filesystem.", +) FLAGS = flags.FLAGS @@ -83,16 +116,17 @@ def AddRunnerArgument(command, runner_name): command.append('-DintegrationTestRunner=dataflow') -def AddRunnerPipelineOption(beam_pipeline_options, runner_name, - runner_option_override): +def AddRunnerPipelineOption( + beam_pipeline_options, runner_name, runner_option_override +): """Add runner to pipeline options.""" runner_pipeline_option = '' if runner_name == 'dataflow': - runner_pipeline_option = ('"--runner=TestDataflowRunner"') + runner_pipeline_option = '"--runner=TestDataflowRunner"' if runner_name == 'direct': - runner_pipeline_option = ('"--runner=DirectRunner"') + runner_pipeline_option = '"--runner=DirectRunner"' if runner_option_override: runner_pipeline_option = '--runner=' + runner_option_override @@ -111,8 +145,9 @@ def AddExtraProperties(command, extra_properties): return if 'integrationTestPipelineOptions=' in extra_properties: - raise ValueError('integrationTestPipelineOptions must not be in ' - 'beam_extra_properties') + raise ValueError( + 'integrationTestPipelineOptions must not be in beam_extra_properties' + ) extra_properties = extra_properties.rstrip(']').lstrip('[').split(',') extra_properties = [p.rstrip('" ').lstrip('" ') for p in extra_properties] @@ -154,9 +189,11 @@ def InitializeBeamRepo(benchmark_spec): vm_util.IssueCommand(git_clone_command, cwd=vm_util.GetTempDir()) elif not os.path.exists(FLAGS.beam_location): - raise errors.Config.InvalidValue('Directory indicated by beam_location ' - 'does not exist: {}.'.format( - FLAGS.beam_location)) + raise errors.Config.InvalidValue( + 'Directory indicated by beam_location does not exist: {}.'.format( + FLAGS.beam_location + ) + ) _PrebuildBeam() @@ -164,7 +201,6 @@ def InitializeBeamRepo(benchmark_spec): def _PrebuildBeam(): """Rebuild beam if it was not build earlier.""" if not FLAGS.beam_prebuilt: - gradle_prebuild_tasks = ['clean', 'assemble'] gradle_prebuild_flags = ['--stacktrace', '--info'] build_command = [_GetGradleCommand()] @@ -222,7 +258,8 @@ def _BuildGradleCommand(classname, job_arguments): if not vm_util.ExecutableOnPath(gradle_executable): raise errors.Setup.MissingExecutableError( - 'Could not find required executable "%s"' % gradle_executable) + 'Could not find required executable "%s"' % gradle_executable + ) cmd.append(gradle_executable) AddTaskArgument(cmd, 'integrationTest', FLAGS.beam_it_module) @@ -231,13 +268,15 @@ def _BuildGradleCommand(classname, job_arguments): beam_args = job_arguments if job_arguments else [] AddRunnerArgument(cmd, FLAGS.beam_runner) - AddRunnerPipelineOption(beam_args, FLAGS.beam_runner, - FLAGS.beam_runner_option) + AddRunnerPipelineOption( + beam_args, FLAGS.beam_runner, FLAGS.beam_runner_option + ) AddFilesystemArgument(cmd, FLAGS.beam_filesystem) AddExtraProperties(cmd, FLAGS.beam_extra_properties) - cmd.append('-DintegrationTestPipelineOptions=' - '[{}]'.format(','.join(beam_args))) + cmd.append( + '-DintegrationTestPipelineOptions=[{}]'.format(','.join(beam_args)) + ) cmd.append('--stacktrace') cmd.append('--info') @@ -274,7 +313,8 @@ def _BuildPythonCommand(benchmark_spec, classname, job_arguments): if not vm_util.ExecutableOnPath(gradle_executable): raise errors.Setup.MissingExecutableError( - 'Could not find required executable "%s"' % gradle_executable) + 'Could not find required executable "%s"' % gradle_executable + ) cmd.append(gradle_executable) AddTaskArgument(cmd, 'integrationTest', FLAGS.beam_it_module) diff --git a/perfkitbenchmarker/beam_pipeline_options.py b/perfkitbenchmarker/beam_pipeline_options.py index 8ac754e8af..a59b26950e 100644 --- a/perfkitbenchmarker/beam_pipeline_options.py +++ b/perfkitbenchmarker/beam_pipeline_options.py @@ -22,8 +22,8 @@ def GetStaticPipelineOptions(options_list): - """ - Takes the dictionary loaded from the yaml configuration file and returns it + """Takes the dictionary loaded from the yaml configuration file and returns it + in a form consistent with the others in GenerateAllPipelineOptions: a list of (pipeline_option_name, pipeline_option_value) tuples. @@ -34,21 +34,23 @@ def GetStaticPipelineOptions(options_list): options = [] for option in options_list: if len(list(option.keys())) != 1: - raise Exception('Each item in static_pipeline_options should only have' - ' 1 key/value') + raise Exception( + 'Each item in static_pipeline_options should only have 1 key/value' + ) option_kv = list(option.items())[0] options.append((option_kv[0], option_kv[1])) return options def EvaluateDynamicPipelineOptions(dynamic_options): - """ - Takes the user's dynamic args and retrieves the information to fill them in. + """Takes the user's dynamic args and retrieves the information to fill them in. - dynamic_args is a python map of argument name -> {type, kubernetesSelector, *format} + dynamic_args is a python map of argument name -> {type, kubernetesSelector, + *format} returns a list of tuples containing (argName, argValue) - if optional format it passed, argValue is equal to format with "{{type}}" being replaced with actual value. + if optional format it passed, argValue is equal to format with "{{type}}" + being replaced with actual value. """ filledOptions = [] for optionDescriptor in dynamic_options: @@ -58,7 +60,8 @@ def EvaluateDynamicPipelineOptions(dynamic_options): if not fillType: raise errors.Config.InvalidValue( - 'For dynamic arguments, you must provide a "type"') + 'For dynamic arguments, you must provide a "type"' + ) if fillType == 'NodePortIp': argValue = RetrieveNodePortIp(optionDescriptor) @@ -68,19 +71,21 @@ def EvaluateDynamicPipelineOptions(dynamic_options): argValue = optionDescriptor['value'] else: raise errors.Config.InvalidValue( - 'Unknown dynamic argument type: %s' % (fillType)) + 'Unknown dynamic argument type: %s' % (fillType) + ) if valueFormat: - argValue = valueFormat.replace("{{" + fillType + "}}", argValue) + argValue = valueFormat.replace('{{' + fillType + '}}', argValue) filledOptions.append((optionName, argValue)) return filledOptions -def GenerateAllPipelineOptions(it_args, it_options, static_pipeline_options, - dynamic_pipeline_options): - """ - :param it_args: options list passed in via FLAGS.beam_it_args +def GenerateAllPipelineOptions( + it_args, it_options, static_pipeline_options, dynamic_pipeline_options +): + """:param it_args: options list passed in via FLAGS.beam_it_args + :param it_options: options list passed in via FLAGS.beam_it_options :param static_pipeline_options: options list loaded from the yaml config file :param dynamic_pipeline_options: options list loaded from the yaml config file @@ -90,29 +95,34 @@ def GenerateAllPipelineOptions(it_args, it_options, static_pipeline_options, user_option_list = [] if it_options is not None and len(it_options) > 0: user_option_list = it_options.rstrip(']').lstrip('[').split(',') - user_option_list = [option.rstrip('" ').lstrip('" ') - for option in user_option_list] + user_option_list = [ + option.rstrip('" ').lstrip('" ') for option in user_option_list + ] # Add static options from the benchmark_spec - benchmark_spec_option_list = ( - EvaluateDynamicPipelineOptions(dynamic_pipeline_options)) + benchmark_spec_option_list = EvaluateDynamicPipelineOptions( + dynamic_pipeline_options + ) benchmark_spec_option_list.extend( - GetStaticPipelineOptions(static_pipeline_options)) - option_list = ['--{}={}'.format(t[0], t[1]) - for t in benchmark_spec_option_list] + GetStaticPipelineOptions(static_pipeline_options) + ) + option_list = [ + '--{}={}'.format(t[0], t[1]) for t in benchmark_spec_option_list + ] # beam_it_args is the old way of passing parameters args_list = [] if it_args is not None and len(it_args) > 0: args_list = it_args.split(',') - return ['"{}"'.format(arg) - for arg in args_list + user_option_list + option_list] + return [ + '"{}"'.format(arg) for arg in args_list + user_option_list + option_list + ] def ReadPipelineOptionConfigFile(): - """ - Reads the path to the config file from FLAGS, then loads the static and + """Reads the path to the config file from FLAGS, then loads the static and + dynamic pipeline options from it. """ dynamic_pipeline_options = [] @@ -130,24 +140,28 @@ def ReadPipelineOptionConfigFile(): def RetrieveNodePortIp(argDescriptor): jsonSelector = argDescriptor['podLabel'] if not jsonSelector: - raise errors.Config.InvalidValue('For NodePortIp arguments, you must' - ' provide a "selector"') + raise errors.Config.InvalidValue( + 'For NodePortIp arguments, you must provide a "selector"' + ) ip = kubernetes_helper.GetWithWaitForContents( - 'pods', '', jsonSelector, '.items[0].status.podIP') + 'pods', '', jsonSelector, '.items[0].status.podIP' + ) if len(ip) == 0: - raise "Could not retrieve NodePort IP address" - logging.info("Using NodePort IP Address: " + ip) + raise 'Could not retrieve NodePort IP address' + logging.info('Using NodePort IP Address: ' + ip) return ip def RetrieveLoadBalancerIp(argDescriptor): serviceName = argDescriptor['serviceName'] if not serviceName: - raise errors.Config.InvalidValue('For LoadBalancerIp arguments, you must' - 'provide a "serviceName"') + raise errors.Config.InvalidValue( + 'For LoadBalancerIp arguments, you mustprovide a "serviceName"' + ) ip = kubernetes_helper.GetWithWaitForContents( - 'svc', serviceName, '', '.status.loadBalancer.ingress[0].ip') + 'svc', serviceName, '', '.status.loadBalancer.ingress[0].ip' + ) if len(ip) == 0: - raise "Could not retrieve LoadBalancer IP address" - logging.info("Using LoadBalancer IP Address: " + ip) + raise 'Could not retrieve LoadBalancer IP address' + logging.info('Using LoadBalancer IP Address: ' + ip) return ip diff --git a/perfkitbenchmarker/benchmark_lookup.py b/perfkitbenchmarker/benchmark_lookup.py index 10335f8066..0d949c804c 100644 --- a/perfkitbenchmarker/benchmark_lookup.py +++ b/perfkitbenchmarker/benchmark_lookup.py @@ -44,5 +44,6 @@ def BenchmarkModule(benchmark_name): """ if not _global_benchmark_module_function: raise errors.Setup.InvalidSetupError( - 'Cannot call benchmark_lookup.py; Was SetUpPKB called?') + 'Cannot call benchmark_lookup.py; Was SetUpPKB called?' + ) return _global_benchmark_module_function(benchmark_name) diff --git a/perfkitbenchmarker/benchmark_sets.py b/perfkitbenchmarker/benchmark_sets.py index ca9f4fed15..0e8a3e0889 100644 --- a/perfkitbenchmarker/benchmark_sets.py +++ b/perfkitbenchmarker/benchmark_sets.py @@ -31,20 +31,25 @@ FLAGS = flags.FLAGS -flags.DEFINE_string('flag_matrix', None, - 'The name of the flag matrix to run.') -flags.DEFINE_string('flag_zip', None, - 'The name of the flag zip to run.') -flags.DEFINE_integer('num_benchmark_copies', 1, - 'The number of copies of each benchmark config to run.') +flags.DEFINE_string('flag_matrix', None, 'The name of the flag matrix to run.') +flags.DEFINE_string('flag_zip', None, 'The name of the flag zip to run.') +flags.DEFINE_integer( + 'num_benchmark_copies', + 1, + 'The number of copies of each benchmark config to run.', +) LINUX = 'linux' WINDOWS = 'windows' CONTAINER = 'container' -flags.DEFINE_enum('benchmark_os_type', None, [LINUX, WINDOWS, CONTAINER], - 'The benchmark os type, decides if pkb searches for a ' - 'benchmark from linux_benchmarks directory or ' - 'windows_benchmarks directory. If not set, defaults to that ' - 'of the os_type flag.') +flags.DEFINE_enum( + 'benchmark_os_type', + None, + [LINUX, WINDOWS, CONTAINER], + 'The benchmark os type, decides if pkb searches for a ' + 'benchmark from linux_benchmarks directory or ' + 'windows_benchmarks directory. If not set, defaults to that ' + 'of the os_type flag.', +) MESSAGE = 'message' BENCHMARK_LIST = 'benchmark_list' @@ -53,8 +58,10 @@ BENCHMARK_SETS = { STANDARD_SET: { - MESSAGE: ('The standard_set is a community agreed upon set of ' - 'benchmarks to measure Cloud performance.'), + MESSAGE: ( + 'The standard_set is a community agreed upon set of ' + 'benchmarks to measure Cloud performance.' + ), BENCHMARK_LIST: [ 'aerospike', 'block_storage_workload', @@ -75,35 +82,33 @@ 'speccpu2006', 'sysbench', 'unixbench', - ] + ], }, COMBO_SET: { - MESSAGE: ('The combo_set is a set of benchmarks that run multiple ' - 'benchmarks in parallel to measure Cloud performance.'), + MESSAGE: ( + 'The combo_set is a set of benchmarks that run multiple ' + 'benchmarks in parallel to measure Cloud performance.' + ), BENCHMARK_LIST: [ 'fio_netperf', 'netperf_hammerdbcli', - ] - }, - 'arm_set': { - MESSAGE: 'ARM benchmark set.', - BENCHMARK_LIST: [STANDARD_SET] + ], }, + 'arm_set': {MESSAGE: 'ARM benchmark set.', BENCHMARK_LIST: [STANDARD_SET]}, 'alicloud_set': { MESSAGE: 'AliCloud benchmark set.', - BENCHMARK_LIST: [STANDARD_SET] + BENCHMARK_LIST: [STANDARD_SET], }, 'broadcom_set': { MESSAGE: 'Broadcom benchmark set.', - BENCHMARK_LIST: [STANDARD_SET] + BENCHMARK_LIST: [STANDARD_SET], }, 'canonical_set': { MESSAGE: 'Canonical benchmark set.', - BENCHMARK_LIST: [STANDARD_SET] + BENCHMARK_LIST: [STANDARD_SET], }, 'centurylinkcloud_set': { - MESSAGE: - 'This benchmark set is supported on CenturyLink Cloud.', + MESSAGE: 'This benchmark set is supported on CenturyLink Cloud.', BENCHMARK_LIST: [ 'cassandra_stress', 'copy_throughput', @@ -115,26 +120,28 @@ 'redis_memtier', 'sysbench', 'unixbench', - ] + ], }, 'cisco_set': { MESSAGE: 'Cisco benchmark set.', - BENCHMARK_LIST: [STANDARD_SET] + BENCHMARK_LIST: [STANDARD_SET], }, 'cloudharmony_set': { MESSAGE: 'CloudHarmony benchmark set.', BENCHMARK_LIST: [ 'speccpu2006', 'unixbench', - ] + ], }, 'cloudspectator_set': { MESSAGE: 'CloudSpectator benchmark set.', - BENCHMARK_LIST: [STANDARD_SET] + BENCHMARK_LIST: [STANDARD_SET], }, 'google_set': { - MESSAGE: ('This benchmark set is maintained by Google Cloud Platform ' - 'Performance Team.'), + MESSAGE: ( + 'This benchmark set is maintained by Google Cloud Platform ' + 'Performance Team.' + ), BENCHMARK_LIST: [ 'bidirectional_network', 'block_storage_workload', @@ -174,11 +181,10 @@ 'tensorflow_serving', 'tomcat_wrk', 'unixbench', - ] + ], }, 'intel_set': { - MESSAGE: - 'Intel benchmark set.', + MESSAGE: 'Intel benchmark set.', BENCHMARK_LIST: [ 'fio', 'iperf', @@ -189,11 +195,10 @@ 'cassandra_stress', 'object_storage_service', 'sysbench', - ] + ], }, 'kubernetes_set': { - MESSAGE: - 'Kubernetes benchmark set.', + MESSAGE: 'Kubernetes benchmark set.', BENCHMARK_LIST: [ 'block_storage_workload', 'cassandra_ycsb', @@ -206,23 +211,22 @@ 'netperf', 'redis_memtier', 'sysbench', - ] + ], }, 'mellanox_set': { MESSAGE: 'Mellanox benchmark set.', - BENCHMARK_LIST: [STANDARD_SET] + BENCHMARK_LIST: [STANDARD_SET], }, 'microsoft_set': { MESSAGE: 'Microsoft benchmark set.', - BENCHMARK_LIST: [STANDARD_SET] + BENCHMARK_LIST: [STANDARD_SET], }, 'qualcomm_technologies_set': { MESSAGE: 'Qualcomm Technologies, Inc. benchmark set.', - BENCHMARK_LIST: [STANDARD_SET] + BENCHMARK_LIST: [STANDARD_SET], }, 'rackspace_set': { - MESSAGE: - 'Rackspace benchmark set.', + MESSAGE: 'Rackspace benchmark set.', BENCHMARK_LIST: [ 'aerospike', 'block_storage_workload', @@ -241,31 +245,30 @@ 'silo', 'sysbench', 'unixbench', - ] + ], }, 'red_hat_set': { MESSAGE: 'Red Hat benchmark set.', - BENCHMARK_LIST: [STANDARD_SET] + BENCHMARK_LIST: [STANDARD_SET], }, 'tradeworx_set': { MESSAGE: 'Tradeworx Inc. benchmark set.', - BENCHMARK_LIST: [STANDARD_SET] + BENCHMARK_LIST: [STANDARD_SET], }, 'thesys_technologies_set': { MESSAGE: 'Thesys Technologies LLC. benchmark set.', - BENCHMARK_LIST: [STANDARD_SET] + BENCHMARK_LIST: [STANDARD_SET], }, 'stanford_set': { MESSAGE: 'Stanford University benchmark set.', - BENCHMARK_LIST: [STANDARD_SET, 'oldisim'] + BENCHMARK_LIST: [STANDARD_SET, 'oldisim'], }, 'mit_set': { MESSAGE: 'Massachusetts Institute of Technology benchmark set.', - BENCHMARK_LIST: [STANDARD_SET, 'silo'] + BENCHMARK_LIST: [STANDARD_SET, 'silo'], }, 'cloudsuite_set': { - MESSAGE: - 'CloudSuite benchmark set.', + MESSAGE: 'CloudSuite benchmark set.', BENCHMARK_LIST: [ 'cloudsuite_data_analytics', 'cloudsuite_data_caching', @@ -274,8 +277,8 @@ 'cloudsuite_media_streaming', 'cloudsuite_web_search', 'cloudsuite_web_serving', - ] - } + ], + }, } @@ -359,12 +362,19 @@ def _GetBenchmarksFromUserConfig(user_config): benchmark_module = valid_benchmarks[name] except KeyError as exc: benchmark_os_type = _GetBenchmarkOsType() - raise ValueError('Benchmark_os_type flag is %s and os_type flag is %s. ' - 'This resulting benchmark type is %s. ' - 'However, benchmark of name %s is not found within the ' - 'list of %s benchmarks. ' % - (FLAGS.benchmark_os_type, FLAGS.os_type, - benchmark_os_type, name, benchmark_os_type)) from exc + raise ValueError( + 'Benchmark_os_type flag is %s and os_type flag is %s. ' + 'This resulting benchmark type is %s. ' + 'However, benchmark of name %s is not found within the ' + 'list of %s benchmarks. ' + % ( + FLAGS.benchmark_os_type, + FLAGS.os_type, + benchmark_os_type, + name, + benchmark_os_type, + ) + ) from exc benchmark_config_list.append((benchmark_module, user_config)) return benchmark_config_list @@ -387,9 +397,9 @@ def _AssertZipAxesHaveSameLength(axes): raise ValueError('flag_zip axes must all be the same length') -def _AssertFlagMatrixAndZipDefsExist(benchmark_config, - flag_matrix_name, - flag_zip_name): +def _AssertFlagMatrixAndZipDefsExist( + benchmark_config, flag_matrix_name, flag_zip_name +): """Asserts that specified flag_matrix and flag_zip exist. Both flag_matrix_name and flag_zip_name can be None, meaning that the user @@ -408,16 +418,18 @@ def _AssertFlagMatrixAndZipDefsExist(benchmark_config, FlagZipNotFoundException: if flag_zip_name is not None, and is not found in the flag_zip_defs section of the benchmark_config. """ - if (flag_matrix_name and - flag_matrix_name not in - benchmark_config.get('flag_matrix_defs', {})): - raise FlagMatrixNotFoundException('No flag_matrix with name {0}' - .format(flag_matrix_name)) - if (flag_zip_name and - flag_zip_name not in - benchmark_config.get('flag_zip_defs', {})): - raise FlagZipNotFoundException('No flag_zip with name {0}' - .format(flag_zip_name)) + if flag_matrix_name and flag_matrix_name not in benchmark_config.get( + 'flag_matrix_defs', {} + ): + raise FlagMatrixNotFoundException( + 'No flag_matrix with name {0}'.format(flag_matrix_name) + ) + if flag_zip_name and flag_zip_name not in benchmark_config.get( + 'flag_zip_defs', {} + ): + raise FlagZipNotFoundException( + 'No flag_zip with name {0}'.format(flag_zip_name) + ) def GetBenchmarksFromFlags(): @@ -460,23 +472,27 @@ def GetBenchmarksFromFlags(): if benchmark_module is None: benchmark_os_type = _GetBenchmarkOsType() - raise ValueError('Benchmark_os_type flag is %s and os_type flag is %s. ' - 'This resulting benchmark type is %s. ' - 'However, benchmark of name %s is not found within the ' - 'list of %s benchmarks. ' % - (FLAGS.benchmark_os_type, FLAGS.os_type, - benchmark_os_type, benchmark_name, - benchmark_os_type)) - - flag_matrix_name = ( - FLAGS.flag_matrix or benchmark_config.get('flag_matrix', None) + raise ValueError( + 'Benchmark_os_type flag is %s and os_type flag is %s. ' + 'This resulting benchmark type is %s. ' + 'However, benchmark of name %s is not found within the ' + 'list of %s benchmarks. ' + % ( + FLAGS.benchmark_os_type, + FLAGS.os_type, + benchmark_os_type, + benchmark_name, + benchmark_os_type, + ) + ) + + flag_matrix_name = FLAGS.flag_matrix or benchmark_config.get( + 'flag_matrix', None ) - flag_zip_name = ( - FLAGS.flag_zip or benchmark_config.get('flag_zip', None) + flag_zip_name = FLAGS.flag_zip or benchmark_config.get('flag_zip', None) + _AssertFlagMatrixAndZipDefsExist( + benchmark_config, flag_matrix_name, flag_zip_name ) - _AssertFlagMatrixAndZipDefsExist(benchmark_config, - flag_matrix_name, - flag_zip_name) # We need to remove the 'flag_matrix', 'flag_matrix_defs', 'flag_zip', # 'flag_zip_defs', and 'flag_matrix_filters' keys from the config @@ -485,12 +501,13 @@ def GetBenchmarksFromFlags(): benchmark_config.pop('flag_matrix', None) benchmark_config.pop('flag_zip', None) - flag_matrix = benchmark_config.pop( - 'flag_matrix_defs', {}).get(flag_matrix_name, {}) - flag_matrix_filter = benchmark_config.pop( - 'flag_matrix_filters', {}).get(flag_matrix_name, {}) - flag_zip = benchmark_config.pop( - 'flag_zip_defs', {}).get(flag_zip_name, {}) + flag_matrix = benchmark_config.pop('flag_matrix_defs', {}).get( + flag_matrix_name, {} + ) + flag_matrix_filter = benchmark_config.pop('flag_matrix_filters', {}).get( + flag_matrix_name, {} + ) + flag_zip = benchmark_config.pop('flag_zip_defs', {}).get(flag_zip_name, {}) zipped_axes = [] crossed_axes = [] @@ -505,19 +522,22 @@ def GetBenchmarksFromFlags(): config = _GetConfigForAxis(benchmark_config, flag_config) zipped_axes.append((benchmark_module, config)) - crossed_axes.append([benchmark_tuple[1]['flags'] for - benchmark_tuple in zipped_axes]) + crossed_axes.append( + [benchmark_tuple[1]['flags'] for benchmark_tuple in zipped_axes] + ) for flag, values in sorted(six.iteritems(flag_matrix)): crossed_axes.append([{flag: v} for v in values]) for flag_config in itertools.product(*crossed_axes): config = _GetConfigForAxis(benchmark_config, flag_config) - if (flag_matrix_filter and not eval( - flag_matrix_filter, {}, config['flags'])): + if flag_matrix_filter and not eval( + flag_matrix_filter, {}, config['flags'] + ): continue - benchmark_config_list.extend([(benchmark_module, config)] * - FLAGS.num_benchmark_copies) + benchmark_config_list.extend( + [(benchmark_module, config)] * FLAGS.num_benchmark_copies + ) return benchmark_config_list diff --git a/perfkitbenchmarker/benchmark_status.py b/perfkitbenchmarker/benchmark_status.py index b2745df19e..cb2bb20adf 100644 --- a/perfkitbenchmarker/benchmark_status.py +++ b/perfkitbenchmarker/benchmark_status.py @@ -27,6 +27,7 @@ class FailedSubstatus(object): """Failure modes for benchmarks.""" + # Failure due to insufficient quota, user preventable QUOTA = 'QUOTA_EXCEEDED' @@ -71,7 +72,11 @@ class FailedSubstatus(object): # logic for freeze/restore is already retried in the BaseResource # Create()/Delete(). RETRYABLE_SUBSTATUSES = [ - QUOTA, INSUFFICIENT_CAPACITY, KNOWN_INTERMITTENT, INTERRUPTED, UNSUPPORTED + QUOTA, + INSUFFICIENT_CAPACITY, + KNOWN_INTERMITTENT, + INTERRUPTED, + UNSUPPORTED, ] @@ -92,25 +97,35 @@ def _CreateSummaryTable(benchmark_specs): cluster_boot cluster_boot0 SKIPPED -------------------------------------------------------- """ - run_status_tuples = [(spec.name, spec.uid, spec.status, - spec.failed_substatus if spec.failed_substatus else '') - for spec in benchmark_specs] - assert run_status_tuples, ('run_status_tuples must contain at least one ' - 'element.') + run_status_tuples = [ + ( + spec.name, + spec.uid, + spec.status, + spec.failed_substatus if spec.failed_substatus else '', + ) + for spec in benchmark_specs + ] + assert ( + run_status_tuples + ), 'run_status_tuples must contain at least one element.' col_headers = 'Name', 'UID', 'Status', 'Failed Substatus' col_lengths = [] - for col_header, col_entries in zip(col_headers, - list(zip(*run_status_tuples))): + for col_header, col_entries in zip( + col_headers, list(zip(*run_status_tuples)) + ): max_col_content_length = max(len(entry) for entry in col_entries) col_lengths.append(max(len(col_header), max_col_content_length)) line_length = (len(col_headers) - 1) * len(_COL_SEPARATOR) + sum(col_lengths) dash_line = '-' * line_length line_format = _COL_SEPARATOR.join( '{{{0}:<{1}s}}'.format(col_index, col_length) - for col_index, col_length in enumerate(col_lengths)) + for col_index, col_length in enumerate(col_lengths) + ) msg = [dash_line, line_format.format(*col_headers), dash_line] - msg.extend(line_format.format(*row_entries) - for row_entries in run_status_tuples) + msg.extend( + line_format.format(*row_entries) for row_entries in run_status_tuples + ) msg.append(dash_line) return os.linesep.join(msg) @@ -134,16 +149,22 @@ def CreateSummary(benchmark_specs): -------------------------------------------------------- Success rate: 25.00% (1/4) """ - run_status_tuples = [(spec.name, spec.uid, spec.status) - for spec in benchmark_specs] - assert run_status_tuples, ('run_status_tuples must contain at least one ' - 'element.') + run_status_tuples = [ + (spec.name, spec.uid, spec.status) for spec in benchmark_specs + ] + assert ( + run_status_tuples + ), 'run_status_tuples must contain at least one element.' benchmark_count = len(run_status_tuples) - successful_benchmark_count = sum(1 for _, _, status in run_status_tuples - if status == SUCCEEDED) + successful_benchmark_count = sum( + 1 for _, _, status in run_status_tuples if status == SUCCEEDED + ) return os.linesep.join(( 'Benchmark run statuses:', _CreateSummaryTable(benchmark_specs), 'Success rate: {0:.2f}% ({1}/{2})'.format( - 100. * successful_benchmark_count / benchmark_count, - successful_benchmark_count, benchmark_count))) + 100.0 * successful_benchmark_count / benchmark_count, + successful_benchmark_count, + benchmark_count, + ), + )) diff --git a/perfkitbenchmarker/capacity_reservation.py b/perfkitbenchmarker/capacity_reservation.py index dec04b532e..3c68e6ac68 100644 --- a/perfkitbenchmarker/capacity_reservation.py +++ b/perfkitbenchmarker/capacity_reservation.py @@ -19,9 +19,12 @@ FLAGS = flags.FLAGS -flags.DEFINE_bool('use_capacity_reservations', False, - 'Whether to use capacity reservations for virtual ' - 'machines. Only supported on AWS.') +flags.DEFINE_bool( + 'use_capacity_reservations', + False, + 'Whether to use capacity reservations for virtual ' + 'machines. Only supported on AWS.', +) def GetResourceClass(cloud): diff --git a/perfkitbenchmarker/cloud_harmony_util.py b/perfkitbenchmarker/cloud_harmony_util.py index 81fd8b0a05..b3d3fcf40f 100644 --- a/perfkitbenchmarker/cloud_harmony_util.py +++ b/perfkitbenchmarker/cloud_harmony_util.py @@ -15,34 +15,52 @@ FLAGS = flags.FLAGS -flags.DEFINE_boolean('ch_store_results', False, - 'Whether to store cloudharmony benchmark reports. ' - 'Defaults to False, can be turned on for production runs. ' - 'This flag is used to produce benchmark reports.') -STORE = flags.DEFINE_string('ch_results_store', None, - 'Storage to store cloudharmony benchmark reports. ' - 'Used if ch_store_results is set to True.') -BUCKET = flags.DEFINE_string('ch_results_bucket', None, - 'Bucket to store cloudharmony benchmark reports. ' - 'Used if ch_store_results is set to True.') +flags.DEFINE_boolean( + 'ch_store_results', + False, + 'Whether to store cloudharmony benchmark reports. ' + 'Defaults to False, can be turned on for production runs. ' + 'This flag is used to produce benchmark reports.', +) +STORE = flags.DEFINE_string( + 'ch_results_store', + None, + 'Storage to store cloudharmony benchmark reports. ' + 'Used if ch_store_results is set to True.', +) +BUCKET = flags.DEFINE_string( + 'ch_results_bucket', + None, + 'Bucket to store cloudharmony benchmark reports. ' + 'Used if ch_store_results is set to True.', +) KEY = flags.DEFINE_string( - 'ch_results_key', None, + 'ch_results_key', + None, 'Access key to store cloudharmony benchmark reports. ' - 'Used in conjunction with ch_results_bucket') + 'Used in conjunction with ch_results_bucket', +) SECRET = flags.DEFINE_string( - 'ch_results_secret', None, + 'ch_results_secret', + None, 'Access secret to store cloudharmony benchmark reports. ' - 'Used in conjunction with ch_results_bucket') + 'Used in conjunction with ch_results_bucket', +) ITERATIONS = flags.DEFINE_integer( - 'ch_iterations', 1, 'The number of times to run the test. Multiple test ' - 'iterations will be grouped and saved in the same results resport.') + 'ch_iterations', + 1, + 'The number of times to run the test. Multiple test ' + 'iterations will be grouped and saved in the same results resport.', +) def GetSaveCommand() -> str: """Returns the cloudharmony command to save benchmark reports.""" - return (f' --db_and_csv --store {STORE.value} --store_key {KEY.value} ' - f' --store_secret {SECRET.value} --store_container {BUCKET.value} ') + return ( + f' --db_and_csv --store {STORE.value} --store_key {KEY.value} ' + f' --store_secret {SECRET.value} --store_container {BUCKET.value} ' + ) def GetRegionFromZone(zone: str) -> str: @@ -53,8 +71,9 @@ def GetRegionFromZone(zone: str) -> str: return zone -def ParseCsvResultsIntoMetadata(vm: virtual_machine.BaseVirtualMachine, - path: str) -> List[Dict[str, Any]]: +def ParseCsvResultsIntoMetadata( + vm: virtual_machine.BaseVirtualMachine, path: str +) -> List[Dict[str, Any]]: """Loads the CSV created by cloud harmony at path in the VM into metadata. The CSV located by path inside of virtual machine VM will be loaded. For each @@ -64,6 +83,7 @@ def ParseCsvResultsIntoMetadata(vm: virtual_machine.BaseVirtualMachine, Args: vm: the Virtual Machine that has run a cloud harmony benchmark path: The path inside of VM which has the CSV file which should be loaded + Returns: A list of metadata outputs that should be appended to the samples that are produced by a cloud harmony benchmark. @@ -73,8 +93,9 @@ def ParseCsvResultsIntoMetadata(vm: virtual_machine.BaseVirtualMachine, return ParseCsvResultsFromString(csv_string) -def ParseCsvResultsFromString(csv_string: str, - prefix: str = '') -> List[Dict[str, Any]]: +def ParseCsvResultsFromString( + csv_string: str, prefix: str = '' +) -> List[Dict[str, Any]]: """Loads the CSV created by cloud harmony in csv_string. The CSV will be loaded into a pandas data frame. @@ -85,8 +106,9 @@ def ParseCsvResultsFromString(csv_string: str, Args: csv_string: a string of the CSV which was produced by cloud_harmony prefix: a string prefix to attach to the metadata. Defaults to empty - string. It can be set to a unique string if cloudharmony data is - attached to every sample instead of being its own sample. + string. It can be set to a unique string if cloudharmony data is attached + to every sample instead of being its own sample. + Returns: A list of metadata dictionaries, where each dict represents one row of results (an iteration) in the csv string. @@ -118,7 +140,8 @@ def GetCommonMetadata(custom_metadata: Optional[Dict[str, Any]] = None) -> str: Args: custom_metadata: a dictionary of metadata key value pairs that should - override any flag chosen in the function, or should also be included. + override any flag chosen in the function, or should also be included. + Returns: A string of metadata that should be appended to the cloudharmony benchmark run. @@ -145,7 +168,8 @@ def GetCommonMetadata(custom_metadata: Optional[Dict[str, Any]] = None) -> str: def GetMetadataSamples( - cloud_harmony_metadata: List[Dict[Any, Any]]) -> List[sample.Sample]: + cloud_harmony_metadata: List[Dict[Any, Any]] +) -> List[sample.Sample]: """Returns the cloudharmony metadata as a list of samples. This function is commonly used across all cloudharmony benchmarks. @@ -156,7 +180,6 @@ def GetMetadataSamples( Returns: A list of sample.Sample objects of cloudharmony metadata, where one sample represents one row of csv results (one row = one test iteration). - """ samples = [] for result in cloud_harmony_metadata: diff --git a/perfkitbenchmarker/cloud_tpu.py b/perfkitbenchmarker/cloud_tpu.py index 2ab85f55b8..f2dda1552b 100644 --- a/perfkitbenchmarker/cloud_tpu.py +++ b/perfkitbenchmarker/cloud_tpu.py @@ -19,31 +19,41 @@ from perfkitbenchmarker import resource -flags.DEFINE_string('tpu_cidr_range', None, """CIDR Range for the TPU. The IP +flags.DEFINE_string( + 'tpu_cidr_range', + None, + """CIDR Range for the TPU. The IP range that the TPU will select an IP address from. Must be in CIDR notation and a /29 range, for example 192.168.0.0/29. Errors will occur if the CIDR range has already been used for a currently existing TPU, the CIDR range conflicts with any networks in the user's provided network, or the provided network is peered with another - network that is using that CIDR range.""") -flags.DEFINE_string('tpu_accelerator_type', 'tpu-v2', - 'TPU accelerator type for the TPU.') -flags.DEFINE_string('tpu_description', None, - 'Specifies a text description of the TPU.') -flags.DEFINE_string('tpu_network', None, - 'Specifies the network that this TPU will be a part of.') -flags.DEFINE_string('tpu_tf_version', None, - 'TensorFlow version for the TPU.') -flags.DEFINE_string('tpu_zone', None, - 'The zone of the tpu to create. Zone in which TPU lives.') -flags.DEFINE_string('tpu_name', None, - 'The name of the TPU to create.') -flags.DEFINE_boolean('tpu_preemptible', False, - 'Use preemptible TPU or not.') -flags.DEFINE_integer('tpu_cores_per_donut', 8, - 'The number of cores per TPU donut. This is 8 because each' - ' TPU has 4 chips each with 2 cores.') + network that is using that CIDR range.""", +) +flags.DEFINE_string( + 'tpu_accelerator_type', 'tpu-v2', 'TPU accelerator type for the TPU.' +) +flags.DEFINE_string( + 'tpu_description', None, 'Specifies a text description of the TPU.' +) +flags.DEFINE_string( + 'tpu_network', + None, + 'Specifies the network that this TPU will be a part of.', +) +flags.DEFINE_string('tpu_tf_version', None, 'TensorFlow version for the TPU.') +flags.DEFINE_string( + 'tpu_zone', None, 'The zone of the tpu to create. Zone in which TPU lives.' +) +flags.DEFINE_string('tpu_name', None, 'The name of the TPU to create.') +flags.DEFINE_boolean('tpu_preemptible', False, 'Use preemptible TPU or not.') +flags.DEFINE_integer( + 'tpu_cores_per_donut', + 8, + 'The number of cores per TPU donut. This is 8 because each' + ' TPU has 4 chips each with 2 cores.', +) FLAGS = flags.FLAGS @@ -119,6 +129,6 @@ def GetResourceMetadata(self): 'tf_version': self.spec.tpu_tf_version, 'zone': self.spec.tpu_zone, 'name': self.spec.tpu_name, - 'preemptible': self.spec.tpu_preemptible + 'preemptible': self.spec.tpu_preemptible, } return metadata diff --git a/perfkitbenchmarker/container_service.py b/perfkitbenchmarker/container_service.py index fa3397cc0f..0bd914d4df 100644 --- a/perfkitbenchmarker/container_service.py +++ b/perfkitbenchmarker/container_service.py @@ -1085,6 +1085,7 @@ def GetEvents(self): @dataclasses.dataclass class KubernetesEventResource: """Holder for Kubernetes event involved objects.""" + kind: str name: str @@ -1097,6 +1098,7 @@ def FromDict(cls, yaml_data: dict[str, Any]) -> 'KubernetesEventResource': @dataclasses.dataclass class KubernetesEvent: """Holder for Kubernetes event data.""" + resource: KubernetesEventResource message: str # Reason is actually more of a machine readable message. diff --git a/perfkitbenchmarker/context.py b/perfkitbenchmarker/context.py index c9dcfb8d24..b5d2f94b57 100644 --- a/perfkitbenchmarker/context.py +++ b/perfkitbenchmarker/context.py @@ -18,6 +18,7 @@ class _ThreadData(threading.local): + def __init__(self): self.benchmark_spec = None diff --git a/perfkitbenchmarker/custom_virtual_machine_spec.py b/perfkitbenchmarker/custom_virtual_machine_spec.py index c278f7694c..b8fca86dd7 100644 --- a/perfkitbenchmarker/custom_virtual_machine_spec.py +++ b/perfkitbenchmarker/custom_virtual_machine_spec.py @@ -11,8 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Classes relating to decoding a custom machine type. -""" +"""Classes relating to decoding a custom machine type.""" import re @@ -39,9 +38,9 @@ def Decode(self, value, component_full_name, flag_values): Args: value: The value specified in the config. component_full_name: string. Fully qualified name of the configurable - component containing the config option. + component containing the config option. flag_values: flags.FlagValues. Runtime flag values to be propagated to - BaseSpec constructors. + BaseSpec constructors. Returns: int. Memory size in MiB. @@ -49,21 +48,27 @@ def Decode(self, value, component_full_name, flag_values): Raises: errors.Config.InvalidValue upon invalid input value. """ - string = super(MemoryDecoder, self).Decode(value, component_full_name, - flag_values) + string = super(MemoryDecoder, self).Decode( + value, component_full_name, flag_values + ) match = self._CONFIG_MEMORY_PATTERN.match(string) if not match: raise errors.Config.InvalidValue( 'Invalid {0} value: "{1}". Examples of valid values: "1280MiB", ' - '"7.5GiB".'.format(self._GetOptionFullName(component_full_name), - string)) + '"7.5GiB".'.format( + self._GetOptionFullName(component_full_name), string + ) + ) try: memory_value = float(match.group(1)) except ValueError: raise errors.Config.InvalidValue( 'Invalid {0} value: "{1}". "{2}" is not a valid float.'.format( - self._GetOptionFullName(component_full_name), string, - match.group(1))) + self._GetOptionFullName(component_full_name), + string, + match.group(1), + ) + ) memory_units = match.group(2) if memory_units == 'GiB': memory_value *= 1024 @@ -71,8 +76,10 @@ def Decode(self, value, component_full_name, flag_values): if memory_value != memory_mib_int: raise errors.Config.InvalidValue( 'Invalid {0} value: "{1}". The specified size must be an integer ' - 'number of MiB.'.format(self._GetOptionFullName(component_full_name), - string)) + 'number of MiB.'.format( + self._GetOptionFullName(component_full_name), string + ) + ) return memory_mib_int @@ -82,7 +89,7 @@ class CustomMachineTypeSpec(spec.BaseSpec): Attributes: cpus: int. Number of vCPUs. memory: string. Representation of the size of memory, expressed in MiB or - GiB. Must be an integer number of MiB (e.g. "1280MiB", "7.5GiB"). + GiB. Must be an integer number of MiB (e.g. "1280MiB", "7.5GiB"). """ @classmethod @@ -95,8 +102,10 @@ def _GetOptionDecoderConstructions(cls): arguments to construct in order to decode the named option. """ result = super(CustomMachineTypeSpec, cls)._GetOptionDecoderConstructions() - result.update({'cpus': (option_decoders.IntDecoder, {'min': 1}), - 'memory': (MemoryDecoder, {})}) + result.update({ + 'cpus': (option_decoders.IntDecoder, {'min': 1}), + 'memory': (MemoryDecoder, {}), + }) return result @@ -104,19 +113,20 @@ class MachineTypeDecoder(option_decoders.TypeVerifier): """Decodes the machine_type option of a VM config.""" def __init__(self, **kwargs): - super(MachineTypeDecoder, self).__init__((six.string_types + (dict,)), - **kwargs) + super(MachineTypeDecoder, self).__init__( + (six.string_types + (dict,)), **kwargs + ) def Decode(self, value, component_full_name, flag_values): """Decodes the machine_type option of a VM config. Args: - value: Either a string name of a machine type or a dict containing - 'cpu' and 'memory' keys describing a custom VM. + value: Either a string name of a machine type or a dict containing 'cpu' + and 'memory' keys describing a custom VM. component_full_name: string. Fully qualified name of the configurable - component containing the config option. + component containing the config option. flag_values: flags.FlagValues. Runtime flag values to be propagated to - BaseSpec constructors. + BaseSpec constructors. Returns: If value is a string, returns it unmodified. Otherwise, returns the @@ -125,31 +135,36 @@ def Decode(self, value, component_full_name, flag_values): Raises: errors.Config.InvalidValue upon invalid input value. """ - super(MachineTypeDecoder, self).Decode(value, component_full_name, - flag_values) + super(MachineTypeDecoder, self).Decode( + value, component_full_name, flag_values + ) if isinstance(value, six.string_types): return value - return CustomMachineTypeSpec(self._GetOptionFullName(component_full_name), - flag_values=flag_values, **value) + return CustomMachineTypeSpec( + self._GetOptionFullName(component_full_name), + flag_values=flag_values, + **value + ) class AzureMachineTypeDecoder(option_decoders.TypeVerifier): """Decodes the machine_type option of a VM config.""" def __init__(self, **kwargs): - super(AzureMachineTypeDecoder, self).__init__(six.string_types + (dict,), - **kwargs) + super(AzureMachineTypeDecoder, self).__init__( + six.string_types + (dict,), **kwargs + ) def Decode(self, value, component_full_name, flag_values): """Decodes the machine_type option of a VM config. Args: value: Either a string name of a machine type or a dict containing - 'compute_units' and 'tier' keys describing a machine type. + 'compute_units' and 'tier' keys describing a machine type. component_full_name: string. Fully qualified name of the configurable - component containing the config option. + component containing the config option. flag_values: flags.FlagValues. Runtime flag values to be propagated to - BaseSpec constructors. + BaseSpec constructors. Returns: If value is a string, returns it unmodified. Otherwise, returns the @@ -158,13 +173,16 @@ def Decode(self, value, component_full_name, flag_values): Raises: errors.Config.InvalidValue upon invalid input value. """ - super(AzureMachineTypeDecoder, self).Decode(value, component_full_name, - flag_values) + super(AzureMachineTypeDecoder, self).Decode( + value, component_full_name, flag_values + ) if isinstance(value, six.string_types): return value return AzurePerformanceTierDecoder( self._GetOptionFullName(component_full_name), - flag_values=flag_values, **value) + flag_values=flag_values, + **value + ) class AzurePerformanceTierDecoder(spec.BaseSpec): @@ -185,12 +203,17 @@ def _GetOptionDecoderConstructions(cls): arguments to construct in order to decode the named option. """ result = super( - AzurePerformanceTierDecoder, cls)._GetOptionDecoderConstructions() + AzurePerformanceTierDecoder, cls + )._GetOptionDecoderConstructions() # https://docs.microsoft.com/en-us/azure/virtual-machines/windows/acu # https://docs.microsoft.com/en-us/azure/sql-database/sql-database-service-tiers - result.update({'compute_units': (option_decoders.IntDecoder, {'min': 50}), - 'tier': (option_decoders.EnumDecoder, { - 'valid_values': azure_flags.VALID_TIERS})}) + result.update({ + 'compute_units': (option_decoders.IntDecoder, {'min': 50}), + 'tier': ( + option_decoders.EnumDecoder, + {'valid_values': azure_flags.VALID_TIERS}, + ), + }) return result @classmethod @@ -200,10 +223,10 @@ def _ApplyFlags(cls, config_values, flag_values): Can be overridden by derived classes to add support for specific flags. Args: - config_values: dict mapping config option names to provided values. - May be modified by this function. + config_values: dict mapping config option names to provided values. May be + modified by this function. flag_values: flags.FlagValues. Runtime flags that may override the - provided config values. + provided config values. """ if flag_values['azure_tier'].present: config_values['tier'] = flag_values.azure_tier diff --git a/perfkitbenchmarker/data/__init__.py b/perfkitbenchmarker/data/__init__.py index d4f124bfaf..77a7e16fe1 100644 --- a/perfkitbenchmarker/data/__init__.py +++ b/perfkitbenchmarker/data/__init__.py @@ -34,16 +34,20 @@ FLAGS = flags.FLAGS -flags.DEFINE_multi_string('data_search_paths', ['.'], - 'Additional paths to search for data files. ' - 'These paths will be searched prior to using files ' - 'bundled with PerfKitBenchmarker.') +flags.DEFINE_multi_string( + 'data_search_paths', + ['.'], + 'Additional paths to search for data files. ' + 'These paths will be searched prior to using files ' + 'bundled with PerfKitBenchmarker.', +) _RESOURCES = 'resources' class ResourceNotFound(ValueError): """Error raised when a resource could not be found on the search path.""" + pass @@ -142,23 +146,28 @@ def ResourcePath(self, name): if not os.path.isdir(dir_path): raise with open(path, 'wb') as extracted_file: - shutil.copyfileobj(pkg_resources.resource_stream(self.package, name), - extracted_file) + shutil.copyfileobj( + pkg_resources.resource_stream(self.package, name), extracted_file + ) return path DATA_PACKAGE_NAME = 'perfkitbenchmarker.data' YCSB_WORKLOAD_DIR_NAME = os.path.join( - os.path.dirname(perfkitbenchmarker.__file__), 'data/ycsb') + os.path.dirname(perfkitbenchmarker.__file__), 'data/ycsb' +) EDW_SCRIPT_DIR_NAME = os.path.join( - os.path.dirname(perfkitbenchmarker.__file__), 'data/edw') + os.path.dirname(perfkitbenchmarker.__file__), 'data/edw' +) SCRIPT_PACKAGE_NAME = 'perfkitbenchmarker.scripts' CONFIG_PACKAGE_NAME = 'perfkitbenchmarker.configs' -DEFAULT_RESOURCE_LOADERS = [PackageResourceLoader(DATA_PACKAGE_NAME), - FileResourceLoader(YCSB_WORKLOAD_DIR_NAME), - FileResourceLoader(EDW_SCRIPT_DIR_NAME), - PackageResourceLoader(SCRIPT_PACKAGE_NAME), - PackageResourceLoader(CONFIG_PACKAGE_NAME)] +DEFAULT_RESOURCE_LOADERS = [ + PackageResourceLoader(DATA_PACKAGE_NAME), + FileResourceLoader(YCSB_WORKLOAD_DIR_NAME), + FileResourceLoader(EDW_SCRIPT_DIR_NAME), + PackageResourceLoader(SCRIPT_PACKAGE_NAME), + PackageResourceLoader(CONFIG_PACKAGE_NAME), +] def _GetResourceLoaders(): @@ -198,6 +207,7 @@ def ResourcePath(resource_name, search_user_paths=True): resource_name: string. Name of a resource. search_user_paths: boolean. Whether paths from "--data_search_paths" should be searched before the default paths. + Returns: A path to the resource on the filesystem. Raises: @@ -211,8 +221,7 @@ def ResourcePath(resource_name, search_user_paths=True): if loader.ResourceExists(resource_name): return loader.ResourcePath(resource_name) - raise ResourceNotFound( - '{0} (Searched: {1})'.format(resource_name, loaders)) + raise ResourceNotFound('{0} (Searched: {1})'.format(resource_name, loaders)) def ResourceExists(resource_name, search_user_paths=True): @@ -228,6 +237,7 @@ def ResourceExists(resource_name, search_user_paths=True): resource_name: string. Name of a resource. search_user_paths: boolean. Whether paths from "--data_search_paths" should be searched before the default paths. + Returns: Whether the resource exists. """ diff --git a/perfkitbenchmarker/data/edw/athena/provider_specific_script_driver.py b/perfkitbenchmarker/data/edw/athena/provider_specific_script_driver.py index 7b32e979b6..fdf2008503 100644 --- a/perfkitbenchmarker/data/edw/athena/provider_specific_script_driver.py +++ b/perfkitbenchmarker/data/edw/athena/provider_specific_script_driver.py @@ -9,12 +9,15 @@ from absl import flags -flags.DEFINE_string('database', None, - 'The database within which the command or query executes.') +flags.DEFINE_string( + 'database', None, 'The database within which the command or query executes.' +) flags.DEFINE_string('query_timeout', '600', 'Query timeout in seconds.') flags.DEFINE_string( - 'athena_query_output_bucket', None, - 'Specifies where to save the results of the query execution.') + 'athena_query_output_bucket', + None, + 'Specifies where to save the results of the query execution.', +) flags.DEFINE_string('athena_region', 'us-east-1', 'Region to use Athena in.') flags.mark_flags_as_required(['database', 'athena_query_output_bucket']) @@ -35,7 +38,13 @@ def generate_provider_specific_cmd_list(script, driver, output, error): """ del error cmd_list = [ - driver, script, FLAGS.athena_region, FLAGS.database, FLAGS.query_timeout, - FLAGS.athena_query_output_bucket, output, 'athena.err' + driver, + script, + FLAGS.athena_region, + FLAGS.database, + FLAGS.query_timeout, + FLAGS.athena_query_output_bucket, + output, + 'athena.err', ] return cmd_list diff --git a/perfkitbenchmarker/data/edw/azuresqldatawarehouse/provider_specific_script_driver.py b/perfkitbenchmarker/data/edw/azuresqldatawarehouse/provider_specific_script_driver.py index 5a92840f1d..5e50b770c8 100644 --- a/perfkitbenchmarker/data/edw/azuresqldatawarehouse/provider_specific_script_driver.py +++ b/perfkitbenchmarker/data/edw/azuresqldatawarehouse/provider_specific_script_driver.py @@ -47,6 +47,13 @@ def generate_provider_specific_cmd_list(script, driver, output, error): Command list to execute the supplied script. """ return [ - driver, FLAGS.server, FLAGS.database, FLAGS.user, FLAGS.password, - FLAGS.query_timeout, script, output, error + driver, + FLAGS.server, + FLAGS.database, + FLAGS.user, + FLAGS.password, + FLAGS.query_timeout, + script, + output, + error, ] diff --git a/perfkitbenchmarker/data/edw/bigquery/provider_specific_script_driver.py b/perfkitbenchmarker/data/edw/bigquery/provider_specific_script_driver.py index 01a02c588d..216bf10b08 100644 --- a/perfkitbenchmarker/data/edw/bigquery/provider_specific_script_driver.py +++ b/perfkitbenchmarker/data/edw/bigquery/provider_specific_script_driver.py @@ -9,10 +9,14 @@ from absl import flags -flags.DEFINE_string('bq_project_id', None, 'Project Id which contains the query' - ' dataset and table.') -flags.DEFINE_string('bq_dataset_id', None, 'Dataset Id which contains the query' - ' table.') +flags.DEFINE_string( + 'bq_project_id', + None, + 'Project Id which contains the query dataset and table.', +) +flags.DEFINE_string( + 'bq_dataset_id', None, 'Dataset Id which contains the query table.' +) flags.mark_flags_as_required(['bq_project_id', 'bq_dataset_id']) FLAGS = flags.FLAGS @@ -30,6 +34,12 @@ def generate_provider_specific_cmd_list(script, driver, output, error): Returns: Command list to execute the supplied script. """ - cmd_list = [driver, FLAGS.bq_project_id, FLAGS.bq_dataset_id, - script, output, error] + cmd_list = [ + driver, + FLAGS.bq_project_id, + FLAGS.bq_dataset_id, + script, + output, + error, + ] return cmd_list diff --git a/perfkitbenchmarker/data/edw/multistream_multiprofile_driver.py b/perfkitbenchmarker/data/edw/multistream_multiprofile_driver.py index 9f46a3f33a..eb775ad2fd 100644 --- a/perfkitbenchmarker/data/edw/multistream_multiprofile_driver.py +++ b/perfkitbenchmarker/data/edw/multistream_multiprofile_driver.py @@ -4,7 +4,6 @@ profile_details module. """ - __author__ = 'p3rf@google.com' import json @@ -17,9 +16,13 @@ import unistream_profile_driver -flags.DEFINE_list('profile_list', None, 'List of profiles. Each will be run on ' - 'its own process to simulate ' - 'concurrency.') +flags.DEFINE_list( + 'profile_list', + None, + 'List of profiles. Each will be run on ' + 'its own process to simulate ' + 'concurrency.', +) flags.mark_flags_as_required(['profile_list']) @@ -55,8 +58,13 @@ def manage_streams(): start_time = time.time() for profile in FLAGS.profile_list: - profile_handling_process = Process(target=process_profile, - args=(profile, profile_performance,)) + profile_handling_process = Process( + target=process_profile, + args=( + profile, + profile_performance, + ), + ) profile_handling_process.start() profile_handling_process_list.append(profile_handling_process) diff --git a/perfkitbenchmarker/data/edw/redshift/provider_specific_script_driver.py b/perfkitbenchmarker/data/edw/redshift/provider_specific_script_driver.py index fbc056daaf..7a59188b6f 100644 --- a/perfkitbenchmarker/data/edw/redshift/provider_specific_script_driver.py +++ b/perfkitbenchmarker/data/edw/redshift/provider_specific_script_driver.py @@ -31,5 +31,13 @@ def generate_provider_specific_cmd_list(script, driver, output, error): Returns: Command list to execute the supplied script. """ - return [driver, FLAGS.host, FLAGS.database, FLAGS.user, FLAGS.password, - script, output, error] + return [ + driver, + FLAGS.host, + FLAGS.database, + FLAGS.user, + FLAGS.password, + script, + output, + error, + ] diff --git a/perfkitbenchmarker/data/edw/script_driver.py b/perfkitbenchmarker/data/edw/script_driver.py index 157bcb6666..49f8fc76e8 100644 --- a/perfkitbenchmarker/data/edw/script_driver.py +++ b/perfkitbenchmarker/data/edw/script_driver.py @@ -4,7 +4,6 @@ time taken to execute the script in seconds or -1 if the script fails. """ - import json import logging from subprocess import call @@ -17,11 +16,14 @@ __author__ = 'p3rf@google.com' flags.DEFINE_string('script', None, 'SQL script which contains the query.') -flags.DEFINE_string('logfile_suffix', 'log', 'Suffix to use for the output and ' - 'error files.') -flags.DEFINE_multi_string('failing_scripts', [], - 'List of failing scripts whose execution should be ' - 'skipped.') +flags.DEFINE_string( + 'logfile_suffix', 'log', 'Suffix to use for the output and error files.' +) +flags.DEFINE_multi_string( + 'failing_scripts', + [], + 'List of failing scripts whose execution should be skipped.', +) FLAGS = flags.FLAGS @@ -56,11 +58,13 @@ def execute_script(script, logfile_suffix): if script not in FLAGS.failing_scripts: output, error = default_logfile_names(script, logfile_suffix) cmd = provider_specific_script_driver.generate_provider_specific_cmd_list( - script, DRIVER_NAME, output, error) + script, DRIVER_NAME, output, error + ) start_time = time.time() response_status = call(cmd) - execution_time = -1 if (response_status != 0) else round((time.time() - - start_time), 2) + execution_time = ( + -1 if (response_status != 0) else round((time.time() - start_time), 2) + ) try: with open(API_LOG_FILE) as fp: line = fp.readline() @@ -80,8 +84,10 @@ def execute_script(script, logfile_suffix): except IOError: pass - script_execution_details = {'execution_time': execution_time, - 'job_id': job_id} + script_execution_details = { + 'execution_time': execution_time, + 'job_id': job_id, + } try: with open(ATHENA_ERROR_FILE) as fp: diff --git a/perfkitbenchmarker/data/edw/snowflake_aws/provider_specific_script_driver.py b/perfkitbenchmarker/data/edw/snowflake_aws/provider_specific_script_driver.py index b161eda241..69d1efe900 100644 --- a/perfkitbenchmarker/data/edw/snowflake_aws/provider_specific_script_driver.py +++ b/perfkitbenchmarker/data/edw/snowflake_aws/provider_specific_script_driver.py @@ -10,8 +10,11 @@ from absl import flags flags.DEFINE_string( - 'connection', None, - 'Named Snowflake connection to use. See https://docs.snowflake.net/manuals/user-guide/snowsql-start.html#using-named-connections for more details.' + 'connection', + None, + 'Named Snowflake connection to use. See' + ' https://docs.snowflake.net/manuals/user-guide/snowsql-start.html#using-named-connections' + ' for more details.', ) FLAGS = flags.FLAGS diff --git a/perfkitbenchmarker/data/edw/spectrum/provider_specific_script_driver.py b/perfkitbenchmarker/data/edw/spectrum/provider_specific_script_driver.py index 5fecff0cb6..279c9efb87 100644 --- a/perfkitbenchmarker/data/edw/spectrum/provider_specific_script_driver.py +++ b/perfkitbenchmarker/data/edw/spectrum/provider_specific_script_driver.py @@ -46,5 +46,13 @@ def generate_provider_specific_cmd_list(script, driver, output, error): Returns: Command list to execute the supplied script. """ - return [driver, FLAGS.host, FLAGS.database, FLAGS.user, FLAGS.password, - script, output, error] + return [ + driver, + FLAGS.host, + FLAGS.database, + FLAGS.user, + FLAGS.password, + script, + output, + error, + ] diff --git a/perfkitbenchmarker/data/edw/tpc_profile_details.py b/perfkitbenchmarker/data/edw/tpc_profile_details.py index 73ea89e233..4275ce82d8 100644 --- a/perfkitbenchmarker/data/edw/tpc_profile_details.py +++ b/perfkitbenchmarker/data/edw/tpc_profile_details.py @@ -4,287 +4,3111 @@ http://www.tpc.org/tpc_documents_current_versions/current_specifications.asp The order is the same for all scale factors. """ + profile_dictionary = { - 'tpc_ds_power': [96, 7, 75, 44, 39, 80, 32, 19, 25, 78, 86, 1, 91, 21, 43, - 27, 94, 45, 58, 64, 36, 33, 46, 62, 16, 10, 63, 69, 60, 59, - 37, 98, 85, 70, 67, 28, 81, 97, 66, 90, 17, 47, 95, 92, 3, - 51, 35, 49, 9, 31, 11, 93, 29, 38, 22, 89, 15, 6, 52, 50, - 42, 41, 8, 12, 20, 88, 82, 23, 14, 57, 65, 71, 34, 48, 30, - 74, 87, 77, 73, 84, 54, 55, 56, 2, 26, 40, 72, 53, 79, 18, - 13, 24, 4, 99, 68, 83, 61, 5, 76], - 'tpc_ds_throughput_1': [83, 32, 30, 92, 66, 84, 98, 58, 16, 77, 40, 96, 13, - 36, 95, 63, 99, 3, 6, 12, 28, 85, 51, 41, 27, 78, 8, - 14, 50, 52, 81, 5, 26, 57, 82, 69, 54, 61, 88, 18, - 94, 35, 68, 24, 75, 11, 67, 9, 25, 37, 86, 4, 60, - 97, - 33, 79, 43, 80, 93, 31, 47, 17, 19, 1, 64, 53, 55, - 46, 21, 15, 20, 65, 70, 49, 59, 48, 72, 87, 34, 2, - 38, 22, 89, 7, 10, 90, 71, 29, 73, 45, 91, 62, 44, - 76, 23, 56, 42, 39, 74], - 'tpc_ds_throughput_2': [56, 98, 59, 24, 88, 2, 5, 6, 27, 87, 90, 83, 91, 28, - 68, 8, 76, 75, 80, 1, 69, 26, 11, 17, 63, 77, 19, - 21, - 31, 93, 54, 39, 10, 15, 55, 14, 38, 42, 53, 45, 99, - 67, 23, 62, 30, 86, 82, 25, 16, 81, 40, 44, 50, 61, - 85, 73, 95, 84, 4, 37, 35, 94, 58, 96, 12, 29, 22, - 51, 36, 43, 64, 20, 57, 9, 52, 49, 71, 72, 70, 7, - 97, - 33, 79, 32, 78, 18, 65, 60, 34, 3, 13, 41, 92, 74, - 46, 89, 47, 66, 48], - 'tpc_ds_throughput_3': [89, 5, 52, 62, 53, 7, 39, 80, 63, 72, 18, 56, 13, - 69, - 23, 19, 74, 30, 84, 96, 14, 10, 86, 94, 8, 87, 58, - 36, 37, 4, 38, 66, 78, 43, 22, 21, 97, 47, 29, 3, - 76, - 82, 46, 41, 59, 40, 55, 16, 27, 54, 90, 92, 31, 42, - 26, 34, 68, 2, 44, 81, 67, 99, 6, 83, 1, 60, 33, - 11, 28, 95, 12, 64, 15, 25, 93, 9, 65, 71, 57, 32, - 61, 85, 73, 98, 77, 45, 20, 50, 70, 75, 91, 17, 24, - 48, 51, 79, 35, 88, 49], - 'tpc_ds_throughput_4': [79, 39, 93, 41, 29, 32, 66, 84, 8, 71, 45, 89, 91, - 14, 46, 58, 48, 59, 2, 83, 21, 78, 40, 99, 19, 72, - 6, - 28, 81, 44, 97, 88, 77, 95, 33, 36, 61, 35, 60, 75, - 74, 55, 51, 17, 52, 90, 22, 27, 63, 38, 18, 24, 37, - 47, 10, 70, 23, 7, 92, 54, 82, 76, 80, 56, 96, 50, - 85, 86, 69, 68, 1, 12, 43, 16, 4, 25, 20, 65, 15, - 98, - 42, 26, 34, 5, 87, 3, 64, 31, 57, 30, 13, 94, 62, - 49, 11, 73, 67, 53, 9], - 'tpc_ds_throughput_5': [73, 66, 4, 17, 60, 98, 88, 2, 19, 65, 3, 79, 13, 21, - 51, 6, 49, 52, 7, 56, 36, 77, 90, 76, 58, 71, 80, - 69, - 54, 92, 61, 53, 87, 68, 85, 28, 42, 67, 50, 30, 48, - 22, 11, 94, 93, 18, 33, 63, 8, 97, 45, 62, 81, 35, - 78, 57, 46, 32, 24, 38, 55, 74, 84, 89, 83, 31, 26, - 40, 14, 23, 96, 1, 95, 27, 44, 16, 64, 20, 43, 5, - 47, - 10, 70, 39, 72, 75, 12, 37, 15, 59, 91, 99, 41, 9, - 86, 34, 82, 29, 25], - 'tpc_ds_throughput_6': [34, 88, 44, 94, 50, 5, 53, 7, 58, 20, 75, 73, 91, - 36, - 11, 80, 9, 93, 32, 89, 28, 87, 18, 74, 6, 65, 84, - 14, 38, 24, 42, 29, 72, 23, 26, 69, 47, 82, 31, 59, - 49, 33, 86, 99, 4, 45, 85, 8, 19, 61, 3, 41, 54, 67, - 77, 15, 51, 98, 62, 97, 22, 48, 2, 79, 56, 37, 10, - 90, 21, 46, 83, 96, 68, 63, 92, 27, 12, 64, 95, 39, - 35, 78, 57, 66, 71, 30, 1, 81, 43, 52, 13, 76, 17, - 25, 40, 70, 55, 60, 16], - 'tpc_ds_throughput_7': [70, 53, 92, 99, 31, 39, 29, 32, 6, 64, 30, 34, 13, - 28, 86, 84, 25, 4, 98, 79, 69, 72, 45, 48, 80, 20, - 2, - 21, 97, 62, 47, 60, 71, 46, 10, 14, 35, 55, 37, 52, - 9, 85, 40, 76, 44, 3, 26, 19, 58, 42, 75, 17, 38, - 82, 87, 43, 11, 5, 41, 61, 33, 49, 7, 73, 89, 81, - 78, - 18, 36, 51, 56, 83, 23, 8, 24, 63, 1, 12, 68, 66, - 67, 77, 15, 88, 65, 59, 96, 54, 95, 93, 91, 74, 94, - 16, 90, 57, 22, 50, 27], - 'tpc_ds_throughput_8': [57, 29, 24, 76, 37, 66, 60, 98, 80, 12, 59, 70, 91, - 69, 40, 2, 16, 44, 5, 73, 14, 71, 3, 49, 84, 64, 7, - 36, 61, 41, 35, 50, 65, 51, 78, 21, 67, 22, 81, 93, - 25, 26, 90, 74, 92, 75, 10, 58, 6, 47, 30, 94, 97, - 55, 72, 95, 86, 39, 17, 42, 85, 9, 32, 34, 79, 54, - 77, 45, 28, 11, 89, 56, 46, 19, 62, 8, 96, 1, 23, - 88, - 82, 87, 43, 53, 20, 52, 83, 38, 68, 4, 13, 48, 99, - 27, 18, 15, 33, 31, 63], - 'tpc_ds_throughput_9': [15, 60, 62, 74, 81, 88, 50, 5, 84, 1, 52, 57, 13, - 14, - 90, 7, 27, 92, 39, 34, 21, 65, 75, 9, 2, 12, 32, - 28, 42, 17, 67, 31, 20, 11, 77, 36, 82, 33, 54, 4, - 16, 10, 18, 48, 24, 30, 78, 6, 80, 35, 59, 99, 61, - 22, 71, 68, 40, 66, 94, 47, 26, 25, 98, 70, 73, 38, - 87, 3, 69, 86, 79, 89, 51, 58, 41, 19, 83, 96, 46, - 53, 55, 72, 95, 29, 64, 93, 56, 97, 23, 44, 91, 49, - 76, 63, 45, 43, 85, 37, 8], - 'tpc_ds_throughput_10': [43, 50, 41, 48, 54, 53, 31, 39, 2, 96, 93, 15, 91, - 21, 18, 32, 63, 24, 66, 70, 36, 20, 30, 25, 7, 1, - 98, 69, 47, 94, 82, 37, 64, 86, 87, 28, 55, 85, 38, - 44, 27, 78, 45, 49, 62, 59, 77, 80, 84, 67, 52, 76, - 42, 33, 65, 23, 90, 88, 99, 35, 10, 16, 5, 57, 34, - 97, 72, 75, 14, 40, 73, 79, 11, 6, 17, 58, 56, 83, - 51, 29, 22, 71, 68, 60, 12, 4, 89, 61, 46, 92, 13, - 9, 74, 8, 3, 95, 26, 81, 19], - 'tpc_ds_throughput_11': [95, 31, 17, 49, 38, 29, 37, 66, 7, 83, 4, 43, 13, - 36, 45, 98, 8, 62, 88, 57, 28, 64, 59, 16, 32, 96, - 5, 14, 35, 99, 55, 81, 12, 40, 72, 69, 22, 26, 97, - 92, 63, 77, 3, 9, 41, 52, 87, 84, 2, 82, 93, 74, - 47, - 85, 20, 46, 18, 53, 76, 67, 78, 27, 39, 15, 70, - 61, 71, 30, 21, 90, 34, 73, 86, 80, 94, 6, 89, 56, - 11, 60, 33, 65, 23, 50, 1, 44, 79, 42, 51, 24, 91, - 25, 48, 19, 75, 68, 10, 54, 58], - 'tpc_ds_throughput_12': [68, 37, 94, 9, 97, 60, 81, 88, 32, 56, 44, 95, 91, - 28, 3, 5, 19, 41, 53, 15, 69, 12, 52, 27, 98, 83, - 39, 21, 67, 76, 22, 54, 1, 90, 71, 14, 33, 10, 61, - 24, 8, 87, 75, 25, 17, 93, 72, 2, 7, 55, 4, 48, 35, - 26, 64, 51, 45, 29, 74, 82, 77, 63, 66, 43, 57, 42, - 65, 59, 36, 18, 70, 34, 40, 84, 99, 80, 79, 89, 86, - 50, 85, 20, 46, 31, 96, 92, 73, 47, 11, 62, 13, 16, - 49, 58, 30, 23, 78, 38, 6], - 'tpc_ds_throughput_13': [23, 81, 99, 25, 61, 50, 54, 53, 98, 89, 92, 68, 13, - 69, 75, 39, 58, 17, 29, 43, 14, 1, 93, 63, 5, 56, - 66, 36, 82, 74, 33, 38, 96, 18, 65, 21, 85, 78, 42, - 62, 19, 72, 30, 16, 94, 4, 71, 7, 32, 22, 44, 49, - 67, 10, 12, 11, 3, 60, 48, 55, 87, 8, 88, 95, 15, - 47, 20, 52, 28, 45, 57, 70, 90, 2, 76, 84, 73, 79, - 40, 31, 26, 64, 51, 37, 83, 24, 34, 35, 86, 41, 91, - 27, 9, 6, 59, 46, 77, 97, 80], - 'tpc_ds_throughput_14': [46, 54, 76, 16, 42, 31, 38, 29, 5, 79, 24, 23, 91, - 14, 30, 66, 6, 94, 60, 95, 21, 96, 4, 8, 39, 89, - 88, - 28, 55, 48, 85, 97, 83, 45, 20, 36, 26, 77, 47, - 41, 58, 71, 59, 27, 99, 44, 65, 32, 98, 33, 92, 9, - 82, 78, 1, 86, 75, 50, 49, 22, 72, 19, 53, 68, 43, - 35, 64, 93, 69, 3, 15, 57, 18, 7, 74, 2, 34, 73, - 90, - 37, 10, 12, 11, 81, 56, 62, 70, 67, 40, 17, 13, - 63, 25, 80, 52, 51, 87, 61, 84], - 'tpc_ds_throughput_15': [51, 38, 74, 27, 47, 37, 97, 60, 39, 73, 62, 46, 13, - 21, 59, 88, 80, 99, 50, 68, 36, 83, 44, 19, 66, 79, - 53, 69, 22, 49, 26, 61, 56, 3, 64, 28, 10, 87, 35, - 17, 6, 65, 52, 63, 76, 92, 20, 98, 5, 85, 24, 25, - 55, 77, 96, 40, 30, 31, 9, 33, 71, 58, 29, 23, 95, - 67, 12, 4, 14, 75, 43, 15, 45, 32, 48, 7, 70, 34, - 18, 81, 78, 1, 86, 54, 89, 41, 57, 82, 90, 94, 91, - 8, 16, 84, 93, 11, 72, 42, 2], - 'tpc_ds_throughput_16': [11, 97, 48, 63, 35, 81, 61, 50, 66, 34, 41, 51, 91, - 36, 52, 53, 84, 76, 31, 23, 28, 56, 92, 58, 88, 73, - 29, 14, 33, 9, 10, 42, 89, 75, 12, 69, 78, 72, 67, - 94, 80, 20, 93, 8, 74, 24, 64, 5, 39, 26, 62, 16, - 22, 87, 83, 90, 59, 37, 25, 85, 65, 6, 60, 46, 68, - 82, 1, 44, 21, 30, 95, 43, 3, 98, 49, 32, 57, 70, - 45, 54, 77, 96, 40, 38, 79, 17, 15, 55, 18, 99, 13, - 19, 27, 2, 4, 86, 71, 47, 7], - 'tpc_ds_throughput_17': [86, 61, 49, 8, 67, 54, 42, 31, 88, 70, 17, 11, 13, - 28, 93, 29, 2, 74, 37, 46, 69, 89, 24, 6, 53, 34, - 60, 21, 85, 25, 78, 47, 79, 30, 1, 14, 77, 71, 82, - 99, 84, 64, 4, 19, 48, 62, 12, 39, 66, 10, 41, 27, - 33, 72, 56, 18, 52, 81, 16, 26, 20, 80, 50, 51, 23, - 55, 96, 92, 36, 59, 68, 95, 75, 5, 9, 98, 15, 57, - 3, - 38, 87, 83, 90, 97, 73, 94, 43, 22, 45, 76, 91, - 58, 63, 7, 44, 40, 65, 35, 32], - 'tpc_ds_throughput_18': [40, 42, 9, 19, 82, 38, 47, 37, 53, 57, 94, 86, 91, - 69, 4, 60, 7, 48, 81, 51, 14, 79, 62, 80, 29, 70, - 50, 36, 26, 16, 77, 35, 73, 59, 96, 21, 87, 65, 55, - 76, 2, 12, 44, 58, 49, 41, 1, 66, 88, 78, 17, 63, - 85, 71, 89, 45, 93, 54, 27, 10, 64, 84, 31, 11, 46, - 22, 83, 24, 28, 52, 23, 68, 30, 39, 25, 5, 43, 15, - 75, 97, 72, 56, 18, 61, 34, 99, 95, 33, 3, 74, 13, - 6, 8, 32, 92, 90, 20, 67, 98], - 'tpc_ds_throughput_19': [90, 47, 25, 58, 55, 97, 35, 81, 29, 15, 99, 40, 13, - 14, 44, 50, 32, 49, 54, 11, 21, 73, 41, 84, 60, 57, - 31, 28, 10, 27, 87, 67, 34, 52, 83, 36, 72, 20, 22, - 74, 7, 1, 92, 6, 9, 17, 96, 88, 53, 77, 94, 8, 26, - 65, 79, 3, 4, 38, 63, 78, 12, 2, 37, 86, 51, 33, - 56, - 62, 69, 93, 46, 23, 59, 66, 16, 39, 95, 43, 30, - 61, 71, 89, 45, 42, 70, 76, 68, 85, 75, 48, 91, 80, - 19, 98, 24, 18, 64, 82, 5], - 'tpc_ds_throughput_20': [18, 35, 16, 6, 22, 61, 67, 54, 60, 43, 76, 90, 91, - 21, 92, 31, 98, 9, 38, 86, 36, 34, 17, 2, 50, 15, - 37, 69, 78, 63, 72, 82, 70, 93, 56, 28, 71, 64, 33, - 48, 32, 96, 24, 80, 25, 94, 83, 53, 29, 87, 99, 19, - 10, 20, 73, 75, 44, 97, 8, 77, 1, 7, 81, 40, 11, - 85, - 89, 41, 14, 4, 51, 46, 52, 88, 27, 66, 68, 95, 59, - 42, 65, 79, 3, 47, 57, 74, 23, 26, 30, 49, 13, 84, - 58, 5, 62, 45, 12, 55, 39], - 'tpc_h_power': [14, 2, 9, 20, 6, 17, 18, 8, 21, 13, 3, 22, 16, 4, 11, 15, 1, - 10, 19, 5, 7, 12], - 'tpc_h_throughput_1': [21, 3, 18, 5, 11, 7, 6, 20, 17, 12, 16, 15, 13, 10, - 2, - 8, 14, 19, 9, 22, 1, 4], - 'tpc_h_throughput_2': [6, 17, 14, 16, 19, 10, 9, 2, 15, 8, 5, 22, 12, 7, 13, - 18, 1, 4, 20, 3, 11, 21], - 'tpc_h_throughput_3': [8, 5, 4, 6, 17, 7, 1, 18, 22, 14, 9, 10, 15, 11, 20, - 2, 21, 19, 13, 16, 12, 3], - 'tpc_h_throughput_4': [5, 21, 14, 19, 15, 17, 12, 6, 4, 9, 8, 16, 11, 2, 10, - 18, 1, 13, 7, 22, 3, 20], - 'tpc_h_throughput_5': [21, 15, 4, 6, 7, 16, 19, 18, 14, 22, 11, 13, 3, 1, 2, - 5, 8, 20, 12, 17, 10, 9], - 'tpc_h_throughput_6': [10, 3, 15, 13, 6, 8, 9, 7, 4, 11, 22, 18, 12, 1, 5, - 16, 2, 14, 19, 20, 17, 21], - 'tpc_h_throughput_7': [18, 8, 20, 21, 2, 4, 22, 17, 1, 11, 9, 19, 3, 13, 5, - 7, 10, 16, 6, 14, 15, 12], - 'tpc_h_throughput_8': [19, 1, 15, 17, 5, 8, 9, 12, 14, 7, 4, 3, 20, 16, 6, - 22, 10, 13, 2, 21, 18, 11], - 'tpc_h_throughput_9': [8, 13, 2, 20, 17, 3, 6, 21, 18, 11, 19, 10, 15, 4, - 22, - 1, 7, 12, 9, 14, 5, 16], - 'tpc_h_throughput_10': [6, 15, 18, 17, 12, 1, 7, 2, 22, 13, 21, 10, 14, 9, - 3, - 16, 20, 19, 11, 4, 8, 5], - 'tpc_h_throughput_11': [15, 14, 18, 17, 10, 20, 16, 11, 1, 8, 4, 22, 5, 12, - 3, 9, 21, 2, 13, 6, 19, 7], - 'tpc_h_throughput_12': [1, 7, 16, 17, 18, 22, 12, 6, 8, 9, 11, 4, 2, 5, 20, - 21, 13, 10, 19, 3, 14, 15], - 'tpc_h_throughput_13': [21, 17, 7, 3, 1, 10, 12, 22, 9, 16, 6, 11, 2, 4, 5, - 14, 8, 20, 13, 18, 15, 19], - 'tpc_h_throughput_14': [2, 9, 5, 4, 18, 1, 20, 15, 16, 17, 7, 21, 13, 14, - 19, - 8, 22, 11, 10, 3, 12, 6], - 'tpc_h_throughput_15': [16, 9, 17, 8, 14, 11, 10, 12, 6, 21, 7, 3, 15, 5, - 22, - 20, 1, 13, 19, 2, 4, 18], - 'tpc_h_throughput_16': [1, 3, 6, 5, 2, 16, 14, 22, 17, 20, 4, 9, 10, 11, 15, - 8, 12, 19, 18, 13, 7, 21], - 'tpc_h_throughput_17': [3, 16, 5, 11, 21, 9, 2, 15, 10, 18, 17, 7, 8, 19, - 14, - 13, 1, 4, 22, 20, 6, 12], - 'tpc_h_throughput_18': [14, 4, 13, 5, 21, 11, 8, 6, 3, 17, 2, 20, 1, 19, 10, - 9, 12, 18, 15, 7, 22, 16], - 'tpc_h_throughput_19': [4, 12, 22, 14, 5, 15, 16, 2, 8, 10, 17, 9, 21, 7, 3, - 6, 13, 18, 11, 20, 19, 1], - 'tpc_h_throughput_20': [16, 15, 14, 13, 4, 22, 18, 19, 7, 1, 12, 17, 5, 10, - 20, 3, 9, 21, 11, 2, 6, 8], - 'tpc_h_throughput_21': [20, 14, 21, 12, 15, 17, 4, 19, 13, 10, 11, 1, 16, 5, - 18, 7, 8, 22, 9, 6, 3, 2], - 'tpc_h_throughput_22': [16, 14, 13, 2, 21, 10, 11, 4, 1, 22, 18, 12, 19, 5, - 7, 8, 6, 3, 15, 20, 9, 17], - 'tpc_h_throughput_23': [18, 15, 9, 14, 12, 2, 8, 11, 22, 21, 16, 1, 6, 17, - 5, - 10, 19, 4, 20, 13, 3, 7], - 'tpc_h_throughput_24': [7, 3, 10, 14, 13, 21, 18, 6, 20, 4, 9, 8, 22, 15, 2, - 1, 5, 12, 19, 17, 11, 16], - 'tpc_h_throughput_25': [18, 1, 13, 7, 16, 10, 14, 2, 19, 5, 21, 11, 22, 15, - 8, 17, 20, 3, 4, 12, 6, 9], - 'tpc_h_throughput_26': [13, 2, 22, 5, 11, 21, 20, 14, 7, 10, 4, 9, 19, 18, - 6, - 3, 1, 8, 15, 12, 17, 16], - 'tpc_h_throughput_27': [14, 17, 21, 8, 2, 9, 6, 4, 5, 13, 22, 7, 15, 3, 1, - 18, 16, 11, 10, 12, 20, 19], - 'tpc_h_throughput_28': [10, 22, 1, 12, 13, 18, 21, 20, 2, 14, 16, 7, 15, 3, - 4, 17, 5, 19, 6, 8, 9, 11], - 'tpc_h_throughput_29': [10, 8, 9, 18, 12, 6, 1, 5, 20, 11, 17, 22, 16, 3, - 13, - 2, 15, 21, 14, 19, 7, 4], - 'tpc_h_throughput_30': [7, 17, 22, 5, 3, 10, 13, 18, 9, 1, 14, 15, 21, 19, - 16, 12, 8, 6, 11, 20, 4, 2], - 'tpc_h_throughput_31': [2, 9, 21, 3, 4, 7, 1, 11, 16, 5, 20, 19, 18, 8, 17, - 13, 10, 12, 15, 6, 14, 22], - 'tpc_h_throughput_32': [15, 12, 8, 4, 22, 13, 16, 17, 18, 3, 7, 5, 6, 1, 9, - 11, 21, 10, 14, 20, 19, 2], - 'tpc_h_throughput_33': [15, 16, 2, 11, 17, 7, 5, 14, 20, 4, 21, 3, 10, 9, - 12, - 8, 13, 6, 18, 19, 22, 1], - 'tpc_h_throughput_34': [1, 13, 11, 3, 4, 21, 6, 14, 15, 22, 18, 9, 7, 5, 10, - 20, 12, 16, 17, 8, 19, 2], - 'tpc_h_throughput_35': [14, 17, 22, 20, 8, 16, 5, 10, 1, 13, 2, 21, 12, 9, - 4, - 18, 3, 7, 6, 19, 15, 11], - 'tpc_h_throughput_36': [9, 17, 7, 4, 5, 13, 21, 18, 11, 3, 22, 1, 6, 16, 20, - 14, 15, 10, 8, 2, 12, 19], - 'tpc_h_throughput_37': [13, 14, 5, 22, 19, 11, 9, 6, 18, 15, 8, 10, 7, 4, - 17, - 16, 3, 1, 12, 2, 21, 20], - 'tpc_h_throughput_38': [20, 5, 4, 14, 11, 1, 6, 16, 8, 22, 7, 3, 2, 12, 21, - 19, 17, 13, 10, 15, 18, 9], - 'tpc_h_throughput_39': [3, 7, 14, 15, 6, 5, 21, 20, 18, 10, 4, 16, 19, 1, - 13, - 9, 8, 17, 11, 12, 22, 2], - 'tpc_h_throughput_40': [13, 15, 17, 1, 22, 11, 3, 4, 7, 20, 14, 21, 9, 8, 2, - 18, 16, 6, 10, 12, 5, 19] + 'tpc_ds_power': [ + 96, + 7, + 75, + 44, + 39, + 80, + 32, + 19, + 25, + 78, + 86, + 1, + 91, + 21, + 43, + 27, + 94, + 45, + 58, + 64, + 36, + 33, + 46, + 62, + 16, + 10, + 63, + 69, + 60, + 59, + 37, + 98, + 85, + 70, + 67, + 28, + 81, + 97, + 66, + 90, + 17, + 47, + 95, + 92, + 3, + 51, + 35, + 49, + 9, + 31, + 11, + 93, + 29, + 38, + 22, + 89, + 15, + 6, + 52, + 50, + 42, + 41, + 8, + 12, + 20, + 88, + 82, + 23, + 14, + 57, + 65, + 71, + 34, + 48, + 30, + 74, + 87, + 77, + 73, + 84, + 54, + 55, + 56, + 2, + 26, + 40, + 72, + 53, + 79, + 18, + 13, + 24, + 4, + 99, + 68, + 83, + 61, + 5, + 76, + ], + 'tpc_ds_throughput_1': [ + 83, + 32, + 30, + 92, + 66, + 84, + 98, + 58, + 16, + 77, + 40, + 96, + 13, + 36, + 95, + 63, + 99, + 3, + 6, + 12, + 28, + 85, + 51, + 41, + 27, + 78, + 8, + 14, + 50, + 52, + 81, + 5, + 26, + 57, + 82, + 69, + 54, + 61, + 88, + 18, + 94, + 35, + 68, + 24, + 75, + 11, + 67, + 9, + 25, + 37, + 86, + 4, + 60, + 97, + 33, + 79, + 43, + 80, + 93, + 31, + 47, + 17, + 19, + 1, + 64, + 53, + 55, + 46, + 21, + 15, + 20, + 65, + 70, + 49, + 59, + 48, + 72, + 87, + 34, + 2, + 38, + 22, + 89, + 7, + 10, + 90, + 71, + 29, + 73, + 45, + 91, + 62, + 44, + 76, + 23, + 56, + 42, + 39, + 74, + ], + 'tpc_ds_throughput_2': [ + 56, + 98, + 59, + 24, + 88, + 2, + 5, + 6, + 27, + 87, + 90, + 83, + 91, + 28, + 68, + 8, + 76, + 75, + 80, + 1, + 69, + 26, + 11, + 17, + 63, + 77, + 19, + 21, + 31, + 93, + 54, + 39, + 10, + 15, + 55, + 14, + 38, + 42, + 53, + 45, + 99, + 67, + 23, + 62, + 30, + 86, + 82, + 25, + 16, + 81, + 40, + 44, + 50, + 61, + 85, + 73, + 95, + 84, + 4, + 37, + 35, + 94, + 58, + 96, + 12, + 29, + 22, + 51, + 36, + 43, + 64, + 20, + 57, + 9, + 52, + 49, + 71, + 72, + 70, + 7, + 97, + 33, + 79, + 32, + 78, + 18, + 65, + 60, + 34, + 3, + 13, + 41, + 92, + 74, + 46, + 89, + 47, + 66, + 48, + ], + 'tpc_ds_throughput_3': [ + 89, + 5, + 52, + 62, + 53, + 7, + 39, + 80, + 63, + 72, + 18, + 56, + 13, + 69, + 23, + 19, + 74, + 30, + 84, + 96, + 14, + 10, + 86, + 94, + 8, + 87, + 58, + 36, + 37, + 4, + 38, + 66, + 78, + 43, + 22, + 21, + 97, + 47, + 29, + 3, + 76, + 82, + 46, + 41, + 59, + 40, + 55, + 16, + 27, + 54, + 90, + 92, + 31, + 42, + 26, + 34, + 68, + 2, + 44, + 81, + 67, + 99, + 6, + 83, + 1, + 60, + 33, + 11, + 28, + 95, + 12, + 64, + 15, + 25, + 93, + 9, + 65, + 71, + 57, + 32, + 61, + 85, + 73, + 98, + 77, + 45, + 20, + 50, + 70, + 75, + 91, + 17, + 24, + 48, + 51, + 79, + 35, + 88, + 49, + ], + 'tpc_ds_throughput_4': [ + 79, + 39, + 93, + 41, + 29, + 32, + 66, + 84, + 8, + 71, + 45, + 89, + 91, + 14, + 46, + 58, + 48, + 59, + 2, + 83, + 21, + 78, + 40, + 99, + 19, + 72, + 6, + 28, + 81, + 44, + 97, + 88, + 77, + 95, + 33, + 36, + 61, + 35, + 60, + 75, + 74, + 55, + 51, + 17, + 52, + 90, + 22, + 27, + 63, + 38, + 18, + 24, + 37, + 47, + 10, + 70, + 23, + 7, + 92, + 54, + 82, + 76, + 80, + 56, + 96, + 50, + 85, + 86, + 69, + 68, + 1, + 12, + 43, + 16, + 4, + 25, + 20, + 65, + 15, + 98, + 42, + 26, + 34, + 5, + 87, + 3, + 64, + 31, + 57, + 30, + 13, + 94, + 62, + 49, + 11, + 73, + 67, + 53, + 9, + ], + 'tpc_ds_throughput_5': [ + 73, + 66, + 4, + 17, + 60, + 98, + 88, + 2, + 19, + 65, + 3, + 79, + 13, + 21, + 51, + 6, + 49, + 52, + 7, + 56, + 36, + 77, + 90, + 76, + 58, + 71, + 80, + 69, + 54, + 92, + 61, + 53, + 87, + 68, + 85, + 28, + 42, + 67, + 50, + 30, + 48, + 22, + 11, + 94, + 93, + 18, + 33, + 63, + 8, + 97, + 45, + 62, + 81, + 35, + 78, + 57, + 46, + 32, + 24, + 38, + 55, + 74, + 84, + 89, + 83, + 31, + 26, + 40, + 14, + 23, + 96, + 1, + 95, + 27, + 44, + 16, + 64, + 20, + 43, + 5, + 47, + 10, + 70, + 39, + 72, + 75, + 12, + 37, + 15, + 59, + 91, + 99, + 41, + 9, + 86, + 34, + 82, + 29, + 25, + ], + 'tpc_ds_throughput_6': [ + 34, + 88, + 44, + 94, + 50, + 5, + 53, + 7, + 58, + 20, + 75, + 73, + 91, + 36, + 11, + 80, + 9, + 93, + 32, + 89, + 28, + 87, + 18, + 74, + 6, + 65, + 84, + 14, + 38, + 24, + 42, + 29, + 72, + 23, + 26, + 69, + 47, + 82, + 31, + 59, + 49, + 33, + 86, + 99, + 4, + 45, + 85, + 8, + 19, + 61, + 3, + 41, + 54, + 67, + 77, + 15, + 51, + 98, + 62, + 97, + 22, + 48, + 2, + 79, + 56, + 37, + 10, + 90, + 21, + 46, + 83, + 96, + 68, + 63, + 92, + 27, + 12, + 64, + 95, + 39, + 35, + 78, + 57, + 66, + 71, + 30, + 1, + 81, + 43, + 52, + 13, + 76, + 17, + 25, + 40, + 70, + 55, + 60, + 16, + ], + 'tpc_ds_throughput_7': [ + 70, + 53, + 92, + 99, + 31, + 39, + 29, + 32, + 6, + 64, + 30, + 34, + 13, + 28, + 86, + 84, + 25, + 4, + 98, + 79, + 69, + 72, + 45, + 48, + 80, + 20, + 2, + 21, + 97, + 62, + 47, + 60, + 71, + 46, + 10, + 14, + 35, + 55, + 37, + 52, + 9, + 85, + 40, + 76, + 44, + 3, + 26, + 19, + 58, + 42, + 75, + 17, + 38, + 82, + 87, + 43, + 11, + 5, + 41, + 61, + 33, + 49, + 7, + 73, + 89, + 81, + 78, + 18, + 36, + 51, + 56, + 83, + 23, + 8, + 24, + 63, + 1, + 12, + 68, + 66, + 67, + 77, + 15, + 88, + 65, + 59, + 96, + 54, + 95, + 93, + 91, + 74, + 94, + 16, + 90, + 57, + 22, + 50, + 27, + ], + 'tpc_ds_throughput_8': [ + 57, + 29, + 24, + 76, + 37, + 66, + 60, + 98, + 80, + 12, + 59, + 70, + 91, + 69, + 40, + 2, + 16, + 44, + 5, + 73, + 14, + 71, + 3, + 49, + 84, + 64, + 7, + 36, + 61, + 41, + 35, + 50, + 65, + 51, + 78, + 21, + 67, + 22, + 81, + 93, + 25, + 26, + 90, + 74, + 92, + 75, + 10, + 58, + 6, + 47, + 30, + 94, + 97, + 55, + 72, + 95, + 86, + 39, + 17, + 42, + 85, + 9, + 32, + 34, + 79, + 54, + 77, + 45, + 28, + 11, + 89, + 56, + 46, + 19, + 62, + 8, + 96, + 1, + 23, + 88, + 82, + 87, + 43, + 53, + 20, + 52, + 83, + 38, + 68, + 4, + 13, + 48, + 99, + 27, + 18, + 15, + 33, + 31, + 63, + ], + 'tpc_ds_throughput_9': [ + 15, + 60, + 62, + 74, + 81, + 88, + 50, + 5, + 84, + 1, + 52, + 57, + 13, + 14, + 90, + 7, + 27, + 92, + 39, + 34, + 21, + 65, + 75, + 9, + 2, + 12, + 32, + 28, + 42, + 17, + 67, + 31, + 20, + 11, + 77, + 36, + 82, + 33, + 54, + 4, + 16, + 10, + 18, + 48, + 24, + 30, + 78, + 6, + 80, + 35, + 59, + 99, + 61, + 22, + 71, + 68, + 40, + 66, + 94, + 47, + 26, + 25, + 98, + 70, + 73, + 38, + 87, + 3, + 69, + 86, + 79, + 89, + 51, + 58, + 41, + 19, + 83, + 96, + 46, + 53, + 55, + 72, + 95, + 29, + 64, + 93, + 56, + 97, + 23, + 44, + 91, + 49, + 76, + 63, + 45, + 43, + 85, + 37, + 8, + ], + 'tpc_ds_throughput_10': [ + 43, + 50, + 41, + 48, + 54, + 53, + 31, + 39, + 2, + 96, + 93, + 15, + 91, + 21, + 18, + 32, + 63, + 24, + 66, + 70, + 36, + 20, + 30, + 25, + 7, + 1, + 98, + 69, + 47, + 94, + 82, + 37, + 64, + 86, + 87, + 28, + 55, + 85, + 38, + 44, + 27, + 78, + 45, + 49, + 62, + 59, + 77, + 80, + 84, + 67, + 52, + 76, + 42, + 33, + 65, + 23, + 90, + 88, + 99, + 35, + 10, + 16, + 5, + 57, + 34, + 97, + 72, + 75, + 14, + 40, + 73, + 79, + 11, + 6, + 17, + 58, + 56, + 83, + 51, + 29, + 22, + 71, + 68, + 60, + 12, + 4, + 89, + 61, + 46, + 92, + 13, + 9, + 74, + 8, + 3, + 95, + 26, + 81, + 19, + ], + 'tpc_ds_throughput_11': [ + 95, + 31, + 17, + 49, + 38, + 29, + 37, + 66, + 7, + 83, + 4, + 43, + 13, + 36, + 45, + 98, + 8, + 62, + 88, + 57, + 28, + 64, + 59, + 16, + 32, + 96, + 5, + 14, + 35, + 99, + 55, + 81, + 12, + 40, + 72, + 69, + 22, + 26, + 97, + 92, + 63, + 77, + 3, + 9, + 41, + 52, + 87, + 84, + 2, + 82, + 93, + 74, + 47, + 85, + 20, + 46, + 18, + 53, + 76, + 67, + 78, + 27, + 39, + 15, + 70, + 61, + 71, + 30, + 21, + 90, + 34, + 73, + 86, + 80, + 94, + 6, + 89, + 56, + 11, + 60, + 33, + 65, + 23, + 50, + 1, + 44, + 79, + 42, + 51, + 24, + 91, + 25, + 48, + 19, + 75, + 68, + 10, + 54, + 58, + ], + 'tpc_ds_throughput_12': [ + 68, + 37, + 94, + 9, + 97, + 60, + 81, + 88, + 32, + 56, + 44, + 95, + 91, + 28, + 3, + 5, + 19, + 41, + 53, + 15, + 69, + 12, + 52, + 27, + 98, + 83, + 39, + 21, + 67, + 76, + 22, + 54, + 1, + 90, + 71, + 14, + 33, + 10, + 61, + 24, + 8, + 87, + 75, + 25, + 17, + 93, + 72, + 2, + 7, + 55, + 4, + 48, + 35, + 26, + 64, + 51, + 45, + 29, + 74, + 82, + 77, + 63, + 66, + 43, + 57, + 42, + 65, + 59, + 36, + 18, + 70, + 34, + 40, + 84, + 99, + 80, + 79, + 89, + 86, + 50, + 85, + 20, + 46, + 31, + 96, + 92, + 73, + 47, + 11, + 62, + 13, + 16, + 49, + 58, + 30, + 23, + 78, + 38, + 6, + ], + 'tpc_ds_throughput_13': [ + 23, + 81, + 99, + 25, + 61, + 50, + 54, + 53, + 98, + 89, + 92, + 68, + 13, + 69, + 75, + 39, + 58, + 17, + 29, + 43, + 14, + 1, + 93, + 63, + 5, + 56, + 66, + 36, + 82, + 74, + 33, + 38, + 96, + 18, + 65, + 21, + 85, + 78, + 42, + 62, + 19, + 72, + 30, + 16, + 94, + 4, + 71, + 7, + 32, + 22, + 44, + 49, + 67, + 10, + 12, + 11, + 3, + 60, + 48, + 55, + 87, + 8, + 88, + 95, + 15, + 47, + 20, + 52, + 28, + 45, + 57, + 70, + 90, + 2, + 76, + 84, + 73, + 79, + 40, + 31, + 26, + 64, + 51, + 37, + 83, + 24, + 34, + 35, + 86, + 41, + 91, + 27, + 9, + 6, + 59, + 46, + 77, + 97, + 80, + ], + 'tpc_ds_throughput_14': [ + 46, + 54, + 76, + 16, + 42, + 31, + 38, + 29, + 5, + 79, + 24, + 23, + 91, + 14, + 30, + 66, + 6, + 94, + 60, + 95, + 21, + 96, + 4, + 8, + 39, + 89, + 88, + 28, + 55, + 48, + 85, + 97, + 83, + 45, + 20, + 36, + 26, + 77, + 47, + 41, + 58, + 71, + 59, + 27, + 99, + 44, + 65, + 32, + 98, + 33, + 92, + 9, + 82, + 78, + 1, + 86, + 75, + 50, + 49, + 22, + 72, + 19, + 53, + 68, + 43, + 35, + 64, + 93, + 69, + 3, + 15, + 57, + 18, + 7, + 74, + 2, + 34, + 73, + 90, + 37, + 10, + 12, + 11, + 81, + 56, + 62, + 70, + 67, + 40, + 17, + 13, + 63, + 25, + 80, + 52, + 51, + 87, + 61, + 84, + ], + 'tpc_ds_throughput_15': [ + 51, + 38, + 74, + 27, + 47, + 37, + 97, + 60, + 39, + 73, + 62, + 46, + 13, + 21, + 59, + 88, + 80, + 99, + 50, + 68, + 36, + 83, + 44, + 19, + 66, + 79, + 53, + 69, + 22, + 49, + 26, + 61, + 56, + 3, + 64, + 28, + 10, + 87, + 35, + 17, + 6, + 65, + 52, + 63, + 76, + 92, + 20, + 98, + 5, + 85, + 24, + 25, + 55, + 77, + 96, + 40, + 30, + 31, + 9, + 33, + 71, + 58, + 29, + 23, + 95, + 67, + 12, + 4, + 14, + 75, + 43, + 15, + 45, + 32, + 48, + 7, + 70, + 34, + 18, + 81, + 78, + 1, + 86, + 54, + 89, + 41, + 57, + 82, + 90, + 94, + 91, + 8, + 16, + 84, + 93, + 11, + 72, + 42, + 2, + ], + 'tpc_ds_throughput_16': [ + 11, + 97, + 48, + 63, + 35, + 81, + 61, + 50, + 66, + 34, + 41, + 51, + 91, + 36, + 52, + 53, + 84, + 76, + 31, + 23, + 28, + 56, + 92, + 58, + 88, + 73, + 29, + 14, + 33, + 9, + 10, + 42, + 89, + 75, + 12, + 69, + 78, + 72, + 67, + 94, + 80, + 20, + 93, + 8, + 74, + 24, + 64, + 5, + 39, + 26, + 62, + 16, + 22, + 87, + 83, + 90, + 59, + 37, + 25, + 85, + 65, + 6, + 60, + 46, + 68, + 82, + 1, + 44, + 21, + 30, + 95, + 43, + 3, + 98, + 49, + 32, + 57, + 70, + 45, + 54, + 77, + 96, + 40, + 38, + 79, + 17, + 15, + 55, + 18, + 99, + 13, + 19, + 27, + 2, + 4, + 86, + 71, + 47, + 7, + ], + 'tpc_ds_throughput_17': [ + 86, + 61, + 49, + 8, + 67, + 54, + 42, + 31, + 88, + 70, + 17, + 11, + 13, + 28, + 93, + 29, + 2, + 74, + 37, + 46, + 69, + 89, + 24, + 6, + 53, + 34, + 60, + 21, + 85, + 25, + 78, + 47, + 79, + 30, + 1, + 14, + 77, + 71, + 82, + 99, + 84, + 64, + 4, + 19, + 48, + 62, + 12, + 39, + 66, + 10, + 41, + 27, + 33, + 72, + 56, + 18, + 52, + 81, + 16, + 26, + 20, + 80, + 50, + 51, + 23, + 55, + 96, + 92, + 36, + 59, + 68, + 95, + 75, + 5, + 9, + 98, + 15, + 57, + 3, + 38, + 87, + 83, + 90, + 97, + 73, + 94, + 43, + 22, + 45, + 76, + 91, + 58, + 63, + 7, + 44, + 40, + 65, + 35, + 32, + ], + 'tpc_ds_throughput_18': [ + 40, + 42, + 9, + 19, + 82, + 38, + 47, + 37, + 53, + 57, + 94, + 86, + 91, + 69, + 4, + 60, + 7, + 48, + 81, + 51, + 14, + 79, + 62, + 80, + 29, + 70, + 50, + 36, + 26, + 16, + 77, + 35, + 73, + 59, + 96, + 21, + 87, + 65, + 55, + 76, + 2, + 12, + 44, + 58, + 49, + 41, + 1, + 66, + 88, + 78, + 17, + 63, + 85, + 71, + 89, + 45, + 93, + 54, + 27, + 10, + 64, + 84, + 31, + 11, + 46, + 22, + 83, + 24, + 28, + 52, + 23, + 68, + 30, + 39, + 25, + 5, + 43, + 15, + 75, + 97, + 72, + 56, + 18, + 61, + 34, + 99, + 95, + 33, + 3, + 74, + 13, + 6, + 8, + 32, + 92, + 90, + 20, + 67, + 98, + ], + 'tpc_ds_throughput_19': [ + 90, + 47, + 25, + 58, + 55, + 97, + 35, + 81, + 29, + 15, + 99, + 40, + 13, + 14, + 44, + 50, + 32, + 49, + 54, + 11, + 21, + 73, + 41, + 84, + 60, + 57, + 31, + 28, + 10, + 27, + 87, + 67, + 34, + 52, + 83, + 36, + 72, + 20, + 22, + 74, + 7, + 1, + 92, + 6, + 9, + 17, + 96, + 88, + 53, + 77, + 94, + 8, + 26, + 65, + 79, + 3, + 4, + 38, + 63, + 78, + 12, + 2, + 37, + 86, + 51, + 33, + 56, + 62, + 69, + 93, + 46, + 23, + 59, + 66, + 16, + 39, + 95, + 43, + 30, + 61, + 71, + 89, + 45, + 42, + 70, + 76, + 68, + 85, + 75, + 48, + 91, + 80, + 19, + 98, + 24, + 18, + 64, + 82, + 5, + ], + 'tpc_ds_throughput_20': [ + 18, + 35, + 16, + 6, + 22, + 61, + 67, + 54, + 60, + 43, + 76, + 90, + 91, + 21, + 92, + 31, + 98, + 9, + 38, + 86, + 36, + 34, + 17, + 2, + 50, + 15, + 37, + 69, + 78, + 63, + 72, + 82, + 70, + 93, + 56, + 28, + 71, + 64, + 33, + 48, + 32, + 96, + 24, + 80, + 25, + 94, + 83, + 53, + 29, + 87, + 99, + 19, + 10, + 20, + 73, + 75, + 44, + 97, + 8, + 77, + 1, + 7, + 81, + 40, + 11, + 85, + 89, + 41, + 14, + 4, + 51, + 46, + 52, + 88, + 27, + 66, + 68, + 95, + 59, + 42, + 65, + 79, + 3, + 47, + 57, + 74, + 23, + 26, + 30, + 49, + 13, + 84, + 58, + 5, + 62, + 45, + 12, + 55, + 39, + ], + 'tpc_h_power': [ + 14, + 2, + 9, + 20, + 6, + 17, + 18, + 8, + 21, + 13, + 3, + 22, + 16, + 4, + 11, + 15, + 1, + 10, + 19, + 5, + 7, + 12, + ], + 'tpc_h_throughput_1': [ + 21, + 3, + 18, + 5, + 11, + 7, + 6, + 20, + 17, + 12, + 16, + 15, + 13, + 10, + 2, + 8, + 14, + 19, + 9, + 22, + 1, + 4, + ], + 'tpc_h_throughput_2': [ + 6, + 17, + 14, + 16, + 19, + 10, + 9, + 2, + 15, + 8, + 5, + 22, + 12, + 7, + 13, + 18, + 1, + 4, + 20, + 3, + 11, + 21, + ], + 'tpc_h_throughput_3': [ + 8, + 5, + 4, + 6, + 17, + 7, + 1, + 18, + 22, + 14, + 9, + 10, + 15, + 11, + 20, + 2, + 21, + 19, + 13, + 16, + 12, + 3, + ], + 'tpc_h_throughput_4': [ + 5, + 21, + 14, + 19, + 15, + 17, + 12, + 6, + 4, + 9, + 8, + 16, + 11, + 2, + 10, + 18, + 1, + 13, + 7, + 22, + 3, + 20, + ], + 'tpc_h_throughput_5': [ + 21, + 15, + 4, + 6, + 7, + 16, + 19, + 18, + 14, + 22, + 11, + 13, + 3, + 1, + 2, + 5, + 8, + 20, + 12, + 17, + 10, + 9, + ], + 'tpc_h_throughput_6': [ + 10, + 3, + 15, + 13, + 6, + 8, + 9, + 7, + 4, + 11, + 22, + 18, + 12, + 1, + 5, + 16, + 2, + 14, + 19, + 20, + 17, + 21, + ], + 'tpc_h_throughput_7': [ + 18, + 8, + 20, + 21, + 2, + 4, + 22, + 17, + 1, + 11, + 9, + 19, + 3, + 13, + 5, + 7, + 10, + 16, + 6, + 14, + 15, + 12, + ], + 'tpc_h_throughput_8': [ + 19, + 1, + 15, + 17, + 5, + 8, + 9, + 12, + 14, + 7, + 4, + 3, + 20, + 16, + 6, + 22, + 10, + 13, + 2, + 21, + 18, + 11, + ], + 'tpc_h_throughput_9': [ + 8, + 13, + 2, + 20, + 17, + 3, + 6, + 21, + 18, + 11, + 19, + 10, + 15, + 4, + 22, + 1, + 7, + 12, + 9, + 14, + 5, + 16, + ], + 'tpc_h_throughput_10': [ + 6, + 15, + 18, + 17, + 12, + 1, + 7, + 2, + 22, + 13, + 21, + 10, + 14, + 9, + 3, + 16, + 20, + 19, + 11, + 4, + 8, + 5, + ], + 'tpc_h_throughput_11': [ + 15, + 14, + 18, + 17, + 10, + 20, + 16, + 11, + 1, + 8, + 4, + 22, + 5, + 12, + 3, + 9, + 21, + 2, + 13, + 6, + 19, + 7, + ], + 'tpc_h_throughput_12': [ + 1, + 7, + 16, + 17, + 18, + 22, + 12, + 6, + 8, + 9, + 11, + 4, + 2, + 5, + 20, + 21, + 13, + 10, + 19, + 3, + 14, + 15, + ], + 'tpc_h_throughput_13': [ + 21, + 17, + 7, + 3, + 1, + 10, + 12, + 22, + 9, + 16, + 6, + 11, + 2, + 4, + 5, + 14, + 8, + 20, + 13, + 18, + 15, + 19, + ], + 'tpc_h_throughput_14': [ + 2, + 9, + 5, + 4, + 18, + 1, + 20, + 15, + 16, + 17, + 7, + 21, + 13, + 14, + 19, + 8, + 22, + 11, + 10, + 3, + 12, + 6, + ], + 'tpc_h_throughput_15': [ + 16, + 9, + 17, + 8, + 14, + 11, + 10, + 12, + 6, + 21, + 7, + 3, + 15, + 5, + 22, + 20, + 1, + 13, + 19, + 2, + 4, + 18, + ], + 'tpc_h_throughput_16': [ + 1, + 3, + 6, + 5, + 2, + 16, + 14, + 22, + 17, + 20, + 4, + 9, + 10, + 11, + 15, + 8, + 12, + 19, + 18, + 13, + 7, + 21, + ], + 'tpc_h_throughput_17': [ + 3, + 16, + 5, + 11, + 21, + 9, + 2, + 15, + 10, + 18, + 17, + 7, + 8, + 19, + 14, + 13, + 1, + 4, + 22, + 20, + 6, + 12, + ], + 'tpc_h_throughput_18': [ + 14, + 4, + 13, + 5, + 21, + 11, + 8, + 6, + 3, + 17, + 2, + 20, + 1, + 19, + 10, + 9, + 12, + 18, + 15, + 7, + 22, + 16, + ], + 'tpc_h_throughput_19': [ + 4, + 12, + 22, + 14, + 5, + 15, + 16, + 2, + 8, + 10, + 17, + 9, + 21, + 7, + 3, + 6, + 13, + 18, + 11, + 20, + 19, + 1, + ], + 'tpc_h_throughput_20': [ + 16, + 15, + 14, + 13, + 4, + 22, + 18, + 19, + 7, + 1, + 12, + 17, + 5, + 10, + 20, + 3, + 9, + 21, + 11, + 2, + 6, + 8, + ], + 'tpc_h_throughput_21': [ + 20, + 14, + 21, + 12, + 15, + 17, + 4, + 19, + 13, + 10, + 11, + 1, + 16, + 5, + 18, + 7, + 8, + 22, + 9, + 6, + 3, + 2, + ], + 'tpc_h_throughput_22': [ + 16, + 14, + 13, + 2, + 21, + 10, + 11, + 4, + 1, + 22, + 18, + 12, + 19, + 5, + 7, + 8, + 6, + 3, + 15, + 20, + 9, + 17, + ], + 'tpc_h_throughput_23': [ + 18, + 15, + 9, + 14, + 12, + 2, + 8, + 11, + 22, + 21, + 16, + 1, + 6, + 17, + 5, + 10, + 19, + 4, + 20, + 13, + 3, + 7, + ], + 'tpc_h_throughput_24': [ + 7, + 3, + 10, + 14, + 13, + 21, + 18, + 6, + 20, + 4, + 9, + 8, + 22, + 15, + 2, + 1, + 5, + 12, + 19, + 17, + 11, + 16, + ], + 'tpc_h_throughput_25': [ + 18, + 1, + 13, + 7, + 16, + 10, + 14, + 2, + 19, + 5, + 21, + 11, + 22, + 15, + 8, + 17, + 20, + 3, + 4, + 12, + 6, + 9, + ], + 'tpc_h_throughput_26': [ + 13, + 2, + 22, + 5, + 11, + 21, + 20, + 14, + 7, + 10, + 4, + 9, + 19, + 18, + 6, + 3, + 1, + 8, + 15, + 12, + 17, + 16, + ], + 'tpc_h_throughput_27': [ + 14, + 17, + 21, + 8, + 2, + 9, + 6, + 4, + 5, + 13, + 22, + 7, + 15, + 3, + 1, + 18, + 16, + 11, + 10, + 12, + 20, + 19, + ], + 'tpc_h_throughput_28': [ + 10, + 22, + 1, + 12, + 13, + 18, + 21, + 20, + 2, + 14, + 16, + 7, + 15, + 3, + 4, + 17, + 5, + 19, + 6, + 8, + 9, + 11, + ], + 'tpc_h_throughput_29': [ + 10, + 8, + 9, + 18, + 12, + 6, + 1, + 5, + 20, + 11, + 17, + 22, + 16, + 3, + 13, + 2, + 15, + 21, + 14, + 19, + 7, + 4, + ], + 'tpc_h_throughput_30': [ + 7, + 17, + 22, + 5, + 3, + 10, + 13, + 18, + 9, + 1, + 14, + 15, + 21, + 19, + 16, + 12, + 8, + 6, + 11, + 20, + 4, + 2, + ], + 'tpc_h_throughput_31': [ + 2, + 9, + 21, + 3, + 4, + 7, + 1, + 11, + 16, + 5, + 20, + 19, + 18, + 8, + 17, + 13, + 10, + 12, + 15, + 6, + 14, + 22, + ], + 'tpc_h_throughput_32': [ + 15, + 12, + 8, + 4, + 22, + 13, + 16, + 17, + 18, + 3, + 7, + 5, + 6, + 1, + 9, + 11, + 21, + 10, + 14, + 20, + 19, + 2, + ], + 'tpc_h_throughput_33': [ + 15, + 16, + 2, + 11, + 17, + 7, + 5, + 14, + 20, + 4, + 21, + 3, + 10, + 9, + 12, + 8, + 13, + 6, + 18, + 19, + 22, + 1, + ], + 'tpc_h_throughput_34': [ + 1, + 13, + 11, + 3, + 4, + 21, + 6, + 14, + 15, + 22, + 18, + 9, + 7, + 5, + 10, + 20, + 12, + 16, + 17, + 8, + 19, + 2, + ], + 'tpc_h_throughput_35': [ + 14, + 17, + 22, + 20, + 8, + 16, + 5, + 10, + 1, + 13, + 2, + 21, + 12, + 9, + 4, + 18, + 3, + 7, + 6, + 19, + 15, + 11, + ], + 'tpc_h_throughput_36': [ + 9, + 17, + 7, + 4, + 5, + 13, + 21, + 18, + 11, + 3, + 22, + 1, + 6, + 16, + 20, + 14, + 15, + 10, + 8, + 2, + 12, + 19, + ], + 'tpc_h_throughput_37': [ + 13, + 14, + 5, + 22, + 19, + 11, + 9, + 6, + 18, + 15, + 8, + 10, + 7, + 4, + 17, + 16, + 3, + 1, + 12, + 2, + 21, + 20, + ], + 'tpc_h_throughput_38': [ + 20, + 5, + 4, + 14, + 11, + 1, + 6, + 16, + 8, + 22, + 7, + 3, + 2, + 12, + 21, + 19, + 17, + 13, + 10, + 15, + 18, + 9, + ], + 'tpc_h_throughput_39': [ + 3, + 7, + 14, + 15, + 6, + 5, + 21, + 20, + 18, + 10, + 4, + 16, + 19, + 1, + 13, + 9, + 8, + 17, + 11, + 12, + 22, + 2, + ], + 'tpc_h_throughput_40': [ + 13, + 15, + 17, + 1, + 22, + 11, + 3, + 4, + 7, + 20, + 14, + 21, + 9, + 8, + 2, + 18, + 16, + 6, + 10, + 12, + 5, + 19, + ], } diff --git a/perfkitbenchmarker/data/edw/unistream_profile_driver.py b/perfkitbenchmarker/data/edw/unistream_profile_driver.py index 35b5819ce0..b977795f13 100644 --- a/perfkitbenchmarker/data/edw/unistream_profile_driver.py +++ b/perfkitbenchmarker/data/edw/unistream_profile_driver.py @@ -4,7 +4,6 @@ tpc_profile_details module. """ - __author__ = 'p3rf@google.com' import json @@ -15,8 +14,9 @@ import script_driver import tpc_profile_details -flags.DEFINE_string('profile', None, 'Profile to identify the sequence of ' - 'scripts.') +flags.DEFINE_string( + 'profile', None, 'Profile to identify the sequence of scripts.' +) FLAGS = flags.FLAGS @@ -45,8 +45,10 @@ def execute_profile(profile): script_performance = script_driver.execute_script(script, logfile_suffix) execution_times.update(json.loads(script_performance)) profile_execution_wall_time = round((time.time() - start_time), 2) - execution_times['wall_time'] = {'execution_time': profile_execution_wall_time, - 'job_id': 'undefined_job'} + execution_times['wall_time'] = { + 'execution_time': profile_execution_wall_time, + 'job_id': 'undefined_job', + } return json.dumps(execution_times) diff --git a/perfkitbenchmarker/data/large_scale_boot/listener_server.py b/perfkitbenchmarker/data/large_scale_boot/listener_server.py index 751bfdedec..27c662dc03 100644 --- a/perfkitbenchmarker/data/large_scale_boot/listener_server.py +++ b/perfkitbenchmarker/data/large_scale_boot/listener_server.py @@ -53,20 +53,25 @@ def ConfirmIPAccessible(client_host, port, timeout=MAX_TIME_SECONDS): """Confirm the given host's port is accessible and return the access time.""" netcat_command = 'nc -zv -w 1 {client} {port}'.format( - client=client_host, - port=port) + client=client_host, port=port + ) start_time = time.time() while time.time() <= (start_time + timeout): - p = subprocess.Popen(netcat_command, shell=True, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + p = subprocess.Popen( + netcat_command, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) _, stderr = p.communicate() # different versions of netcat uses different stderr strings. if any(word in stderr.decode('utf-8') for word in ['open', 'succeeded']): # return the system time in nanoseconds return 'Pass:%s:%d' % (client_host, time.time() * NANO) - logging.warning('Could not netcat to port %s on client vm %s.', - port, client_host) + logging.warning( + 'Could not netcat to port %s on client vm %s.', port, client_host + ) return 'Fail:%s:%d' % (client_host, time.time() * NANO) @@ -86,8 +91,13 @@ def WaitForRunningStatus(client_host, timeout=MAX_TIME_SECONDS): command = reader.read() start_time = time.time() while time.time() <= (start_time + timeout): - p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, - universal_newlines=True, stderr=subprocess.PIPE) + p = subprocess.Popen( + command, + shell=True, + stdout=subprocess.PIPE, + universal_newlines=True, + stderr=subprocess.PIPE, + ) status, _ = p.communicate() if 'running' in status.lower(): return 'Running:%s:%d' % (client_host, time.time() * NANO) @@ -134,14 +144,18 @@ def BuildHostNames(name_pattern, count, use_public_ip): public_dns = name_pattern.split('_')[-2] start_vm_index = int(name_pattern.split('_')[-1]) if public_dns: - return [public_dns.replace('VMID', str(vm_id)) - for vm_id in range(start_vm_index, count + start_vm_index)] + return [ + public_dns.replace('VMID', str(vm_id)) + for vm_id in range(start_vm_index, count + start_vm_index) + ] else: return GenerateHostIPs(start_vm_index, count) else: - return [name_pattern.replace('VM_ID', str(vm_id)) - for vm_id in range(1, count + 1)] + return [ + name_pattern.replace('VM_ID', str(vm_id)) + for vm_id in range(1, count + 1) + ] def WaitForHostNames(use_public_ip, timeout=MAX_TIME_SECONDS_NO_CALLING): @@ -159,6 +173,7 @@ def WaitForHostNames(use_public_ip, timeout=MAX_TIME_SECONDS_NO_CALLING): Args: use_public_ip: whether to use public_ip hostname. timeout: Amount of time in seconds to wait for boot. + Returns: hosts to netcat. """ @@ -180,17 +195,19 @@ def WaitForHostNames(use_public_ip, timeout=MAX_TIME_SECONDS_NO_CALLING): if not use_public_ip and 'PRIVATEIPADDRESSES' in line: hostnames.append(line.split()[2]) return set(hostnames) - raise ValueError('Boot did not complete successfully before timeout of %s ' - 'seconds.' % MAX_TIME_SECONDS_NO_CALLING) + raise ValueError( + 'Boot did not complete successfully before timeout of %s seconds.' + % MAX_TIME_SECONDS_NO_CALLING + ) def GenerateHostIPs(boot_vm_index, count): """Logic must be aligned with large_scale_boot/boot_script.sh.""" hostnames = [] for vm_id in range(boot_vm_index, boot_vm_index + count): - hostnames.append('10.0.{octet3}.{octet4}'.format( - octet3=vm_id // 256, - octet4=vm_id % 256)) + hostnames.append( + '10.0.{octet3}.{octet4}'.format(octet3=vm_id // 256, octet4=vm_id % 256) + ) return hostnames @@ -201,14 +218,23 @@ def ActAsClient(pool, queue, port, name_pattern, vms_count, use_public_ip): for host_name in BuildHostNames(name_pattern, vms_count, use_public_ip): job = pool.apply_async( ConfirmIPAccessible, - args=(host_name, port, MAX_TIME_SECONDS_NO_CALLING,), - callback=store_results) + args=( + host_name, + port, + MAX_TIME_SECONDS_NO_CALLING, + ), + callback=store_results, + ) all_jobs.append(job) if vms_count == 1: status_job = pool.apply_async( WaitForRunningStatus, - args=(host_name, MAX_TIME_SECONDS_NO_CALLING,), - callback=store_results) + args=( + host_name, + MAX_TIME_SECONDS_NO_CALLING, + ), + callback=store_results, + ) all_jobs.append(status_job) logging.info([async_job.get() for async_job in all_jobs]) queue.put(_STOP_QUEUE_ENTRY) @@ -267,9 +293,14 @@ def do_GET(self): # pylint: disable=g-bad-name # Process this client logging.info(client_host) store_results_func = functools.partial(StoreResult, queue=self.timing_queue) - self.process_pool.apply_async(ConfirmIPAccessible, - args=(client_host, self.access_port,), - callback=store_results_func) + self.process_pool.apply_async( + ConfirmIPAccessible, + args=( + client_host, + self.access_port, + ), + callback=store_results_func, + ) def shutdown(self): """Shut down the server.""" @@ -281,16 +312,18 @@ def shutdown(self): if __name__ == '__main__': logging.basicConfig(level=logging.INFO) if len(sys.argv) != 9: - raise ValueError('Got unexpected number of command-line arguments. ' - 'There should be at most 7 command-line arguments: ' - '1. name of the server vm, ' - '2. server port, ' - '3. results file, ' - '4. port to access the boot VMs, ' - '5. whether to use the listening server, ' - '6. launched vm naming pattern, ' - '7. number of launched vms.' - '8. whether to use public ip address.') + raise ValueError( + 'Got unexpected number of command-line arguments. ' + 'There should be at most 7 command-line arguments: ' + '1. name of the server vm, ' + '2. server port, ' + '3. results file, ' + '4. port to access the boot VMs, ' + '5. whether to use the listening server, ' + '6. launched vm naming pattern, ' + '7. number of launched vms.' + '8. whether to use public ip address.' + ) hostname = sys.argv[1] server_address = ('', int(sys.argv[2])) results_file_path = sys.argv[3] @@ -304,13 +337,25 @@ def shutdown(self): timing_queue = multiprocessing_manager.Queue() # Start the worker to move results from queue to file first. - process_pool.apply_async(WriteResultsToFile, - args=(results_file_path, timing_queue,)) + process_pool.apply_async( + WriteResultsToFile, + args=( + results_file_path, + timing_queue, + ), + ) if use_listening_server: - ActAsServer(process_pool, timing_queue, clients_port, hostname, - server_address) + ActAsServer( + process_pool, timing_queue, clients_port, hostname, server_address + ) # The start the server to listen and put results on queue. else: - ActAsClient(process_pool, timing_queue, clients_port, - vms_name_pattern, num_vms, using_public_ip) + ActAsClient( + process_pool, + timing_queue, + clients_port, + vms_name_pattern, + num_vms, + using_public_ip, + ) diff --git a/perfkitbenchmarker/data/mlperf_inference_custom_list.py b/perfkitbenchmarker/data/mlperf_inference_custom_list.py index 680713a75e..fa3eb88390 100644 --- a/perfkitbenchmarker/data/mlperf_inference_custom_list.py +++ b/perfkitbenchmarker/data/mlperf_inference_custom_list.py @@ -42,7 +42,7 @@ def gpu( memory_quantity: int, max_power_limit: float, pci_id: str, - compute_cm: int + compute_cm: int, ) -> GPU: return GPU( name=name, @@ -55,6 +55,7 @@ def gpu( compute_sm=compute_cm, ) + CloudT4 = gpu("Tesla T4", 15, 70.0, "0x1EB810DE", 75) CloudL4 = gpu("NVIDIA L4", 22, 72.0, "0x27B810DE", 89) A10G = gpu("NVIDIA A10G", 22, 300.0, "0x223710DE", 86) diff --git a/perfkitbenchmarker/data/mlperf_inference_dlrm_server_custom.py b/perfkitbenchmarker/data/mlperf_inference_dlrm_server_custom.py index 414032ccdf..86e38a3df5 100644 --- a/perfkitbenchmarker/data/mlperf_inference_dlrm_server_custom.py +++ b/perfkitbenchmarker/data/mlperf_inference_dlrm_server_custom.py @@ -408,6 +408,7 @@ class A10x2HighAccuracyTriton(A10x2Triton): ) class H100x1(ServerGPUBaseConfig): """Base DLRM H100 config.""" + system = KnownSystem.H100x1 use_small_tile_gemm_plugin = False use_jemalloc = True diff --git a/perfkitbenchmarker/data/spark_io/dml_script.py b/perfkitbenchmarker/data/spark_io/dml_script.py index e4b76862cf..cab63bfadb 100644 --- a/perfkitbenchmarker/data/spark_io/dml_script.py +++ b/perfkitbenchmarker/data/spark_io/dml_script.py @@ -1,15 +1,15 @@ """Spark application to create a Spark table for IO intensive benchmarking.""" - import sys from pyspark.sql import SparkSession def main(): - spark = (SparkSession.builder - .appName('Setup Spark table') - .enableHiveSupport() - .getOrCreate()) + spark = ( + SparkSession.builder.appName('Setup Spark table') + .enableHiveSupport() + .getOrCreate() + ) table = 'warehouse' table_dir = sys.argv[1] # clean up previous table @@ -17,5 +17,6 @@ def main(): # register new table spark.catalog.createTable(table, table_dir, source='parquet') + if __name__ == '__main__': main() diff --git a/perfkitbenchmarker/db_util.py b/perfkitbenchmarker/db_util.py index fa631b691b..f172440e2b 100644 --- a/perfkitbenchmarker/db_util.py +++ b/perfkitbenchmarker/db_util.py @@ -19,6 +19,6 @@ def GenerateRandomDbPassword(): prefix = [ random.choice(string.ascii_lowercase), random.choice(string.ascii_uppercase), - random.choice(string.digits) + random.choice(string.digits), ] return ''.join(prefix) + str(uuid.uuid4())[:10] diff --git a/perfkitbenchmarker/disk.py b/perfkitbenchmarker/disk.py index 315e3db5e0..f76d8ad855 100644 --- a/perfkitbenchmarker/disk.py +++ b/perfkitbenchmarker/disk.py @@ -24,38 +24,52 @@ from perfkitbenchmarker.configs import spec import six -flags.DEFINE_boolean('nfs_timeout_hard', True, - 'Whether to use hard or soft for NFS mount.') +flags.DEFINE_boolean( + 'nfs_timeout_hard', True, 'Whether to use hard or soft for NFS mount.' +) flags.DEFINE_integer('nfs_rsize', 1048576, 'NFS read size.') flags.DEFINE_integer('nfs_wsize', 1048576, 'NFS write size.') flags.DEFINE_integer('nfs_timeout', 60, 'NFS timeout.') flags.DEFINE_integer('nfs_retries', 2, 'NFS Retries.') flags.DEFINE_integer( - 'nfs_nconnect', None, 'Number of connections that each NFS client should ' - 'establish to the server.') + 'nfs_nconnect', + None, + 'Number of connections that each NFS client should ' + 'establish to the server.', +) flags.DEFINE_boolean( - 'nfs_noresvport', False, + 'nfs_noresvport', + False, 'Whether the NFS client should use a non-privileged ' - 'source port. Suggested to use with EFS') + 'source port. Suggested to use with EFS', +) flags.DEFINE_boolean( - 'nfs_managed', True, + 'nfs_managed', + True, 'Use a managed NFS service if using NFS disks. Otherwise ' - 'start an NFS server on the first VM.') + 'start an NFS server on the first VM.', +) flags.DEFINE_string( - 'nfs_ip_address', None, + 'nfs_ip_address', + None, 'If specified, PKB will target this ip address when ' 'mounting NFS "disks" rather than provisioning an NFS ' - 'Service for the corresponding cloud.') + 'Service for the corresponding cloud.', +) flags.DEFINE_string( - 'nfs_directory', None, + 'nfs_directory', + None, 'Directory to mount if using a StaticNfsService. This ' 'corresponds to the "VOLUME_NAME" of other NfsService ' - 'classes.') + 'classes.', +) flags.DEFINE_string('smb_version', '3.0', 'SMB version.') -flags.DEFINE_list('mount_options', [], - 'Additional arguments to supply when mounting.') -flags.DEFINE_list('fstab_options', [], - 'Additional arguments to supply to fstab.') +flags.DEFINE_list( + 'mount_options', [], 'Additional arguments to supply when mounting.' +) +flags.DEFINE_list( + 'fstab_options', [], 'Additional arguments to supply to fstab.' +) FLAGS = flags.FLAGS @@ -476,7 +490,8 @@ def _GetNetworkDiskMountOptionsDict(self): def mount_options(self): opts = [] for key, value in sorted( - six.iteritems(self._GetNetworkDiskMountOptionsDict())): + six.iteritems(self._GetNetworkDiskMountOptionsDict()) + ): opts.append('%s' % key if value is None else '%s=%s' % (key, value)) return ','.join(opts) @@ -509,11 +524,13 @@ class NfsDisk(NetworkDisk): nfs_tier: The NFS tier / performance level of the server. """ - def __init__(self, - disk_spec, - remote_mount_address, - default_nfs_version=None, - nfs_tier=None): + def __init__( + self, + disk_spec, + remote_mount_address, + default_nfs_version=None, + nfs_tier=None, + ): super(NfsDisk, self).__init__(disk_spec) self.nfs_version = disk_spec.nfs_version or default_nfs_version self.nfs_timeout_hard = disk_spec.nfs_timeout_hard @@ -564,12 +581,14 @@ class SmbDisk(NetworkDisk): smb_tier: The SMB tier / performance level of the server. """ - def __init__(self, - disk_spec, - remote_mount_address, - storage_account_and_key, - default_smb_version=None, - smb_tier=None): + def __init__( + self, + disk_spec, + remote_mount_address, + storage_account_and_key, + default_smb_version=None, + smb_tier=None, + ): super(SmbDisk, self).__init__(disk_spec) self.smb_version = disk_spec.smb_version self.device_path = remote_mount_address diff --git a/perfkitbenchmarker/disk_iops_to_capacity.py b/perfkitbenchmarker/disk_iops_to_capacity.py index 1b414c8449..5e4cb93295 100644 --- a/perfkitbenchmarker/disk_iops_to_capacity.py +++ b/perfkitbenchmarker/disk_iops_to_capacity.py @@ -82,13 +82,13 @@ class InvalidStorageTypeError(Exception): AWS: { MAX_IOPS: 75000, DEFAULT_STORAGE_TYPE: 'ebs-gp2', - VALID_STORAGE_TYPES: ['ebs-gp2'] + VALID_STORAGE_TYPES: ['ebs-gp2'], }, GCP: { MAX_IOPS: 30000, DEFAULT_STORAGE_TYPE: 'pd-ssd', VALID_STORAGE_TYPES: ['pd-ssd'], - } + }, } @@ -135,16 +135,18 @@ def _ValidateStorageType(self): TODO: When support other types of storage types (i.e. when this class supports ebs-piops for AWS or pd-hhd for gcp), will need to update VALID_STORAGE_TYPES in CLOUD_PROVIDERS_INFO dictionary. - """ if self._storage_type: self._storage_type = self._storage_type.lower() - if (self._storage_type is - not CLOUD_PROVIDERS_INFO[self._provider][DEFAULT_STORAGE_TYPE]): + if ( + self._storage_type + is not CLOUD_PROVIDERS_INFO[self._provider][DEFAULT_STORAGE_TYPE] + ): raise InvalidStorageTypeError() else: self._storage_type = CLOUD_PROVIDERS_INFO[self._provider][ - DEFAULT_STORAGE_TYPE] + DEFAULT_STORAGE_TYPE + ] def _ValidateProvider(self): """Validate provider to be GCP or AWS, throw exception if invalid. @@ -153,8 +155,9 @@ def _ValidateProvider(self): InvalidProviderError: Incorrect provider type given. """ if self._provider not in list(CLOUD_PROVIDERS_INFO.keys()): - raise InvalidProviderError('Provider given is not supported by ' - 'storage_utility.') + raise InvalidProviderError( + 'Provider given is not supported by storage_utility.' + ) def _ValidateIOPS(self): """Validate IOPS to be within valid limits, throw exception if invalid. @@ -165,13 +168,17 @@ def _ValidateIOPS(self): Raises: InvalidIOPSError: Invalid IOPS parameter given. """ - if (self._iops < 1 or - self._iops > CLOUD_PROVIDERS_INFO[self._provider][MAX_IOPS]): + if ( + self._iops < 1 + or self._iops > CLOUD_PROVIDERS_INFO[self._provider][MAX_IOPS] + ): raise InvalidIOPSError( 'Invalid IOPS parameter, must be positive number less than ' 'the maximum achievable for given cloud provider. ' 'The maximum for {} is {}.'.format( - self._provider, CLOUD_PROVIDERS_INFO[self._provider][MAX_IOPS])) + self._provider, CLOUD_PROVIDERS_INFO[self._provider][MAX_IOPS] + ) + ) def _PopulateConfigs(self): """Populate Storage Configurations.""" @@ -181,11 +188,17 @@ def _PopulateConfigs(self): def PrintConfigs(self): """Print out necessary configs.""" - vm_config = ('For {} IOPS using {}, the following is required:\n\tStorage ' - 'Size (GB): {}\n\tCPU Count: {}\n\tNumber of ' - 'Disks: {}').format(self._iops, - self._provider.upper(), self._size, - self._cpu_count, self._number_disks) + vm_config = ( + 'For {} IOPS using {}, the following is required:\n\tStorage ' + 'Size (GB): {}\n\tCPU Count: {}\n\tNumber of ' + 'Disks: {}' + ).format( + self._iops, + self._provider.upper(), + self._size, + self._cpu_count, + self._number_disks, + ) print(vm_config) def _SetSize(self): @@ -207,11 +220,20 @@ def _SetSize(self): value = self._iops value = numpy.array(value) self._size = int( - numpy.piecewise([value], [[value <= 100], [(value > 100) & ( - value <= 9999)], [value > 9999]], [ + numpy.piecewise( + [value], + [ + [value <= 100], + [(value > 100) & (value <= 9999)], + [value > 9999], + ], + [ lambda x: int(math.ceil(1.07374)), lambda x: int(math.ceil(3 * value)), - lambda x: int(math.ceil(3579.855))])) + lambda x: int(math.ceil(3579.855)), + ], + ) + ) def GetSize(self): """Return storage size. @@ -231,14 +253,20 @@ def _SetCPUCount(self): optimized is *.large VM types (e.g., c4.large), those comes with 2 cores. ratings from http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSOptimized.html - """ if self._provider == GCP: value = self._iops self._cpu_count = int( - numpy.piecewise([value], [[value <= 15000], [ - (value > 15000) & (value <= 25000) - ], [value > 25000]], [lambda x: 1, lambda x: 16, lambda x: 32])) + numpy.piecewise( + [value], + [ + [value <= 15000], + [(value > 15000) & (value <= 25000)], + [value > 25000], + ], + [lambda x: 1, lambda x: 16, lambda x: 32], + ) + ) elif self._provider == AWS: self._cpu_count = 2 diff --git a/perfkitbenchmarker/disk_strategies.py b/perfkitbenchmarker/disk_strategies.py index b9d77153fd..e615d678d8 100644 --- a/perfkitbenchmarker/disk_strategies.py +++ b/perfkitbenchmarker/disk_strategies.py @@ -47,8 +47,12 @@ class CreateDiskStrategy: disk_specs: Duplicated disk_spec by disk_count """ - def __init__(self, vm: 'virtual_machine. BaseVirtualMachine', - disk_spec: disk.BaseDiskSpec, disk_count: int): + def __init__( + self, + vm: 'virtual_machine. BaseVirtualMachine', + disk_spec: disk.BaseDiskSpec, + disk_count: int, + ): self.vm = vm self.disk_spec = disk_spec self.disk_count = disk_count @@ -99,7 +103,7 @@ def _CreateAndAttachDisk(self) -> None: """Does nothing.""" return - def DiskCreatedOnVMCreation(self)-> bool: + def DiskCreatedOnVMCreation(self) -> bool: return True diff --git a/perfkitbenchmarker/dpb_service.py b/perfkitbenchmarker/dpb_service.py index 129c267fd0..d56718ec47 100644 --- a/perfkitbenchmarker/dpb_service.py +++ b/perfkitbenchmarker/dpb_service.py @@ -290,7 +290,6 @@ def SubmitJob( pass def _WaitForJob(self, job_id, timeout, poll_interval): - if poll_interval is None: poll_interval = FLAGS.dpb_job_poll_interval_secs diff --git a/perfkitbenchmarker/dpb_sparksql_benchmark_helper.py b/perfkitbenchmarker/dpb_sparksql_benchmark_helper.py index 9c7133be8e..630f9fa7bb 100644 --- a/perfkitbenchmarker/dpb_sparksql_benchmark_helper.py +++ b/perfkitbenchmarker/dpb_sparksql_benchmark_helper.py @@ -26,79 +26,105 @@ from perfkitbenchmarker import temp_dir from perfkitbenchmarker import vm_util -BENCHMARK_NAMES = { - 'tpcds_2_4': 'TPC-DS', - 'tpch': 'TPC-H' -} +BENCHMARK_NAMES = {'tpcds_2_4': 'TPC-DS', 'tpch': 'TPC-H'} flags.DEFINE_string( - 'dpb_sparksql_data', None, - 'The HCFS based dataset to run Spark SQL query ' - 'against') -flags.DEFINE_bool('dpb_sparksql_create_hive_tables', False, - 'Whether to load dpb_sparksql_data into external hive tables ' - 'or not.') -flags.DEFINE_bool('dpb_sparksql_simultaneous', False, - 'Run all queries simultaneously instead of one by one. ' - 'Depending on the service type and cluster shape, it might ' - 'fail if too many queries are to be run. This flag and ' - '--dpb_sparksql_streams are mutually exclusive.') + 'dpb_sparksql_data', + None, + 'The HCFS based dataset to run Spark SQL query against', +) +flags.DEFINE_bool( + 'dpb_sparksql_create_hive_tables', + False, + 'Whether to load dpb_sparksql_data into external hive tables or not.', +) +flags.DEFINE_bool( + 'dpb_sparksql_simultaneous', + False, + 'Run all queries simultaneously instead of one by one. ' + 'Depending on the service type and cluster shape, it might ' + 'fail if too many queries are to be run. This flag and ' + '--dpb_sparksql_streams are mutually exclusive.', +) flags.DEFINE_string( - 'dpb_sparksql_database', None, + 'dpb_sparksql_database', + None, 'Name of preprovisioned Hive database to look for data in ' - '(https://spark.apache.org/docs/latest/sql-data-sources-hive-tables.html).') + '(https://spark.apache.org/docs/latest/sql-data-sources-hive-tables.html).', +) flags.DEFINE_string( - 'dpb_sparksql_data_format', None, + 'dpb_sparksql_data_format', + None, "Format of data to load. Assumed to be 'parquet' for HCFS " - "and 'bigquery' for bigquery if unspecified.") + "and 'bigquery' for bigquery if unspecified.", +) flags.DEFINE_string( - 'dpb_sparksql_data_compression', None, + 'dpb_sparksql_data_compression', + None, 'Compression format of the data to load. Since for most formats available ' 'in Spark this may only be specified when writing data, for most cases ' 'this option is a no-op and only serves the purpose of tagging the results ' 'reported by PKB with the appropriate compression format. One notable ' 'exception though is when the dpb_sparksql_copy_to_hdfs flag is passed. In ' 'that case the compression data passed will be used to write data into ' - 'HDFS.') -flags.DEFINE_string('dpb_sparksql_csv_delimiter', ',', - 'CSV delimiter to load the CSV file.') -flags.DEFINE_enum('dpb_sparksql_query', 'tpcds_2_4', BENCHMARK_NAMES.keys(), - 'A list of query to run on dpb_sparksql_data') + 'HDFS.', +) +flags.DEFINE_string( + 'dpb_sparksql_csv_delimiter', ',', 'CSV delimiter to load the CSV file.' +) +flags.DEFINE_enum( + 'dpb_sparksql_query', + 'tpcds_2_4', + BENCHMARK_NAMES.keys(), + 'A list of query to run on dpb_sparksql_data', +) flags.DEFINE_list( - 'dpb_sparksql_order', [], + 'dpb_sparksql_order', + [], 'The names (numbers) of the queries to run in order in a POWER run or all ' 'at the same time in a SIMULTANEOUS run. For DPB SparkSQL benchmarks ' - 'either this flag or --dpb_sparksql_streams must be set.') + 'either this flag or --dpb_sparksql_streams must be set.', +) _STREAMS = flags.DEFINE_multi_string( - 'dpb_sparksql_streams', [], + 'dpb_sparksql_streams', + [], 'List of all query streams to execute for a TPC-DS/H THROUGHPUT run. Each ' 'stream should be passed in separately and the queries should be comma ' 'separated, e.g. --dpb_sparksql_streams=1,2,3 ' '--dpb_sparksql_streams=3,2,1. For DPB SparkSQL benchmarks either this ' - 'flag or --dpb_sparksql_order must be set.') + 'flag or --dpb_sparksql_order must be set.', +) flags.DEFINE_bool( - 'dpb_sparksql_copy_to_hdfs', False, - 'Instead of reading the data directly, copy into HDFS and read from there.') + 'dpb_sparksql_copy_to_hdfs', + False, + 'Instead of reading the data directly, copy into HDFS and read from there.', +) flags.DEFINE_enum( - 'dpb_sparksql_table_cache', None, ['eager', 'lazy'], + 'dpb_sparksql_table_cache', + None, + ['eager', 'lazy'], 'Optionally tell Spark to cache all tables to memory and spilling to disk. ' 'Eager cache will prefetch all tables in lexicographic order. ' 'Lazy will cache tables as they are read. This might have some ' 'counter-intuitive results and Spark reads more data than necessary to ' - "populate it's cache.") + "populate it's cache.", +) _QUERIES_URL = flags.DEFINE_string( - 'dpb_sparksql_queries_url', None, + 'dpb_sparksql_queries_url', + None, 'Object Storage (e.g. GCS or S3) directory URL where the benchmark query ' 'files are contained. Their name must be their query number alone (e.g: ' '"1" or "14a") without any prefix or extension. If omitted, queries will ' 'be fetched from databricks/spark-sql-perf Github repo and use the ' '--dpb_sparksql_query flag to decide whether to get TPC-DS or TPC-H ' - 'queries.') + 'queries.', +) DUMP_SPARK_CONF = flags.DEFINE_bool( - 'dpb_sparksql_dump_spark_conf', False, + 'dpb_sparksql_dump_spark_conf', + False, "Dump job's spark configuration properties to the DPB service's bucket. " - 'For debugging purposes.' + 'For debugging purposes.', ) FLAGS = flags.FLAGS @@ -151,15 +177,18 @@ def GetTableMetadata(benchmark_spec): option_params['header'] = 'true' option_params['delimiter'] = FLAGS.dpb_sparksql_csv_delimiter - metadata[subdir] = (FLAGS.dpb_sparksql_data_format or - 'parquet', option_params) + metadata[subdir] = ( + FLAGS.dpb_sparksql_data_format or 'parquet', + option_params, + ) return metadata def StageMetadata( json_metadata: Any, storage_service: object_storage_service.ObjectStorageService, - staged_file: str): + staged_file: str, +): """Write JSON metadata to object storage.""" # Write computed metadata to object storage. temp_run_dir = temp_dir.GetRunDirPath() @@ -181,11 +210,14 @@ def Prepare(benchmark_spec): cluster = benchmark_spec.dpb_service storage_service = cluster.storage_service benchmark_spec.query_dir = LoadAndStageQueries( - storage_service, cluster.base_dir) + storage_service, cluster.base_dir + ) benchmark_spec.query_streams = GetStreams() scripts_to_upload = [ - SPARK_SQL_DISTCP_SCRIPT, SPARK_TABLE_SCRIPT, SPARK_SQL_RUNNER_SCRIPT + SPARK_SQL_DISTCP_SCRIPT, + SPARK_TABLE_SCRIPT, + SPARK_SQL_RUNNER_SCRIPT, ] + cluster.GetServiceWrapperScriptsToUpload() for script in scripts_to_upload: src_url = data.ResourcePath(script) @@ -199,14 +231,15 @@ def Prepare(benchmark_spec): # GCS will sometimes list the directory itself. if line and line != table_dir: benchmark_spec.table_subdirs.append( - re.split(' |/', line.rstrip('/')).pop()) + re.split(' |/', line.rstrip('/')).pop() + ) benchmark_spec.data_dir = FLAGS.dpb_sparksql_data def LoadAndStageQueries( - storage_service: object_storage_service.ObjectStorageService, - base_dir: str) -> str: + storage_service: object_storage_service.ObjectStorageService, base_dir: str +) -> str: """Loads queries stages them in object storage if needed. Queries are selected using --dpb_sparksql_query and --dpb_sparksql_order. @@ -232,7 +265,8 @@ def LoadAndStageQueries( def _GetQueryFilesFromUrl( storage_service: object_storage_service.ObjectStorageService, - queries_url: str) -> None: + queries_url: str, +) -> None: """Checks if relevant query files from queries_url exist. Args: @@ -240,9 +274,7 @@ def _GetQueryFilesFromUrl( queries_url: Object Storage directory URL where the benchmark queries are contained. """ - query_paths = { - q: os.path.join(queries_url, q) for q in GetQueryIdsToStage() - } + query_paths = {q: os.path.join(queries_url, q) for q in GetQueryIdsToStage()} queries_missing = set() for q in query_paths: try: @@ -251,12 +283,13 @@ def _GetQueryFilesFromUrl( queries_missing.add(q) if queries_missing: raise errors.Benchmarks.PrepareException( - 'Could not find queries {}'.format(', '.join(sorted(queries_missing)))) + 'Could not find queries {}'.format(', '.join(sorted(queries_missing))) + ) def _StageQueriesFromRepo( - storage_service: object_storage_service.ObjectStorageService, - base_dir: str) -> None: + storage_service: object_storage_service.ObjectStorageService, base_dir: str +) -> None: """Copies queries from default Github repo to object storage. Args: @@ -268,10 +301,12 @@ def _StageQueriesFromRepo( # Clone repo vm_util.IssueCommand(['git', 'clone', SPARK_SQL_PERF_GIT, spark_sql_perf_dir]) - vm_util.IssueCommand(['git', 'checkout', SPARK_SQL_PERF_GIT_COMMIT], - cwd=spark_sql_perf_dir) - query_dir = os.path.join(spark_sql_perf_dir, 'src', 'main', 'resources', - FLAGS.dpb_sparksql_query) + vm_util.IssueCommand( + ['git', 'checkout', SPARK_SQL_PERF_GIT_COMMIT], cwd=spark_sql_perf_dir + ) + query_dir = os.path.join( + spark_sql_perf_dir, 'src', 'main', 'resources', FLAGS.dpb_sparksql_query + ) # Search repo for queries query_file = {} # map query -> staged file @@ -292,4 +327,5 @@ def _StageQueriesFromRepo( missing_queries = set(queries_to_stage) - set(query_file.keys()) if missing_queries: raise errors.Benchmarks.PrepareException( - 'Could not find queries {}'.format(missing_queries)) + 'Could not find queries {}'.format(missing_queries) + ) diff --git a/perfkitbenchmarker/edw_benchmark_results_aggregator.py b/perfkitbenchmarker/edw_benchmark_results_aggregator.py index 8a167f2950..5e4365da78 100644 --- a/perfkitbenchmarker/edw_benchmark_results_aggregator.py +++ b/perfkitbenchmarker/edw_benchmark_results_aggregator.py @@ -11,6 +11,7 @@ f. Raw geo mean performance for each iteration g. Aggregated geo mean performance using the aggregated query performances """ + import copy import enum import functools @@ -23,9 +24,12 @@ from perfkitbenchmarker import sample -flags.DEFINE_bool('edw_generate_aggregated_metrics', True, - 'Whether the benchmark generates aggregated_metrics such as ' - 'geomean. Query performance metrics are still generated.') +flags.DEFINE_bool( + 'edw_generate_aggregated_metrics', + True, + 'Whether the benchmark generates aggregated_metrics such as ' + 'geomean. Query performance metrics are still generated.', +) FLAGS = flags.FLAGS @@ -47,7 +51,7 @@ def geometric_mean(iterable: List[float]) -> float: EdwPerformanceAggregationError: If an invalid performance value was included for aggregation. """ - if (not iterable or any(perf <= 0.0 for perf in iterable)): + if not iterable or any(perf <= 0.0 for perf in iterable): raise EdwPerformanceAggregationError('Invalid values cannot be aggregated.') a = np.array(iterable) return a.prod() ** (1.0 / len(a)) @@ -60,6 +64,7 @@ class EdwQueryExecutionStatus(enum.Enum): FAILED: Indicates that the query execution failed. SUCCESSFUL: Indicates that the query execution succeeded. """ + FAILED = 'query_execution_failed' SUCCESSFUL = 'query_execution_successful' @@ -70,20 +75,23 @@ class EdwQueryPerformance(object): Attributes: name: A string name of the query that was executed performance: A Float variable set to the query's completion time in secs. - -1.0 is used as a sentinel value implying the query failed. For a successful - query the value is expected to be positive. + -1.0 is used as a sentinel value implying the query failed. For a + successful query the value is expected to be positive. execution_status: An EdwQueryExecutionStatus enum indicating success/failure metadata: A dictionary of query execution attributes (job_id, etc.) """ - def __init__(self, query_name: Text, performance: float, - metadata: Dict[str, str]): + def __init__( + self, query_name: Text, performance: float, metadata: Dict[str, str] + ): # TODO(user): add query start and query end as attributes. self.name = query_name self.performance = performance - self.execution_status = (EdwQueryExecutionStatus.FAILED - if performance == -1.0 - else EdwQueryExecutionStatus.SUCCESSFUL) + self.execution_status = ( + EdwQueryExecutionStatus.FAILED + if performance == -1.0 + else EdwQueryExecutionStatus.SUCCESSFUL + ) self.metadata = metadata @classmethod @@ -108,16 +116,18 @@ def from_json(cls, serialized_performance: str): metadata = {} if results['query_wall_time_in_secs'] == -1: logging.warning('Query %s failed.', results['query']) - return cls(query_name=results['query'], - performance=results['query_wall_time_in_secs'], - metadata=metadata) + return cls( + query_name=results['query'], + performance=results['query_wall_time_in_secs'], + metadata=metadata, + ) def get_performance_sample(self, metadata: Dict[str, str]) -> sample.Sample: """Method to generate a sample for the query performance. Args: metadata: A dictionary of execution attributes to be merged with the query - execution attributes, for eg. tpc suite, scale of dataset, etc. + execution attributes, for eg. tpc suite, scale of dataset, etc. Returns: A sample for the edw query performance. @@ -126,8 +136,9 @@ def get_performance_sample(self, metadata: Dict[str, str]) -> sample.Sample: query_metadata['query'] = self.name query_metadata['execution_status'] = self.execution_status query_metadata.update(self.metadata) - return sample.Sample('edw_raw_query_time', self.performance, 'seconds', - query_metadata) + return sample.Sample( + 'edw_raw_query_time', self.performance, 'seconds', query_metadata + ) def get_performance_value(self) -> float: """Method to get the query's completion time in secs. @@ -179,8 +190,9 @@ def __init__(self, iteration_id: Text, total_queries: int): self.end_time = -1 self.wall_time: float = 0.0 - def add_query_performance(self, query_name: Text, performance: float, - metadata: Dict[str, str]): + def add_query_performance( + self, query_name: Text, performance: float, metadata: Dict[str, str] + ): """Creates and populates a query performance from the input results. Updates the iteration's performance map with the query performance. @@ -201,12 +213,14 @@ def add_query_performance(self, query_name: Text, performance: float, """ query_metadata = copy.copy(metadata) query_performance = EdwQueryPerformance( - query_name=query_name, performance=performance, metadata=query_metadata) + query_name=query_name, performance=performance, metadata=query_metadata + ) if query_performance.name in self.performance: - raise EdwPerformanceAggregationError('Attempting to aggregate a ' - 'duplicate query: %s.' % - query_performance.name) + raise EdwPerformanceAggregationError( + 'Attempting to aggregate a duplicate query: %s.' + % query_performance.name + ) self.performance[query_performance.name] = query_performance if query_performance.is_successful(): self.successful_count += 1 @@ -258,8 +272,9 @@ def get_query_metadata(self, query_name: Text) -> Dict[str, Any]: EdwPerformanceAggregationError: If the query failed. """ if not self.is_query_successful(query_name): - raise EdwPerformanceAggregationError('Cannot aggregate invalid / failed' - ' query' + query_name) + raise EdwPerformanceAggregationError( + 'Cannot aggregate invalid / failed query' + query_name + ) return self.performance.get(query_name).metadata def get_all_queries_in_iteration(self) -> List[Text]: @@ -271,7 +286,8 @@ def get_all_queries_in_iteration(self) -> List[Text]: return self.performance.keys() def get_all_query_performance_samples( - self, metadata: Dict[str, str]) -> List[sample.Sample]: + self, metadata: Dict[str, str] + ) -> List[sample.Sample]: """Gets a list of samples for all queries in the iteration. Args: @@ -291,7 +307,7 @@ def add_start_time(self, start_time: int): Args: start_time: The UNIX timestamp, in milliseconds, at which the iteration - was started + was started """ self.start_time = start_time @@ -300,7 +316,7 @@ def add_end_time(self, end_time: int): Args: end_time: The UNIX timestamp, in milliseconds, at which the iteration - completed + completed """ self.end_time = end_time @@ -314,8 +330,9 @@ def get_wall_time(self) -> float: """ return self.wall_time - def get_wall_time_performance_sample(self, metadata: Dict[ - str, str]) -> sample.Sample: + def get_wall_time_performance_sample( + self, metadata: Dict[str, str] + ) -> sample.Sample: """Gets a sample for wall time performance of the iteration. Args: @@ -328,13 +345,15 @@ def get_wall_time_performance_sample(self, metadata: Dict[ wall_time_metadata = copy.copy(metadata) wall_time_metadata['iteration_start_time'] = self.start_time wall_time_metadata['iteration_end_time'] = self.end_time - return sample.Sample('edw_iteration_wall_time', self.wall_time, 'seconds', - wall_time_metadata) + return sample.Sample( + 'edw_iteration_wall_time', self.wall_time, 'seconds', wall_time_metadata + ) def is_successful(self, expected_queries: List[Text]) -> bool: """Check if all the expected queries ran and all succeeded.""" - all_queries_ran = set( - self.get_all_queries_in_iteration()) == set(expected_queries) + all_queries_ran = set(self.get_all_queries_in_iteration()) == set( + expected_queries + ) all_queries_were_successful = self.total_count == self.successful_count return all_queries_ran and all_queries_were_successful @@ -354,8 +373,8 @@ def get_queries_geomean(self) -> float: ]) def get_queries_geomean_performance_sample( - self, expected_queries: List[Text], metadata: Dict[str, - str]) -> sample.Sample: + self, expected_queries: List[Text], metadata: Dict[str, str] + ) -> sample.Sample: """Gets a sample for geomean of all queries in the iteration. Args: @@ -375,8 +394,9 @@ def get_queries_geomean_performance_sample( raise EdwPerformanceAggregationError('Failed executions in iteration.') raw_geo_mean = self.get_queries_geomean() geo_mean_metadata = copy.copy(metadata) - return sample.Sample('edw_iteration_geomean_time', raw_geo_mean, 'seconds', - geo_mean_metadata) + return sample.Sample( + 'edw_iteration_geomean_time', raw_geo_mean, 'seconds', geo_mean_metadata + ) class EdwSimultaneousIterationPerformance(EdwBaseIterationPerformance): @@ -392,10 +412,15 @@ class EdwSimultaneousIterationPerformance(EdwBaseIterationPerformance): all_queries_succeeded: Whether all queries in the iteration were successful. """ - def __init__(self, iteration_id: Text, iteration_start_time: int, - iteration_end_time: int, iteration_wall_time: float, - iteration_performance: Dict[str, EdwQueryPerformance], - all_queries_succeeded: bool): + def __init__( + self, + iteration_id: Text, + iteration_start_time: int, + iteration_end_time: int, + iteration_wall_time: float, + iteration_performance: Dict[str, EdwQueryPerformance], + all_queries_succeeded: bool, + ): self.id = iteration_id self.start_time = iteration_start_time self.end_time = iteration_end_time @@ -429,19 +454,23 @@ def from_json(cls, iteration_id: str, serialized_performance: str): if all_queries_succeeded: for query_perf_json in results['all_queries_performance_array']: query_perf = EdwQueryPerformance.from_json( - serialized_performance=(json.dumps(query_perf_json))) + serialized_performance=(json.dumps(query_perf_json)) + ) query_performance_map[query_perf.name] = query_perf else: logging.warning('Failure reported. Reason: %s', results['failure_reason']) return cls( iteration_id=iteration_id, - iteration_start_time=(results['simultaneous_start'] - if all_queries_succeeded else -1), - iteration_end_time=(results['simultaneous_end'] - if all_queries_succeeded else -1), + iteration_start_time=( + results['simultaneous_start'] if all_queries_succeeded else -1 + ), + iteration_end_time=( + results['simultaneous_end'] if all_queries_succeeded else -1 + ), iteration_wall_time=results['simultaneous_wall_time_in_secs'], iteration_performance=query_performance_map, - all_queries_succeeded=all_queries_succeeded) + all_queries_succeeded=all_queries_succeeded, + ) def get_wall_time(self) -> float: """Gets the total wall time, in seconds, for the iteration. @@ -454,8 +483,9 @@ def get_wall_time(self) -> float: """ return self.wall_time - def get_wall_time_performance_sample(self, metadata: Dict[ - str, str]) -> sample.Sample: + def get_wall_time_performance_sample( + self, metadata: Dict[str, str] + ) -> sample.Sample: """Gets a sample for wall time performance of the iteration. Args: @@ -469,11 +499,13 @@ def get_wall_time_performance_sample(self, metadata: Dict[ wall_time_metadata = copy.copy(metadata) wall_time_metadata['iteration_start_time'] = self.start_time wall_time_metadata['iteration_end_time'] = self.end_time - return sample.Sample('edw_iteration_wall_time', wall_time, 'seconds', - wall_time_metadata) + return sample.Sample( + 'edw_iteration_wall_time', wall_time, 'seconds', wall_time_metadata + ) def get_all_query_performance_samples( - self, metadata: Dict[str, str]) -> List[sample.Sample]: + self, metadata: Dict[str, str] + ) -> List[sample.Sample]: """Gets a list of samples for all queries in the iteration. Args: @@ -542,8 +574,9 @@ def get_query_metadata(self, query_name: Text) -> Dict[str, Any]: EdwPerformanceAggregationError: If the query failed in the iteration. """ if not self.is_query_successful(query_name): - raise EdwPerformanceAggregationError('Cannot aggregate invalid / failed' - ' query' + query_name) + raise EdwPerformanceAggregationError( + 'Cannot aggregate invalid / failed query' + query_name + ) return self.performance.get(query_name).metadata def get_queries_geomean(self) -> float: @@ -562,8 +595,8 @@ def get_queries_geomean(self) -> float: ]) def get_queries_geomean_performance_sample( - self, expected_queries: List[Text], metadata: Dict[str, - str]) -> sample.Sample: + self, expected_queries: List[Text], metadata: Dict[str, str] + ) -> sample.Sample: """Gets a sample for geomean of all queries in the iteration. Args: @@ -583,8 +616,9 @@ def get_queries_geomean_performance_sample( raise EdwPerformanceAggregationError('Failed executions in iteration.') raw_geo_mean = self.get_queries_geomean() geo_mean_metadata = copy.copy(metadata) - return sample.Sample('edw_iteration_geomean_time', raw_geo_mean, 'seconds', - geo_mean_metadata) + return sample.Sample( + 'edw_iteration_geomean_time', raw_geo_mean, 'seconds', geo_mean_metadata + ) class EdwThroughputIterationPerformance(EdwBaseIterationPerformance): @@ -600,10 +634,14 @@ class EdwThroughputIterationPerformance(EdwBaseIterationPerformance): EdwQueryPerformance instances. """ - def __init__(self, iteration_id: Text, iteration_start_time: int, - iteration_end_time: int, iteration_wall_time: float, - iteration_performance: Dict[str, Dict[str, - EdwQueryPerformance]]): + def __init__( + self, + iteration_id: Text, + iteration_start_time: int, + iteration_end_time: int, + iteration_wall_time: float, + iteration_performance: Dict[str, Dict[str, EdwQueryPerformance]], + ): self.id = iteration_id self.start_time = iteration_start_time self.end_time = iteration_end_time @@ -652,24 +690,29 @@ def from_json(cls, iteration_id: str, serialized_performance: str): all_queries_succeeded = 'failure_reason' not in results if all_queries_succeeded: for stream_id, stream_perf_json in enumerate( - results['all_streams_performance_array']): + results['all_streams_performance_array'] + ): stream_id = str(stream_id) stream_performance_map = {} for query_perf_json in stream_perf_json['stream_performance_array']: query_perf = EdwQueryPerformance.from_json( - serialized_performance=(json.dumps(query_perf_json))) + serialized_performance=(json.dumps(query_perf_json)) + ) stream_performance_map[query_perf.name] = query_perf stream_performances.update({stream_id: stream_performance_map}) else: logging.warning('Failure reported. Reason: %s', results['failure_reason']) return cls( iteration_id=iteration_id, - iteration_start_time=(results['throughput_start'] - if all_queries_succeeded else -1), - iteration_end_time=(results['throughput_end'] - if all_queries_succeeded else -1), + iteration_start_time=( + results['throughput_start'] if all_queries_succeeded else -1 + ), + iteration_end_time=( + results['throughput_end'] if all_queries_succeeded else -1 + ), iteration_wall_time=results['throughput_wall_time_in_secs'], - iteration_performance=stream_performances) + iteration_performance=stream_performances, + ) def has_query_performance(self, query_name: Text) -> bool: """Returns whether the query was run at least once in the iteration. @@ -742,7 +785,8 @@ def get_query_metadata(self, query_name: Text) -> Dict[str, Any]: return result def get_all_query_performance_samples( - self, metadata: Dict[str, str]) -> List[sample.Sample]: + self, metadata: Dict[str, str] + ) -> List[sample.Sample]: """Gets a list of samples for all queries in all streams of the iteration. Args: @@ -763,7 +807,8 @@ def get_all_query_performance_samples( return all_query_performances def all_streams_ran_all_expected_queries( - self, expected_queries: List[Text]) -> bool: + self, expected_queries: List[Text] + ) -> bool: """Checks that the same set of expected queries ran in all streams.""" for stream in self.performance.values(): if set(stream.keys()) != set(expected_queries): @@ -803,11 +848,16 @@ def is_successful(self, expected_queries: List[Text]) -> bool: """ non_zero_streams = len(self.performance) >= 1 all_streams_ran_all_queries = self.all_streams_ran_all_expected_queries( - expected_queries) + expected_queries + ) no_duplicate_queries = self.no_duplicate_queries() all_queries_succeeded = self.all_queries_succeeded() - return (non_zero_streams and all_streams_ran_all_queries and - no_duplicate_queries and all_queries_succeeded) + return ( + non_zero_streams + and all_streams_ran_all_queries + and no_duplicate_queries + and all_queries_succeeded + ) def get_queries_geomean(self) -> float: """Gets the geometric mean of all queries in all streams of the iteration. @@ -827,8 +877,8 @@ def get_queries_geomean(self) -> float: return geometric_mean(query_performances) def get_queries_geomean_performance_sample( - self, expected_queries: List[Text], metadata: Dict[str, - str]) -> sample.Sample: + self, expected_queries: List[Text], metadata: Dict[str, str] + ) -> sample.Sample: """Gets a sample for geomean of all queries in all streams of the iteration. Args: @@ -848,8 +898,9 @@ def get_queries_geomean_performance_sample( raise EdwPerformanceAggregationError('Failed executions in iteration.') raw_geo_mean = self.get_queries_geomean() geo_mean_metadata = copy.copy(metadata) - return sample.Sample('edw_iteration_geomean_time', raw_geo_mean, 'seconds', - geo_mean_metadata) + return sample.Sample( + 'edw_iteration_geomean_time', raw_geo_mean, 'seconds', geo_mean_metadata + ) def get_wall_time(self) -> float: """Gets the total wall time, in seconds, for the iteration. @@ -863,7 +914,8 @@ def get_wall_time(self) -> float: return self.wall_time def get_wall_time_performance_sample( - self, metadata: Dict[str, str]) -> sample.Sample: + self, metadata: Dict[str, str] + ) -> sample.Sample: """Gets a sample for total wall time performance of the iteration. Args: @@ -876,8 +928,9 @@ def get_wall_time_performance_sample( wall_time_metadata = copy.copy(metadata) wall_time_metadata['iteration_start_time'] = self.start_time wall_time_metadata['iteration_end_time'] = self.end_time - return sample.Sample('edw_iteration_wall_time', self.wall_time, 'seconds', - wall_time_metadata) + return sample.Sample( + 'edw_iteration_wall_time', self.wall_time, 'seconds', wall_time_metadata + ) class EdwBenchmarkPerformance(object): @@ -908,16 +961,20 @@ def add_iteration_performance(self, performance: EdwBaseIterationPerformance): """ iteration_id = performance.id if iteration_id in self.iteration_performances: - raise EdwPerformanceAggregationError('Attempting to aggregate a duplicate' - ' iteration: %s.' % iteration_id) + raise EdwPerformanceAggregationError( + 'Attempting to aggregate a duplicate iteration: %s.' % iteration_id + ) self.iteration_performances[iteration_id] = performance def is_successful(self) -> bool: """Check a benchmark's success, only if all the iterations succeed.""" - return functools.reduce((lambda x, y: x and y), [ - iteration_performance.is_successful(self.expected_queries) - for iteration_performance in self.iteration_performances.values() - ]) + return functools.reduce( + (lambda x, y: x and y), + [ + iteration_performance.is_successful(self.expected_queries) + for iteration_performance in self.iteration_performances.values() + ], + ) def aggregated_query_status(self, query_name: Text) -> bool: """Gets the status of query aggregated across all iterations. @@ -953,8 +1010,9 @@ def aggregated_query_execution_time(self, query_name: Text) -> float: iterations """ if not self.aggregated_query_status(query_name): - raise EdwPerformanceAggregationError('Cannot aggregate invalid / failed ' - 'query ' + query_name) + raise EdwPerformanceAggregationError( + 'Cannot aggregate invalid / failed query ' + query_name + ) query_performances = [ iteration_performance.get_query_performance(query_name) for iteration_performance in self.iteration_performances.values() @@ -976,20 +1034,25 @@ def aggregated_query_metadata(self, query_name: Text) -> Dict[str, Any]: iterations """ if not self.aggregated_query_status(query_name): - raise EdwPerformanceAggregationError('Cannot aggregate invalid / failed ' - 'query ' + query_name) + raise EdwPerformanceAggregationError( + 'Cannot aggregate invalid / failed query ' + query_name + ) result = {} - for iteration_id, iteration_performance in ( - self.iteration_performances.items()): + for ( + iteration_id, + iteration_performance, + ) in self.iteration_performances.items(): result.update({ iteration_id + '_' + k: v for (k, v) in iteration_performance.get_query_metadata( - query_name).items() + query_name + ).items() }) return result def get_aggregated_query_performance_sample( - self, query_name: Text, metadata: Dict[str, str]) -> sample.Sample: + self, query_name: Text, metadata: Dict[str, str] + ) -> sample.Sample: """Gets the performance of query aggregated across all iterations. Args: @@ -1010,11 +1073,13 @@ def get_aggregated_query_performance_sample( agg_md = self.aggregated_query_metadata(query_name=query_name) query_metadata['execution_status'] = exec_status query_metadata.update(agg_md) - return sample.Sample('edw_aggregated_query_time', perf, 'seconds', - query_metadata) + return sample.Sample( + 'edw_aggregated_query_time', perf, 'seconds', query_metadata + ) - def get_all_query_performance_samples(self, metadata: Dict[str, str]) -> List[ - sample.Sample]: + def get_all_query_performance_samples( + self, metadata: Dict[str, str] + ) -> List[sample.Sample]: """Generates samples for all query performances. Benchmark relies on iteration runs to generate the raw query performance @@ -1033,17 +1098,21 @@ def get_all_query_performance_samples(self, metadata: Dict[str, str]) -> List[ for iteration, performance in self.iteration_performances.items(): iteration_metadata = copy.copy(metadata) iteration_metadata['iteration'] = iteration - results.extend(performance.get_all_query_performance_samples( - iteration_metadata)) + results.extend( + performance.get_all_query_performance_samples(iteration_metadata) + ) # Aggregated query performance samples for query in self.expected_queries: - results.append(self.get_aggregated_query_performance_sample( - query_name=query, metadata=metadata)) + results.append( + self.get_aggregated_query_performance_sample( + query_name=query, metadata=metadata + ) + ) return results - def get_aggregated_wall_time_performance_sample(self, - metadata: Dict[str, str] - ) -> sample.Sample: + def get_aggregated_wall_time_performance_sample( + self, metadata: Dict[str, str] + ) -> sample.Sample: """Gets the wall time performance aggregated across all iterations. Args: @@ -1060,8 +1129,12 @@ def get_aggregated_wall_time_performance_sample(self, aggregated_wall_time = sum(wall_times) / self.total_iterations wall_time_metadata = copy.copy(metadata) wall_time_metadata['aggregation_method'] = 'mean' - return sample.Sample('edw_aggregated_wall_time', aggregated_wall_time, - 'seconds', wall_time_metadata) + return sample.Sample( + 'edw_aggregated_wall_time', + aggregated_wall_time, + 'seconds', + wall_time_metadata, + ) def get_wall_time_performance_samples(self, metadata: Dict[str, str]): """Generates samples for all wall time performances. @@ -1082,16 +1155,17 @@ def get_wall_time_performance_samples(self, metadata: Dict[str, str]): for iteration, performance in self.iteration_performances.items(): iteration_metadata = copy.copy(metadata) iteration_metadata['iteration'] = iteration - results.append(performance.get_wall_time_performance_sample( - iteration_metadata)) - results.append(self.get_aggregated_wall_time_performance_sample( - metadata=metadata)) + results.append( + performance.get_wall_time_performance_sample(iteration_metadata) + ) + results.append( + self.get_aggregated_wall_time_performance_sample(metadata=metadata) + ) return results - def get_aggregated_geomean_performance_sample(self, - metadata: - Dict[str, - str]) -> sample.Sample: + def get_aggregated_geomean_performance_sample( + self, metadata: Dict[str, str] + ) -> sample.Sample: """Gets the geomean performance aggregated across all iterations. Args: @@ -1115,11 +1189,16 @@ def get_aggregated_geomean_performance_sample(self, geomean_metadata = copy.copy(metadata) geomean_metadata['intra_query_aggregation_method'] = 'mean' geomean_metadata['inter_query_aggregation_method'] = 'geomean' - return sample.Sample('edw_aggregated_geomean', aggregated_geo_mean, - 'seconds', geomean_metadata) - - def get_queries_geomean_performance_samples(self, metadata: Dict[str, str] - ) -> List[sample.Sample]: + return sample.Sample( + 'edw_aggregated_geomean', + aggregated_geo_mean, + 'seconds', + geomean_metadata, + ) + + def get_queries_geomean_performance_samples( + self, metadata: Dict[str, str] + ) -> List[sample.Sample]: """Generates samples for all geomean performances. Benchmark relies on iteration runs to generate the raw geomean performance @@ -1146,8 +1225,11 @@ def get_queries_geomean_performance_samples(self, metadata: Dict[str, str] iteration_metadata['iteration'] = iteration results.append( performance.get_queries_geomean_performance_sample( - self.expected_queries, iteration_metadata)) + self.expected_queries, iteration_metadata + ) + ) - results.append(self.get_aggregated_geomean_performance_sample( - metadata=metadata)) + results.append( + self.get_aggregated_geomean_performance_sample(metadata=metadata) + ) return results diff --git a/perfkitbenchmarker/edw_service.py b/perfkitbenchmarker/edw_service.py index 522e195abc..580af6319b 100644 --- a/perfkitbenchmarker/edw_service.py +++ b/perfkitbenchmarker/edw_service.py @@ -22,86 +22,150 @@ from absl import flags from perfkitbenchmarker import resource -flags.DEFINE_integer('edw_service_cluster_concurrency', 5, - 'Number of queries to run concurrently on the cluster.') -flags.DEFINE_string('edw_service_cluster_snapshot', None, - 'If set, the snapshot to restore as cluster.') -flags.DEFINE_string('edw_service_cluster_identifier', None, - 'If set, the preprovisioned edw cluster.') -flags.DEFINE_string('edw_service_endpoint', None, - 'If set, the preprovisioned edw cluster endpoint.') +flags.DEFINE_integer( + 'edw_service_cluster_concurrency', + 5, + 'Number of queries to run concurrently on the cluster.', +) flags.DEFINE_string( - 'edw_service_cluster_db', None, + 'edw_service_cluster_snapshot', + None, + 'If set, the snapshot to restore as cluster.', +) +flags.DEFINE_string( + 'edw_service_cluster_identifier', + None, + 'If set, the preprovisioned edw cluster.', +) +flags.DEFINE_string( + 'edw_service_endpoint', + None, + 'If set, the preprovisioned edw cluster endpoint.', +) +flags.DEFINE_string( + 'edw_service_cluster_db', + None, 'If set, the db on cluster to use during the benchmark (' - 'only applicable when using snapshots).') + 'only applicable when using snapshots).', +) flags.DEFINE_string( - 'edw_service_cluster_user', None, + 'edw_service_cluster_user', + None, 'If set, the user authorized on cluster (only applicable ' - 'when using snapshots).') + 'when using snapshots).', +) flags.DEFINE_string( - 'edw_service_cluster_password', None, + 'edw_service_cluster_password', + None, 'If set, the password authorized on cluster (only ' - 'applicable when using snapshots).') -flags.DEFINE_string('snowflake_snowsql_config_override_file', None, - 'The SnowSQL configuration to use.' - 'https://docs.snowflake.net/manuals/user-guide/snowsql-config.html#snowsql-config-file') # pylint: disable=line-too-long -flags.DEFINE_string('snowflake_connection', None, - 'Named Snowflake connection defined in SnowSQL config file.' - 'https://docs.snowflake.net/manuals/user-guide/snowsql-start.html#using-named-connections') # pylint: disable=line-too-long -flags.DEFINE_integer('edw_suite_iterations', 1, - 'Number of suite iterations to perform.') -flags.DEFINE_integer('edw_suite_warmup_iterations', None, - 'Number of suite warmup iterations to perform.') + 'applicable when using snapshots).', +) +flags.DEFINE_string( + 'snowflake_snowsql_config_override_file', + None, + 'The SnowSQL configuration to use.' + 'https://docs.snowflake.net/manuals/user-guide/snowsql-config.html#snowsql-config-file', +) # pylint: disable=line-too-long +flags.DEFINE_string( + 'snowflake_connection', + None, + 'Named Snowflake connection defined in SnowSQL config file.' + 'https://docs.snowflake.net/manuals/user-guide/snowsql-start.html#using-named-connections', +) # pylint: disable=line-too-long +flags.DEFINE_integer( + 'edw_suite_iterations', 1, 'Number of suite iterations to perform.' +) +flags.DEFINE_integer( + 'edw_suite_warmup_iterations', + None, + 'Number of suite warmup iterations to perform.', +) # TODO(user): Revisit flags for accepting query lists. -flags.DEFINE_string('edw_simultaneous_queries', None, - 'CSV list of simultaneous queries to benchmark.') -flags.DEFINE_integer('edw_simultaneous_query_submission_interval', '0', - 'Simultaneous query submission interval in milliseconds.') -flags.DEFINE_string('edw_power_queries', None, - 'CSV list of power queries to benchmark.') +flags.DEFINE_string( + 'edw_simultaneous_queries', + None, + 'CSV list of simultaneous queries to benchmark.', +) +flags.DEFINE_integer( + 'edw_simultaneous_query_submission_interval', + '0', + 'Simultaneous query submission interval in milliseconds.', +) +flags.DEFINE_string( + 'edw_power_queries', None, 'CSV list of power queries to benchmark.' +) flags.DEFINE_multi_string( - 'concurrency_streams', [], 'List of all query streams to execute. Each ' + 'concurrency_streams', + [], + 'List of all query streams to execute. Each ' 'stream should be passed in separately and the queries should be comma ' - 'separated, e.g. --concurrency_streams=1,2,3 --concurrency_streams=3,2,1') -flags.DEFINE_string('snowflake_warehouse', None, - 'A virtual warehouse, often referred to simply as a - ' - 'warehouse, is a cluster of compute in Snowflake. ' - 'https://docs.snowflake.com/en/user-guide/warehouses.html') # pylint: disable=line-too-long + 'separated, e.g. --concurrency_streams=1,2,3 --concurrency_streams=3,2,1', +) flags.DEFINE_string( - 'snowflake_database', None, - 'The hosted snowflake database to use during the benchmark.') + 'snowflake_warehouse', + None, + 'A virtual warehouse, often referred to simply as a - ' + 'warehouse, is a cluster of compute in Snowflake. ' + 'https://docs.snowflake.com/en/user-guide/warehouses.html', +) # pylint: disable=line-too-long flags.DEFINE_string( - 'snowflake_schema', None, - 'The schema of the hosted snowflake database to use during the benchmark.') + 'snowflake_database', + None, + 'The hosted snowflake database to use during the benchmark.', +) +flags.DEFINE_string( + 'snowflake_schema', + None, + 'The schema of the hosted snowflake database to use during the benchmark.', +) flags.DEFINE_enum( - 'snowflake_client_interface', 'JDBC', ['JDBC'], - 'The Runtime Interface used when interacting with Snowflake.') + 'snowflake_client_interface', + 'JDBC', + ['JDBC'], + 'The Runtime Interface used when interacting with Snowflake.', +) FLAGS = flags.FLAGS -TYPE_2_PROVIDER = dict([('athena', 'aws'), ('redshift', 'aws'), - ('spectrum', 'aws'), ('snowflake_aws', 'aws'), - ('snowflake_azure', 'azure'), - ('snowflakeexternal_aws', 'aws'), - ('snowflakeexternal_azure', 'azure'), - ('bigquery', 'gcp'), ('endor', 'gcp'), - ('endorazure', 'gcp'), ('bqfederated', 'gcp'), - ('azuresqldatawarehouse', 'azure')]) +TYPE_2_PROVIDER = dict([ + ('athena', 'aws'), + ('redshift', 'aws'), + ('spectrum', 'aws'), + ('snowflake_aws', 'aws'), + ('snowflake_azure', 'azure'), + ('snowflakeexternal_aws', 'aws'), + ('snowflakeexternal_azure', 'azure'), + ('bigquery', 'gcp'), + ('endor', 'gcp'), + ('endorazure', 'gcp'), + ('bqfederated', 'gcp'), + ('azuresqldatawarehouse', 'azure'), +]) TYPE_2_MODULE = dict([ ('athena', 'perfkitbenchmarker.providers.aws.athena'), ('redshift', 'perfkitbenchmarker.providers.aws.redshift'), ('spectrum', 'perfkitbenchmarker.providers.aws.spectrum'), ('snowflake_aws', 'perfkitbenchmarker.providers.aws.snowflake_aws'), - ('snowflake_azure', 'perfkitbenchmarker.providers.azure.snowflake_azure'), - ('snowflakeexternal_aws', 'perfkitbenchmarker.providers.aws.snowflake_aws'), - ('snowflakeexternal_azure', - 'perfkitbenchmarker.providers.azure.snowflake_azure'), + ( + 'snowflake_azure', + 'perfkitbenchmarker.providers.azure.snowflake_azure', + ), + ( + 'snowflakeexternal_aws', + 'perfkitbenchmarker.providers.aws.snowflake_aws', + ), + ( + 'snowflakeexternal_azure', + 'perfkitbenchmarker.providers.azure.snowflake_azure', + ), ('bigquery', 'perfkitbenchmarker.providers.gcp.bigquery'), ('endor', 'perfkitbenchmarker.providers.gcp.bigquery'), ('endorazure', 'perfkitbenchmarker.providers.gcp.bigquery'), ('bqfederated', 'perfkitbenchmarker.providers.gcp.bigquery'), - ('azuresqldatawarehouse', - 'perfkitbenchmarker.providers.azure.azure_sql_data_warehouse') + ( + 'azuresqldatawarehouse', + 'perfkitbenchmarker.providers.azure.azure_sql_data_warehouse', + ), ]) DEFAULT_NUMBER_OF_NODES = 1 # The order of stages is important to the successful lifecycle completion. @@ -132,8 +196,9 @@ def __init__(self): def SetProvisionedAttributes(self, benchmark_spec): """Sets any attributes that were unknown during initialization.""" self.client_vm = benchmark_spec.vms[0] - self.client_vm.RemoteCommand('echo "\nMaxSessions 100" | ' - 'sudo tee -a /etc/ssh/sshd_config') + self.client_vm.RemoteCommand( + 'echo "\nMaxSessions 100" | sudo tee -a /etc/ssh/sshd_config' + ) def Prepare(self, package_name: Text) -> None: """Prepares the client vm to execute query. @@ -162,8 +227,9 @@ def ExecuteQuery(self, query_name: Text) -> Tuple[float, Dict[str, str]]: """ raise NotImplementedError - def ExecuteSimultaneous(self, submission_interval: int, - queries: List[str]) -> str: + def ExecuteSimultaneous( + self, submission_interval: int, queries: List[str] + ) -> str: """Executes queries simultaneously on client and return performance details. Simultaneous app expects queries as white space separated query file names. @@ -318,7 +384,7 @@ def GetMetadata(self): 'edw_service_type': self.spec.type, 'edw_cluster_identifier': self.cluster_identifier, 'edw_cluster_node_type': self.node_type, - 'edw_cluster_node_count': self.node_count + 'edw_cluster_node_count': self.node_count, } return basic_data @@ -332,7 +398,8 @@ def GenerateLifecycleStageScriptName(self, lifecycle_stage): script name for implementing the argument lifecycle_stage. """ return os.path.basename( - os.path.normpath('database_%s.sql' % lifecycle_stage)) + os.path.normpath('database_%s.sql' % lifecycle_stage) + ) def Cleanup(self): """Cleans up any temporary resources created for the service.""" @@ -342,11 +409,9 @@ def GetDatasetLastUpdatedTime(self, dataset=None): """Get the formatted last modified timestamp of the dataset.""" raise NotImplementedError - def ExtractDataset(self, - dest_bucket, - dataset=None, - tables=None, - dest_format='CSV'): + def ExtractDataset( + self, dest_bucket, dataset=None, tables=None, dest_format='CSV' + ): """Extract all tables in a dataset to object storage. Args: diff --git a/perfkitbenchmarker/errors.py b/perfkitbenchmarker/errors.py index a1373e86a0..8322ba4d17 100644 --- a/perfkitbenchmarker/errors.py +++ b/perfkitbenchmarker/errors.py @@ -26,34 +26,42 @@ class Setup(object): class PythonPackageRequirementUnfulfilled(Error): """Error raised when a Python package requirement is unfulfilled.""" + pass class MissingExecutableError(Error): """Error raised when we cannot find an executable we need.""" + pass class NoRunURIError(Error): """Error raised when we were not given a run_uri and cannot infer it.""" + pass class BadRunURIError(Error): """Error raised when the given run_uri is invalid.""" + pass class BadPreprovisionedDataError(Error): """Error raised when the pre-provisioned data is invalid.""" + pass class InvalidSetupError(Error): """Error raised when SetUpPKB was not called correctly.""" + pass class InvalidFlagConfigurationError(Error): """Error raised when the set of command line flags is invalid.""" + pass class InvalidConfigurationError(Error): """Error raised when configuration is invalid.""" + pass @@ -62,6 +70,7 @@ class VirtualMachine(object): class RemoteCommandError(Error): """Error raised when a Remote Command or Remote Copy fails.""" + pass class RemoteExceptionError(Error): @@ -69,6 +78,7 @@ class RemoteExceptionError(Error): class AuthError(Error): """Error raised when one VM cannot access another VM.""" + pass class VirtualMachineError(Error): @@ -80,7 +90,7 @@ def FromDebugInfo(cls, info, error_message): Args: info: A dictionary containing debug information (such as traceroute - info). + info). error_message: the error message from the originating code. Returns: @@ -91,10 +101,12 @@ def FromDebugInfo(cls, info, error_message): """ if isinstance(info, dict): info = VirtualMachine.VirtualMachineError.FormatDebugInfo( - info, error_message) + info, error_message + ) return cls(info) - raise TypeError('The argument of FromDebugInfo should be an instance ' - 'of dictionary.') + raise TypeError( + 'The argument of FromDebugInfo should be an instance of dictionary.' + ) @staticmethod def FormatDebugInfo(info, error_message): @@ -102,7 +114,7 @@ def FormatDebugInfo(info, error_message): Args: info: A dictionary containing debug information (such as traceroute - info). + info). error_message: the error message from the originating code. Returns: @@ -112,7 +124,8 @@ def FormatDebugInfo(info, error_message): def AddHeader(error, header, message): error += '{sep}{header}\n{message}\n'.format( - sep=sep, header=header, message=message) + sep=sep, header=header, message=message + ) return error def AddKeyIfExists(result, header, key): @@ -121,8 +134,11 @@ def AddKeyIfExists(result, header, key): del info[key] return result - result = AddHeader('', 'error_message:', - error_message) if error_message else '' + result = ( + AddHeader('', 'error_message:', error_message) + if error_message + else '' + ) result = AddKeyIfExists(result, 'traceroute:', 'traceroute') return AddHeader(result, 'Debug Info:', pprint.pformat(info)) @@ -191,6 +207,7 @@ class UnsupportedConfigError(Error): This is a retryable error when --zone=any. """ + pass @@ -199,6 +216,7 @@ class Resource(object): class CreationError(Error): """An error on creation which is not retryable.""" + pass class CreationInternalError(Error): @@ -207,6 +225,7 @@ class CreationInternalError(Error): These errors should not be retried, but instead cleanly retry creating the resource (probably by rerunning the benchmark). """ + pass class CleanupError(Error): @@ -220,6 +239,7 @@ class RetryableDeletionError(Error): class GetError(Error): """An error on get which is not retryable.""" + pass class RetryableGetError(Error): @@ -227,6 +247,7 @@ class RetryableGetError(Error): class UpdateError(Error): """An error on update.""" + pass class SubclassNotFoundError(Error): @@ -234,14 +255,17 @@ class SubclassNotFoundError(Error): class RestoreError(Error): """Errors while restoring a resource.""" + pass class FreezeError(Error): """Errors while freezing a resource.""" + pass class ProvisionTimeoutError(Error): """Timeout errors while provisioning a resource.""" + pass @@ -250,18 +274,22 @@ class Config(object): class InvalidValue(Error): """User provided an invalid value for a config option.""" + pass class MissingOption(Error): """User did not provide a value for a required config option.""" + pass class ParseError(Error): """Error raised when a config can't be loaded properly.""" + pass class UnrecognizedOption(Error): """User provided a value for an unrecognized config option.""" + pass diff --git a/perfkitbenchmarker/events.py b/perfkitbenchmarker/events.py index 1fdaf85ceb..59b956f7c3 100644 --- a/perfkitbenchmarker/events.py +++ b/perfkitbenchmarker/events.py @@ -34,70 +34,99 @@ _events = blinker.Namespace() -initialization_complete = _events.signal('system-ready', doc=""" +initialization_complete = _events.signal( + 'system-ready', + doc=""" Signal sent once after the system is initialized (command-line flags parsed, temporary directory initialized, run_uri set). Sender: None -Payload: parsed_flags, the parsed FLAGS object.""") +Payload: parsed_flags, the parsed FLAGS object.""", +) -provider_imported = _events.signal('provider-imported', doc=""" +provider_imported = _events.signal( + 'provider-imported', + doc=""" Signal sent after a cloud provider's modules have been imported. -Sender: string. Cloud provider name chosen from provider_info.VALID_CLOUDS.""") +Sender: string. Cloud provider name chosen from provider_info.VALID_CLOUDS.""", +) -benchmark_start = _events.signal('benchmark-start', doc=""" +benchmark_start = _events.signal( + 'benchmark-start', + doc=""" Signal sent at the beginning of a benchmark before any resources are provisioned. Sender: None -Payload: benchmark_spec.""") +Payload: benchmark_spec.""", +) -on_vm_startup = _events.signal('on-vm-startup', doc=""" +on_vm_startup = _events.signal( + 'on-vm-startup', + doc=""" Signal sent on vm startup. Sender: None -Payload: vm (VirtualMachine object).""") +Payload: vm (VirtualMachine object).""", +) -benchmark_end = _events.signal('benchmark-end', doc=""" +benchmark_end = _events.signal( + 'benchmark-end', + doc=""" Signal sent at the end of a benchmark after any resources have been torn down (if run_stage includes teardown). Sender: None -Payload: benchmark_spec.""") +Payload: benchmark_spec.""", +) -before_phase = _events.signal('before-phase', doc=""" +before_phase = _events.signal( + 'before-phase', + doc=""" Signal sent before run phase and trigger phase. Sender: the stage. -Payload: benchmark_spec.""") +Payload: benchmark_spec.""", +) -trigger_phase = _events.signal('trigger-phase', doc=""" +trigger_phase = _events.signal( + 'trigger-phase', + doc=""" Signal sent immediately before run phase. This is used for short running command right before the run phase. Before run phase can be slow, time sensitive function should use this event insteand. -Payload: benchmark_spec.""") +Payload: benchmark_spec.""", +) -after_phase = _events.signal('after-phase', doc=""" +after_phase = _events.signal( + 'after-phase', + doc=""" Signal sent immediately after a phase runs, regardless of whether it was successful. Sender: the stage. -Payload: benchmark_spec.""") +Payload: benchmark_spec.""", +) -benchmark_samples_created = _events.signal('benchmark-samples-created', doc=""" +benchmark_samples_created = _events.signal( + 'benchmark-samples-created', + doc=""" Called with samples list and benchmark spec. Signal sent immediately after a sample is created. Sample's should be added into this phase. Any global metadata should be added in the samples finalized. -Payload: benchmark_spec (BenchmarkSpec), samples (list of sample.Sample).""") +Payload: benchmark_spec (BenchmarkSpec), samples (list of sample.Sample).""", +) -all_samples_created = _events.signal('all_samples_created', doc=""" +all_samples_created = _events.signal( + 'all_samples_created', + doc=""" Called with samples list and benchmark spec. Signal sent immediately after a benchmark_samples_created is called. @@ -107,9 +136,12 @@ Sender: the phase. Currently only stages.RUN. -Payload: benchmark_spec (BenchmarkSpec), samples (list of sample.Sample).""") +Payload: benchmark_spec (BenchmarkSpec), samples (list of sample.Sample).""", +) -record_event = _events.signal('record-event', doc=""" +record_event = _events.signal( + 'record-event', + doc=""" Signal sent when an event is recorded. Signal sent after an event occurred. Record start, end timestamp and metadata @@ -117,7 +149,8 @@ Sender: None Payload: event (string), start_timestamp (float), end_timestamp (float), -metadata (dict).""") +metadata (dict).""", +) def RegisterTracingEvents(): @@ -148,7 +181,8 @@ def __init__(self, sender, event, start_timestamp, end_timestamp, metadata): def AddEvent(sender, event, start_timestamp, end_timestamp, metadata): """Record a TracingEvent.""" TracingEvent.events.append( - TracingEvent(sender, event, start_timestamp, end_timestamp, metadata)) + TracingEvent(sender, event, start_timestamp, end_timestamp, metadata) + ) @on_vm_startup.connect @@ -157,20 +191,34 @@ def _RunStartupScript(unused_sender, vm): if FLAGS.startup_script: vm.RemoteCopy(data.ResourcePath(FLAGS.startup_script)) vm.startup_script_output = vm.RemoteCommand( - './%s' % os.path.basename(FLAGS.startup_script)) + './%s' % os.path.basename(FLAGS.startup_script) + ) @benchmark_samples_created.connect def _AddScriptSamples(unused_sender, benchmark_spec, samples): def _ScriptResultToMetadata(out): return {'stdout': out[0], 'stderr': out[1]} + for vm in benchmark_spec.vms: if FLAGS.startup_script: - samples.append(sample.Sample( - 'startup', 0, '', _ScriptResultToMetadata(vm.startup_script_output))) + samples.append( + sample.Sample( + 'startup', + 0, + '', + _ScriptResultToMetadata(vm.startup_script_output), + ) + ) if FLAGS.postrun_script: - samples.append(sample.Sample( - 'postrun', 0, '', _ScriptResultToMetadata(vm.postrun_script_output))) + samples.append( + sample.Sample( + 'postrun', + 0, + '', + _ScriptResultToMetadata(vm.postrun_script_output), + ) + ) @after_phase.connect @@ -178,9 +226,12 @@ def _RunPostRunScript(sender, benchmark_spec): if sender != stages.RUN: logging.info( 'Receive after_phase signal from :%s, not ' - 'triggering _RunPostRunScript.', sender) + 'triggering _RunPostRunScript.', + sender, + ) if FLAGS.postrun_script: for vm in benchmark_spec.vms: vm.RemoteCopy(FLAGS.postrun_script) vm.postrun_script_output = vm.RemoteCommand( - './%s' % os.path.basename(FLAGS.postrun_script)) + './%s' % os.path.basename(FLAGS.postrun_script) + ) diff --git a/perfkitbenchmarker/flag_alias.py b/perfkitbenchmarker/flag_alias.py index d584984c0c..426656c445 100644 --- a/perfkitbenchmarker/flag_alias.py +++ b/perfkitbenchmarker/flag_alias.py @@ -22,7 +22,7 @@ 'scratch_disk_type': 'data_disk_type', 'scratch_disk_iops': 'aws_provisioned_iops', 'scratch_disk_throughput': 'aws_provisioned_throughput', - 'scratch_disk_size': 'data_disk_size' + 'scratch_disk_size': 'data_disk_size', } # Added 10th Nov 2022 @@ -46,15 +46,13 @@ LIST_TO_MULTISTRING_TRANSLATIONS = {'zones': 'zone', 'extra_zones': 'zone'} -SYSBENCH_TRANSLATIONS = { - 'sysbench_thread_counts': 'sysbench_run_threads' -} +SYSBENCH_TRANSLATIONS = {'sysbench_thread_counts': 'sysbench_run_threads'} ALL_TRANSLATIONS = [ DISK_FLAGS_TO_TRANSLATE, RELATIONAL_DB_FLAGS_TO_TRANSLATE, LIST_TO_MULTISTRING_TRANSLATIONS, - SYSBENCH_TRANSLATIONS + SYSBENCH_TRANSLATIONS, ] # Make sure the regex only matches the argument instead of value @@ -82,8 +80,8 @@ def _GetMultiStringFromList(flag: str, args: str) -> List[str]: # pylint: disable=dangerous-default-value def AliasFlagsFromArgs( - argv: List[str], - alias_dict: List[Dict[str, str]] = ALL_TRANSLATIONS) -> List[str]: + argv: List[str], alias_dict: List[Dict[str, str]] = ALL_TRANSLATIONS +) -> List[str]: """Alias flags to support backwards compatibility.""" original_to_translation = _FlattenTranslationsDicts(alias_dict) new_argv = [] @@ -114,7 +112,7 @@ def AliasFlagsFromArgs( # pylint: disable=dangerous-default-value def AliasFlagsFromYaml( config: Optional[Dict[str, Any]], - alias_dict: List[Dict[str, str]] = ALL_TRANSLATIONS + alias_dict: List[Dict[str, str]] = ALL_TRANSLATIONS, ) -> Optional[Dict[str, Any]]: """Alias flags to support backwards compatibility.""" if not config: diff --git a/perfkitbenchmarker/flag_util.py b/perfkitbenchmarker/flag_util.py index ac64f8ffe3..c082995c09 100644 --- a/perfkitbenchmarker/flag_util.py +++ b/perfkitbenchmarker/flag_util.py @@ -134,6 +134,7 @@ def _IsNonIncreasing(result, val): Args: result: list integers and/or range tuples. val: integer or range tuple to append. + Returns: bool indicating if the appended list is non-increasing. """ @@ -170,8 +171,10 @@ class IntegerListParser(flags.ArgumentParser): to [-3, -1, 1, 3]. """ - syntactic_help = ('A comma-separated list of integers or integer ' - 'ranges. Ex: -1,3,5:7 is read as -1,3,5,6,7.') + syntactic_help = ( + 'A comma-separated list of integers or integer ' + 'ranges. Ex: -1,3,5:7 is read as -1,3,5,6,7.' + ) WARN = 'warn' EXCEPTION = 'exception' @@ -212,7 +215,8 @@ def HandleNonIncreasing(): for group in groups: match = INTEGER_GROUP_REGEXP.match( - group) or INTEGER_GROUP_REGEXP_COLONS.match(group) + group + ) or INTEGER_GROUP_REGEXP_COLONS.match(group) if match is None: raise ValueError('Invalid integer list %s' % inp) elif match.group(2) is None: @@ -246,13 +250,17 @@ def _SerializeRange(self, val): return separator.join(str(item) for item in val) def serialize(self, il): - return ','.join([str(val) if isinstance(val, six.integer_types) - else self._SerializeRange(val) - for val in il.groups]) + return ','.join([ + str(val) + if isinstance(val, six.integer_types) + else self._SerializeRange(val) + for val in il.groups + ]) -def DEFINE_integerlist(name, default, help, on_nonincreasing=None, - flag_values=FLAGS, **kwargs): +def DEFINE_integerlist( + name, default, help, on_nonincreasing=None, flag_values=FLAGS, **kwargs +): """Register a flag whose value must be an integer list.""" parser = IntegerListParser(on_nonincreasing=on_nonincreasing) @@ -264,15 +272,16 @@ def DEFINE_integerlist(name, default, help, on_nonincreasing=None, class OverrideFlags(object): """Context manager that applies any config_dict overrides to flag_values.""" - def __init__(self, flag_values, config_dict, - alias=flag_alias.ALL_TRANSLATIONS): + def __init__( + self, flag_values, config_dict, alias=flag_alias.ALL_TRANSLATIONS + ): """Initializes an OverrideFlags context manager. Args: flag_values: FlagValues that is temporarily modified so that any options in config_dict that are not 'present' in flag_values are applied to - flag_values. - Upon exit, flag_values will be restored to its original state. + flag_values. Upon exit, flag_values will be restored to its original + state. config_dict: Merged config flags from the benchmark config and benchmark configuration yaml file. alias: Alias to rename the flags to. @@ -290,7 +299,8 @@ def __enter__(self): if key not in self._flag_values: raise errors.Config.UnrecognizedOption( 'Unrecognized option {0}.{1}. Each option within {0} must ' - 'correspond to a valid command-line flag.'.format('flags', key)) + 'correspond to a valid command-line flag.'.format('flags', key) + ) if not self._flag_values[key].present: self._flags_to_reapply[key] = self._flag_values[key].value try: @@ -298,8 +308,9 @@ def __enter__(self): except flags.IllegalFlagValueError as e: raise errors.Config.InvalidValue( 'Invalid {0}.{1} value: "{2}" (of type "{3}").{4}{5}'.format( - 'flags', key, value, - value.__class__.__name__, os.linesep, e)) + 'flags', key, value, value.__class__.__name__, os.linesep, e + ) + ) def __exit__(self, *unused_args, **unused_kwargs): """Restores flag_values to its original state.""" @@ -315,23 +326,22 @@ class UnitsParser(flags.ArgumentParser): Attributes: convertible_to: list of units.Unit instances. A parsed expression must be - convertible to at least one of the Units in this list. For example, - if the parser requires that its inputs are convertible to bits, then - values expressed in KiB and GB are valid, but values expressed in meters - are not. + convertible to at least one of the Units in this list. For example, if the + parser requires that its inputs are convertible to bits, then values + expressed in KiB and GB are valid, but values expressed in meters are not. """ - syntactic_help = ('A quantity with a unit. Ex: 12.3MB.') + syntactic_help = 'A quantity with a unit. Ex: 12.3MB.' def __init__(self, convertible_to): """Initialize the UnitsParser. Args: convertible_to: Either an individual unit specification or a series of - unit specifications, where each unit specification is either a string - (e.g. 'byte') or a units.Unit. The parser input must be convertible to - at least one of the specified Units, or the parse() method will raise - a ValueError. + unit specifications, where each unit specification is either a string + (e.g. 'byte') or a units.Unit. The parser input must be convertible to + at least one of the specified Units, or the parse() method will raise a + ValueError. """ if isinstance(convertible_to, (six.string_types, units.Unit)): self.convertible_to = [units.Unit(convertible_to)] @@ -343,7 +353,7 @@ def parse(self, inp): Args: inp: a string or a units.Quantity. If a string, it has the format - "", as in "12KB", or "2.5GB". + "", as in "12KB", or "2.5GB". Returns: A units.Quantity. @@ -358,8 +368,9 @@ def parse(self, inp): try: quantity = units.ParseExpression(inp) except Exception as e: - raise ValueError("Couldn't parse unit expression %r: %s" % - (inp, str(e))) + raise ValueError( + "Couldn't parse unit expression %r: %s" % (inp, str(e)) + ) if not isinstance(quantity, units.Quantity): raise ValueError('Expression %r evaluates to a unitless value.' % inp) @@ -372,18 +383,21 @@ def parse(self, inp): else: raise ValueError( 'Expression {0!r} is not convertible to an acceptable unit ' - '({1}).'.format(inp, ', '.join(str(u) for u in self.convertible_to))) + '({1}).'.format(inp, ', '.join(str(u) for u in self.convertible_to)) + ) return quantity class UnitsSerializer(flags.ArgumentSerializer): + def serialize(self, units): return str(units) -def DEFINE_units(name, default, help, convertible_to, - flag_values=flags.FLAGS, **kwargs): +def DEFINE_units( + name, default, help, convertible_to, flag_values=flags.FLAGS, **kwargs +): """Register a flag whose value is a units expression. Args: @@ -391,9 +405,9 @@ def DEFINE_units(name, default, help, convertible_to, default: units.Quantity. The default value. help: string. A help message for the user. convertible_to: Either an individual unit specification or a series of unit - specifications, where each unit specification is either a string (e.g. - 'byte') or a units.Unit. The flag value must be convertible to at least - one of the specified Units to be considered valid. + specifications, where each unit specification is either a string (e.g. + 'byte') or a units.Unit. The flag value must be convertible to at least + one of the specified Units to be considered valid. flag_values: the absl.flags.FlagValues object to define the flag in. """ parser = UnitsParser(convertible_to=convertible_to) @@ -426,14 +440,15 @@ def StringToBytes(string): try: bytes = quantity.m_as(units.byte) except units.DimensionalityError: - raise ValueError("Quantity %s is not a size" % string) + raise ValueError('Quantity %s is not a size' % string) if bytes != int(bytes): - raise ValueError("Size %s has a non-integer number (%s) of bytes!" % - (string, bytes)) + raise ValueError( + 'Size %s has a non-integer number (%s) of bytes!' % (string, bytes) + ) if bytes < 0: - raise ValueError("Size %s has a negative number of bytes!" % string) + raise ValueError('Size %s has a negative number of bytes!' % string) return int(bytes) @@ -480,8 +495,8 @@ def parse(self, inp): """Parse the input. Args: - inp: A string or the result of yaml.safe_load. If a string, should be - a valid YAML document. + inp: A string or the result of yaml.safe_load. If a string, should be a + valid YAML document. """ if isinstance(inp, six.string_types): @@ -496,8 +511,7 @@ def parse(self, inp): try: return yaml.safe_load(inp) except yaml.YAMLError as e: - raise ValueError("Couldn't parse YAML string '%s': %s" % - (inp, str(e))) + raise ValueError("Couldn't parse YAML string '%s': %s" % (inp, str(e))) else: return inp @@ -555,6 +569,7 @@ def GetProvidedCommandLineFlags(): Returns: A dictionary of provided flags in the form: {flag_name: flag_value}. """ + def _GetSerializeableValue(v): if isinstance(v, enum.Enum): return v.name diff --git a/perfkitbenchmarker/flags.py b/perfkitbenchmarker/flags.py index 7f8de68415..e01716fb88 100644 --- a/perfkitbenchmarker/flags.py +++ b/perfkitbenchmarker/flags.py @@ -45,277 +45,428 @@ def GetCurrentUser(): ) flags.DEFINE_list('ssh_options', [], 'Additional options to pass to ssh.') flags.DEFINE_boolean('use_ipv6', False, 'Whether to use ipv6 for ssh/scp.') -flags.DEFINE_list('benchmarks', ['cluster_boot'], - 'Benchmarks and/or benchmark sets that should be run. The ' - 'default is cluster_boot. For more information about ' - 'benchmarks and benchmark sets, see the README and ' - 'benchmark_sets.py.') -flags.DEFINE_boolean('multi_os_benchmark', False, 'Whether is benchmark will ' - 'involve multiple os types.') -flags.DEFINE_string('archive_bucket', None, - 'Archive results to the given S3/GCS bucket.' - 'Must include gs:// or s3:// prefix.') -flags.DEFINE_string('project', None, 'GCP project ID under which ' - 'to create the virtual machines') +flags.DEFINE_list( + 'benchmarks', + ['cluster_boot'], + 'Benchmarks and/or benchmark sets that should be run. The ' + 'default is cluster_boot. For more information about ' + 'benchmarks and benchmark sets, see the README and ' + 'benchmark_sets.py.', +) +flags.DEFINE_boolean( + 'multi_os_benchmark', + False, + 'Whether is benchmark will involve multiple os types.', +) +flags.DEFINE_string( + 'archive_bucket', + None, + 'Archive results to the given S3/GCS bucket.' + 'Must include gs:// or s3:// prefix.', +) +flags.DEFINE_string( + 'project', None, 'GCP project ID under which to create the virtual machines' +) flags.DEFINE_multi_string( - 'zone', [], + 'zone', + [], 'Similar to the --zones flag, but allows the flag to be specified ' 'multiple times on the commandline. For example, --zone=a --zone=b is ' 'equivalent to --zones=a,b. Furthermore, any values specified by --zone ' - 'will be appended to those specified by --zones.') + 'will be appended to those specified by --zones.', +) flags.DEFINE_list( - 'zones', [], + 'zones', + [], 'A list of zones within which to run PerfKitBenchmarker. ' 'This is specific to the cloud provider you are running on. ' 'If multiple zones are given, PerfKitBenchmarker will create 1 VM in ' 'zone, until enough VMs are created as specified in each ' 'benchmark. The order in which this flag is applied to VMs is ' - 'undefined.') + 'undefined.', +) flags.DEFINE_list( - 'extra_zones', [], + 'extra_zones', + [], 'Zones that will be appended to the "zones" list. This is functionally ' - 'the same, but allows flag matrices to have two zone axes.') + 'the same, but allows flag matrices to have two zone axes.', +) # TODO(user): note that this is currently very GCE specific. Need to create a # module which can translate from some generic types to provider specific # nomenclature. -flags.DEFINE_string('machine_type', None, 'Machine ' - 'types that will be created for benchmarks that don\'t ' - 'require a particular type.') -flags.DEFINE_integer('num_vms', 1, 'For benchmarks which can make use of a ' - 'variable number of machines, the number of VMs to use.') -flags.DEFINE_string('image', None, 'Default image that will be ' - 'linked to the VM') -flags.DEFINE_string('run_uri', None, 'Name of the Run. If provided, this ' - 'should be alphanumeric and less than or equal to %d ' - 'characters in length.' % MAX_RUN_URI_LENGTH) -flags.DEFINE_boolean('use_pkb_logging', True, 'Whether to use PKB-specific ' - 'logging handlers. Disabling this will use the standard ' - 'ABSL logging directly.') -flags.DEFINE_boolean('log_dmesg', False, 'Whether to log dmesg from ' - 'each VM to the PKB log file before the VM is deleted.') -flags.DEFINE_boolean('always_teardown_on_exception', False, 'Whether to tear ' - 'down VMs when there is exception during the PKB run. If' - 'enabled, VMs will be torn down even if FLAGS.run_stage ' - 'does not specify teardown.') -RESTORE_PATH = flags.DEFINE_string('restore', None, - 'Path to restore resources from.') -FREEZE_PATH = flags.DEFINE_string('freeze', None, - 'Path to freeze resources to.') -COLLECT_MEMINFO = flags.DEFINE_bool('collect_meminfo', False, - 'Whether to collect /proc/meminfo stats.') +flags.DEFINE_string( + 'machine_type', + None, + 'Machine ' + "types that will be created for benchmarks that don't " + 'require a particular type.', +) +flags.DEFINE_integer( + 'num_vms', + 1, + 'For benchmarks which can make use of a ' + 'variable number of machines, the number of VMs to use.', +) +flags.DEFINE_string( + 'image', None, 'Default image that will be linked to the VM' +) +flags.DEFINE_string( + 'run_uri', + None, + 'Name of the Run. If provided, this ' + 'should be alphanumeric and less than or equal to %d ' + 'characters in length.' % MAX_RUN_URI_LENGTH, +) +flags.DEFINE_boolean( + 'use_pkb_logging', + True, + 'Whether to use PKB-specific ' + 'logging handlers. Disabling this will use the standard ' + 'ABSL logging directly.', +) +flags.DEFINE_boolean( + 'log_dmesg', + False, + 'Whether to log dmesg from ' + 'each VM to the PKB log file before the VM is deleted.', +) +flags.DEFINE_boolean( + 'always_teardown_on_exception', + False, + 'Whether to tear ' + 'down VMs when there is exception during the PKB run. If' + 'enabled, VMs will be torn down even if FLAGS.run_stage ' + 'does not specify teardown.', +) +RESTORE_PATH = flags.DEFINE_string( + 'restore', None, 'Path to restore resources from.' +) +FREEZE_PATH = flags.DEFINE_string( + 'freeze', None, 'Path to freeze resources to.' +) +COLLECT_MEMINFO = flags.DEFINE_bool( + 'collect_meminfo', False, 'Whether to collect /proc/meminfo stats.' +) OWNER = flags.DEFINE_string( - 'owner', GetCurrentUser(), 'Owner name. ' - 'Used to tag created resources and performance records.') + 'owner', + GetCurrentUser(), + 'Owner name. Used to tag created resources and performance records.', +) flags.DEFINE_enum( - 'log_level', log_util.INFO, + 'log_level', + log_util.INFO, list(log_util.LOG_LEVELS.keys()), - 'The log level to run at.') + 'The log level to run at.', +) flags.DEFINE_enum( - 'file_log_level', log_util.DEBUG, list(log_util.LOG_LEVELS.keys()), - 'Anything logged at this level or higher will be written to the log file.') -flags.DEFINE_integer('duration_in_seconds', None, - 'duration of benchmarks. ' - '(only valid for mesh_benchmark)') -flags.DEFINE_string('static_vm_file', None, - 'The file path for the Static Machine file. See ' - 'static_virtual_machine.py for a description of this file.') -flags.DEFINE_boolean('version', False, 'Display the version and exit.', - allow_override_cpp=True) + 'file_log_level', + log_util.DEBUG, + list(log_util.LOG_LEVELS.keys()), + 'Anything logged at this level or higher will be written to the log file.', +) +flags.DEFINE_integer( + 'duration_in_seconds', + None, + 'duration of benchmarks. (only valid for mesh_benchmark)', +) +flags.DEFINE_string( + 'static_vm_file', + None, + 'The file path for the Static Machine file. See ' + 'static_virtual_machine.py for a description of this file.', +) +flags.DEFINE_boolean( + 'version', False, 'Display the version and exit.', allow_override_cpp=True +) flags.DEFINE_boolean('time_commands', False, 'Times each command issued.') flags.DEFINE_string( - 'data_disk_type', None, + 'data_disk_type', + None, 'Type for all data disks. If a provider keeps the operating system and ' 'user data on separate disks, this only affects the user data disk(s).' 'If the provider has OS and user data on the same disk, this flag affects' - 'that disk.') + 'that disk.', +) flags.DEFINE_integer('data_disk_size', None, 'Size, in gb, for all data disks.') -flags.DEFINE_integer('num_striped_disks', None, - 'The number of data disks to stripe together to form one ' - '"logical" data disk. This defaults to 1, ' - 'which means no striping. ', lower_bound=1) -flags.DEFINE_bool('install_packages', None, - 'Override for determining whether packages should be ' - 'installed. If this is false, no packages will be installed ' - 'on any VMs. This option should probably only ever be used ' - 'if you have already created an image with all relevant ' - 'packages installed.') +flags.DEFINE_integer( + 'num_striped_disks', + None, + 'The number of data disks to stripe together to form one ' + '"logical" data disk. This defaults to 1, ' + 'which means no striping. ', + lower_bound=1, +) flags.DEFINE_bool( - 'stop_after_benchmark_failure', False, + 'install_packages', + None, + 'Override for determining whether packages should be ' + 'installed. If this is false, no packages will be installed ' + 'on any VMs. This option should probably only ever be used ' + 'if you have already created an image with all relevant ' + 'packages installed.', +) +flags.DEFINE_bool( + 'stop_after_benchmark_failure', + False, 'Determines response when running multiple benchmarks serially and a ' 'benchmark run fails. When True, no further benchmarks are scheduled, and ' 'execution ends. When False, benchmarks continue to be scheduled. Does not ' 'apply to keyboard interrupts, which will always prevent further ' - 'benchmarks from being scheduled.') + 'benchmarks from being scheduled.', +) flags.DEFINE_boolean( - 'ignore_package_requirements', False, - 'Disables Python package requirement runtime checks.') + 'ignore_package_requirements', + False, + 'Disables Python package requirement runtime checks.', +) flags.DEFINE_boolean( - 'publish_after_run', False, + 'publish_after_run', + False, 'If true, PKB will publish all samples available immediately after running ' 'each benchmark. This may be useful in scenarios where the PKB run time ' - 'for all benchmarks is much greater than a single benchmark.') + 'for all benchmarks is much greater than a single benchmark.', +) flags.DEFINE_integer( - 'publish_period', None, + 'publish_period', + None, 'The period in seconds to publish samples from repeated run stages. ' - 'This will only publish samples if publish_after_run is True.') + 'This will only publish samples if publish_after_run is True.', +) flags.DEFINE_integer( - 'run_stage_time', 0, + 'run_stage_time', + 0, 'PKB will run/re-run the run stage of each benchmark until it has spent ' 'at least this many seconds. It defaults to 0, so benchmarks will only ' 'be run once unless some other value is specified. This flag and ' - 'run_stage_iterations are mutually exclusive.') + 'run_stage_iterations are mutually exclusive.', +) flags.DEFINE_integer( - 'run_stage_iterations', 1, + 'run_stage_iterations', + 1, 'PKB will run/re-run the run stage of each benchmark this many times. ' 'It defaults to 1, so benchmarks will only be run once unless some other ' - 'value is specified. This flag and run_stage_time are mutually exclusive.') + 'value is specified. This flag and run_stage_time are mutually exclusive.', +) flags.DEFINE_integer( - 'run_stage_retries', 0, + 'run_stage_retries', + 0, 'The number of allowable consecutive failures during the run stage. After ' 'this number of failures any exceptions will cause benchmark termination. ' 'If run_stage_time is exceeded, the run stage will not be retried even if ' - 'the number of failures is less than the value of this flag.') + 'the number of failures is less than the value of this flag.', +) MAX_RETRIES = flags.DEFINE_integer( - 'retries', 0, 'The amount of times PKB should retry each benchmark.' + 'retries', + 0, + 'The amount of times PKB should retry each benchmark.' 'Use with --retry_substatuses to specify which failure substatuses to ' - 'retry on. Defaults to all valid substatuses.') + 'retry on. Defaults to all valid substatuses.', +) RETRY_DELAY_SECONDS = flags.DEFINE_integer( - 'retry_delay_seconds', 0, 'The time to wait in between retries.') + 'retry_delay_seconds', 0, 'The time to wait in between retries.' +) # Retries could also allow for a dict of failed_substatus: 'zone'|'region' # retry method which would make the retry functionality more customizable. SMART_QUOTA_RETRY = flags.DEFINE_bool( - 'smart_quota_retry', False, + 'smart_quota_retry', + False, 'If True, causes the benchmark to rerun in a zone in a different region ' 'in the same geo on a quota exception. Currently only works for benchmarks ' 'that specify a single zone (via --zone or --zones). The zone is selected ' 'at random and overrides the --zones flag or the --zone flag, depending on ' 'which is provided. QUOTA_EXCEEDED must be in the list of retry ' - 'substatuses for this to work.') + 'substatuses for this to work.', +) SMART_CAPACITY_RETRY = flags.DEFINE_bool( - 'smart_capacity_retry', False, + 'smart_capacity_retry', + False, 'If True, causes the benchmark to rerun in a different zone in the same ' 'region on a capacity/config exception. Currently only works for ' 'benchmarks that specify a single zone (via --zone or --zones). The zone ' 'is selected at random and overrides the --zones flag or the --zone flag, ' 'depending on which is provided. INSUFFICIENT_CAPACITY and UNSUPPORTED ' - 'must be in the list of retry substatuses for this to work.') + 'must be in the list of retry substatuses for this to work.', +) flags.DEFINE_boolean( - 'boot_samples', False, - 'Whether to publish boot time samples for all tests.') + 'boot_samples', False, 'Whether to publish boot time samples for all tests.' +) MEASURE_DELETE = flags.DEFINE_boolean( - 'collect_delete_samples', False, - 'Whether to publish delete time samples for all tests.') + 'collect_delete_samples', + False, + 'Whether to publish delete time samples for all tests.', +) flags.DEFINE_boolean( - 'gpu_samples', False, - 'Whether to publish GPU memcpy bandwidth samples for GPU tests.') + 'gpu_samples', + False, + 'Whether to publish GPU memcpy bandwidth samples for GPU tests.', +) flags.DEFINE_integer( - 'run_processes', None, + 'run_processes', + None, 'The number of parallel processes to use to run benchmarks.', - lower_bound=1) + lower_bound=1, +) flags.DEFINE_float( - 'run_processes_delay', None, - 'The delay in seconds between parallel processes\' invocation. ' + 'run_processes_delay', + None, + "The delay in seconds between parallel processes' invocation. " 'Increasing this value may reduce provider throttling issues.', - lower_bound=0) + lower_bound=0, +) flags.DEFINE_string( - 'completion_status_file', None, + 'completion_status_file', + None, 'If specified, this file will contain the completion status of each ' 'benchmark that ran (SUCCEEDED, FAILED, or SKIPPED). The file has one json ' 'object per line, each with the following format:\n' '{ "name": , "flags": , ' - '"status": }') + '"status": }', +) flags.DEFINE_string( - 'helpmatch', '', + 'helpmatch', + '', 'Shows only flags defined in a module whose name matches the given regex.', - allow_override_cpp=True) + allow_override_cpp=True, +) flags.DEFINE_string( - 'helpmatchmd', '', + 'helpmatchmd', + '', 'helpmatch query with markdown friendly output. ' 'Shows only flags defined in a module whose name matches the given regex.', - allow_override_cpp=True) + allow_override_cpp=True, +) flags.DEFINE_boolean( - 'create_failed_run_samples', False, + 'create_failed_run_samples', + False, 'If true, PKB will create a sample specifying that a run stage failed. ' 'This sample will include metadata specifying the run stage that ' 'failed, the exception that occurred, as well as all the flags that ' - 'were provided to PKB on the command line.') + 'were provided to PKB on the command line.', +) CREATE_STARTED_RUN_SAMPLE = flags.DEFINE_boolean( - 'create_started_run_sample', False, + 'create_started_run_sample', + False, 'Whether PKB will create a sample at the start of the provision phase of ' - 'the benchmark run.') + 'the benchmark run.', +) CREATE_STARTED_STAGE_SAMPLES = flags.DEFINE_boolean( - 'create_started_stage_samples', False, + 'create_started_stage_samples', + False, 'Whether PKB will create a sample at the start of the each stage of ' - 'the benchmark run.') + 'the benchmark run.', +) flags.DEFINE_integer( - 'failed_run_samples_error_length', 10240, + 'failed_run_samples_error_length', + 10240, 'If create_failed_run_samples is true, PKB will truncate any error ' - 'messages at failed_run_samples_error_length.') + 'messages at failed_run_samples_error_length.', +) flags.DEFINE_boolean( - 'dry_run', False, + 'dry_run', + False, 'If true, PKB will print the flags configurations to be run and exit. ' 'The configurations are generated from the command line flags, the ' - 'flag_matrix, and flag_zip.') + 'flag_matrix, and flag_zip.', +) flags.DEFINE_string( - 'skip_pending_runs_file', None, - 'If file exists, any pending runs will be not be executed.') -flags.DEFINE_boolean( - 'use_vpn', False, - 'Creates VPN tunnels between vm_groups') + 'skip_pending_runs_file', + None, + 'If file exists, any pending runs will be not be executed.', +) +flags.DEFINE_boolean('use_vpn', False, 'Creates VPN tunnels between vm_groups') flags.DEFINE_integer( - 'after_prepare_sleep_time', 0, + 'after_prepare_sleep_time', + 0, 'The time in seconds to sleep after the prepare phase. This can be useful ' - 'for letting burst tokens accumulate.') + 'for letting burst tokens accumulate.', +) flags.DEFINE_integer( - 'after_run_sleep_time', 0, + 'after_run_sleep_time', + 0, 'The time in seconds to sleep after the run phase. This can be useful ' - 'for letting the VM sit idle after the bechmarking phase is complete.') + 'for letting the VM sit idle after the bechmarking phase is complete.', +) flags.DEFINE_bool( - 'before_run_pause', False, + 'before_run_pause', + False, 'If true, wait for command line input before executing the run phase. ' - 'This is useful for debugging benchmarks during development.') + 'This is useful for debugging benchmarks during development.', +) flags.DEFINE_bool( - 'before_cleanup_pause', False, + 'before_cleanup_pause', + False, 'If true, wait for command line input before executing the cleanup phase. ' - 'This is useful for debugging benchmarks during development.') + 'This is useful for debugging benchmarks during development.', +) flags.DEFINE_integer( - 'timeout_minutes', 240, + 'timeout_minutes', + 240, 'An upper bound on the time in minutes that the benchmark is expected to ' 'run. This time is annotated or tagged on the resources of cloud ' - 'providers. Note that for retries, this applies to each individual retry.') + 'providers. Note that for retries, this applies to each individual retry.', +) flags.DEFINE_integer( - 'persistent_timeout_minutes', 240, + 'persistent_timeout_minutes', + 240, 'An upper bound on the time in minutes that resources left behind by the ' 'benchmark. Some benchmarks purposefully create resources for other ' 'benchmarks to use. Persistent timeout specifies how long these shared ' - 'resources should live.') -flags.DEFINE_bool('disable_interrupt_moderation', False, - 'Turn off the interrupt moderation networking feature') -flags.DEFINE_bool('disable_rss', False, - 'Whether or not to disable the Receive Side Scaling feature.') -flags.DEFINE_boolean('record_lscpu', True, - 'Whether to record the lscpu output in a sample') -flags.DEFINE_boolean('record_proccpu', True, - 'Whether to record the /proc/cpuinfo output in a sample') -flags.DEFINE_boolean('record_cpu_vuln', True, - 'Whether to record the CPU vulnerabilities on linux VMs') -flags.DEFINE_boolean('record_gcc', True, - 'Whether to record the gcc version in a sample') -flags.DEFINE_boolean('record_glibc', True, - 'Whether to record the glibc version in a sample') + 'resources should live.', +) +flags.DEFINE_bool( + 'disable_interrupt_moderation', + False, + 'Turn off the interrupt moderation networking feature', +) +flags.DEFINE_bool( + 'disable_rss', + False, + 'Whether or not to disable the Receive Side Scaling feature.', +) +flags.DEFINE_boolean( + 'record_lscpu', True, 'Whether to record the lscpu output in a sample' +) +flags.DEFINE_boolean( + 'record_proccpu', + True, + 'Whether to record the /proc/cpuinfo output in a sample', +) +flags.DEFINE_boolean( + 'record_cpu_vuln', + True, + 'Whether to record the CPU vulnerabilities on linux VMs', +) +flags.DEFINE_boolean( + 'record_gcc', True, 'Whether to record the gcc version in a sample' +) +flags.DEFINE_boolean( + 'record_glibc', True, 'Whether to record the glibc version in a sample' +) # Support for using a proxy in the cloud environment. -flags.DEFINE_string('http_proxy', '', - 'Specify a proxy for HTTP in the form ' - '[user:passwd@]proxy.server:port.') -flags.DEFINE_string('https_proxy', '', - 'Specify a proxy for HTTPS in the form ' - '[user:passwd@]proxy.server:port.') -flags.DEFINE_string('ftp_proxy', '', - 'Specify a proxy for FTP in the form ' - '[user:passwd@]proxy.server:port.') -flags.DEFINE_string('no_proxy', '', - 'Specify host(s) to exclude from proxy, e.g. ' - '--no_proxy=localhost,.example.com,192.168.0.1') -flags.DEFINE_bool('randomize_run_order', False, - 'When running with more than one benchmarks, ' - 'randomize order of the benchmarks.') +flags.DEFINE_string( + 'http_proxy', + '', + 'Specify a proxy for HTTP in the form [user:passwd@]proxy.server:port.', +) +flags.DEFINE_string( + 'https_proxy', + '', + 'Specify a proxy for HTTPS in the form [user:passwd@]proxy.server:port.', +) +flags.DEFINE_string( + 'ftp_proxy', + '', + 'Specify a proxy for FTP in the form [user:passwd@]proxy.server:port.', +) +flags.DEFINE_string( + 'no_proxy', + '', + 'Specify host(s) to exclude from proxy, e.g. ' + '--no_proxy=localhost,.example.com,192.168.0.1', +) +flags.DEFINE_bool( + 'randomize_run_order', + False, + 'When running with more than one benchmarks, ' + 'randomize order of the benchmarks.', +) diff --git a/perfkitbenchmarker/hpc_util.py b/perfkitbenchmarker/hpc_util.py index 0dd6f04cea..31d1341d83 100644 --- a/perfkitbenchmarker/hpc_util.py +++ b/perfkitbenchmarker/hpc_util.py @@ -17,14 +17,19 @@ from absl import flags from perfkitbenchmarker import vm_util -flags.DEFINE_boolean('mpirun_allow_run_as_root', False, - 'Whether to allow mpirun to be run by the root user.') +flags.DEFINE_boolean( + 'mpirun_allow_run_as_root', + False, + 'Whether to allow mpirun to be run by the root user.', +) -def CreateMachineFile(vms, - num_slots=lambda vm: vm.NumCpusForBenchmark(), - remote_path='MACHINEFILE', - mpi_vendor='openmpi'): +def CreateMachineFile( + vms, + num_slots=lambda vm: vm.NumCpusForBenchmark(), + remote_path='MACHINEFILE', + mpi_vendor='openmpi', +): """Create a file with the IP of each machine in the cluster on its own line. The file is then pushed to the provided path on the master vm. @@ -39,8 +44,8 @@ def CreateMachineFile(vms, Args: vms: The list of vms which will be in the cluster. - num_slots: The function to use to calculate the number of slots - for each vm. Defaults to vm.NumCpusForBenchmark() + num_slots: The function to use to calculate the number of slots for each vm. + Defaults to vm.NumCpusForBenchmark() remote_path: remote path of the machine file. Defaults to MACHINEFILE mpi_vendor: Implementation of MPI. Can be openmpi or intel. """ diff --git a/perfkitbenchmarker/iaas_relational_db.py b/perfkitbenchmarker/iaas_relational_db.py index d7b8197886..f4ecd75510 100644 --- a/perfkitbenchmarker/iaas_relational_db.py +++ b/perfkitbenchmarker/iaas_relational_db.py @@ -30,6 +30,7 @@ class IAASRelationalDb(relational_db.BaseRelationalDb): """Object representing a IAAS relational database Service.""" + IS_MANAGED = False REQUIRED_ATTRS = ['CLOUD', 'IS_MANAGED', 'ENGINE'] DEFAULT_ENGINE_VERSION = None @@ -59,8 +60,9 @@ def server_vm(self): The server_vm. """ if not hasattr(self, '_server_vm'): - raise relational_db.RelationalDbPropertyNotSetError('server_vm is ' - 'not set') + raise relational_db.RelationalDbPropertyNotSetError( + 'server_vm is not set' + ) return self._server_vm @server_vm.setter @@ -71,10 +73,16 @@ def server_vm(self, server_vm): def server_vm_query_tools(self): if not hasattr(self, '_server_vm_query_tools'): connection_properties = sql_engine_utils.DbConnectionProperties( - self.spec.engine, self.spec.engine_version, 'localhost', self.port, - self.spec.database_username, self.spec.database_password) + self.spec.engine, + self.spec.engine_version, + 'localhost', + self.port, + self.spec.database_username, + self.spec.database_password, + ) self._server_vm_query_tools = sql_engine_utils.GetQueryToolsByEngine( - self.server_vm, connection_properties) + self.server_vm, connection_properties + ) return self._server_vm_query_tools def SetVms(self, vm_groups): diff --git a/perfkitbenchmarker/import_util.py b/perfkitbenchmarker/import_util.py index 2352646780..94b1d5eae0 100644 --- a/perfkitbenchmarker/import_util.py +++ b/perfkitbenchmarker/import_util.py @@ -35,6 +35,7 @@ def LoadModulesForPath(path, package_prefix=None): """ prefix = package_prefix + '.' if package_prefix else '' + def LogImportError(modname): logging.exception('Error importing module %s', modname) diff --git a/perfkitbenchmarker/key.py b/perfkitbenchmarker/key.py index c6d25a4cfb..fc68c64ab6 100644 --- a/perfkitbenchmarker/key.py +++ b/perfkitbenchmarker/key.py @@ -14,14 +14,17 @@ @dataclasses.dataclass class BaseKeySpec(spec.BaseSpec): """Configurable options of a cryptographic key.""" + # Needed for registering the spec class and its subclasses. See BaseSpec. SPEC_TYPE = 'BaseKeySpec' CLOUD = None - def __init__(self, - component_full_name: str, - flag_values: Optional[flags.FlagValues] = None, - **kwargs): + def __init__( + self, + component_full_name: str, + flag_values: Optional[flags.FlagValues] = None, + **kwargs + ): super().__init__(component_full_name, flag_values=flag_values, **kwargs) self.cloud: str @@ -30,17 +33,21 @@ def _GetOptionDecoderConstructions(cls): """Gets decoder classes and constructor args for each configurable option.""" result = super()._GetOptionDecoderConstructions() result.update({ - 'cloud': (option_decoders.EnumDecoder, { - # Uncomment when there are other cloud implementations - # 'valid_values': provider_info.VALID_CLOUDS - 'valid_values': ['GCP'] - }), + 'cloud': ( + option_decoders.EnumDecoder, + { + # Uncomment when there are other cloud implementations + # 'valid_values': provider_info.VALID_CLOUDS + 'valid_values': ['GCP'] + }, + ), }) return result class BaseKey(resource.BaseResource): """Object representing a cryptographic key.""" + RESOURCE_TYPE = 'BaseKey' CLOUD = None diff --git a/perfkitbenchmarker/kubernetes_helper.py b/perfkitbenchmarker/kubernetes_helper.py index 26370f0281..5f6f9a55fa 100644 --- a/perfkitbenchmarker/kubernetes_helper.py +++ b/perfkitbenchmarker/kubernetes_helper.py @@ -21,32 +21,51 @@ from perfkitbenchmarker import vm_util FLAGS = flags.FLAGS -flags.DEFINE_integer('k8s_get_retry_count', 18, - 'Maximum number of waits for getting LoadBalancer external IP') -flags.DEFINE_integer('k8s_get_wait_interval', 10, - 'Wait interval for getting LoadBalancer external IP') +flags.DEFINE_integer( + 'k8s_get_retry_count', + 18, + 'Maximum number of waits for getting LoadBalancer external IP', +) +flags.DEFINE_integer( + 'k8s_get_wait_interval', + 10, + 'Wait interval for getting LoadBalancer external IP', +) def checkKubernetesFlags(): if not FLAGS.kubectl: - raise Exception('Please provide path to kubectl tool using --kubectl ' - 'flag. Exiting.') + raise Exception( + 'Please provide path to kubectl tool using --kubectl flag. Exiting.' + ) if not FLAGS.kubeconfig: - raise Exception('Please provide path to kubeconfig using --kubeconfig ' - 'flag. Exiting.') + raise Exception( + 'Please provide path to kubeconfig using --kubeconfig flag. Exiting.' + ) def CreateFromFile(file_name): checkKubernetesFlags() - create_cmd = [FLAGS.kubectl, '--kubeconfig=%s' % FLAGS.kubeconfig, 'create', - '-f', file_name] + create_cmd = [ + FLAGS.kubectl, + '--kubeconfig=%s' % FLAGS.kubeconfig, + 'create', + '-f', + file_name, + ] vm_util.IssueRetryableCommand(create_cmd) def DeleteFromFile(file_name): checkKubernetesFlags() - delete_cmd = [FLAGS.kubectl, '--kubeconfig=%s' % FLAGS.kubeconfig, 'delete', - '-f', file_name, '--ignore-not-found'] + delete_cmd = [ + FLAGS.kubectl, + '--kubeconfig=%s' % FLAGS.kubeconfig, + 'delete', + '-f', + file_name, + '--ignore-not-found', + ] vm_util.IssueRetryableCommand(delete_cmd) @@ -62,8 +81,12 @@ def CreateAllFiles(file_list): def Get(resource, resourceInstanceName, labelFilter, jsonSelector): checkKubernetesFlags() - get_pod_cmd = [FLAGS.kubectl, '--kubeconfig=%s' % FLAGS.kubeconfig, - 'get', resource] + get_pod_cmd = [ + FLAGS.kubectl, + '--kubeconfig=%s' % FLAGS.kubeconfig, + 'get', + resource, + ] if len(resourceInstanceName) > 0: get_pod_cmd.append(resourceInstanceName) if len(labelFilter) > 0: @@ -71,7 +94,7 @@ def Get(resource, resourceInstanceName, labelFilter, jsonSelector): get_pod_cmd.append('-ojsonpath={{{}}}'.format(jsonSelector)) stdout, stderr, _ = vm_util.IssueCommand(get_pod_cmd, raise_on_failure=False) if len(stderr) > 0: - raise Exception("Error received from kubectl get: " + stderr) + raise Exception('Error received from kubectl get: ' + stderr) return stdout diff --git a/perfkitbenchmarker/linux_benchmarks/ch_block_storage_benchmark.py b/perfkitbenchmarker/linux_benchmarks/ch_block_storage_benchmark.py index 53d0e81fb1..2e525cc074 100644 --- a/perfkitbenchmarker/linux_benchmarks/ch_block_storage_benchmark.py +++ b/perfkitbenchmarker/linux_benchmarks/ch_block_storage_benchmark.py @@ -90,12 +90,10 @@ def _LocateFioJson(vm, outdir, test): def Run(benchmark_spec): """Runs cloudharmony block storage and reports the results.""" vm = benchmark_spec.vms[0] - target = ' '.join( - [ - '--target=%s' % _PrepareDevicePath(vm, dev.GetDevicePath()) - for dev in vm.scratch_disks - ] - ) + target = ' '.join([ + '--target=%s' % _PrepareDevicePath(vm, dev.GetDevicePath()) + for dev in vm.scratch_disks + ]) tests = ' '.join(['--test=%s' % test for test in FLAGS.ch_block_tests]) args = ' '.join(['--%s' % param for param in FLAGS.ch_params]) outdir = vm_util.VM_TMP_DIR diff --git a/perfkitbenchmarker/linux_benchmarks/cloud_bigtable_ycsb_benchmark.py b/perfkitbenchmarker/linux_benchmarks/cloud_bigtable_ycsb_benchmark.py index c669e3517d..26acdd0897 100644 --- a/perfkitbenchmarker/linux_benchmarks/cloud_bigtable_ycsb_benchmark.py +++ b/perfkitbenchmarker/linux_benchmarks/cloud_bigtable_ycsb_benchmark.py @@ -529,12 +529,10 @@ def _CreateTable(benchmark_spec: bm_spec.BenchmarkSpec): """ instance: _Bigtable = benchmark_spec.non_relational_db vm = benchmark_spec.vms[0] - splits = ','.join( - [ - f'user{1000 + i * (9999 - 1000) // hbase_ycsb.TABLE_SPLIT_COUNT}' - for i in range(hbase_ycsb.TABLE_SPLIT_COUNT) - ] - ) + splits = ','.join([ + f'user{1000 + i * (9999 - 1000) // hbase_ycsb.TABLE_SPLIT_COUNT}' + for i in range(hbase_ycsb.TABLE_SPLIT_COUNT) + ]) command = [ google_cloud_cbt.CBT_BIN, f'-project={FLAGS.project or _GetDefaultProject()}', diff --git a/perfkitbenchmarker/linux_benchmarks/cloudharmony_network_benchmark.py b/perfkitbenchmarker/linux_benchmarks/cloudharmony_network_benchmark.py index 189d786d16..df4d03a9c3 100644 --- a/perfkitbenchmarker/linux_benchmarks/cloudharmony_network_benchmark.py +++ b/perfkitbenchmarker/linux_benchmarks/cloudharmony_network_benchmark.py @@ -437,12 +437,10 @@ def _Run(benchmark_spec, test): vms = vm_groups['server'] # use GCP zonal internal DNS, # but maybe should add domain to vm's data attributes? - endpoints = ' '.join( - [ - f'--test_endpoint={vm.name}.{vm.zone}.c.{vm.project}.internal' - for vm in vms - ] - ) + endpoints = ' '.join([ + f'--test_endpoint={vm.name}.{vm.zone}.c.{vm.project}.internal' + for vm in vms + ]) _AddComputeMetadata(client, vms[0], metadata) if FLAGS.ch_network_throughput_https: diff --git a/perfkitbenchmarker/linux_benchmarks/horovod_benchmark.py b/perfkitbenchmarker/linux_benchmarks/horovod_benchmark.py index 4863d7d465..e895925445 100644 --- a/perfkitbenchmarker/linux_benchmarks/horovod_benchmark.py +++ b/perfkitbenchmarker/linux_benchmarks/horovod_benchmark.py @@ -463,14 +463,10 @@ def RunWithVMs(vms, extra_envs=None): run_command += ( 'DeepLearningExamples/TensorFlow/Classification/ConvNets/main.py ' ) - run_command += ' '.join( - [ - '--{}'.format(key) - if value is None - else '--{}={}'.format(key, value) - for key, value in sorted(run_flags.items()) - ] - ) + run_command += ' '.join([ + '--{}'.format(key) if value is None else '--{}={}'.format(key, value) + for key, value in sorted(run_flags.items()) + ]) elif FLAGS.horovod_model == 'resnext-101': run_flags = { 'arch': 'resnext101-32x4d', @@ -500,14 +496,10 @@ def RunWithVMs(vms, extra_envs=None): run_command += ( 'DeepLearningExamples/TensorFlow/Classification/ConvNets/main.py ' ) - run_command += ' '.join( - [ - '--{}'.format(key) - if value is None - else '--{}={}'.format(key, value) - for key, value in sorted(run_flags.items()) - ] - ) + run_command += ' '.join([ + '--{}'.format(key) if value is None else '--{}={}'.format(key, value) + for key, value in sorted(run_flags.items()) + ]) elif FLAGS.horovod_model.startswith('bert'): # bert if not FLAGS.horovod_bert_finetune: raise NotImplementedError('BERT pretraining is not supported.') @@ -540,14 +532,10 @@ def RunWithVMs(vms, extra_envs=None): run_command += ( 'DeepLearningExamples/TensorFlow/LanguageModeling/BERT/run_squad.py ' ) - run_command += ' '.join( - [ - '--{}'.format(key) - if value is None - else '--{}={}'.format(key, value) - for key, value in sorted(run_flags.items()) - ] - ) + run_command += ' '.join([ + '--{}'.format(key) if value is None else '--{}={}'.format(key, value) + for key, value in sorted(run_flags.items()) + ]) else: run_command += ( 'tensorpack/examples/FasterRCNN/train.py --config ' diff --git a/perfkitbenchmarker/linux_benchmarks/hpcc_benchmark.py b/perfkitbenchmarker/linux_benchmarks/hpcc_benchmark.py index bd4b64b634..64d9cf2e45 100644 --- a/perfkitbenchmarker/linux_benchmarks/hpcc_benchmark.py +++ b/perfkitbenchmarker/linux_benchmarks/hpcc_benchmark.py @@ -494,13 +494,11 @@ def RunHpccSource( """Returns the parsed output from running the compiled from source HPCC.""" headnode_vm = vms[0] # backup existing HPCC output, if any - headnode_vm.RemoteCommand( - ( - 'if [ -f hpccoutf.txt ]; then ' - 'mv hpccoutf.txt hpccoutf-$(date +%s).txt; ' - 'fi' - ) - ) + headnode_vm.RemoteCommand(( + 'if [ -f hpccoutf.txt ]; then ' + 'mv hpccoutf.txt hpccoutf-$(date +%s).txt; ' + 'fi' + )) num_processes = len(vms) * headnode_vm.NumCpusForBenchmark() run_as_root = '--allow-run-as-root' if FLAGS.mpirun_allow_run_as_root else '' mpi_flags = ( diff --git a/perfkitbenchmarker/linux_benchmarks/iperf_benchmark.py b/perfkitbenchmarker/linux_benchmarks/iperf_benchmark.py index ec9b9b5a9c..6c7ced0d68 100644 --- a/perfkitbenchmarker/linux_benchmarks/iperf_benchmark.py +++ b/perfkitbenchmarker/linux_benchmarks/iperf_benchmark.py @@ -246,14 +246,12 @@ def _RunIperf( total_stats = {} if thread_count == 1: - r = re.compile( - ( - r'\[\s*(?P\d+)\]\s+(?P\d+\.\d+-\d+\.\d+)\s+sec\s+' - r'(?P\d+\.?\d*)\s\w+\s+(?P\d+\.?\d*)\sMbits\/sec\s+' - r'(?P\d+)\/(?P\d+)\s+(?P\d+)\s+(?P-?\d+)(?P\w*)\/' - r'(?P\d+)\s+(?P\w+)\s+(?P\d+\.?\d*)' - ) - ) + r = re.compile(( + r'\[\s*(?P\d+)\]\s+(?P\d+\.\d+-\d+\.\d+)\s+sec\s+' + r'(?P\d+\.?\d*)\s\w+\s+(?P\d+\.?\d*)\sMbits\/sec\s+' + r'(?P\d+)\/(?P\d+)\s+(?P\d+)\s+(?P-?\d+)(?P\w*)\/' + r'(?P\d+)\s+(?P\w+)\s+(?P\d+\.?\d*)' + )) interval_stats = [m.groupdict() for m in r.finditer(stdout)] for count in range(0, len(interval_stats) - 1): @@ -272,24 +270,20 @@ def _RunIperf( elif thread_count > 1: # Parse aggregates of multiple sending threads for each report interval - r = re.compile( - ( - r'\[SUM\]\s+(?P\d+\.\d+-\d+\.\d+)\s\w+\s+(?P\d+\.?\d*)' - r'\s\w+\s+(?P\d+\.?\d*)\sMbits\/sec\s+(?P\d+)' - r'\/(?P\d+)\s+(?P\d+)' - ) - ) + r = re.compile(( + r'\[SUM\]\s+(?P\d+\.\d+-\d+\.\d+)\s\w+\s+(?P\d+\.?\d*)' + r'\s\w+\s+(?P\d+\.?\d*)\sMbits\/sec\s+(?P\d+)' + r'\/(?P\d+)\s+(?P\d+)' + )) interval_sums = [m.groupdict() for m in r.finditer(stdout)] # Parse output for each individual thread for each report interval - r = re.compile( - ( - r'\[\s*(?P\d+)\]\s+(?P\d+\.\d+-\d+\.\d+)\s+sec\s+' - r'(?P\d+\.?\d*)\s\w+\s+(?P\d+\.?\d*)\sMbits\/sec\s+(?P\d+)\/' - r'(?P\d+)\s+(?P\d+)\s+(?P-?\d+)(?P\w*)\/(?P\d+)\s+' - r'(?P\w+)\s+(?P\d+\.?\d*)' - ) - ) + r = re.compile(( + r'\[\s*(?P\d+)\]\s+(?P\d+\.\d+-\d+\.\d+)\s+sec\s+' + r'(?P\d+\.?\d*)\s\w+\s+(?P\d+\.?\d*)\sMbits\/sec\s+(?P\d+)\/' + r'(?P\d+)\s+(?P\d+)\s+(?P-?\d+)(?P\w*)\/(?P\d+)\s+' + r'(?P\w+)\s+(?P\d+\.?\d*)' + )) interval_threads = [m.groupdict() for m in r.finditer(stdout)] # sum and average across threads for each interval report @@ -392,13 +386,11 @@ def _RunIperf( err = int(match.group('err')) retry = int(match.group('retry')) - r = re.compile( - ( - r'\d+ Mbits\/sec\s+' - r' \d+\/\d+\s+\d+\s+(?P-*\d+)(?P\w+)\/(?P\d+)' - r'\s+(?P\w+)\s+(?P\d+\.\d+)' - ) - ) + r = re.compile(( + r'\d+ Mbits\/sec\s+' + r' \d+\/\d+\s+\d+\s+(?P-*\d+)(?P\w+)\/(?P\d+)' + r'\s+(?P\w+)\s+(?P\d+\.\d+)' + )) match = [m.groupdict() for m in r.finditer(stdout)] cwnd = sum(float(i['cwnd']) for i in match) / len(match) diff --git a/perfkitbenchmarker/linux_benchmarks/lmbench_benchmark.py b/perfkitbenchmarker/linux_benchmarks/lmbench_benchmark.py index d012151749..c8470b3a56 100644 --- a/perfkitbenchmarker/linux_benchmarks/lmbench_benchmark.py +++ b/perfkitbenchmarker/linux_benchmarks/lmbench_benchmark.py @@ -195,9 +195,9 @@ def _ParseSections(lmbench_output, parse_section_func_dict, metadata, results): if title is None: break function = parse_section_func_dict[title] - reading_buffer = list(itertools.takewhile( - lambda line: line != stop_parsing, lines_iter - )) + reading_buffer = list( + itertools.takewhile(lambda line: line != stop_parsing, lines_iter) + ) function(reading_buffer, title, metadata, results) diff --git a/perfkitbenchmarker/linux_benchmarks/netperf_benchmark.py b/perfkitbenchmarker/linux_benchmarks/netperf_benchmark.py index 6f66803136..47891c7f3a 100644 --- a/perfkitbenchmarker/linux_benchmarks/netperf_benchmark.py +++ b/perfkitbenchmarker/linux_benchmarks/netperf_benchmark.py @@ -683,8 +683,11 @@ def RunClientServerVMs(client_vm, server_vm): for netperf_benchmark in FLAGS.netperf_benchmarks: if vm_util.ShouldRunOnExternalIpAddress(): external_ip_results = RunNetperf( - client_vm, netperf_benchmark, [server_vm.ip_address], num_streams, - [client_vm.ip_address] + client_vm, + netperf_benchmark, + [server_vm.ip_address], + num_streams, + [client_vm.ip_address], ) for external_ip_result in external_ip_results: external_ip_result.metadata['ip_type'] = ( @@ -699,7 +702,7 @@ def RunClientServerVMs(client_vm, server_vm): netperf_benchmark, server_vm.GetInternalIPs(), num_streams, - client_vm.GetInternalIPs() + client_vm.GetInternalIPs(), ) for internal_ip_result in internal_ip_results: internal_ip_result.metadata.update(metadata) diff --git a/perfkitbenchmarker/linux_benchmarks/object_storage_service_benchmark.py b/perfkitbenchmarker/linux_benchmarks/object_storage_service_benchmark.py index 950e5b1fb6..aba0be4ab5 100644 --- a/perfkitbenchmarker/linux_benchmarks/object_storage_service_benchmark.py +++ b/perfkitbenchmarker/linux_benchmarks/object_storage_service_benchmark.py @@ -718,14 +718,12 @@ def ProcessMultiStreamResults( results.append( sample.Sample( 'Multi-stream ' + operation + ' net throughput', - np.sum( - ( - size / active_time * 8 - for size, active_time in zip( - total_active_sizes, total_active_times - ) + np.sum(( + size / active_time * 8 + for size, active_time in zip( + total_active_sizes, total_active_times ) - ), + )), 'bit / second', metadata=distribution_metadata, ) @@ -733,14 +731,10 @@ def ProcessMultiStreamResults( results.append( sample.Sample( 'Multi-stream ' + operation + ' net throughput (with gap)', - np.sum( - ( - size / duration * 8 - for size, duration in zip( - total_active_sizes, active_durations - ) - ) - ), + np.sum(( + size / duration * 8 + for size, duration in zip(total_active_sizes, active_durations) + )), 'bit / second', metadata=distribution_metadata, ) @@ -775,14 +769,12 @@ def ProcessMultiStreamResults( ) # Statistics about benchmarking overhead - gap_time = sum( - ( - active_duration - active_time - for active_duration, active_time in zip( - active_durations, total_active_times - ) + gap_time = sum(( + active_duration - active_time + for active_duration, active_time in zip( + active_durations, total_active_times ) - ) + )) results.append( sample.Sample( 'Multi-stream ' + operation + ' total gap time', diff --git a/perfkitbenchmarker/linux_benchmarks/provisioning_benchmarks/provision_container_cluster_benchmark.py b/perfkitbenchmarker/linux_benchmarks/provisioning_benchmarks/provision_container_cluster_benchmark.py index f495a493bb..84e07d88ff 100644 --- a/perfkitbenchmarker/linux_benchmarks/provisioning_benchmarks/provision_container_cluster_benchmark.py +++ b/perfkitbenchmarker/linux_benchmarks/provisioning_benchmarks/provision_container_cluster_benchmark.py @@ -146,6 +146,7 @@ def GetNode() -> str: if new_nodes: return new_nodes.pop() return '' + new_node = PollForData( GetNode, poll_message='Node not found waiting...', diff --git a/perfkitbenchmarker/linux_benchmarks/specsfs2014_benchmark.py b/perfkitbenchmarker/linux_benchmarks/specsfs2014_benchmark.py index a54d966814..47ff6179be 100644 --- a/perfkitbenchmarker/linux_benchmarks/specsfs2014_benchmark.py +++ b/perfkitbenchmarker/linux_benchmarks/specsfs2014_benchmark.py @@ -209,12 +209,10 @@ def _ConfigureSpec( } # Any special characters in the overrides dictionary should be escaped so # that they don't interfere with sed. - sed_expressions = ' '.join( - [ - '-e "s/{0}=.*/{0}={1}/"'.format(k, v) - for k, v in six.iteritems(configuration_overrides) - ] - ) + sed_expressions = ' '.join([ + '-e "s/{0}=.*/{0}={1}/"'.format(k, v) + for k, v in six.iteritems(configuration_overrides) + ]) sed_cmd = 'sudo sed -i {0} {1}'.format(sed_expressions, config_path) prime_client.RemoteCommand(sed_cmd) diff --git a/perfkitbenchmarker/linux_benchmarks/vbench_transcoding_benchmark.py b/perfkitbenchmarker/linux_benchmarks/vbench_transcoding_benchmark.py index 4a3e828c4b..8645c6ada3 100644 --- a/perfkitbenchmarker/linux_benchmarks/vbench_transcoding_benchmark.py +++ b/perfkitbenchmarker/linux_benchmarks/vbench_transcoding_benchmark.py @@ -200,26 +200,22 @@ def RunParallel(spec: benchmark_spec.BenchmarkSpec) -> List[sample.Sample]: ) vm.RemoteCommand(f'cd {input_videos_dir} && rm -rf *.out.mkv') - samples.extend( - [ - sample.Sample( - 'Total Transcode Time', - total_runtime, - 'seconds', - metadata={ - 'test': 'upload', - 'codec': codec, - 'num_files': 15, # TODO(spencerkim): Count *.out* files. - 'parallelism': jobs, - 'threads': threads, - 'ffmpeg_compiled_from_source': ( - FLAGS.build_ffmpeg_from_source - ), - 'video_copies': 1, - }, - ) - ] - ) + samples.extend([ + sample.Sample( + 'Total Transcode Time', + total_runtime, + 'seconds', + metadata={ + 'test': 'upload', + 'codec': codec, + 'num_files': 15, # TODO(spencerkim): Count *.out* files. + 'parallelism': jobs, + 'threads': threads, + 'ffmpeg_compiled_from_source': FLAGS.build_ffmpeg_from_source, + 'video_copies': 1, + }, + ) + ]) return samples diff --git a/perfkitbenchmarker/linux_packages/__init__.py b/perfkitbenchmarker/linux_packages/__init__.py index e03a05c0a5..defd419485 100644 --- a/perfkitbenchmarker/linux_packages/__init__.py +++ b/perfkitbenchmarker/linux_packages/__init__.py @@ -44,8 +44,10 @@ def _LoadPackages(): - packages = dict([(module.__name__.split('.')[-1], module) for module in - import_util.LoadModulesForPath(__path__, __name__)]) + packages = dict([ + (module.__name__.split('.')[-1], module) + for module in import_util.LoadModulesForPath(__path__, __name__) + ]) packages.update(packages['docker'].CreateImagePackages()) return packages diff --git a/perfkitbenchmarker/linux_packages/act.py b/perfkitbenchmarker/linux_packages/act.py index 150cc12206..d2eebf9e6b 100644 --- a/perfkitbenchmarker/linux_packages/act.py +++ b/perfkitbenchmarker/linux_packages/act.py @@ -29,23 +29,32 @@ GIT_REPO = 'https://github.com/aerospike/act.git' ACT_DIR = '%s/act' % linux_packages.INSTALL_DIR -flags.DEFINE_list('act_load', ['1.0'], - 'Load multiplier for act test per device.') -flags.DEFINE_boolean('act_parallel', False, - 'Run act tools in parallel. One copy per device.') +flags.DEFINE_list( + 'act_load', ['1.0'], 'Load multiplier for act test per device.' +) +flags.DEFINE_boolean( + 'act_parallel', False, 'Run act tools in parallel. One copy per device.' +) flags.DEFINE_integer('act_duration', 86400, 'Duration of act test in seconds.') -flags.DEFINE_integer('act_reserved_partitions', 0, - 'Number of partitions reserved (not being used by act).') flags.DEFINE_integer( - 'act_service_threads', None, + 'act_reserved_partitions', + 0, + 'Number of partitions reserved (not being used by act).', +) +flags.DEFINE_integer( + 'act_service_threads', + None, 'Total number of service threads on which requests are ' - 'generated and done. Default is 5x the number of CPUs.') + 'generated and done. Default is 5x the number of CPUs.', +) flags.DEFINE_integer( - 'act_max_lags_sec', 10, + 'act_max_lags_sec', + 10, 'How much the large-block operations (act_storage) or ' 'cache-thread operations (act_index) are allowed to lag ' 'behind their target rates before the ACT test fails. ' - 'Default is 10.') + 'Default is 10.', +) # TODO(user): Support user provided config file. ACT_CONFIG_TEMPLATE = """ device-names: {devices} @@ -64,8 +73,9 @@ def _Install(vm): vm.Install('build_tools') vm.Install('openssl') vm.RemoteCommand('git clone {0} {1}'.format(GIT_REPO, ACT_DIR)) - vm.RemoteCommand('cd {0} && git checkout {1} && make'.format( - ACT_DIR, ACT_COMMIT)) + vm.RemoteCommand( + 'cd {0} && git checkout {1} && make'.format(ACT_DIR, ACT_COMMIT) + ) def YumInstall(vm): @@ -88,11 +98,15 @@ def RunActPrep(vm): """Runs actprep binary to initialize the drive.""" def _RunActPrep(device): - vm.RobustRemoteCommand('cd {0} && sudo ./target/bin/act_prep {1}'.format( - ACT_DIR, device.GetDevicePath())) - - assert len(vm.scratch_disks) > FLAGS.act_reserved_partitions, ( - 'More reserved partition than total partitions available.') + vm.RobustRemoteCommand( + 'cd {0} && sudo ./target/bin/act_prep {1}'.format( + ACT_DIR, device.GetDevicePath() + ) + ) + + assert ( + len(vm.scratch_disks) > FLAGS.act_reserved_partitions + ), 'More reserved partition than total partitions available.' # Only salt partitions will be used. background_tasks.RunThreaded( _RunActPrep, vm.scratch_disks[FLAGS.act_reserved_partitions :] @@ -104,7 +118,7 @@ def PrepActConfig(vm, load, index=None): if index is None: disk_lst = vm.scratch_disks # Treat first few partitions as reserved. - disk_lst = disk_lst[FLAGS.act_reserved_partitions:] + disk_lst = disk_lst[FLAGS.act_reserved_partitions :] config_file = 'actconfig_{0}.txt'.format(load) else: disk_lst = [vm.scratch_disks[index]] @@ -117,7 +131,8 @@ def PrepActConfig(vm, load, index=None): duration=FLAGS.act_duration, read_iops=_CalculateReadIops(num_disk, load), write_iops=_CalculateWriteIops(num_disk, load), - max_lags=FLAGS.act_max_lags_sec) + max_lags=FLAGS.act_max_lags_sec, + ) if FLAGS.act_service_threads: content += 'service-threads: %d\n' % FLAGS.act_service_threads logging.info('ACT config: %s', content) @@ -140,21 +155,26 @@ def RunAct(vm, load, index=None): # Push config file to remote VM. vm.RobustRemoteCommand( 'cd {0} && sudo ./target/bin/act_storage ~/{1} > ~/{2}'.format( - ACT_DIR, config, output)) + ACT_DIR, config, output + ) + ) # Shows 1,2,4,8,..,64. out, _ = vm.RemoteCommand( 'cd {0} ; python3 ./analysis/act_latency.py -n 7 -e 1 -x -l ~/{1}; ' 'exit 0'.format(ACT_DIR, output), - ignore_failure=True) + ignore_failure=True, + ) samples = ParseRunAct(out) last_output_block, _ = vm.RemoteCommand('tail -n 100 ~/{0}'.format(output)) # Early termination. - if 'drive(s) can\'t keep up - test stopped' in last_output_block: + if "drive(s) can't keep up - test stopped" in last_output_block: act_config_metadata['ERROR'] = 'cannot keep up' act_config_metadata.update( GetActMetadata( - len(vm.scratch_disks) - FLAGS.act_reserved_partitions, load)) + len(vm.scratch_disks) - FLAGS.act_reserved_partitions, load + ) + ) for s in samples: s.metadata.update(act_config_metadata) return samples @@ -183,8 +203,7 @@ def ParseRunAct(out): """ ret = [] if 'could not find 3600 seconds of data' in out: - ret.append(sample.Sample('Failed:NotEnoughSample', 0, '', - {})) + ret.append(sample.Sample('Failed:NotEnoughSample', 0, '', {})) return ret lines = out.split('\n') buckets = [] @@ -204,13 +223,21 @@ def ParseRunAct(out): num_buckets = len(vals) - 1 for i in range(num_buckets - 1): ret.append( - sample.Sample('reads' + matrix, float(vals[i + 1]), '%>(ms)', - {'slice': vals[0], - 'bucket': int(buckets[i])})) + sample.Sample( + 'reads' + matrix, + float(vals[i + 1]), + '%>(ms)', + {'slice': vals[0], 'bucket': int(buckets[i])}, + ) + ) ret.append( - sample.Sample('read_rate' + matrix, - float(vals[num_buckets]), 'iops', - {'slice': vals[0]})) + sample.Sample( + 'read_rate' + matrix, + float(vals[num_buckets]), + 'iops', + {'slice': vals[0]}, + ) + ) return ret @@ -230,7 +257,7 @@ def GetActMetadata(num_disk, load): 'write-reqs-per-sec': _CalculateWriteIops(num_disk, load), 'max-lag-sec': FLAGS.act_max_lags_sec, 'microsecond-histograms': 'no', - 'scheduler-mode': 'noop' + 'scheduler-mode': 'noop', } metadata['service-threads'] = FLAGS.act_service_threads or 'default' # num-queues & threads-per-queues are deprecated in act v6+, diff --git a/perfkitbenchmarker/linux_packages/aerospike_client.py b/perfkitbenchmarker/linux_packages/aerospike_client.py index fbb4e020cd..e751708fa9 100644 --- a/perfkitbenchmarker/linux_packages/aerospike_client.py +++ b/perfkitbenchmarker/linux_packages/aerospike_client.py @@ -30,8 +30,9 @@ FLAGS = flags.FLAGS PATH = 'aerospike-tools-7.0.5-ubuntu20.04' TAR_FILE = f'{PATH}.tgz' -DOWNLOAD_URL = ('https://download.aerospike.com/' - f'artifacts/aerospike-tools/7.0.5/{TAR_FILE}') +DOWNLOAD_URL = ( + f'https://download.aerospike.com/artifacts/aerospike-tools/7.0.5/{TAR_FILE}' +) STDOUT_START = 'Stage 1: default config' SUM = lambda x, y: x + y METADATA_AGGREGATOR = { @@ -78,7 +79,7 @@ def YumInstall(vm): def _ExtractResultLines(lines): for line_idx in range(len(lines)): if lines[line_idx].startswith(STDOUT_START): # ignore everyhing before. - return lines[line_idx + 1:] + return lines[line_idx + 1 :] def ParseDate(date: str) -> datetime.datetime: @@ -140,7 +141,7 @@ def ParseAsbenchStdout(output): f'{op}_p90': float(values[8][:-1]), f'{op}_p99': float(values[9][:-1]), f'{op}_p99.9': float(values[10][:-1]), - f'{op}_p99.99': float(values[11]) + f'{op}_p99.99': float(values[11]), }) elif 'AEROSPIKE_ERR_TIMEOUT' in line or 'AEROSPIKE_ERR_CONNECTION' in line: continue @@ -148,7 +149,8 @@ def ParseAsbenchStdout(output): continue else: aggregation_metrics = regex_util.ExtractAllFloatMetrics( - regex_util.ExtractExactlyOneMatch(r'total\(.*\)', line)) + regex_util.ExtractExactlyOneMatch(r'total\(.*\)', line) + ) metadata.update(aggregation_metrics) samples.append( @@ -156,7 +158,9 @@ def ParseAsbenchStdout(output): 'throughput', aggregation_metrics['tps'], 'transaction_per_second', - metadata=copy.deepcopy(metadata))) + metadata=copy.deepcopy(metadata), + ) + ) metadata.clear() return samples @@ -181,11 +185,13 @@ def AggregateAsbenchSamples(raw_samples): else: new_value = aggregated_samples[timestamp].value + s.value current_sample = aggregated_samples[timestamp] - aggregated_samples[timestamp] = sample.Sample(current_sample.metric, - new_value, - current_sample.unit, - current_sample.metadata, - current_sample.timestamp) + aggregated_samples[timestamp] = sample.Sample( + current_sample.metric, + new_value, + current_sample.unit, + current_sample.metadata, + current_sample.timestamp, + ) def _AggregateMetadata(agg_metadata, metadata): # Iterate on the copy of metadata, so we can drop keys at runtime. @@ -193,8 +199,9 @@ def _AggregateMetadata(agg_metadata, metadata): # find aggregator for regex in METADATA_AGGREGATOR: if re.search(regex, key): - agg_metadata[key] = METADATA_AGGREGATOR[regex](value, - metadata[key]) + agg_metadata[key] = METADATA_AGGREGATOR[regex]( + value, metadata[key] + ) break else: # drop metadata if we do not know how to aggregate @@ -395,6 +402,7 @@ def ParseAsbenchHistogram(result_files: List[str]) -> List[sample.Sample]: @dataclasses.dataclass class AsbenchResult: """Class that represents Asbench results.""" + ops: float timestamp: float read_min: float @@ -405,8 +413,9 @@ class AsbenchResult: def __init__(self, s: sample.Sample): super(AsbenchResult, self).__init__() self.ops = s.value - self.timestamp = (s.metadata['start_timestamp'] + s.metadata['window'] - - 1) * 1000 + self.timestamp = ( + s.metadata['start_timestamp'] + s.metadata['window'] - 1 + ) * 1000 self.read_min = s.metadata.get('read_min', None) self.read_max = s.metadata.get('read_max', None) self.write_min = s.metadata.get('write_min', None) @@ -430,51 +439,61 @@ def CreateTimeSeriesSample(samples: List[sample.Sample]) -> List[sample.Sample]: results = sorted(results, key=lambda r: r.timestamp) rampup_end_time = min(r.timestamp for r in results) + RAMPUP_TIME_IN_MS ts_samples = [ - sample.CreateTimeSeriesSample([r.ops for r in results], - [r.timestamp for r in results], - sample.OPS_TIME_SERIES, - 'ops', - 1, - ramp_up_ends=rampup_end_time, - additional_metadata={}) + sample.CreateTimeSeriesSample( + [r.ops for r in results], + [r.timestamp for r in results], + sample.OPS_TIME_SERIES, + 'ops', + 1, + ramp_up_ends=rampup_end_time, + additional_metadata={}, + ) ] - total_ops = sample.Sample('total_ops', - sum([r.ops for r in results]) / len(results), 'ops', - {}) + total_ops = sample.Sample( + 'total_ops', sum([r.ops for r in results]) / len(results), 'ops', {} + ) if results[0].read_min: # the workload does read operations ts_samples.extend([ - sample.CreateTimeSeriesSample([r.read_min for r in results], - [r.timestamp for r in results], - f'Read_Min_{sample.LATENCY_TIME_SERIES}', - 'us', - 1, - ramp_up_ends=rampup_end_time, - additional_metadata={}), - sample.CreateTimeSeriesSample([r.read_max for r in results], - [r.timestamp for r in results], - f'Read_Max_{sample.LATENCY_TIME_SERIES}', - 'us', - 1, - ramp_up_ends=rampup_end_time, - additional_metadata={}) + sample.CreateTimeSeriesSample( + [r.read_min for r in results], + [r.timestamp for r in results], + f'Read_Min_{sample.LATENCY_TIME_SERIES}', + 'us', + 1, + ramp_up_ends=rampup_end_time, + additional_metadata={}, + ), + sample.CreateTimeSeriesSample( + [r.read_max for r in results], + [r.timestamp for r in results], + f'Read_Max_{sample.LATENCY_TIME_SERIES}', + 'us', + 1, + ramp_up_ends=rampup_end_time, + additional_metadata={}, + ), ]) if results[0].write_min: # The workload has write operations ts_samples.extend([ - sample.CreateTimeSeriesSample([r.write_min for r in results], - [r.timestamp for r in results], - f'Write_Min_{sample.LATENCY_TIME_SERIES}', - 'us', - 1, - ramp_up_ends=rampup_end_time, - additional_metadata={}), - sample.CreateTimeSeriesSample([r.write_max for r in results], - [r.timestamp for r in results], - f'Write_Max_{sample.LATENCY_TIME_SERIES}', - 'us', - 1, - ramp_up_ends=rampup_end_time, - additional_metadata={}) + sample.CreateTimeSeriesSample( + [r.write_min for r in results], + [r.timestamp for r in results], + f'Write_Min_{sample.LATENCY_TIME_SERIES}', + 'us', + 1, + ramp_up_ends=rampup_end_time, + additional_metadata={}, + ), + sample.CreateTimeSeriesSample( + [r.write_max for r in results], + [r.timestamp for r in results], + f'Write_Max_{sample.LATENCY_TIME_SERIES}', + 'us', + 1, + ramp_up_ends=rampup_end_time, + additional_metadata={}, + ), ]) return ts_samples + [total_ops] diff --git a/perfkitbenchmarker/linux_packages/aerospike_server.py b/perfkitbenchmarker/linux_packages/aerospike_server.py index decb33ba03..dae8f358eb 100644 --- a/perfkitbenchmarker/linux_packages/aerospike_server.py +++ b/perfkitbenchmarker/linux_packages/aerospike_server.py @@ -84,12 +84,17 @@ class AerospikeEdition(enum.Enum): ' to zeroize the header. If set to true, it will wipe the full disk.' ), ) -flags.DEFINE_integer('aerospike_replication_factor', 1, - 'Replication factor for aerospike server.') -flags.DEFINE_integer('aerospike_service_threads', 4, - 'Number of threads per transaction queue.') -flags.DEFINE_integer('aerospike_vms', 1, - 'Number of vms (nodes) for aerospike server.') +flags.DEFINE_integer( + 'aerospike_replication_factor', + 1, + 'Replication factor for aerospike server.', +) +flags.DEFINE_integer( + 'aerospike_service_threads', 4, 'Number of threads per transaction queue.' +) +flags.DEFINE_integer( + 'aerospike_vms', 1, 'Number of vms (nodes) for aerospike server.' +) MIN_FREE_KBYTES = 1160000 @@ -125,10 +130,13 @@ def _InstallFromGit(vm): 'cd {0} && git checkout {1} && git submodule update --init ' '&& sed -i "s/COMMON_CFLAGS += -Werror/# $COMMON_CFLAGS += -Werror/" ' '{0}/make_in/Makefile.in ' - '&& make'.format(_GetAerospikeDir(), GIT_TAG)) + '&& make'.format(_GetAerospikeDir(), GIT_TAG) + ) for idx in range(FLAGS.aerospike_instances): - vm.RemoteCommand(f'mkdir {linux_packages.INSTALL_DIR}/{idx}; ' - f'cp -rf {_GetAerospikeDir()} {_GetAerospikeDir(idx)}') + vm.RemoteCommand( + f'mkdir {linux_packages.INSTALL_DIR}/{idx}; ' + f'cp -rf {_GetAerospikeDir()} {_GetAerospikeDir(idx)}' + ) def _InstallFromPackage(vm): @@ -181,8 +189,11 @@ def AptInstall(vm): _Install(vm) -@vm_util.Retry(poll_interval=5, timeout=300, - retryable_exceptions=(errors.Resource.RetryableCreationError)) +@vm_util.Retry( + poll_interval=5, + timeout=300, + retryable_exceptions=(errors.Resource.RetryableCreationError), +) def _WaitForServerUp(server, idx=None): """Block until the Aerospike server is up and responsive. @@ -209,6 +220,7 @@ def _WaitForServerUp(server, idx=None): logging.info('Trying to connect to Aerospike at %s:%s', server, port) try: + def _NetcatPrefix(): _, stderr = server.RemoteCommand('nc -h', ignore_failure=True) if '-q' in stderr: @@ -217,17 +229,20 @@ def _NetcatPrefix(): return 'nc -i 1' out, _ = server.RemoteCommand( - '(echo -e "status\n" ; sleep 1)| %s %s %s' % ( - _NetcatPrefix(), address, port)) + '(echo -e "status\n" ; sleep 1)| %s %s %s' + % (_NetcatPrefix(), address, port) + ) if out.startswith('ok'): logging.info('Aerospike server status is OK. Server up and running.') return except errors.VirtualMachine.RemoteCommandError as e: raise errors.Resource.RetryableCreationError( - 'Aerospike server not up yet: %s.' % str(e)) + 'Aerospike server not up yet: %s.' % str(e) + ) else: raise errors.Resource.RetryableCreationError( - "Aerospike server not up yet. Expected 'ok' but got '%s'." % out) + "Aerospike server not up yet. Expected 'ok' but got '%s'." % out + ) def WipeDisk(server, devices): @@ -311,15 +326,18 @@ def ConfigureAndStart(server, seed_node_ips=None): # Linux best practice based on: # https://docs.aerospike.com/server/operations/install/linux/bestpractices#linux-best-practices - server.RemoteCommand(f'echo {MIN_FREE_KBYTES * FLAGS.aerospike_instances} ' - '| sudo tee /proc/sys/vm/min_free_kbytes') + server.RemoteCommand( + f'echo {MIN_FREE_KBYTES * FLAGS.aerospike_instances} ' + '| sudo tee /proc/sys/vm/min_free_kbytes' + ) server.RemoteCommand('echo 0 | sudo tee /proc/sys/vm/swappiness') for idx in range(FLAGS.aerospike_instances): current_devices = [] if devices: num_device_per_instance = int(len(devices) / FLAGS.aerospike_instances) - current_devices = devices[idx * num_device_per_instance:(idx + 1) * - num_device_per_instance] + current_devices = devices[ + idx * num_device_per_instance : (idx + 1) * num_device_per_instance + ] server.RenderTemplate( data.ResourcePath(_AEROSPIKE_CONFIGS[_AEROSPIKE_EDITION.value]), _GetAerospikeConfig(idx), diff --git a/perfkitbenchmarker/linux_packages/amdblis.py b/perfkitbenchmarker/linux_packages/amdblis.py index 4995a4a5b4..1d12cd4ac4 100644 --- a/perfkitbenchmarker/linux_packages/amdblis.py +++ b/perfkitbenchmarker/linux_packages/amdblis.py @@ -27,9 +27,11 @@ def _Install(vm): vm.RemoteCommand('git clone {0} {1}'.format(GIT_REPO, AMDBLIS_DIR)) vm.RemoteCommand('cd {0} && git checkout {1}'.format(AMDBLIS_DIR, GIT_TAG)) vm.RemoteCommand( - 'cd {0} && ./configure --enable-cblas zen'.format(AMDBLIS_DIR)) - vm.RemoteCommand('sudo ln -s /usr/bin/python3 /usr/bin/python', - ignore_failure=True) + 'cd {0} && ./configure --enable-cblas zen'.format(AMDBLIS_DIR) + ) + vm.RemoteCommand( + 'sudo ln -s /usr/bin/python3 /usr/bin/python', ignore_failure=True + ) vm.RemoteCommand('cd {0} && make -j'.format(AMDBLIS_DIR)) diff --git a/perfkitbenchmarker/linux_packages/amdgpu_driver.py b/perfkitbenchmarker/linux_packages/amdgpu_driver.py index 56f170923c..23b0477d51 100644 --- a/perfkitbenchmarker/linux_packages/amdgpu_driver.py +++ b/perfkitbenchmarker/linux_packages/amdgpu_driver.py @@ -31,17 +31,20 @@ def AptInstall(vm: virtual_machine.BaseVirtualMachine) -> None: tmp_dir = vm.scratch_disks[0].mount_point vm.DownloadPreprovisionedData(tmp_dir, 'amdgpu', _AMD_DRIVER) vm.RemoteCommand( - f'tar -xf {posixpath.join(tmp_dir, _AMD_DRIVER)} -C {tmp_dir}') + f'tar -xf {posixpath.join(tmp_dir, _AMD_DRIVER)} -C {tmp_dir}' + ) vm.InstallPackages('linux-modules-extra-$(uname -r)') - install_file = posixpath.join(tmp_dir, - 'amdgpu-pro-20.20-1184451-ubuntu-18.04', - 'amdgpu-pro-install') + install_file = posixpath.join( + tmp_dir, 'amdgpu-pro-20.20-1184451-ubuntu-18.04', 'amdgpu-pro-install' + ) vm.RemoteCommand(f'{install_file} -y --opencl=pal,legacy') vm.RemoteCommand( - 'echo -e "Section \\\"Device\\\"\n Identifier \\\"AMD\\\"\n' - ' Driver \\\"amdgpu\\\"\nEndSection" | sudo tee /etc/X11/xorg.conf') + 'echo -e "Section \\"Device\\"\n Identifier \\"AMD\\"\n' + ' Driver \\"amdgpu\\"\nEndSection" | sudo tee /etc/X11/xorg.conf' + ) vm.RemoteCommand( - 'echo "allowed_users=anybody" | sudo tee /etc/X11/Xwrapper.config') + 'echo "allowed_users=anybody" | sudo tee /etc/X11/Xwrapper.config' + ) vm.InstallPackages('x11vnc xorg xserver-xorg-video-amdgpu') vm.RemoteCommand( 'wget https://github.com/GPUOpen-Drivers/AMDVLK/releases/download/v-2021.Q2.5/amdvlk_2021.Q2.5_amd64.deb' @@ -52,8 +55,10 @@ def AptInstall(vm: virtual_machine.BaseVirtualMachine) -> None: vm.RemoteCommand('sudo modprobe amdgpu') vm.RemoteCommand('sudo modprobe drm') vm.RemoteCommand('sudo chown ubuntu /dev/tty2') - vm.RemoteCommand('sudo nohup Xorg :0 -verbose 2 vt2 ' - '1> /tmp/stdout.log 2> /tmp/stderr.log &') + vm.RemoteCommand( + 'sudo nohup Xorg :0 -verbose 2 vt2 ' + '1> /tmp/stdout.log 2> /tmp/stderr.log &' + ) def YumInstall(vm: virtual_machine.BaseVirtualMachine) -> None: diff --git a/perfkitbenchmarker/linux_packages/ant.py b/perfkitbenchmarker/linux_packages/ant.py index 92ad3a234b..27f5d373a0 100644 --- a/perfkitbenchmarker/linux_packages/ant.py +++ b/perfkitbenchmarker/linux_packages/ant.py @@ -24,8 +24,8 @@ PACKAGE_NAME = 'ant' PREPROVISIONED_DATA = { - ANT_TAR: - '90d28c0202871bd9875a5da6d982f362bb3114d346b9d8ae58860b8d3312c21c'} + ANT_TAR: '90d28c0202871bd9875a5da6d982f362bb3114d346b9d8ae58860b8d3312c21c' +} PACKAGE_DATA_URL = {ANT_TAR: ANT_TAR_URL} ANT_HOME_DIR = posixpath.join(linux_packages.INSTALL_DIR, PACKAGE_NAME) @@ -33,11 +33,15 @@ def _Install(vm): """Installs the Ant package on the VM.""" vm.Install('wget') - vm.InstallPreprovisionedPackageData(PACKAGE_NAME, PREPROVISIONED_DATA.keys(), - linux_packages.INSTALL_DIR) - vm.RemoteCommand('cd {0} && tar -zxf apache-ant-1.9.6-bin.tar.gz && ' - 'ln -s {0}/apache-ant-1.9.6/ {1}'.format( - linux_packages.INSTALL_DIR, ANT_HOME_DIR)) + vm.InstallPreprovisionedPackageData( + PACKAGE_NAME, PREPROVISIONED_DATA.keys(), linux_packages.INSTALL_DIR + ) + vm.RemoteCommand( + 'cd {0} && tar -zxf apache-ant-1.9.6-bin.tar.gz && ' + 'ln -s {0}/apache-ant-1.9.6/ {1}'.format( + linux_packages.INSTALL_DIR, ANT_HOME_DIR + ) + ) def YumInstall(vm): diff --git a/perfkitbenchmarker/linux_packages/apache2_server.py b/perfkitbenchmarker/linux_packages/apache2_server.py index f6237c54e5..fb526ace50 100644 --- a/perfkitbenchmarker/linux_packages/apache2_server.py +++ b/perfkitbenchmarker/linux_packages/apache2_server.py @@ -31,14 +31,14 @@ def SetupServer(vm: virtual_machine.VirtualMachine, content_bytes: int) -> None: Args: vm: The VM that will have the apache server set up. content_bytes: The number of bytes the content that the apache server will - serve will be. + serve will be. """ # Set up access to vm/var/www vm.RemoteCommand('sudo chown -R $USER:$USER /var/www') # Set up basic HTML webpage with tempfile.NamedTemporaryFile() as tmp: - tmp.write(f''' + tmp.write(f""" @@ -50,7 +50,7 @@ def SetupServer(vm: virtual_machine.VirtualMachine, content_bytes: int) -> None:

{'a' * content_bytes}

- '''.encode()) + """.encode()) vm.PushFile(tmp.name, '/var/www/html/index.html') # Set up read access to index.html @@ -58,7 +58,8 @@ def SetupServer(vm: virtual_machine.VirtualMachine, content_bytes: int) -> None: # Download sample image to serve vm.RemoteCommand( - 'wget --output-document=/var/www/html/image.png https://http.cat/100') + 'wget --output-document=/var/www/html/image.png https://http.cat/100' + ) # Enable status module if not already enabled for Apache server monitoring output, _ = vm.RemoteCommand('ls /etc/apache2/mods-enabled | grep status*') @@ -81,8 +82,7 @@ def StartServer(vm: virtual_machine.VirtualMachine) -> None: def GetApacheCPUSeconds(vm: virtual_machine.VirtualMachine) -> float: """Returns the total number of CPU seconds Apache has used.""" output, _ = vm.RemoteCommand( - 'curl -s http://localhost/server-status?auto | ' - 'grep \'CPUSystem\\|CPUUser\'' + "curl -s http://localhost/server-status?auto | grep 'CPUSystem\\|CPUUser'" ) cpu_use = 0.0 diff --git a/perfkitbenchmarker/linux_packages/aws_credentials.py b/perfkitbenchmarker/linux_packages/aws_credentials.py index 422d5cb265..6baa1fc59e 100644 --- a/perfkitbenchmarker/linux_packages/aws_credentials.py +++ b/perfkitbenchmarker/linux_packages/aws_credentials.py @@ -34,18 +34,24 @@ FLAGS = flags.FLAGS flags.DEFINE_string( - 'aws_credentials_local_path', os.path.join('~', '.aws'), - 'Path where the AWS credential files can be found on the local machine.') + 'aws_credentials_local_path', + os.path.join('~', '.aws'), + 'Path where the AWS credential files can be found on the local machine.', +) flags.DEFINE_string( - 'aws_credentials_remote_path', '.aws', - 'Path where the AWS credential files will be written on remote machines.') + 'aws_credentials_remote_path', + '.aws', + 'Path where the AWS credential files will be written on remote machines.', +) flags.DEFINE_boolean( - 'aws_credentials_overwrite', False, + 'aws_credentials_overwrite', + False, 'When set, if an AWS credential file already exists at the destination ' 'specified by --aws_credentials_remote_path, it will be overwritten during ' - 'AWS credential file installation.') + 'AWS credential file installation.', +) flags.DEFINE_string('aws_s3_region', None, 'Region for the S3 bucket') @@ -89,7 +95,8 @@ def CheckPrerequisites(): local_path = _GetLocalPath() if not os.path.exists(local_path): raise data.ResourceNotFound( - 'AWS credential files were not found at {0}'.format(local_path)) + 'AWS credential files were not found at {0}'.format(local_path) + ) def Install(vm): diff --git a/perfkitbenchmarker/linux_packages/azure_cli.py b/perfkitbenchmarker/linux_packages/azure_cli.py index f68d0dc22f..f5b297af78 100644 --- a/perfkitbenchmarker/linux_packages/azure_cli.py +++ b/perfkitbenchmarker/linux_packages/azure_cli.py @@ -30,8 +30,10 @@ # Debian info _DEB_REPO_FILE = '/etc/apt/sources.list.d/azure-cli.list' _DEB_REPO_KEY = 'https://packages.microsoft.com/keys/microsoft.asc' -_DEB_REPO = ('deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ ' - '{az_repo} main') +_DEB_REPO = ( + 'deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ ' + '{az_repo} main' +) # RedHat info _YUM_REPO_FILE = '/etc/yum.repos.d/azure-cli.repo' @@ -47,7 +49,7 @@ """ -@vm_util.Retry(poll_interval=15, timeout=5*60) +@vm_util.Retry(poll_interval=15, timeout=5 * 60) def ZypperInstall(vm): """Installs the azure-cli package on the VM for SUSE systems. @@ -57,13 +59,15 @@ def ZypperInstall(vm): # Work-around to remove conflicting python packages. See # https://github.com/Azure/azure-cli/issues/13209 vm.RemoteCommand( - 'sudo zypper install -y --oldpackage azure-cli-2.0.45-4.22.noarch') + 'sudo zypper install -y --oldpackage azure-cli-2.0.45-4.22.noarch' + ) vm.RemoteCommand('sudo zypper rm -y --clean-deps azure-cli') vm.Install('curl') vm.RemoteCommand('sudo rpm --import {key}'.format(key=_YUM_REPO_KEY)) vm.RemoteCommand( f'sudo zypper addrepo --name "{_YUM_REPO_NAME}" ' - f'--check {_YUM_REPO_URL} azure-cli') + f'--check {_YUM_REPO_URL} azure-cli' + ) vm.RemoteCommand('sudo zypper install -y --from azure-cli azure-cli') @@ -93,7 +97,8 @@ def AptInstall(vm): az_repo, _ = vm.RemoteCommand('lsb_release -cs') _CreateFile(vm, _DEB_REPO.format(az_repo=az_repo.strip()), _DEB_REPO_FILE) vm.RemoteCommand( - 'curl -L {key} | sudo apt-key add -'.format(key=_DEB_REPO_KEY)) + 'curl -L {key} | sudo apt-key add -'.format(key=_DEB_REPO_KEY) + ) vm.RemoteCommand('sudo apt-get update') vm.InstallPackages('azure-cli') @@ -117,5 +122,8 @@ def _CreateFile(vm, content, file_path): content: Text to put into the file. file_path: Path to text output file. """ - vm.RemoteCommand('echo "{content}" | sudo tee {file_path}'.format( - content=content, file_path=file_path)) + vm.RemoteCommand( + 'echo "{content}" | sudo tee {file_path}'.format( + content=content, file_path=file_path + ) + ) diff --git a/perfkitbenchmarker/linux_packages/azure_credentials.py b/perfkitbenchmarker/linux_packages/azure_credentials.py index 0997b0c177..dce889d160 100644 --- a/perfkitbenchmarker/linux_packages/azure_credentials.py +++ b/perfkitbenchmarker/linux_packages/azure_credentials.py @@ -18,7 +18,6 @@ from absl import logging from packaging import version - from perfkitbenchmarker import errors from perfkitbenchmarker import object_storage_service from perfkitbenchmarker import vm_util @@ -28,8 +27,9 @@ # Necessary for user login with MSAL TOKENS_FILE = os.path.join(AZURE_CREDENTIAL_DIRECTORY, 'msal_token_cache.json') # Necessary for service_principal login with MSAL -SERVICE_PRINCIPAL_FILE = os.path.join(AZURE_CREDENTIAL_DIRECTORY, - 'service_principal_entries.json') +SERVICE_PRINCIPAL_FILE = os.path.join( + AZURE_CREDENTIAL_DIRECTORY, 'service_principal_entries.json' +) # Determines which login is active PROFILE_FILE = os.path.join(AZURE_CREDENTIAL_DIRECTORY, 'azureProfile.json') @@ -41,13 +41,15 @@ def Install(vm): """Copies Azure credentials to the VM.""" # Validate local azure-cli uses MSAL stdout, _, _ = vm_util.IssueCommand( - [azure.AZURE_PATH, 'version', '--query', '"azure-cli"']) + [azure.AZURE_PATH, 'version', '--query', '"azure-cli"'] + ) az_version = version.Version(stdout.strip('"\n')) if az_version < version.Version(_REQUIRED_MSAL_CLI_VERSION): raise errors.Benchmarks.MissingObjectCredentialException( f'Local Azure CLI version must be at least {_REQUIRED_MSAL_CLI_VERSION}' f' to copy credentials into a VM. Found version {az_version}. ' - 'The recent CLI on the VM will not be able to use your credentials.') + 'The recent CLI on the VM will not be able to use your credentials.' + ) # Install CLI to validate credentials vm.Install('azure_cli') @@ -55,7 +57,8 @@ def Install(vm): # Copy credentials to VM vm.RemoteCommand('mkdir -p {0}'.format(AZURE_CREDENTIAL_DIRECTORY)) vm.PushFile( - object_storage_service.FindCredentialFile(PROFILE_FILE), PROFILE_FILE) + object_storage_service.FindCredentialFile(PROFILE_FILE), PROFILE_FILE + ) for file in [SERVICE_PRINCIPAL_FILE, TOKENS_FILE]: try: vm.PushFile(object_storage_service.FindCredentialFile(file), file) @@ -69,4 +72,5 @@ def Install(vm): vm.RemoteCommand('az account get-access-token') except errors.VirtualMachine.RemoteCommandError as e: raise errors.Benchmarks.MissingObjectCredentialException( - 'Failed to install azure_credentials on VM.') from e + 'Failed to install azure_credentials on VM.' + ) from e diff --git a/perfkitbenchmarker/linux_packages/azure_sdk.py b/perfkitbenchmarker/linux_packages/azure_sdk.py index 1d07d8fbc5..ad00ae6487 100644 --- a/perfkitbenchmarker/linux_packages/azure_sdk.py +++ b/perfkitbenchmarker/linux_packages/azure_sdk.py @@ -16,8 +16,11 @@ from absl import flags -flags.DEFINE_string('azure_lib_version', '1.0.3', - 'Use a particular version of azure client lib, e.g.: 1.0.2') +flags.DEFINE_string( + 'azure_lib_version', + '1.0.3', + 'Use a particular version of azure client lib, e.g.: 1.0.2', +) FLAGS = flags.FLAGS diff --git a/perfkitbenchmarker/linux_packages/blaze.py b/perfkitbenchmarker/linux_packages/blaze.py index e2ad0781f3..3b5370ed0b 100644 --- a/perfkitbenchmarker/linux_packages/blaze.py +++ b/perfkitbenchmarker/linux_packages/blaze.py @@ -21,8 +21,7 @@ BLAZE_VERSION = '3.0' BLAZE_TAR = 'blaze-%s.tar.gz' % BLAZE_VERSION BLAZE_DIR = '%s/blaze-%s' % (vm_util.VM_TMP_DIR, BLAZE_VERSION) -BLAZE_TAR_URL = ( - 'https://bitbucket.org/blaze-lib/blaze/downloads/%s' % BLAZE_TAR) +BLAZE_TAR_URL = 'https://bitbucket.org/blaze-lib/blaze/downloads/%s' % BLAZE_TAR CONFIG_TEMPLATE = 'blaze_config.j2' CONFIG = 'config' MAX_BLAZE_CACHE_SIZE_IN_B = 100000000 @@ -37,30 +36,42 @@ def _Configure(vm): vm.RenderTemplate( data.ResourcePath(CONFIG_TEMPLATE), os.path.join(BLAZE_DIR, CONFIG), - {'compiler': 'g++-5', - 'compile_flags': ' -DBLAZE_USE_BOOST_THREADS --std=c++14'}) + { + 'compiler': 'g++-5', + 'compile_flags': ' -DBLAZE_USE_BOOST_THREADS --std=c++14', + }, + ) # Adjust cache size cache_in_KB, _ = vm.RemoteCommand( - 'cat /proc/cpuinfo | grep "cache size" | awk \'{print $4}\'') + 'cat /proc/cpuinfo | grep "cache size" | awk \'{print $4}\'' + ) cache_in_B = int(1024 * float(cache_in_KB.split()[0])) vm.RemoteCommand( - 'sed -i \'s/constexpr size_t cacheSize = 3145728UL;/constexpr ' - 'size_t cacheSize = %sUL;/g\' %s' % ( - min(cache_in_B, MAX_BLAZE_CACHE_SIZE_IN_B - 1), os.path.join( - BLAZE_DIR, 'blaze', 'config', 'CacheSize.h'))) - vm.RemoteCommand('cd %s; ./configure %s; make -j %s' % ( - BLAZE_DIR, CONFIG, vm.NumCpusForBenchmark())) + "sed -i 's/constexpr size_t cacheSize = 3145728UL;/constexpr " + "size_t cacheSize = %sUL;/g' %s" + % ( + min(cache_in_B, MAX_BLAZE_CACHE_SIZE_IN_B - 1), + os.path.join(BLAZE_DIR, 'blaze', 'config', 'CacheSize.h'), + ) + ) + vm.RemoteCommand( + 'cd %s; ./configure %s; make -j %s' + % (BLAZE_DIR, CONFIG, vm.NumCpusForBenchmark()) + ) def _Install(vm): """Installs the blaze package on the VM.""" vm.RemoteCommand( 'cd {tmp_dir}; wget {tar_url}; tar xzvf {tar}'.format( - tmp_dir=vm_util.VM_TMP_DIR, - tar_url=BLAZE_TAR_URL, - tar=BLAZE_TAR)) - vm.RemoteCommand('sudo cp -r {blaze_dir}/blaze /usr/local/include'.format( - blaze_dir=BLAZE_DIR)) + tmp_dir=vm_util.VM_TMP_DIR, tar_url=BLAZE_TAR_URL, tar=BLAZE_TAR + ) + ) + vm.RemoteCommand( + 'sudo cp -r {blaze_dir}/blaze /usr/local/include'.format( + blaze_dir=BLAZE_DIR + ) + ) _Configure(vm) diff --git a/perfkitbenchmarker/linux_packages/blazemark.py b/perfkitbenchmarker/linux_packages/blazemark.py index 2adca49710..dc4942ec99 100644 --- a/perfkitbenchmarker/linux_packages/blazemark.py +++ b/perfkitbenchmarker/linux_packages/blazemark.py @@ -31,42 +31,136 @@ THROUGHPUT_HEADER_REGEX = ( r'(\w+[\w\- ]+\w+)\s*(\([0-9.]+% filled\))*\s*[\[\(]' - '([\w/]+)[\]\)]:([0-9\s.e\-+]+)') + '([\w/]+)[\]\)]:([0-9\s.e\-+]+)' +) THROUGHPUT_RESULT_REGEX = r'([0-9]+)\s*([0-9.e\-+]+)' FILLED_REGEX = r'([0-9.]+)% filled' LIBS = frozenset([ - 'C-like', 'Classic', 'Blaze', 'Boost uBLAS', 'Blitz++', - 'GMM++', 'Armadillo', 'MTL', 'Eigen']) + 'C-like', + 'Classic', + 'Blaze', + 'Boost uBLAS', + 'Blitz++', + 'GMM++', + 'Armadillo', + 'MTL', + 'Eigen', +]) BLAZEMARK_BINARIES = frozenset([ - 'cg', 'daxpy', 'dmatsvecmult', 'dvecdvecsub', 'mat3mat3mult', - 'smatdmatmult', 'smattsmatadd', 'svectdvecmult', 'tdmattdmatmult', - 'tmat3mat3mult', 'tsmatdmatmult', 'tsvecdmatmult', 'tvec6tmat6mult', - 'complex1', 'dmatdmatadd', 'dmattdmatadd', 'dvecnorm', 'mat3tmat3mult', - 'smatdvecmult', 'smattsmatmult', 'svectsvecmult', 'tdmattsmatadd', - 'tmat3tmat3add', 'tsmatdvecmult', 'tsvecdvecmult', 'vec3vec3add', - 'complex2', 'dmatdmatmult', 'dmattdmatmult', 'dvecscalarmult', - 'mat3vec3mult', 'smatscalarmult', 'svecdvecadd', 'tdmatdmatadd', - 'tdmattsmatmult', 'tmat3tmat3mult', 'tsmatsmatadd', 'tsvecsmatmult', - 'vec6vec6add', 'complex3', 'dmatdmatsub', 'dmattrans', 'dvecsvecadd', - 'mat6mat6add', 'smatsmatadd', 'svecdveccross', 'tdmatdmatmult', - 'tdvecdmatmult', 'tmat3vec3mult', 'tsmatsmatmult', 'tsvecsvecmult', - 'complex4', 'dmatdvecmult', 'dmattsmatadd', 'dvecsveccross', 'mat6mat6mult', - 'smatsmatmult', 'svecdvecmult', 'tdmatdvecmult', 'tdvecdvecmult', - 'tmat6mat6mult', 'tsmatsvecmult', 'tsvectdmatmult', 'complex5', 'dmatinv', - 'dmattsmatmult', 'dvecsvecmult', 'mat6tmat6mult', 'smatsvecmult', - 'svecscalarmult', 'tdmatsmatadd', 'tdvecsmatmult', 'tmat6tmat6add', - 'tsmattdmatadd', 'tsvectsmatmult', 'complex6', 'dmatscalarmult', - 'dvecdvecadd', 'dvectdvecmult', 'mat6vec6mult', 'smattdmatadd', - 'svecsvecadd', 'tdmatsmatmult', 'tdvecsvecmult', 'tmat6tmat6mult', - 'tsmattdmatmult', 'tvec3mat3mult', 'complex7', 'dmatsmatadd', - 'dvecdveccross', 'dvectsvecmult', 'memorysweep', 'smattdmatmult', - 'svecsveccross', 'tdmatsvecmult', 'tdvectdmatmult', 'tmat6vec6mult', - 'tsmattsmatadd', 'tvec3tmat3mult', 'complex8', 'dmatsmatmult', - 'dvecdvecmult', 'mat3mat3add', 'smatdmatadd', 'smattrans', - 'svecsvecmult', 'tdmattdmatadd', 'tdvectsmatmult', 'tsmatdmatadd', - 'tsmattsmatmult', 'tvec6mat6mult']) + 'cg', + 'daxpy', + 'dmatsvecmult', + 'dvecdvecsub', + 'mat3mat3mult', + 'smatdmatmult', + 'smattsmatadd', + 'svectdvecmult', + 'tdmattdmatmult', + 'tmat3mat3mult', + 'tsmatdmatmult', + 'tsvecdmatmult', + 'tvec6tmat6mult', + 'complex1', + 'dmatdmatadd', + 'dmattdmatadd', + 'dvecnorm', + 'mat3tmat3mult', + 'smatdvecmult', + 'smattsmatmult', + 'svectsvecmult', + 'tdmattsmatadd', + 'tmat3tmat3add', + 'tsmatdvecmult', + 'tsvecdvecmult', + 'vec3vec3add', + 'complex2', + 'dmatdmatmult', + 'dmattdmatmult', + 'dvecscalarmult', + 'mat3vec3mult', + 'smatscalarmult', + 'svecdvecadd', + 'tdmatdmatadd', + 'tdmattsmatmult', + 'tmat3tmat3mult', + 'tsmatsmatadd', + 'tsvecsmatmult', + 'vec6vec6add', + 'complex3', + 'dmatdmatsub', + 'dmattrans', + 'dvecsvecadd', + 'mat6mat6add', + 'smatsmatadd', + 'svecdveccross', + 'tdmatdmatmult', + 'tdvecdmatmult', + 'tmat3vec3mult', + 'tsmatsmatmult', + 'tsvecsvecmult', + 'complex4', + 'dmatdvecmult', + 'dmattsmatadd', + 'dvecsveccross', + 'mat6mat6mult', + 'smatsmatmult', + 'svecdvecmult', + 'tdmatdvecmult', + 'tdvecdvecmult', + 'tmat6mat6mult', + 'tsmatsvecmult', + 'tsvectdmatmult', + 'complex5', + 'dmatinv', + 'dmattsmatmult', + 'dvecsvecmult', + 'mat6tmat6mult', + 'smatsvecmult', + 'svecscalarmult', + 'tdmatsmatadd', + 'tdvecsmatmult', + 'tmat6tmat6add', + 'tsmattdmatadd', + 'tsvectsmatmult', + 'complex6', + 'dmatscalarmult', + 'dvecdvecadd', + 'dvectdvecmult', + 'mat6vec6mult', + 'smattdmatadd', + 'svecsvecadd', + 'tdmatsmatmult', + 'tdvecsvecmult', + 'tmat6tmat6mult', + 'tsmattdmatmult', + 'tvec3mat3mult', + 'complex7', + 'dmatsmatadd', + 'dvecdveccross', + 'dvectsvecmult', + 'memorysweep', + 'smattdmatmult', + 'svecsveccross', + 'tdmatsvecmult', + 'tdvectdmatmult', + 'tmat6vec6mult', + 'tsmattsmatadd', + 'tvec3tmat3mult', + 'complex8', + 'dmatsmatmult', + 'dvecdvecmult', + 'mat3mat3add', + 'smatdmatadd', + 'smattrans', + 'svecsvecmult', + 'tdmattdmatadd', + 'tdvectsmatmult', + 'tsmatdmatadd', + 'tsmattsmatmult', + 'tvec6mat6mult', +]) def GetBinaries(): @@ -162,11 +256,14 @@ def _ParseResult(out, test): unit = m[-2] for v in regex_util.ExtractAllMatches(THROUGHPUT_RESULT_REGEX, m[-1]): metadata['N'] = int(v[0]) - results.append(sample.Sample( - '_'.join([test, lib, 'Throughput']), # Metric name - float(v[1]), # Value - unit, # Unit - copy.deepcopy(metadata))) # Metadata + results.append( + sample.Sample( + '_'.join([test, lib, 'Throughput']), # Metric name + float(v[1]), # Value + unit, # Unit + copy.deepcopy(metadata), + ) + ) # Metadata logging.info('Results for %s:\n %s', test, results) return results @@ -183,8 +280,9 @@ def RunTest(vm, test): metadata). """ out, _ = vm.RemoteCommand( - 'cd %s; export BLAZE_NUM_THREADS=%s; ./%s -only-blaze' % ( - os.path.join(BLAZEMARK_DIR, 'bin'), vm.NumCpusForBenchmark(), test)) + 'cd %s; export BLAZE_NUM_THREADS=%s; ./%s -only-blaze' + % (os.path.join(BLAZEMARK_DIR, 'bin'), vm.NumCpusForBenchmark(), test) + ) ret = [] try: ret = _ParseResult(out, test) @@ -198,16 +296,22 @@ def _Configure(vm): vm.RenderTemplate( data.ResourcePath(CONFIG_TEMPLATE), os.path.join(BLAZEMARK_DIR, CONFIG), - {'compiler': '"g++-5"', - 'compiler_flags': ( - '"-Wall -Wextra -Werror -Wshadow -Woverloaded-virtual -ansi -O3 ' - '-mavx -DNDEBUG -fpermissive -ansi -O3 -DNDEBUG ' - '-DBLAZE_USE_BOOST_THREADS --std=c++14"'), - 'lapack_path': '"/tmp/pkb/lapack-3.6.1/lib"', - 'lapack_libs': '"-llapack -lblas -L%s -lgfortran"' - % os.path.dirname(fortran.GetLibPath(vm))}) - vm.RemoteCommand('cd %s; ./configure %s; make -j %s' % ( - BLAZEMARK_DIR, CONFIG, vm.num_cpus)) + { + 'compiler': '"g++-5"', + 'compiler_flags': ( + '"-Wall -Wextra -Werror -Wshadow -Woverloaded-virtual -ansi -O3 ' + '-mavx -DNDEBUG -fpermissive -ansi -O3 -DNDEBUG ' + '-DBLAZE_USE_BOOST_THREADS --std=c++14"' + ), + 'lapack_path': '"/tmp/pkb/lapack-3.6.1/lib"', + 'lapack_libs': '"-llapack -lblas -L%s -lgfortran"' % os.path.dirname( + fortran.GetLibPath(vm) + ), + }, + ) + vm.RemoteCommand( + 'cd %s; ./configure %s; make -j %s' % (BLAZEMARK_DIR, CONFIG, vm.num_cpus) + ) def _Install(vm): diff --git a/perfkitbenchmarker/linux_packages/boto3.py b/perfkitbenchmarker/linux_packages/boto3.py index 515f3a22fe..fb0a93d9b1 100644 --- a/perfkitbenchmarker/linux_packages/boto3.py +++ b/perfkitbenchmarker/linux_packages/boto3.py @@ -18,4 +18,3 @@ def Install(vm): """Installs the boto package on the VM.""" vm.Install('pip3') vm.RemoteCommand('sudo pip3 install boto3') - diff --git a/perfkitbenchmarker/linux_packages/build_tools.py b/perfkitbenchmarker/linux_packages/build_tools.py index 60b5b4189b..4729906928 100644 --- a/perfkitbenchmarker/linux_packages/build_tools.py +++ b/perfkitbenchmarker/linux_packages/build_tools.py @@ -18,20 +18,29 @@ FLAGS = flags.FLAGS flags.DEFINE_string( - 'gcc_version', None, 'Version of gcc to use. Benchmarks ' + 'gcc_version', + None, + 'Version of gcc to use. Benchmarks ' 'that utilize gcc compilation should ensure reinstallation ' - 'of GCC. Default is set by the OS package manager.') -flags.DEFINE_boolean('force_build_gcc_from_source', False, 'Whether to force ' - 'building GCC from source.') + 'of GCC. Default is set by the OS package manager.', +) flags.DEFINE_boolean( - 'build_fortran', False, 'Whether to build fortran ' - 'alongside c and c++ when building GCC.') + 'force_build_gcc_from_source', + False, + 'Whether to force building GCC from source.', +) +flags.DEFINE_boolean( + 'build_fortran', + False, + 'Whether to build fortran alongside c and c++ when building GCC.', +) GCC_TAR = 'gcc-{version}.tar.gz' GCC_URL = 'https://ftp.gnu.org/gnu/gcc/gcc-{version}/' + GCC_TAR PREPROVISIONED_DATA = { - GCC_TAR.format(version='9.2.0'): - 'a931a750d6feadacbeecb321d73925cd5ebb6dfa7eff0802984af3aef63759f4' + GCC_TAR.format( + version='9.2.0' + ): 'a931a750d6feadacbeecb321d73925cd5ebb6dfa7eff0802984af3aef63759f4' } PACKAGE_DATA_URL = { GCC_TAR.format(version='9.2.0'): GCC_URL.format(version='9.2.0') @@ -62,6 +71,7 @@ def BuildGccFromSource(vm, gcc_version): Args: vm: VirtualMachine object. gcc_version: string. GCC version. + Taken from: https://gist.github.com/nchaigne/ad06bc867f911a3c0d32939f1e930a11 """ if gcc_version == '9' or gcc_version == '9.2': @@ -72,62 +82,79 @@ def BuildGccFromSource(vm, gcc_version): build_dir = vm.GetScratchDir() if vm.scratch_disks else '~/' gcc_tar = GCC_TAR.format(version=gcc_version) if gcc_tar in PREPROVISIONED_DATA: - vm.InstallPreprovisionedPackageData('build_tools', - PREPROVISIONED_DATA.keys(), build_dir) + vm.InstallPreprovisionedPackageData( + 'build_tools', PREPROVISIONED_DATA.keys(), build_dir + ) else: vm.InstallPackages('wget') try: - vm.RemoteCommand(f'cd {build_dir} && ' - f'wget {GCC_URL.format(version=gcc_version)}') + vm.RemoteCommand( + f'cd {build_dir} && wget {GCC_URL.format(version=gcc_version)}' + ) # Some older OS images have outdated certificates; use curl instead except errors.VirtualMachine.RemoteCommandError as e: logging.info('Failed to wget, trying curl instead. Error: %s', str(e)) - vm.RemoteCommand(f'cd {build_dir} && ' - f'curl {GCC_URL.format(version=gcc_version)} ' - f'-L -o {gcc_tar}') + vm.RemoteCommand( + f'cd {build_dir} && ' + f'curl {GCC_URL.format(version=gcc_version)} ' + f'-L -o {gcc_tar}' + ) vm.RemoteCommand(f'cd {build_dir} && tar xzvf {gcc_tar}') vm.RemoteCommand(f'cd {build_dir} && mkdir -p obj.gcc-{gcc_version}') - vm.RemoteCommand(f'cd {build_dir}/gcc-{gcc_version} && ' - './contrib/download_prerequisites') + vm.RemoteCommand( + f'cd {build_dir}/gcc-{gcc_version} && ./contrib/download_prerequisites' + ) enable_languages = 'c,c++' + (',fortran' if FLAGS.build_fortran else '') - vm.RemoteCommand(f'cd {build_dir}/obj.gcc-{gcc_version} && ' - f'../gcc-{gcc_version}/configure ' - f'--disable-multilib --enable-languages={enable_languages}') + vm.RemoteCommand( + f'cd {build_dir}/obj.gcc-{gcc_version} && ' + f'../gcc-{gcc_version}/configure ' + f'--disable-multilib --enable-languages={enable_languages}' + ) # TODO(user): Measure GCC compilation time as a benchmark. - vm.RemoteCommand(f'cd {build_dir}/obj.gcc-{gcc_version} && ' - f'time make -j {vm.NumCpusForBenchmark()}') + vm.RemoteCommand( + f'cd {build_dir}/obj.gcc-{gcc_version} && ' + f'time make -j {vm.NumCpusForBenchmark()}' + ) vm.RemoteCommand(f'cd {build_dir}/obj.gcc-{gcc_version} && sudo make install') - vm.RemoteCommand('sudo rm -rf /usr/bin/gcc && ' - 'sudo ln -s /usr/local/bin/gcc /usr/bin/gcc') - vm.RemoteCommand('sudo rm -rf /usr/bin/g++ && ' - 'sudo ln -s /usr/local/bin/g++ /usr/bin/g++') + vm.RemoteCommand( + 'sudo rm -rf /usr/bin/gcc && sudo ln -s /usr/local/bin/gcc /usr/bin/gcc' + ) + vm.RemoteCommand( + 'sudo rm -rf /usr/bin/g++ && sudo ln -s /usr/local/bin/g++ /usr/bin/g++' + ) if FLAGS.build_fortran: - vm.RemoteCommand('sudo rm -rf /usr/bin/gfortran && ' - 'sudo ln -s /usr/local/bin/gfortran /usr/bin/gfortran') + vm.RemoteCommand( + 'sudo rm -rf /usr/bin/gfortran && ' + 'sudo ln -s /usr/local/bin/gfortran /usr/bin/gfortran' + ) if gcc_version.startswith('11'): # https://stackoverflow.com/a/65384705 vm.RemoteCommand( f'sudo cp {build_dir}/obj.gcc-{gcc_version}/x86_64-pc-linux-gnu/' 'libstdc++-v3/src/.libs/* /usr/lib/x86_64-linux-gnu/', - ignore_failure=True) + ignore_failure=True, + ) vm.RemoteCommand( f'sudo cp {build_dir}/obj.gcc-{gcc_version}/aarch64-unknown-linux-gnu/' 'libstdc++-v3/src/.libs/* /usr/lib/aarch64-linux-gnu/', - ignore_failure=True) + ignore_failure=True, + ) def GetVersion(vm, pkg): """Get version of package using -dumpversion.""" out, _ = vm.RemoteCommand( - '{pkg} -dumpfullversion'.format(pkg=pkg), ignore_failure=True) + '{pkg} -dumpfullversion'.format(pkg=pkg), ignore_failure=True + ) return out.rstrip() def GetVersionInfo(vm, pkg): """Get compiler version info for package using --version.""" out, _ = vm.RemoteCommand( - '{pkg} --version'.format(pkg=pkg), ignore_failure=True) + '{pkg} --version'.format(pkg=pkg), ignore_failure=True + ) # return first line of pkg --version return out.splitlines()[0] if out else None @@ -151,12 +178,22 @@ def Reinstall(vm, version: str): continue else: logging.info( - 'Built-in version of %s is incorrect: expected %s, but found %s', pkg, - version_string, version) + 'Built-in version of %s is incorrect: expected %s, but found %s', + pkg, + version_string, + version, + ) new_pkg = pkg + '-' + version vm.InstallPackages(new_pkg) vm.RemoteCommand('sudo rm -f /usr/bin/{pkg}'.format(pkg=pkg)) - vm.RemoteCommand('sudo ln -s /usr/bin/{new_pkg} /usr/bin/{pkg}'.format( - new_pkg=new_pkg, pkg=pkg)) - logging.info('Updated version of %s: Old: %s New: %s', pkg, - version_string, GetVersion(vm, pkg)) + vm.RemoteCommand( + 'sudo ln -s /usr/bin/{new_pkg} /usr/bin/{pkg}'.format( + new_pkg=new_pkg, pkg=pkg + ) + ) + logging.info( + 'Updated version of %s: Old: %s New: %s', + pkg, + version_string, + GetVersion(vm, pkg), + ) diff --git a/perfkitbenchmarker/linux_packages/cassandra.py b/perfkitbenchmarker/linux_packages/cassandra.py index 89e861dbf8..7c01c9e0de 100644 --- a/perfkitbenchmarker/linux_packages/cassandra.py +++ b/perfkitbenchmarker/linux_packages/cassandra.py @@ -34,8 +34,10 @@ from six.moves import range -JNA_JAR_URL = ('https://maven.java.net/content/repositories/releases/' - 'net/java/dev/jna/jna/4.1.0/jna-4.1.0.jar') +JNA_JAR_URL = ( + 'https://maven.java.net/content/repositories/releases/' + 'net/java/dev/jna/jna/4.1.0/jna-4.1.0.jar' +) CASSANDRA_GIT_REPRO = 'https://github.com/apache/cassandra.git' CASSANDRA_VERSION = 'cassandra-2.1' CASSANDRA_YAML_TEMPLATE = 'cassandra/cassandra.yaml.j2' @@ -60,14 +62,18 @@ FLAGS = flags.FLAGS flags.DEFINE_integer('cassandra_replication_factor', 3, 'Num of replicas.') -flags.DEFINE_integer('cassandra_concurrent_reads', 32, - 'Concurrent read requests each server accepts.') +flags.DEFINE_integer( + 'cassandra_concurrent_reads', + 32, + 'Concurrent read requests each server accepts.', +) # Partial list of known mirrors: # https://repo.maven.apache.org/maven2/.meta/repository-metadata.xml # See instructions for setting up own mirror: # https://maven.apache.org/guides/mini/guide-mirror-settings.html -flags.DEFINE_boolean('cassandra_maven_repo_url', None, - 'Optional maven repo mirror to use.') +flags.DEFINE_boolean( + 'cassandra_maven_repo_url', None, 'Optional maven repo mirror to use.' +) def CheckPrerequisites(): @@ -76,8 +82,7 @@ def CheckPrerequisites(): Raises: perfkitbenchmarker.data.ResourceNotFound: On missing resource. """ - for resource in (CASSANDRA_YAML_TEMPLATE, - CASSANDRA_ENV_TEMPLATE): + for resource in (CASSANDRA_YAML_TEMPLATE, CASSANDRA_ENV_TEMPLATE): data.ResourcePath(resource) @@ -87,19 +92,27 @@ def _Install(vm): vm.Install('build_tools') vm.Install('openjdk') vm.Install('curl') - vm.RemoteCommand('cd {0}; git clone {1}; cd {2}; git checkout {3}'.format( - linux_packages.INSTALL_DIR, CASSANDRA_GIT_REPRO, CASSANDRA_DIR, - CASSANDRA_VERSION)) + vm.RemoteCommand( + 'cd {0}; git clone {1}; cd {2}; git checkout {3}'.format( + linux_packages.INSTALL_DIR, + CASSANDRA_GIT_REPRO, + CASSANDRA_DIR, + CASSANDRA_VERSION, + ) + ) if FLAGS.cassandra_maven_repo_url: # sets maven repo properties in the build.properties file_contents = _MAVEN_REPO_PARAMS.format(FLAGS.cassandra_maven_repo_url) - vm.RemoteCommand('echo "{}" > {}/build.properties'.format( - file_contents, CASSANDRA_DIR)) + vm.RemoteCommand( + 'echo "{}" > {}/build.properties'.format(file_contents, CASSANDRA_DIR) + ) vm.RemoteCommand('cd {}; {}/bin/ant'.format(CASSANDRA_DIR, ANT_HOME_DIR)) # Add JNA - vm.RemoteCommand('cd {0} && curl -LJO {1}'.format( - posixpath.join(CASSANDRA_DIR, 'lib'), - JNA_JAR_URL)) + vm.RemoteCommand( + 'cd {0} && curl -LJO {1}'.format( + posixpath.join(CASSANDRA_DIR, 'lib'), JNA_JAR_URL + ) + ) def YumInstall(vm): @@ -119,23 +132,24 @@ def JujuInstall(vm, vm_group_name): # The charm defaults to Cassandra 2.2.x, which has deprecated # cassandra-cli. Specify the sources to downgrade to Cassandra 2.1.x # to match the cassandra benchmark(s) expectations. - sources = ['deb https://www.apache.org/dist/cassandra/debian 21x main', - 'ppa:openjdk-r/ppa', - 'ppa:stub/cassandra'] + sources = [ + 'deb https://www.apache.org/dist/cassandra/debian 21x main', + 'ppa:openjdk-r/ppa', + 'ppa:stub/cassandra', + ] - keys = ['F758CE318D77295D', - 'null', - 'null'] + keys = ['F758CE318D77295D', 'null', 'null'] vm.JujuSet( 'cassandra', [ # Allow authentication from all units 'authenticator=AllowAllAuthenticator', - 'install_sources="[%s]"' % - ', '.join(["'" + x + "'" for x in sources]), - 'install_keys="[%s]"' % ', '.join(keys) - ]) + 'install_sources="[%s]"' + % ', '.join(["'" + x + "'" for x in sources]), + 'install_keys="[%s]"' % ', '.join(keys), + ], + ) # Wait for cassandra to be installed and configured vm.JujuWait() @@ -154,18 +168,22 @@ def Configure(vm, seed_vms): vm: VirtualMachine. The VM to configure. seed_vms: List of VirtualMachine. The seed virtual machine(s). """ - context = {'ip_address': vm.internal_ip, - 'data_path': posixpath.join(vm.GetScratchDir(), 'cassandra'), - 'seeds': ','.join(vm.internal_ip for vm in seed_vms), - 'num_cpus': vm.NumCpusForBenchmark(), - 'cluster_name': 'Test cluster', - 'concurrent_reads': FLAGS.cassandra_concurrent_reads} + context = { + 'ip_address': vm.internal_ip, + 'data_path': posixpath.join(vm.GetScratchDir(), 'cassandra'), + 'seeds': ','.join(vm.internal_ip for vm in seed_vms), + 'num_cpus': vm.NumCpusForBenchmark(), + 'cluster_name': 'Test cluster', + 'concurrent_reads': FLAGS.cassandra_concurrent_reads, + } for config_file in [CASSANDRA_ENV_TEMPLATE, CASSANDRA_YAML_TEMPLATE]: local_path = data.ResourcePath(config_file) remote_path = posixpath.join( - CASSANDRA_DIR, 'conf', - os.path.splitext(os.path.basename(config_file))[0]) + CASSANDRA_DIR, + 'conf', + os.path.splitext(os.path.basename(config_file))[0], + ) vm.RenderTemplate(local_path, remote_path, context=context) @@ -180,7 +198,9 @@ def Start(vm): vm.RemoteCommand( 'nohup {0}/bin/cassandra -p "{1}" 1> {2} 2> {3} &'.format( - CASSANDRA_DIR, CASSANDRA_PID, CASSANDRA_OUT, CASSANDRA_ERR)) + CASSANDRA_DIR, CASSANDRA_PID, CASSANDRA_OUT, CASSANDRA_ERR + ) + ) def Stop(vm): @@ -188,14 +208,14 @@ def Stop(vm): if vm.OS_TYPE == os_types.JUJU: return - vm.RemoteCommand('kill $(cat {0})'.format(CASSANDRA_PID), - ignore_failure=True) + vm.RemoteCommand('kill $(cat {0})'.format(CASSANDRA_PID), ignore_failure=True) def IsRunning(vm): """Returns a boolean indicating whether Cassandra is running on 'vm'.""" - cassandra_pid = vm.RemoteCommand( - 'cat {0} || true'.format(CASSANDRA_PID))[0].strip() + cassandra_pid = vm.RemoteCommand('cat {0} || true'.format(CASSANDRA_PID))[ + 0 + ].strip() if not cassandra_pid: return False @@ -203,11 +223,12 @@ def IsRunning(vm): vm.RemoteCommand('kill -0 {0}'.format(cassandra_pid)) return True except errors.VirtualMachine.RemoteCommandError: - logging.warn('%s: Cassandra is not running. ' - 'Startup STDOUT:\n%s\n\nSTDERR:\n%s', - vm, - vm.RemoteCommand('cat ' + CASSANDRA_OUT), - vm.RemoteCommand('cat ' + CASSANDRA_ERR)) + logging.warn( + '%s: Cassandra is not running. Startup STDOUT:\n%s\n\nSTDERR:\n%s', + vm, + vm.RemoteCommand('cat ' + CASSANDRA_OUT), + vm.RemoteCommand('cat ' + CASSANDRA_ERR), + ) return False @@ -237,8 +258,7 @@ def GetCassandraCliPath(vm): # installed by the cassandra charm. return '/usr/bin/cassandra-cli' - return posixpath.join(CASSANDRA_DIR, 'bin', - 'cassandra-cli') + return posixpath.join(CASSANDRA_DIR, 'bin', 'cassandra-cli') def GetCassandraStressPath(vm): @@ -247,8 +267,7 @@ def GetCassandraStressPath(vm): # installed by the cassandra-stress charm. return '/usr/bin/cassandra-stress' - return posixpath.join(CASSANDRA_DIR, 'tools', 'bin', - 'cassandra-stress') + return posixpath.join(CASSANDRA_DIR, 'tools', 'bin', 'cassandra-stress') def GetNumberOfNodesUp(vm): @@ -257,8 +276,9 @@ def GetNumberOfNodesUp(vm): Args: vm: VirtualMachine. The VM to use to check the cluster status. """ - vms_up = vm.RemoteCommand( - '{0} status | grep -c "^UN"'.format(NODETOOL))[0].strip() + vms_up = vm.RemoteCommand('{0} status | grep -c "^UN"'.format(NODETOOL))[ + 0 + ].strip() return int(vms_up) @@ -288,8 +308,12 @@ def StartCluster(seed_vm, vms): time.sleep(NODE_START_SLEEP) for i in range(5): if not IsRunning(seed_vm): - logging.warn('Seed %s: Cassandra not running yet (try %d). Waiting %ds.', - seed_vm, i, NODE_START_SLEEP) + logging.warn( + 'Seed %s: Cassandra not running yet (try %d). Waiting %ds.', + seed_vm, + i, + NODE_START_SLEEP, + ) time.sleep(NODE_START_SLEEP) else: break @@ -315,8 +339,13 @@ def StartCluster(seed_vm, vms): logging.info('All %d nodes up!', vm_count) break - logging.warn('Try %d: only %s of %s up. Restarting and sleeping %ds', i, - vms_up, vm_count, NODE_START_SLEEP) + logging.warn( + 'Try %d: only %s of %s up. Restarting and sleeping %ds', + i, + vms_up, + vm_count, + NODE_START_SLEEP, + ) background_tasks.RunThreaded(_StartCassandraIfNotRunning, vms) time.sleep(NODE_START_SLEEP) else: diff --git a/perfkitbenchmarker/linux_packages/ch_block_storage.py b/perfkitbenchmarker/linux_packages/ch_block_storage.py index ab326b95b8..c14fd6b3f6 100644 --- a/perfkitbenchmarker/linux_packages/ch_block_storage.py +++ b/perfkitbenchmarker/linux_packages/ch_block_storage.py @@ -24,13 +24,14 @@ flags.DEFINE_list( - 'ch_params', [], + 'ch_params', + [], 'A list of comma seperated "key=value" parameters passed into ' - 'cloud harmony benchmarks.') + 'cloud harmony benchmarks.', +) BENCHMARK = 'block-storage' -INSTALL_PATH = os.path.join(linux_packages.INSTALL_DIR, - BENCHMARK) +INSTALL_PATH = os.path.join(linux_packages.INSTALL_DIR, BENCHMARK) STEADY_STATE_MEASUREMENT_WINDOW = '-ssmw' @@ -39,8 +40,10 @@ def _Install(vm): for deps in ['php', 'build_tools']: vm.Install(deps) vm.RemoteCommand( - ('git clone https://github.com/cloudharmony/{benchmark}.git ' - '{dir}').format(benchmark=BENCHMARK, dir=INSTALL_PATH)) + ( + 'git clone https://github.com/cloudharmony/{benchmark}.git {dir}' + ).format(benchmark=BENCHMARK, dir=INSTALL_PATH) + ) def YumInstall(vm): @@ -69,7 +72,8 @@ def _ParseFioJson(fio_json): # Remove ssmw suffix from job name. try: job['jobname'] = regex_util.Substitute( - STEADY_STATE_MEASUREMENT_WINDOW, '', job['jobname']) + STEADY_STATE_MEASUREMENT_WINDOW, '', job['jobname'] + ) additional_metadata['steady_state'] = True except regex_util.NoMatchError: additional_metadata['steady_state'] = False @@ -77,7 +81,8 @@ def _ParseFioJson(fio_json): # Mock fio_json to reuse fio parser. mock_json = {'jobs': [job]} new_samples = fio.ParseResults( - fio.FioParametersToJob(cmd).__str__(), mock_json) + fio.FioParametersToJob(cmd).__str__(), mock_json + ) for s in new_samples: s.metadata.update(additional_metadata) samples += new_samples @@ -96,8 +101,10 @@ def _ParseCHResultsJson(results_json): """ metadata = {} _ExtractMetadata(metadata) - return [sample.Sample(metric, val, '', metadata) - for metric, val in json.loads(results_json).items()] + return [ + sample.Sample(metric, val, '', metadata) + for metric, val in json.loads(results_json).items() + ] def ParseOutput(results_json, fio_json_list): @@ -106,11 +113,13 @@ def ParseOutput(results_json, fio_json_list): Args: results_json: string. Json output reported by CloudHarmony. fio_json_list: list of string. Json output strings from fio command. + Returns: A list of sample.Sample object. """ return _ParseCHResultsJson(results_json) + [ - s for fio_json in fio_json_list for s in _ParseFioJson(fio_json)] + s for fio_json in fio_json_list for s in _ParseFioJson(fio_json) + ] def _ExtractMetadata(metadata): diff --git a/perfkitbenchmarker/linux_packages/cloud_harmony_network.py b/perfkitbenchmarker/linux_packages/cloud_harmony_network.py index 9c37c67fc9..08d82d9429 100644 --- a/perfkitbenchmarker/linux_packages/cloud_harmony_network.py +++ b/perfkitbenchmarker/linux_packages/cloud_harmony_network.py @@ -13,8 +13,9 @@ WEB_PROBE_TAR = 'probe.tgz' WEB_PROBE = posixpath.join('http://cloudfront.cloudharmony.net', WEB_PROBE_TAR) PREPROVISIONED_DATA = { - WEB_PROBE_TAR: + WEB_PROBE_TAR: ( 'b6e0580a5d5bbcffc6c7fbe48485721727ad18141cde01f86fc9f96a2612eb79' + ) } PACKAGE_DATA_URL = {WEB_PROBE_TAR: WEB_PROBE} PACKAGE_NAME = 'cloud_harmony_network' @@ -24,5 +25,6 @@ def Install(vm): """Install Cloud Harmory network benchmark on VM.""" for deps in ['php', 'build_tools', 'curl']: vm.Install(deps) - vm.RemoteCommand(f'git clone --recurse-submodules {BENCHMARK_GIT_URL} ' - f'{INSTALL_PATH}') + vm.RemoteCommand( + f'git clone --recurse-submodules {BENCHMARK_GIT_URL} {INSTALL_PATH}' + ) diff --git a/perfkitbenchmarker/linux_packages/cloud_tpu_models.py b/perfkitbenchmarker/linux_packages/cloud_tpu_models.py index 448bfb9d32..f59e3ff635 100644 --- a/perfkitbenchmarker/linux_packages/cloud_tpu_models.py +++ b/perfkitbenchmarker/linux_packages/cloud_tpu_models.py @@ -13,20 +13,24 @@ # limitations under the License. """Module containing cloud TPU models installation and cleanup functions.""" from absl import flags + FLAGS = flags.FLAGS CLOUD_TPU_GIT = 'https://github.com/tensorflow/tpu.git' -flags.DEFINE_string('cloud_tpu_commit_hash', - '45f33771f9d9aef6d37386d07b086d7e20be0254', - 'git commit hash of desired cloud TPU models commit.') +flags.DEFINE_string( + 'cloud_tpu_commit_hash', + '45f33771f9d9aef6d37386d07b086d7e20be0254', + 'git commit hash of desired cloud TPU models commit.', +) def Install(vm): """Installs cloud TPU models on the VM.""" vm.InstallPackages('git') vm.RemoteCommand('git clone {}'.format(CLOUD_TPU_GIT)) - vm.RemoteCommand('cd tpu && git checkout {}'.format( - FLAGS.cloud_tpu_commit_hash)) + vm.RemoteCommand( + 'cd tpu && git checkout {}'.format(FLAGS.cloud_tpu_commit_hash) + ) vm.Install('pip') vm.RemoteCommand('sudo pip install absl-py') diff --git a/perfkitbenchmarker/linux_packages/cmake.py b/perfkitbenchmarker/linux_packages/cmake.py index 01cec089b2..efbaf597df 100644 --- a/perfkitbenchmarker/linux_packages/cmake.py +++ b/perfkitbenchmarker/linux_packages/cmake.py @@ -18,9 +18,11 @@ from perfkitbenchmarker import os_types _CMAKE_KITWARE = flags.DEFINE_bool( - 'cmake_kitware', False, + 'cmake_kitware', + False, 'Whether to install cmake from the Kitware repo. Default is to install ' - 'from the (possibily outdated) OS distro.') + 'from the (possibily outdated) OS distro.', +) # Needed to configure the kitware debian repo _UBUNTU_VERSION_NAMES = { @@ -55,8 +57,10 @@ def AptInstall(vm): def _AddCmakeAptRepo(vm): vm.Install('curl') - vm.RemoteCommand(f'curl --silent {_KITWARE_KEY_URL} | gpg --dearmor | ' - 'sudo tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null') + vm.RemoteCommand( + f'curl --silent {_KITWARE_KEY_URL} | gpg --dearmor | ' + 'sudo tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null' + ) repo = _KITWARE_DEB_CONFIG.format(version=_UBUNTU_VERSION_NAMES[vm.OS_TYPE]) vm.RemoteCommand(f'sudo apt-add-repository "{repo}"') vm.AptUpdate() diff --git a/perfkitbenchmarker/linux_packages/collectd.py b/perfkitbenchmarker/linux_packages/collectd.py index 8b06216f76..ef4b8c083b 100644 --- a/perfkitbenchmarker/linux_packages/collectd.py +++ b/perfkitbenchmarker/linux_packages/collectd.py @@ -28,8 +28,9 @@ # TODO: Make collection interval configurable. INTERVAL = 10 SCRIPT_NAME = 'build_collectd.sh.j2' -COLLECTD_URL = ('https://github.com/collectd/collectd/archive/' - 'collectd-5.5.0.tar.gz') +COLLECTD_URL = ( + 'https://github.com/collectd/collectd/archive/collectd-5.5.0.tar.gz' +) BUILD_DIR = posixpath.join(linux_packages.INSTALL_DIR, 'collectd-build') CSV_DIR = posixpath.join(linux_packages.INSTALL_DIR, 'collectd-csv') PREFIX = posixpath.join(linux_packages.INSTALL_DIR, 'collectd') @@ -42,13 +43,16 @@ def _Install(vm): 'build_dir': BUILD_DIR, 'root_dir': PREFIX, 'csv_dir': CSV_DIR, - 'interval': INTERVAL} + 'interval': INTERVAL, + } remote_path = posixpath.join( linux_packages.INSTALL_DIR, - posixpath.splitext(posixpath.basename(SCRIPT_NAME))[0]) - vm.RenderTemplate(data.ResourcePath(SCRIPT_NAME), - remote_path, context=context) + posixpath.splitext(posixpath.basename(SCRIPT_NAME))[0], + ) + vm.RenderTemplate( + data.ResourcePath(SCRIPT_NAME), remote_path, context=context + ) vm.RemoteCommand('bash ' + remote_path) diff --git a/perfkitbenchmarker/linux_packages/coremark.py b/perfkitbenchmarker/linux_packages/coremark.py index 5294e0a5ec..72a6978c9c 100644 --- a/perfkitbenchmarker/linux_packages/coremark.py +++ b/perfkitbenchmarker/linux_packages/coremark.py @@ -26,12 +26,14 @@ def InstallCoremark(remote_command): Args: remote_command: Function to run a remote command on the VM. This allows this - function to be reused by the windows/cygwin version of the coremark test. + function to be reused by the windows/cygwin version of the coremark test. """ - remote_command('wget %s -P %s' % - (COREMARK_TAR_URL, linux_packages.INSTALL_DIR)) - remote_command('cd %s && tar xvfz %s' % - (linux_packages.INSTALL_DIR, COREMARK_TAR)) + remote_command( + 'wget %s -P %s' % (COREMARK_TAR_URL, linux_packages.INSTALL_DIR) + ) + remote_command( + 'cd %s && tar xvfz %s' % (linux_packages.INSTALL_DIR, COREMARK_TAR) + ) def Install(vm): diff --git a/perfkitbenchmarker/linux_packages/cuda_samples.py b/perfkitbenchmarker/linux_packages/cuda_samples.py index 8bf12dd4ba..a0e78fcf6e 100644 --- a/perfkitbenchmarker/linux_packages/cuda_samples.py +++ b/perfkitbenchmarker/linux_packages/cuda_samples.py @@ -16,22 +16,27 @@ from absl import flags from perfkitbenchmarker.linux_packages import cuda_toolkit -_VERSION = flags.DEFINE_enum('cuda_samples_version', None, [ - '9.0', - '10.0', - '10.1', - '10.2', - '11.0', - '11.1', - '11.2', - '11.4', - '11.5', - '11.6', - '11.7', - '11.8', - '12.0', - '12.1', -], 'Version of CUDA samples to install.') +_VERSION = flags.DEFINE_enum( + 'cuda_samples_version', + None, + [ + '9.0', + '10.0', + '10.1', + '10.2', + '11.0', + '11.1', + '11.2', + '11.4', + '11.5', + '11.6', + '11.7', + '11.8', + '12.0', + '12.1', + ], + 'Version of CUDA samples to install.', +) FLAGS = flags.FLAGS @@ -42,8 +47,10 @@ def Install(vm): vm.Install('cuda_toolkit') vm.InstallPackages('libfreeimage3 libfreeimage-dev') version = _VERSION.value or FLAGS.cuda_toolkit_version - vm.RemoteCommand(f'git clone --branch v{version} --depth 1 ' - 'https://github.com/NVIDIA/cuda-samples.git') + vm.RemoteCommand( + f'git clone --branch v{version} --depth 1 ' + 'https://github.com/NVIDIA/cuda-samples.git' + ) vm.RemoteCommand('cd cuda-samples && make') @@ -53,7 +60,9 @@ def GetBandwidthTestPath(vm): return BANDWIDTH_TEST_PATH cpu_arch = cuda_toolkit.GetCpuArchPath(vm) - bandwidth_test_path = f'cuda-samples/bin/{cpu_arch}/linux/release/bandwidthTest' + bandwidth_test_path = ( + f'cuda-samples/bin/{cpu_arch}/linux/release/bandwidthTest' + ) if vm.TryRemoteCommand(f'stat {bandwidth_test_path}'): return bandwidth_test_path diff --git a/perfkitbenchmarker/linux_packages/cuda_toolkit.py b/perfkitbenchmarker/linux_packages/cuda_toolkit.py index b86725906e..05779d6b60 100644 --- a/perfkitbenchmarker/linux_packages/cuda_toolkit.py +++ b/perfkitbenchmarker/linux_packages/cuda_toolkit.py @@ -64,8 +64,10 @@ ) _KEY = flags.DEFINE_string( - 'cuda_toolkit_key', '7fa2af80', - 'The new GPG keys for the CUDA repository. This is Debian-based distros.') + 'cuda_toolkit_key', + '7fa2af80', + 'The new GPG keys for the CUDA repository. This is Debian-based distros.', +) FLAGS = flags.FLAGS @@ -163,14 +165,16 @@ def GetCudaToolkitVersion(vm): NvccParseOutputError: On can not parse nvcc output """ stdout, _ = vm.RemoteCommand( - posixpath.join(CUDA_HOME, 'bin/nvcc') + ' --version', ignore_failure=True) + posixpath.join(CUDA_HOME, 'bin/nvcc') + ' --version', ignore_failure=True + ) if bool(stdout.rstrip()): regex = r'release (\S+),' match = re.search(regex, stdout) if match: return str(match.group(1)) - raise NvccParseOutputError('Unable to parse nvcc version output from {}' - .format(stdout)) + raise NvccParseOutputError( + 'Unable to parse nvcc version output from {}'.format(stdout) + ) else: return None @@ -200,15 +204,15 @@ def _InstallCudaPatch(vm, patch_url): # Need to append .deb to package name because the file downloaded from # NVIDIA is missing the .deb extension. basename = posixpath.basename(patch_url) + '.deb' - vm.RemoteCommand('wget -q %s -O %s' % (patch_url, - basename)) + vm.RemoteCommand('wget -q %s -O %s' % (patch_url, basename)) vm.RemoteCommand('sudo dpkg -i %s' % basename) vm.AptUpdate() # Need to be extra careful on the command below because without these # precautions, it was brining up a menu option about grub's menu.lst # on AWS Ubuntu16.04 and thus causing the RemoteCommand to hang and fail. vm.RemoteCommand( - 'sudo DEBIAN_FRONTEND=noninteractive apt-get upgrade -yq cuda') + 'sudo DEBIAN_FRONTEND=noninteractive apt-get upgrade -yq cuda' + ) def _InstallCuda9Point0(vm): @@ -217,19 +221,34 @@ def _InstallCuda9Point0(vm): Args: vm: VM to install CUDA on """ - basename = posixpath.basename( - CUDA_9_0_TOOLKIT.format(os=_CudaOs(vm.OS_TYPE), - cpu_arch=_GetCpuArch(vm))) + '.deb' - vm.RemoteCommand('wget -q %s -O %s' % (CUDA_9_0_TOOLKIT.format( - os=_CudaOs(vm.OS_TYPE), cpu_arch=_GetCpuArch(vm)), basename)) + basename = ( + posixpath.basename( + CUDA_9_0_TOOLKIT.format( + os=_CudaOs(vm.OS_TYPE), cpu_arch=_GetCpuArch(vm) + ) + ) + + '.deb' + ) + vm.RemoteCommand( + 'wget -q %s -O %s' + % ( + CUDA_9_0_TOOLKIT.format( + os=_CudaOs(vm.OS_TYPE), cpu_arch=_GetCpuArch(vm) + ), + basename, + ) + ) vm.RemoteCommand('sudo dpkg -i %s' % basename) EnrollSigningKey(vm) vm.AptUpdate() - vm.InstallPackages('cuda-toolkit-9-0 cuda-tools-9-0 cuda-libraries-9-0 ' - 'cuda-libraries-dev-9-0') + vm.InstallPackages( + 'cuda-toolkit-9-0 cuda-tools-9-0 cuda-libraries-9-0 ' + 'cuda-libraries-dev-9-0' + ) _InstallCudaPatch( vm, - CUDA_9_0_PATCH.format(os=_CudaOs(vm.OS_TYPE), cpu_arch=_GetCpuArch(vm))) + CUDA_9_0_PATCH.format(os=_CudaOs(vm.OS_TYPE), cpu_arch=_GetCpuArch(vm)), + ) def _InstallCuda10Point0(vm): @@ -242,13 +261,17 @@ def _InstallCuda10Point0(vm): f'{posixpath.basename(CUDA_10_0_TOOLKIT.format(os=_CudaOs(vm.OS_TYPE), cpu_arch=_GetCpuArch(vm)))}.deb' ) vm.RemoteCommand( - f'wget -q {CUDA_10_0_TOOLKIT.format(os=_CudaOs(vm.OS_TYPE), cpu_arch=_GetCpuArch(vm))} -O ' - f'{basename}') + 'wget -q' + f' {CUDA_10_0_TOOLKIT.format(os=_CudaOs(vm.OS_TYPE), cpu_arch=_GetCpuArch(vm))} -O' + f' {basename}' + ) vm.RemoteCommand('sudo dpkg -i %s' % basename) EnrollSigningKey(vm) vm.AptUpdate() - vm.InstallPackages('cuda-toolkit-10-0 cuda-tools-10-0 cuda-libraries-10-0 ' - 'cuda-libraries-dev-10-0') + vm.InstallPackages( + 'cuda-toolkit-10-0 cuda-tools-10-0 cuda-libraries-10-0 ' + 'cuda-libraries-dev-10-0' + ) def _InstallCuda10Point1(vm): @@ -258,20 +281,29 @@ def _InstallCuda10Point1(vm): vm: VM to install CUDA on """ basename = posixpath.basename( - CUDA_10_1_TOOLKIT.format( - os=_CudaOs(vm.OS_TYPE), cpu_arch=_GetCpuArch(vm))) + CUDA_10_1_TOOLKIT.format(os=_CudaOs(vm.OS_TYPE), cpu_arch=_GetCpuArch(vm)) + ) + vm.RemoteCommand( + 'wget -q %s' + % CUDA_PIN.format(os=_CudaOs(vm.OS_TYPE), cpu_arch=GetCpuArchPath(vm)) + ) + vm.RemoteCommand( + f'sudo mv cuda-{_CudaOs(vm.OS_TYPE)}.pin ' + '/etc/apt/preferences.d/cuda-repository-pin-600' + ) vm.RemoteCommand( - 'wget -q %s' % - CUDA_PIN.format(os=_CudaOs(vm.OS_TYPE), cpu_arch=GetCpuArchPath(vm))) - vm.RemoteCommand(f'sudo mv cuda-{_CudaOs(vm.OS_TYPE)}.pin ' - '/etc/apt/preferences.d/cuda-repository-pin-600') - vm.RemoteCommand('wget -q %s' % CUDA_10_1_TOOLKIT.format( - os=_CudaOs(vm.OS_TYPE), cpu_arch=_GetCpuArch(vm))) + 'wget -q %s' + % CUDA_10_1_TOOLKIT.format( + os=_CudaOs(vm.OS_TYPE), cpu_arch=_GetCpuArch(vm) + ) + ) vm.RemoteCommand('sudo dpkg -i %s' % basename) EnrollSigningKey(vm) vm.AptUpdate() - vm.InstallPackages('cuda-toolkit-10-1 cuda-tools-10-1 cuda-libraries-10-1 ' - 'cuda-libraries-dev-10-1') + vm.InstallPackages( + 'cuda-toolkit-10-1 cuda-tools-10-1 cuda-libraries-10-1 ' + 'cuda-libraries-dev-10-1' + ) def _InstallCuda10Point2(vm): @@ -281,20 +313,29 @@ def _InstallCuda10Point2(vm): vm: VM to install CUDA on """ basename = posixpath.basename( - CUDA_10_2_TOOLKIT.format( - os=_CudaOs(vm.OS_TYPE), cpu_arch=_GetCpuArch(vm))) + CUDA_10_2_TOOLKIT.format(os=_CudaOs(vm.OS_TYPE), cpu_arch=_GetCpuArch(vm)) + ) + vm.RemoteCommand( + 'wget -q %s' + % CUDA_PIN.format(os=_CudaOs(vm.OS_TYPE), cpu_arch=GetCpuArchPath(vm)) + ) + vm.RemoteCommand( + f'sudo mv cuda-{_CudaOs(vm.OS_TYPE)}.pin ' + '/etc/apt/preferences.d/cuda-repository-pin-600' + ) vm.RemoteCommand( - 'wget -q %s' % - CUDA_PIN.format(os=_CudaOs(vm.OS_TYPE), cpu_arch=GetCpuArchPath(vm))) - vm.RemoteCommand(f'sudo mv cuda-{_CudaOs(vm.OS_TYPE)}.pin ' - '/etc/apt/preferences.d/cuda-repository-pin-600') - vm.RemoteCommand('wget -q %s' % CUDA_10_2_TOOLKIT.format( - os=_CudaOs(vm.OS_TYPE), cpu_arch=_GetCpuArch(vm))) + 'wget -q %s' + % CUDA_10_2_TOOLKIT.format( + os=_CudaOs(vm.OS_TYPE), cpu_arch=_GetCpuArch(vm) + ) + ) vm.RemoteCommand('sudo dpkg -i %s' % basename) EnrollSigningKey(vm) vm.AptUpdate() - vm.InstallPackages('cuda-toolkit-10-2 cuda-tools-10-2 cuda-libraries-10-2 ' - 'cuda-libraries-dev-10-2') + vm.InstallPackages( + 'cuda-toolkit-10-2 cuda-tools-10-2 cuda-libraries-10-2 ' + 'cuda-libraries-dev-10-2' + ) def _DownloadCuda(vm, toolkit_fmt): @@ -340,17 +381,22 @@ def _InstallCuda11Generic(vm, toolkit_fmt, version_dash): version_dash: Version (ie 11-1) to install """ vm.RemoteCommand( - f'wget -q {CUDA_PIN.format(os=_CudaOs(vm.OS_TYPE), cpu_arch=GetCpuArchPath(vm))}' + 'wget -q' + f' {CUDA_PIN.format(os=_CudaOs(vm.OS_TYPE), cpu_arch=GetCpuArchPath(vm))}' + ) + vm.RemoteCommand( + f'sudo mv cuda-{_CudaOs(vm.OS_TYPE)}.pin ' + '/etc/apt/preferences.d/cuda-repository-pin-600' ) - vm.RemoteCommand(f'sudo mv cuda-{_CudaOs(vm.OS_TYPE)}.pin ' - '/etc/apt/preferences.d/cuda-repository-pin-600') _DownloadCuda(vm, toolkit_fmt) EnrollSigningKey(vm) vm.AptUpdate() - vm.InstallPackages(f'cuda-toolkit-{version_dash} ' - f'cuda-tools-{version_dash} ' - f'cuda-libraries-{version_dash} ' - f'cuda-libraries-dev-{version_dash}') + vm.InstallPackages( + f'cuda-toolkit-{version_dash} ' + f'cuda-tools-{version_dash} ' + f'cuda-libraries-{version_dash} ' + f'cuda-libraries-dev-{version_dash}' + ) def _InstallCuda12Point0(vm): @@ -404,7 +450,7 @@ def _InstallCuda11Point8(vm): def AptInstall(vm): """Installs CUDA toolkit on the VM if not already installed.""" version_to_install = FLAGS.cuda_toolkit_version - if (version_to_install == 'None' or not version_to_install): + if version_to_install == 'None' or not version_to_install: return current_version = GetCudaToolkitVersion(vm) if current_version == version_to_install: diff --git a/perfkitbenchmarker/linux_packages/dacapo.py b/perfkitbenchmarker/linux_packages/dacapo.py index 3a1e83b4dc..afdd0edac4 100644 --- a/perfkitbenchmarker/linux_packages/dacapo.py +++ b/perfkitbenchmarker/linux_packages/dacapo.py @@ -15,8 +15,10 @@ from perfkitbenchmarker import linux_packages -_DACAPO_URL = ('https://sourceforge.net/projects/dacapobench/files/' - '9.12-bach-MR1/dacapo-9.12-MR1-bach.jar') +_DACAPO_URL = ( + 'https://sourceforge.net/projects/dacapobench/files/' + '9.12-bach-MR1/dacapo-9.12-MR1-bach.jar' +) def Install(vm): diff --git a/perfkitbenchmarker/linux_packages/docker.py b/perfkitbenchmarker/linux_packages/docker.py index 741e771c4b..84a963a6d2 100644 --- a/perfkitbenchmarker/linux_packages/docker.py +++ b/perfkitbenchmarker/linux_packages/docker.py @@ -24,11 +24,14 @@ from perfkitbenchmarker import linux_packages from perfkitbenchmarker import virtual_machine -VERSION = flags.DEFINE_string('docker_version', None, - 'Version of docker to install.') +VERSION = flags.DEFINE_string( + 'docker_version', None, 'Version of docker to install.' +) -DOCKER_RPM_URL = ('https://get.docker.com/rpm/1.7.0/centos-6/' - 'RPMS/x86_64/docker-engine-1.7.0-1.el6.x86_64.rpm') +DOCKER_RPM_URL = ( + 'https://get.docker.com/rpm/1.7.0/centos-6/' + 'RPMS/x86_64/docker-engine-1.7.0-1.el6.x86_64.rpm' +) # Docker images that VMs are allowed to install. @@ -81,11 +84,14 @@ def YumInstall(vm): """Installs the docker package on the VM.""" if IsInstalled(vm): return - vm.RemoteHostCommand('curl -o %s/docker.rpm -sSL %s' % - (linux_packages.INSTALL_DIR, DOCKER_RPM_URL)) - vm.RemoteHostCommand('sudo yum localinstall ' - '--nogpgcheck %s/docker.rpm -y' % - linux_packages.INSTALL_DIR) + vm.RemoteHostCommand( + 'curl -o %s/docker.rpm -sSL %s' + % (linux_packages.INSTALL_DIR, DOCKER_RPM_URL) + ) + vm.RemoteHostCommand( + 'sudo yum localinstall --nogpgcheck %s/docker.rpm -y' + % linux_packages.INSTALL_DIR + ) vm.RemoteHostCommand('sudo service docker start') @@ -93,8 +99,9 @@ def AptInstall(vm): """Installs the docker package on the VM.""" if IsInstalled(vm): return - vm.RemoteHostCommand('curl -sSL https://get.docker.com/ | ' - f'VERSION={VERSION.value or ""} sh') + vm.RemoteHostCommand( + f'curl -sSL https://get.docker.com/ | VERSION={VERSION.value or ""} sh' + ) def IsInstalled(vm): diff --git a/perfkitbenchmarker/linux_packages/dstat.py b/perfkitbenchmarker/linux_packages/dstat.py index 593a11ea4b..f7b85eb1b1 100644 --- a/perfkitbenchmarker/linux_packages/dstat.py +++ b/perfkitbenchmarker/linux_packages/dstat.py @@ -61,10 +61,12 @@ def ParseCsvFile(fp: TextIO) -> Tuple[List[str], np.ndarray]: raise ValueError(f'Expected 4 header lines got {len(headers)}\n{headers}') if not headers[0] or 'pcp-dstat' not in headers[0][0]: raise ValueError( - f'Expected first header cell to contain "pcp-dstat"\n{headers[0]}') + f'Expected first header cell to contain "pcp-dstat"\n{headers[0]}' + ) if not headers[2] or 'Host:' not in headers[2][0]: raise ValueError( - f'Expected first cell in third line to be "Host:"\n{headers[2]}') + f'Expected first cell in third line to be "Host:"\n{headers[2]}' + ) categories = next(reader) if not categories: @@ -72,8 +74,9 @@ def ParseCsvFile(fp: TextIO) -> Tuple[List[str], np.ndarray]: categories = next(reader) if not categories or categories[0] != 'epoch': - raise ValueError('Expected first category to "epoch". ' - f'Categories were:\n{categories}') + raise ValueError( + f'Expected first category to "epoch". Categories were:\n{categories}' + ) # Categories are not repeated; copy category name across columns in the # same category @@ -86,7 +89,8 @@ def ParseCsvFile(fp: TextIO) -> Tuple[List[str], np.ndarray]: if len(labels) != len(categories): raise ValueError( f'Number of categories ({len(categories)}) does not match number of ' - f'labels ({len(labels)}\nCategories: {categories}\nLabels:{labels}') + f'labels ({len(labels)}\nCategories: {categories}\nLabels:{labels}' + ) # Generate new column names labels = [f'{label}__{cat}' for label, cat in zip(labels, categories)] @@ -96,8 +100,10 @@ def ParseCsvFile(fp: TextIO) -> Tuple[List[str], np.ndarray]: # Remove the trailing comma if len(row) == len(labels) + 1: if row[-1]: - raise ValueError(f'Expected the last element of row {row} to be empty,' - f' found {row[-1]}') + raise ValueError( + f'Expected the last element of row {row} to be empty,' + f' found {row[-1]}' + ) row = row[:-1] if len(labels) != len(row): @@ -105,7 +111,8 @@ def ParseCsvFile(fp: TextIO) -> Tuple[List[str], np.ndarray]: continue raise ValueError( f'Number of labels ({len(labels)}) does not match number of ' - f'columns ({len(row)}) in row {i}:\n{row}') + f'columns ({len(row)}) in row {i}:\n{row}' + ) data.append(row) return labels, np.array(data, dtype=float) diff --git a/perfkitbenchmarker/linux_packages/ffmpeg.py b/perfkitbenchmarker/linux_packages/ffmpeg.py index e19ddbd9ae..29c9467216 100644 --- a/perfkitbenchmarker/linux_packages/ffmpeg.py +++ b/perfkitbenchmarker/linux_packages/ffmpeg.py @@ -3,20 +3,41 @@ Follows the instructions here (https://trac.ffmpeg.org/wiki/CompilationGuide) for compiling FFmpeg. """ + from absl import flags _BUILD_FFMPEG_FROM_SOURCE = flags.DEFINE_boolean( - 'build_ffmpeg_from_source', False, 'Whether to build ffmpeg from source') + 'build_ffmpeg_from_source', False, 'Whether to build ffmpeg from source' +) FLAGS = flags.FLAGS _APT_DEPS = [ - 'autoconf', 'automake', 'build-essential', 'cmake', 'git-core', - 'libass-dev', 'libfreetype6-dev', 'libgnutls28-dev', 'libsdl2-dev', - 'libtool', 'libva-dev', 'libvdpau-dev', 'libvorbis-dev', 'libxcb1-dev', - 'libxcb-shm0-dev', 'libxcb-xfixes0-dev', 'meson', 'ninja-build', - 'pkg-config', 'texinfo', 'wget', 'yasm', 'zlib1g-dev', 'mercurial', - 'libnuma-dev bc' + 'autoconf', + 'automake', + 'build-essential', + 'cmake', + 'git-core', + 'libass-dev', + 'libfreetype6-dev', + 'libgnutls28-dev', + 'libsdl2-dev', + 'libtool', + 'libva-dev', + 'libvdpau-dev', + 'libvorbis-dev', + 'libxcb1-dev', + 'libxcb-shm0-dev', + 'libxcb-xfixes0-dev', + 'meson', + 'ninja-build', + 'pkg-config', + 'texinfo', + 'wget', + 'yasm', + 'zlib1g-dev', + 'mercurial', + 'libnuma-dev bc', ] @@ -41,15 +62,15 @@ def AptInstall(vm): 'cd nasm-2.15.03 && ./autogen.sh && PATH="$HOME/bin:$PATH" ' './configure --prefix="$HOME/ffmpeg_build" --bindir="$HOME/bin" && ' 'make -j && make install' - ) + ) # Install Yasm vm.RemoteCommand( - 'cd ~/ffmpeg_sources && wget -O yasm-1.3.0.tar.gz ' - 'https://github.com/yasm/yasm/releases/download/v1.3.0/yasm-1.3.0.tar.gz && ' - 'tar xzvf yasm-1.3.0.tar.gz && cd yasm-1.3.0 && ./configure ' - '--prefix="$HOME/ffmpeg_build" --bindir="$HOME/bin" && make -j && ' - 'make install' - ) + 'cd ~/ffmpeg_sources && wget -O yasm-1.3.0.tar.gz' + ' https://github.com/yasm/yasm/releases/download/v1.3.0/yasm-1.3.0.tar.gz' + ' && tar xzvf yasm-1.3.0.tar.gz && cd yasm-1.3.0 && ./configure' + ' --prefix="$HOME/ffmpeg_build" --bindir="$HOME/bin" && make -j && make' + ' install' + ) # Install libx264 vm.RemoteCommand( 'cd ~/ffmpeg_sources && git -C x264 pull 2> /dev/null || git clone ' @@ -58,7 +79,7 @@ def AptInstall(vm): 'pkgconfig" ./configure --prefix="$HOME/ffmpeg_build" ' '--bindir="$HOME/bin" --enable-static --enable-pic && ' 'PATH="$HOME/bin:$PATH" make -j && make install' - ) + ) # Install libx265 vm.RemoteCommand( 'cd ~/ffmpeg_sources && git clone https://github.com/videolan/x265 ' @@ -66,23 +87,23 @@ def AptInstall(vm): '"Unix Makefiles" -DCMAKE_INSTALL_PREFIX="$HOME/ffmpeg_build" ' '-DENABLE_SHARED=off ../../source && PATH="$HOME/bin:$PATH" make -j && ' 'make install' - ) + ) # Install libvpx vm.RemoteCommand( - 'cd ~/ffmpeg_sources && git -C libvpx pull 2> /dev/null || git clone ' - '--depth 1 https://chromium.googlesource.com/webm/libvpx.git && ' - 'cd libvpx && PATH="$HOME/bin:$PATH" ./configure ' - '--prefix="$HOME/ffmpeg_build" --disable-examples --disable-unit-tests ' - '--enable-vp9-highbitdepth --as=yasm && PATH="$HOME/bin:$PATH" make -j && ' - 'make install' - ) + 'cd ~/ffmpeg_sources && git -C libvpx pull 2> /dev/null || git clone' + ' --depth 1 https://chromium.googlesource.com/webm/libvpx.git && cd' + ' libvpx && PATH="$HOME/bin:$PATH" ./configure' + ' --prefix="$HOME/ffmpeg_build" --disable-examples --disable-unit-tests' + ' --enable-vp9-highbitdepth --as=yasm && PATH="$HOME/bin:$PATH" make -j' + ' && make install' + ) # Install libfdk-aac vm.RemoteCommand( 'cd ~/ffmpeg_sources && git -C fdk-aac pull 2> /dev/null || git clone ' '--depth 1 https://github.com/mstorsjo/fdk-aac && cd fdk-aac && ' 'autoreconf -fiv && ./configure --prefix="$HOME/ffmpeg_build" ' '--disable-shared && make -j && make install' - ) + ) # Install libmp3lame vm.RemoteCommand( 'cd ~/ffmpeg_sources && wget -O lame-3.100.tar.gz ' @@ -91,14 +112,14 @@ def AptInstall(vm): 'PATH="$HOME/bin:$PATH" ./configure --prefix="$HOME/ffmpeg_build" ' '--bindir="$HOME/bin" --disable-shared --enable-nasm && ' 'PATH="$HOME/bin:$PATH" make -j && make install' - ) + ) # Install libopus vm.RemoteCommand( 'cd ~/ffmpeg_sources && git -C opus pull 2> /dev/null || git clone ' '--depth 1 https://github.com/xiph/opus.git && cd opus && ' './autogen.sh && ./configure --prefix="$HOME/ffmpeg_build" ' '--disable-shared && make -j && make install' - ) + ) # Skip installation of AV1 libraries: libaom, libsvtav1, libdav1d # Install FFmpeg @@ -115,4 +136,4 @@ def AptInstall(vm): '--enable-libfreetype --enable-libmp3lame --enable-libopus ' '--enable-libvorbis --enable-libvpx --enable-libx264 --enable-libx265 ' '--enable-nonfree && PATH="$HOME/bin:$PATH" make -j && make install' - ) + ) diff --git a/perfkitbenchmarker/linux_packages/fio.py b/perfkitbenchmarker/linux_packages/fio.py index c6ca51bd2b..8f01a14724 100644 --- a/perfkitbenchmarker/linux_packages/fio.py +++ b/perfkitbenchmarker/linux_packages/fio.py @@ -74,11 +74,14 @@ def _Install(vm): if flags.FLAGS.fio_hist_log: vm.PushDataFile(FIO_HIST_LOG_PARSER_PATCH) vm.RemoteCommand( - ('cp {log_parser_path}/{log_parser} ./; ' - 'patch {log_parser} {patch}').format( - log_parser_path=FIO_HIST_LOG_PARSER_PATH, - log_parser=FIO_HIST_LOG_PARSER, - patch=FIO_HIST_LOG_PARSER_PATCH)) + ( + 'cp {log_parser_path}/{log_parser} ./; patch {log_parser} {patch}' + ).format( + log_parser_path=FIO_HIST_LOG_PARSER_PATH, + log_parser=FIO_HIST_LOG_PARSER, + patch=FIO_HIST_LOG_PARSER_PATCH, + ) + ) def YumInstall(vm): @@ -144,17 +147,25 @@ def FioParametersToJob(fio_parameters): """ fio_parameters = fio_parameters.replace(' ', '\n') fio_parameters = regex_util.Substitute( - CMD_SECTION_REGEX, JOB_SECTION_REPL_REGEX, fio_parameters) + CMD_SECTION_REGEX, JOB_SECTION_REPL_REGEX, fio_parameters + ) fio_parameters = '[%s]\n%s' % (GLOBAL, fio_parameters) fio_parameters = regex_util.Substitute( - CMD_PARAMETER_REGEX, CMD_PARAMETER_REPL_REGEX, fio_parameters) - return fio_parameters.replace(CMD_STONEWALL_PARAMETER, - JOB_STONEWALL_PARAMETER) - - -def ParseResults(job_file, fio_json_result, base_metadata=None, - log_file_base='', bin_vals=None, - skip_latency_individual_stats=False): + CMD_PARAMETER_REGEX, CMD_PARAMETER_REPL_REGEX, fio_parameters + ) + return fio_parameters.replace( + CMD_STONEWALL_PARAMETER, JOB_STONEWALL_PARAMETER + ) + + +def ParseResults( + job_file, + fio_json_result, + base_metadata=None, + log_file_base='', + bin_vals=None, + skip_latency_individual_stats=False, +): """Parse fio json output into samples. Args: @@ -162,8 +173,8 @@ def ParseResults(job_file, fio_json_result, base_metadata=None, fio_json_result: Fio results in json format. base_metadata: Extra metadata to annotate the samples with. log_file_base: String. Base name for fio log files. - bin_vals: A 2-D list of int. Each list represents a list of - bin values in histgram log. Calculated from remote VM using + bin_vals: A 2-D list of int. Each list represents a list of bin values in + histgram log. Calculated from remote VM using fio/tools/hist/fiologparser_hist.py skip_latency_individual_stats: Bool. If true, skips pulling latency stats that are not aggregate. @@ -197,12 +208,17 @@ def ParseResults(job_file, fio_json_result, base_metadata=None, 'bw_max': job[mode]['bw_max'], 'bw_dev': job[mode]['bw_dev'], 'bw_agg': job[mode]['bw_agg'], - 'bw_mean': job[mode]['bw_mean']} + 'bw_mean': job[mode]['bw_mean'], + } bw_metadata.update(parameters) samples.append( - sample.Sample('%s:bandwidth' % metric_name, - job[mode]['bw'], - 'KB/s', bw_metadata)) + sample.Sample( + '%s:bandwidth' % metric_name, + job[mode]['bw'], + 'KB/s', + bw_metadata, + ) + ) # There is one sample whose metric is ':latency' # with all of the latency statistics in its metadata, and then @@ -220,53 +236,75 @@ def _ConvertClat(value): else: return value - lat_statistics = [('min', _ConvertClat(clat_section['min'])), - ('max', _ConvertClat(clat_section['max'])), - ('mean', _ConvertClat(clat_section['mean'])), - ('stddev', _ConvertClat(clat_section['stddev']))] + lat_statistics = [ + ('min', _ConvertClat(clat_section['min'])), + ('max', _ConvertClat(clat_section['max'])), + ('mean', _ConvertClat(clat_section['mean'])), + ('stddev', _ConvertClat(clat_section['stddev'])), + ] if not skip_latency_individual_stats: percentiles = clat_section['percentile'] - lat_statistics += [('p1', _ConvertClat(percentiles['1.000000'])), - ('p5', _ConvertClat(percentiles['5.000000'])), - ('p10', _ConvertClat(percentiles['10.000000'])), - ('p20', _ConvertClat(percentiles['20.000000'])), - ('p30', _ConvertClat(percentiles['30.000000'])), - ('p40', _ConvertClat(percentiles['40.000000'])), - ('p50', _ConvertClat(percentiles['50.000000'])), - ('p60', _ConvertClat(percentiles['60.000000'])), - ('p70', _ConvertClat(percentiles['70.000000'])), - ('p80', _ConvertClat(percentiles['80.000000'])), - ('p90', _ConvertClat(percentiles['90.000000'])), - ('p95', _ConvertClat(percentiles['95.000000'])), - ('p99', _ConvertClat(percentiles['99.000000'])), - ('p99.5', _ConvertClat(percentiles['99.500000'])), - ('p99.9', _ConvertClat(percentiles['99.900000'])), - ('p99.95', _ConvertClat(percentiles['99.950000'])), - ('p99.99', _ConvertClat(percentiles['99.990000']))] + lat_statistics += [ + ('p1', _ConvertClat(percentiles['1.000000'])), + ('p5', _ConvertClat(percentiles['5.000000'])), + ('p10', _ConvertClat(percentiles['10.000000'])), + ('p20', _ConvertClat(percentiles['20.000000'])), + ('p30', _ConvertClat(percentiles['30.000000'])), + ('p40', _ConvertClat(percentiles['40.000000'])), + ('p50', _ConvertClat(percentiles['50.000000'])), + ('p60', _ConvertClat(percentiles['60.000000'])), + ('p70', _ConvertClat(percentiles['70.000000'])), + ('p80', _ConvertClat(percentiles['80.000000'])), + ('p90', _ConvertClat(percentiles['90.000000'])), + ('p95', _ConvertClat(percentiles['95.000000'])), + ('p99', _ConvertClat(percentiles['99.000000'])), + ('p99.5', _ConvertClat(percentiles['99.500000'])), + ('p99.9', _ConvertClat(percentiles['99.900000'])), + ('p99.95', _ConvertClat(percentiles['99.950000'])), + ('p99.99', _ConvertClat(percentiles['99.990000'])), + ] lat_metadata = parameters.copy() for name, val in lat_statistics: lat_metadata[name] = val samples.append( - sample.Sample('%s:latency' % metric_name, - _ConvertClat(job[mode][clat_key]['mean']), - 'usec', lat_metadata, timestamp)) + sample.Sample( + '%s:latency' % metric_name, + _ConvertClat(job[mode][clat_key]['mean']), + 'usec', + lat_metadata, + timestamp, + ) + ) for stat_name, stat_val in lat_statistics: samples.append( - sample.Sample('%s:latency:%s' % (metric_name, stat_name), - stat_val, 'usec', parameters, timestamp)) + sample.Sample( + '%s:latency:%s' % (metric_name, stat_name), + stat_val, + 'usec', + parameters, + timestamp, + ) + ) samples.append( - sample.Sample('%s:iops' % metric_name, - job[mode]['iops'], '', parameters, timestamp)) + sample.Sample( + '%s:iops' % metric_name, + job[mode]['iops'], + '', + parameters, + timestamp, + ) + ) if log_file_base and bin_vals: # Parse histograms aggregates = collections.defaultdict(collections.Counter) for _ in range(int(parameters.get('numjobs', 1))): clat_hist_idx += 1 hist_file_path = vm_util.PrependTempDir( - '%s_clat_hist.%s.log' % (log_file_base, str(clat_hist_idx))) + '%s_clat_hist.%s.log' % (log_file_base, str(clat_hist_idx)) + ) hists = _ParseHistogram(hist_file_path, bin_vals[clat_hist_idx - 1]) for key in hists: @@ -288,8 +326,12 @@ def ComputeHistogramBinVals(vm, log_file): A list of float. Representing the mean value of the bin. """ try: - return [float(v) for v in vm.RemoteCommand( - './%s %s' % (FIO_HIST_LOG_PARSER, log_file))[0].split()] + return [ + float(v) + for v in vm.RemoteCommand('./%s %s' % (FIO_HIST_LOG_PARSER, log_file))[ + 0 + ].split() + ] except errors.VirtualMachine.RemoteCommandError: logging.exception('Calculate bin values for %s failed.', log_file) return [] @@ -315,9 +357,8 @@ def _ParseHistogram(hist_log_file, mean_bin_vals): """Parses histogram log file reported by fio. Args: - hist_log_file: String. File name of fio histogram log. Format: - time (msec), data direction (0: read, 1: write, 2: trim), block size, - bin 0, .., etc + hist_log_file: String. File name of fio histogram log. Format: time (msec), + data direction (0: read, 1: write, 2: trim), block size, bin 0, .., etc mean_bin_vals: List of float. Representing the mean value of each bucket. Returns: @@ -345,24 +386,29 @@ def _ParseHistogram(hist_log_file, mean_bin_vals): return aggregates -def _BuildHistogramSamples(aggregates, metric_prefix='', - additional_metadata=None): +def _BuildHistogramSamples( + aggregates, metric_prefix='', additional_metadata=None +): """Builds a sample for a histogram aggregated from several files. - Args: - metric_prefix: String. Prefix of the metric name to use. - additional_metadata: dict. Additional metadata attaching to Sample. + Args: + metric_prefix: String. Prefix of the metric name to use. + additional_metadata: dict. Additional metadata attaching to Sample. - Returns: - samples.Sample object that reports the fio histogram. + Returns: + samples.Sample object that reports the fio histogram. """ samples = [] - for (rw, bs) in aggregates.keys(): + for rw, bs in aggregates.keys(): metadata = {'histogram': json.dumps(aggregates[(rw, bs)])} if additional_metadata: metadata.update(additional_metadata) samples.append( sample.Sample( ':'.join([metric_prefix, str(bs), rw, 'histogram']), - 0, 'us', metadata)) + 0, + 'us', + metadata, + ) + ) return samples diff --git a/perfkitbenchmarker/linux_packages/fortran.py b/perfkitbenchmarker/linux_packages/fortran.py index 7aefeb1046..1a140f6242 100644 --- a/perfkitbenchmarker/linux_packages/fortran.py +++ b/perfkitbenchmarker/linux_packages/fortran.py @@ -16,8 +16,9 @@ import logging from absl import flags -VERSION = flags.DEFINE_integer('fortran_version', None, - 'Version of gfortran to install') +VERSION = flags.DEFINE_integer( + 'fortran_version', None, 'Version of gfortran to install' +) def GetLibPath(vm): @@ -57,7 +58,8 @@ def _YumInstallVersion(vm, version): sym_link = f'/usr/bin/gfortran-{version}' real_path = f'/opt/rh/{devtoolset}/root/usr/bin/gfortran' vm.RemoteCommand( - f'sudo alternatives --install {sym_link} fortran {real_path} 100') + f'sudo alternatives --install {sym_link} fortran {real_path} 100' + ) def _AptInstallVersion(vm, version): @@ -65,8 +67,10 @@ def _AptInstallVersion(vm, version): vm.Install('ubuntu_toolchain') vm.InstallPackages(f'gfortran-{version}') # make symlink so that 'gfortran' will use the newly installed version - vm.RemoteCommand('sudo update-alternatives --install /usr/bin/gfortran ' - f'gfortran /usr/bin/gfortran-{version} 100') + vm.RemoteCommand( + 'sudo update-alternatives --install /usr/bin/gfortran ' + f'gfortran /usr/bin/gfortran-{version} 100' + ) def _LogFortranVersion(vm): diff --git a/perfkitbenchmarker/linux_packages/gce_hpc_tools.py b/perfkitbenchmarker/linux_packages/gce_hpc_tools.py index 6b526ae83a..e6616764fe 100644 --- a/perfkitbenchmarker/linux_packages/gce_hpc_tools.py +++ b/perfkitbenchmarker/linux_packages/gce_hpc_tools.py @@ -34,12 +34,25 @@ # HPC tools tuning script _HPC_SCRIPT = 'mpi-tuning.sh' -flags.DEFINE_string('gce_hpc_tools_tag', None, - 'Github tag of hpc-tools to use. Default is latest.') -flags.DEFINE_list('gce_hpc_tools_tuning', [ - 'hpcprofile', 'tcpmem', 'limits', 'nosmt', 'nofirewalld', 'noselinux', - 'nomitigation', 'reboot' -], 'List of HPC tunings. `bash mpi-tuning.sh` for description.') +flags.DEFINE_string( + 'gce_hpc_tools_tag', + None, + 'Github tag of hpc-tools to use. Default is latest.', +) +flags.DEFINE_list( + 'gce_hpc_tools_tuning', + [ + 'hpcprofile', + 'tcpmem', + 'limits', + 'nosmt', + 'nofirewalld', + 'noselinux', + 'nomitigation', + 'reboot', + ], + 'List of HPC tunings. `bash mpi-tuning.sh` for description.', +) FLAGS = flags.FLAGS @@ -74,10 +87,12 @@ def YumInstall(vm): def _CloneRepo(vm, hpc_tools_tag): """Clones git repo, switches to tag, and returns current commit.""" vm.InstallPackages('git') - vm.RemoteCommand(f'rm -rf {_HPC_REMOTE_DIR}; ' - f'git clone {_HPC_URL} {_HPC_REMOTE_DIR}') + vm.RemoteCommand( + f'rm -rf {_HPC_REMOTE_DIR}; git clone {_HPC_URL} {_HPC_REMOTE_DIR}' + ) if hpc_tools_tag: vm.RemoteCommand(f'cd {_HPC_REMOTE_DIR}; git checkout {hpc_tools_tag}') stdout, _ = vm.RemoteCommand( - f'cd {_HPC_REMOTE_DIR}; git log --pretty=format:"%h" -n 1') + f'cd {_HPC_REMOTE_DIR}; git log --pretty=format:"%h" -n 1' + ) return stdout.splitlines()[-1] diff --git a/perfkitbenchmarker/linux_packages/gcs_boto_plugin.py b/perfkitbenchmarker/linux_packages/gcs_boto_plugin.py index 1d459f8762..4c3fc6ab1d 100644 --- a/perfkitbenchmarker/linux_packages/gcs_boto_plugin.py +++ b/perfkitbenchmarker/linux_packages/gcs_boto_plugin.py @@ -20,7 +20,8 @@ def _Install(vm): """Installs the GCS boto plugin on the VM.""" vm.Install('pip3') vm.RemoteCommand( - 'sudo pip3 install --ignore-installed gcs-oauth2-boto-plugin') + 'sudo pip3 install --ignore-installed gcs-oauth2-boto-plugin' + ) def YumInstall(vm): diff --git a/perfkitbenchmarker/linux_packages/gcsfuse.py b/perfkitbenchmarker/linux_packages/gcsfuse.py index df85faf561..5494393b2c 100644 --- a/perfkitbenchmarker/linux_packages/gcsfuse.py +++ b/perfkitbenchmarker/linux_packages/gcsfuse.py @@ -6,8 +6,11 @@ flags.DEFINE_string('gcsfuse_version', '0.37.0', 'The version of the gcsfuse.') flags.DEFINE_string( - 'gcsfuse_bucket', '', 'The GCS bucket to be mounted. ' - 'If not set, all buckets are mounted as subdirectories.') + 'gcsfuse_bucket', + '', + 'The GCS bucket to be mounted. ' + 'If not set, all buckets are mounted as subdirectories.', +) PACKAGE_LOCAL = '/tmp/gcsfuse.deb' MNT = '/gcs' @@ -15,7 +18,8 @@ def _PackageUrl(): return 'https://github.com/GoogleCloudPlatform/gcsfuse/releases/download/v{v}/gcsfuse_{v}_amd64.deb'.format( - v=FLAGS.gcsfuse_version) + v=FLAGS.gcsfuse_version + ) def AptInstall(vm): @@ -25,8 +29,10 @@ def AptInstall(vm): vm: BaseVirtualMachine. VM to receive the scripts. """ vm.InstallPackages('wget') - vm.RemoteCommand('wget -O {local} {url}'.format( - local=PACKAGE_LOCAL, url=_PackageUrl())) + vm.RemoteCommand( + 'wget -O {local} {url}'.format(local=PACKAGE_LOCAL, url=_PackageUrl()) + ) vm.RemoteCommand( - f'sudo apt-get install -y --allow-downgrades {PACKAGE_LOCAL}') + f'sudo apt-get install -y --allow-downgrades {PACKAGE_LOCAL}' + ) diff --git a/perfkitbenchmarker/linux_packages/glibc.py b/perfkitbenchmarker/linux_packages/glibc.py index aa2462512a..5abe1c9ab3 100644 --- a/perfkitbenchmarker/linux_packages/glibc.py +++ b/perfkitbenchmarker/linux_packages/glibc.py @@ -27,16 +27,18 @@ BINUTILS_VERSION = '2.34' BINUTILS_TAR = 'binutils-{}.tar.gz'.format(BINUTILS_VERSION) PREPROVISIONED_DATA = { - BINUTILS_TAR: - '53537d334820be13eeb8acb326d01c7c81418772d626715c7ae927a7d401cab3', - GLIBC_TAR: + BINUTILS_TAR: ( + '53537d334820be13eeb8acb326d01c7c81418772d626715c7ae927a7d401cab3' + ), + GLIBC_TAR: ( '9246fe44f68feeec8c666bb87973d590ce0137cca145df014c72ec95be9ffd17' + ), } PACKAGE_DATA_URL = { BINUTILS_TAR: posixpath.join( - 'https://ftp.gnu.org/gnu/binutils', BINUTILS_TAR), - GLIBC_TAR: posixpath.join( - 'https://ftp.gnu.org/gnu/libc', GLIBC_TAR) + 'https://ftp.gnu.org/gnu/binutils', BINUTILS_TAR + ), + GLIBC_TAR: posixpath.join('https://ftp.gnu.org/gnu/libc', GLIBC_TAR), } _GCC_VERSION_RE = re.compile(r'gcc\ version\ (.*?)\ ') @@ -61,27 +63,31 @@ def _Install(vm): # bison and texinfo are required for compiling newer versions of glibc > 2.27. vm.InstallPackages('bison texinfo') - vm.RemoteCommand('cd {0} && mkdir binutils'.format( - linux_packages.INSTALL_DIR)) + vm.RemoteCommand( + 'cd {0} && mkdir binutils'.format(linux_packages.INSTALL_DIR) + ) vm.InstallPreprovisionedPackageData( - PACKAGE_NAME, [BINUTILS_TAR], BINUTILS_DIR) + PACKAGE_NAME, [BINUTILS_TAR], BINUTILS_DIR + ) vm.RemoteCommand('cd {0} && tar xvf {1}'.format(BINUTILS_DIR, BINUTILS_TAR)) - vm.RemoteCommand('cd {0} && mkdir binutils-build && ' - 'cd binutils-build/ && ' - '../binutils-{1}/configure --prefix=/opt/binutils && ' - 'make -j 4 && sudo make install'.format( - BINUTILS_DIR, BINUTILS_VERSION)) + vm.RemoteCommand( + 'cd {0} && mkdir binutils-build && ' + 'cd binutils-build/ && ' + '../binutils-{1}/configure --prefix=/opt/binutils && ' + 'make -j 4 && sudo make install'.format(BINUTILS_DIR, BINUTILS_VERSION) + ) vm.RemoteCommand('cd {0} && mkdir glibc'.format(linux_packages.INSTALL_DIR)) - vm.InstallPreprovisionedPackageData( - PACKAGE_NAME, [GLIBC_TAR], GLIBC_DIR) + vm.InstallPreprovisionedPackageData(PACKAGE_NAME, [GLIBC_TAR], GLIBC_DIR) vm.RemoteCommand('cd {0} && tar xvf {1}'.format(GLIBC_DIR, GLIBC_TAR)) vm.RemoteCommand( 'cd {0} && mkdir glibc-build && cd glibc-build && ' '../glibc-{1}/configure --prefix=/usr/local/glibc --disable-profile ' '--enable-add-ons --with-headers=/usr/include ' '--with-binutils=/opt/binutils/bin && make && sudo make install'.format( - GLIBC_DIR, GLIBC_VERSION)) + GLIBC_DIR, GLIBC_VERSION + ) + ) def YumInstall(vm): diff --git a/perfkitbenchmarker/linux_packages/gluster.py b/perfkitbenchmarker/linux_packages/gluster.py index 9fae056d68..0a3f9299e8 100644 --- a/perfkitbenchmarker/linux_packages/gluster.py +++ b/perfkitbenchmarker/linux_packages/gluster.py @@ -23,12 +23,8 @@ FLAGS = flags.FLAGS -flags.DEFINE_integer( - 'gluster_replicas', 3, - 'The number of Gluster replicas.') -flags.DEFINE_integer( - 'gluster_stripes', 1, - 'The number of Gluster stripes.') +flags.DEFINE_integer('gluster_replicas', 3, 'The number of Gluster replicas.') +flags.DEFINE_integer('gluster_stripes', 1, 'The number of Gluster stripes.') def YumInstall(vm): @@ -37,7 +33,8 @@ def YumInstall(vm): if FLAGS.os_type != os_types.CENTOS7: raise NotImplementedError( 'PKB currently only supports installation of gluster on centos7 or ' - 'Debian-based VMs.') + 'Debian-based VMs.' + ) vm.InstallPackages('centos-release-gluster') vm.InstallPackages('glusterfs-server') vm.RemoteCommand('sudo glusterd') @@ -61,10 +58,14 @@ def MountGluster(vm, gluster_server, volume_name, mount_point): """ vm.Install('gluster') volume = '{ip}:/{volume_name}'.format( - ip=gluster_server.internal_ip, volume_name=volume_name) + ip=gluster_server.internal_ip, volume_name=volume_name + ) vm.RemoteCommand('sudo mkdir -p %s' % mount_point) - vm.RemoteCommand('sudo mount -t glusterfs {volume} {mount_point}'.format( - volume=volume, mount_point=mount_point)) + vm.RemoteCommand( + 'sudo mount -t glusterfs {volume} {mount_point}'.format( + volume=volume, mount_point=mount_point + ) + ) def _CreateVolume(vm, bricks, volume_name): @@ -72,24 +73,38 @@ def _CreateVolume(vm, bricks, volume_name): Args: vm: The Virtual Machine to create the volume from. - bricks: A list of strings of the form "ip_address:/path/to/brick" which - will be combined to form the Gluster volume. + bricks: A list of strings of the form "ip_address:/path/to/brick" which will + be combined to form the Gluster volume. volume_name: The name of the volume which is being created. """ - replicas = ('replica %s' % FLAGS.gluster_replicas - if FLAGS.gluster_replicas > 1 else '') - stripes = ('stripe %s' % FLAGS.gluster_stripes - if FLAGS.gluster_stripes > 1 else '') + replicas = ( + 'replica %s' % FLAGS.gluster_replicas + if FLAGS.gluster_replicas > 1 + else '' + ) + stripes = ( + 'stripe %s' % FLAGS.gluster_stripes if FLAGS.gluster_stripes > 1 else '' + ) - vm.RemoteCommand(('sudo gluster volume create {volume_name} ' - '{stripes} {replicas} {bricks}').format( - volume_name=volume_name, replicas=replicas, - stripes=stripes, bricks=' '.join(bricks))) + vm.RemoteCommand( + ( + 'sudo gluster volume create {volume_name} ' + '{stripes} {replicas} {bricks}' + ).format( + volume_name=volume_name, + replicas=replicas, + stripes=stripes, + bricks=' '.join(bricks), + ) + ) def _ProbePeer(vm1, vm2): - vm1.RemoteCommand('sudo gluster peer probe {internal_ip}'.format( - internal_ip=vm2.internal_ip)) + vm1.RemoteCommand( + 'sudo gluster peer probe {internal_ip}'.format( + internal_ip=vm2.internal_ip + ) + ) def ConfigureServers(gluster_servers, volume_name): @@ -119,8 +134,11 @@ def ConfigureServers(gluster_servers, volume_name): for vm in gluster_servers: for disk in vm.scratch_disks: brick_path = posixpath.join(disk.mount_point, 'gluster_brick') - bricks.append('{internal_ip}:{brick_path}'.format( - internal_ip=vm.internal_ip, brick_path=brick_path)) + bricks.append( + '{internal_ip}:{brick_path}'.format( + internal_ip=vm.internal_ip, brick_path=brick_path + ) + ) vm.RemoteCommand('mkdir -p %s' % brick_path) # Gluster servers need to be added to the trusted storage pool @@ -135,13 +153,15 @@ def ConfigureServers(gluster_servers, volume_name): _CreateVolume(gluster_servers[0], bricks, volume_name) gluster_servers[0].RemoteCommand( - 'sudo gluster volume start {volume_name}'.format( - volume_name=volume_name)) + 'sudo gluster volume start {volume_name}'.format(volume_name=volume_name) + ) def DeleteVolume(gluster_server, volume_name): """Stops and deletes a Gluster volume.""" gluster_server.RemoteCommand( - 'yes | sudo gluster volume stop %s' % volume_name) + 'yes | sudo gluster volume stop %s' % volume_name + ) gluster_server.RemoteCommand( - 'yes | sudo gluster volume delete %s' % volume_name) + 'yes | sudo gluster volume delete %s' % volume_name + ) diff --git a/perfkitbenchmarker/linux_packages/go_lang.py b/perfkitbenchmarker/linux_packages/go_lang.py index 74c0558e4d..bc294d5161 100644 --- a/perfkitbenchmarker/linux_packages/go_lang.py +++ b/perfkitbenchmarker/linux_packages/go_lang.py @@ -31,10 +31,13 @@ def Install(vm): """Install go lang package on the VM.""" - vm.InstallPreprovisionedPackageData(PACKAGE_NAME, PREPROVISIONED_DATA.keys(), - linux_packages.INSTALL_DIR) - vm.RemoteCommand('cd %s && sudo tar -C /usr/local -xzf %s' % - (linux_packages.INSTALL_DIR, GO_TAR)) + vm.InstallPreprovisionedPackageData( + PACKAGE_NAME, PREPROVISIONED_DATA.keys(), linux_packages.INSTALL_DIR + ) + vm.RemoteCommand( + 'cd %s && sudo tar -C /usr/local -xzf %s' + % (linux_packages.INSTALL_DIR, GO_TAR) + ) def Uninstall(_): diff --git a/perfkitbenchmarker/linux_packages/google_cloud_bigtable_client.py b/perfkitbenchmarker/linux_packages/google_cloud_bigtable_client.py index c5b02e17d3..18903dc08d 100644 --- a/perfkitbenchmarker/linux_packages/google_cloud_bigtable_client.py +++ b/perfkitbenchmarker/linux_packages/google_cloud_bigtable_client.py @@ -34,10 +34,12 @@ _MVN_DEPENDENCY_PLUGIN = 'org.apache.maven.plugins:maven-dependency-plugin' _MVN_DEPENDENCY_COPY_GOAL = ( f'{_MVN_DEPENDENCY_PLUGIN}:{_MVN_DEPENDENCY_PLUGIN_VERSION}' - ':copy-dependencies') + ':copy-dependencies' +) _CLIENT_CONNECTOR_PROJECT_POM_TEMPLATE = ( - 'cloudbigtable/pom-cbt-client-connector.xml.j2') + 'cloudbigtable/pom-cbt-client-connector.xml.j2' +) @vm_util.Retry() @@ -45,7 +47,8 @@ def Install(vm): """Installs the Cloud Bigtable YCSB HBase client connector on the VM.""" if not CLIENT_VERSION.value: raise errors.Setup.InvalidFlagConfigurationError( - '--google_bigtable_client_version must be set') + '--google_bigtable_client_version must be set' + ) # Ensure YCSB is set up as that's where we copy the artifacts to. vm.Install('ycsb') @@ -61,15 +64,17 @@ def Install(vm): # and all its runtime dependencies the the specified folder. pom_template = data.ResourcePath(_CLIENT_CONNECTOR_PROJECT_POM_TEMPLATE) remote_pom_path = posixpath.join( - ycsb.YCSB_DIR, - os.path.basename(pom_template)[:-3]) # strip .j2 suffix + ycsb.YCSB_DIR, os.path.basename(pom_template)[:-3] + ) # strip .j2 suffix vm.RenderTemplate(pom_template, remote_pom_path, context) - dependency_install_dir = posixpath.join(ycsb.YCSB_DIR, - f'{FLAGS.hbase_binding}-binding', - 'lib') + dependency_install_dir = posixpath.join( + ycsb.YCSB_DIR, f'{FLAGS.hbase_binding}-binding', 'lib' + ) maven_args = [ - '-f', remote_pom_path, _MVN_DEPENDENCY_COPY_GOAL, - f'-DoutputDirectory={dependency_install_dir}' + '-f', + remote_pom_path, + _MVN_DEPENDENCY_COPY_GOAL, + f'-DoutputDirectory={dependency_install_dir}', ] vm.RemoteCommand(maven.GetRunCommand(' '.join(maven_args))) diff --git a/perfkitbenchmarker/linux_packages/google_cloud_cbt.py b/perfkitbenchmarker/linux_packages/google_cloud_cbt.py index 21bbccb4da..2b3a7248b7 100644 --- a/perfkitbenchmarker/linux_packages/google_cloud_cbt.py +++ b/perfkitbenchmarker/linux_packages/google_cloud_cbt.py @@ -25,8 +25,10 @@ FLAGS = flags.FLAGS CBT_CLI_VERSION = flags.DEFINE_string( - 'google_cloud_cbt_cli_version', '9c4f438e783f3af6e8900e80e834725640f1c6a9', - 'Google Cloud Bigtable `cbt` CLI version') + 'google_cloud_cbt_cli_version', + '9c4f438e783f3af6e8900e80e834725640f1c6a9', + 'Google Cloud Bigtable `cbt` CLI version', +) CBT_BIN = os.path.join(linux_packages.INSTALL_DIR, 'cbt/cbt') @@ -35,6 +37,8 @@ def Install(vm): """Installs the cbt CLI.""" vm.Install('go_lang') vm.RemoteCommand(f'cd {linux_packages.INSTALL_DIR} && mkdir -p cbt') - vm.RemoteCommand(f'{go_lang.GO_BIN} install ' - f'cloud.google.com/go/cbt@{CBT_CLI_VERSION.value}') + vm.RemoteCommand( + f'{go_lang.GO_BIN} install ' + f'cloud.google.com/go/cbt@{CBT_CLI_VERSION.value}' + ) vm.RemoteCommand(f'cp go/bin/cbt {CBT_BIN}') diff --git a/perfkitbenchmarker/linux_packages/google_cloud_go.py b/perfkitbenchmarker/linux_packages/google_cloud_go.py index 66ee5330a2..5a61b8810f 100644 --- a/perfkitbenchmarker/linux_packages/google_cloud_go.py +++ b/perfkitbenchmarker/linux_packages/google_cloud_go.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Module containing google.cloud go package installation and cleanup. -""" +"""Module containing google.cloud go package installation and cleanup.""" def Install(vm): diff --git a/perfkitbenchmarker/linux_packages/google_cloud_sdk.py b/perfkitbenchmarker/linux_packages/google_cloud_sdk.py index c7d24a578a..853b41a8be 100644 --- a/perfkitbenchmarker/linux_packages/google_cloud_sdk.py +++ b/perfkitbenchmarker/linux_packages/google_cloud_sdk.py @@ -28,16 +28,21 @@ def RunGcloud(vm, cmd): - return vm.RemoteCommand('export CLOUDSDK_CORE_DISABLE_PROMPTS=1 && %s %s ' - '--project %s --format json' % (GCLOUD_PATH, cmd, - vm.project)) + return vm.RemoteCommand( + 'export CLOUDSDK_CORE_DISABLE_PROMPTS=1 && %s %s ' + '--project %s --format json' % (GCLOUD_PATH, cmd, vm.project) + ) def Install(vm): """Installs google cloud sdk on the VM.""" vm.Install('wget') - vm.RemoteCommand('cd {0} && wget {1} && tar xzf {2} && rm {2}'.format( - vm_util.VM_TMP_DIR, SDK_REPO, os.path.basename(SDK_REPO))) - vm.RemoteCommand('%s --disable-installation-options --usage-report=false ' - '--path-update=false --bash-completion=false' - % SDK_INSTALL_FILE) + vm.RemoteCommand( + 'cd {0} && wget {1} && tar xzf {2} && rm {2}'.format( + vm_util.VM_TMP_DIR, SDK_REPO, os.path.basename(SDK_REPO) + ) + ) + vm.RemoteCommand( + '%s --disable-installation-options --usage-report=false ' + '--path-update=false --bash-completion=false' % SDK_INSTALL_FILE + ) diff --git a/perfkitbenchmarker/linux_packages/hadoop.py b/perfkitbenchmarker/linux_packages/hadoop.py index 841276d6ce..f9ab1f4048 100644 --- a/perfkitbenchmarker/linux_packages/hadoop.py +++ b/perfkitbenchmarker/linux_packages/hadoop.py @@ -38,17 +38,23 @@ _VERSION = flags.DEFINE_string('hadoop_version', None, 'Version of Hadoop.') _URL_OVERRIDE = flags.DEFINE_string( - 'hadoop_bin_url', None, 'Specify to override url from HADOOP_URL_BASE.') + 'hadoop_bin_url', None, 'Specify to override url from HADOOP_URL_BASE.' +) _BLOCKSIZE_OVERRIDE = flags.DEFINE_integer( - 'hadoop_hdfs_blocksize', 128, + 'hadoop_hdfs_blocksize', + 128, 'Blocksize in MiB to be used by the HDFS filesystem. ' - 'This is the chunksize in which the HDFS file will be divided into.') + 'This is the chunksize in which the HDFS file will be divided into.', +) DATA_FILES = [ - 'hadoop/core-site.xml.j2', 'hadoop/yarn-site.xml.j2', - 'hadoop/hdfs-site.xml.j2', 'hadoop/mapred-site.xml.j2', - 'hadoop/hadoop-env.sh.j2', 'hadoop/workers.j2' + 'hadoop/core-site.xml.j2', + 'hadoop/yarn-site.xml.j2', + 'hadoop/hdfs-site.xml.j2', + 'hadoop/mapred-site.xml.j2', + 'hadoop/hadoop-env.sh.j2', + 'hadoop/workers.j2', ] START_HADOOP_SCRIPT = 'hadoop/start-hadoop.sh.j2' @@ -76,17 +82,20 @@ def HadoopVersion() -> version.Version: return version.Version(_VERSION.value) if _URL_OVERRIDE.value: extracted_version = re.search( - r'[0-9]+\.[0-9]+\.[0-9]+', _URL_OVERRIDE.value) + r'[0-9]+\.[0-9]+\.[0-9]+', _URL_OVERRIDE.value + ) if extracted_version: return version.Version(extracted_version.group(0)) else: raise errors.Config.InvalidValue( 'Cannot parse version out of --hadoop_bin_url please pass ' - '--hadoop_version as well.') + '--hadoop_version as well.' + ) response = requests.get(HADOOP_STABLE_URL) if not response.ok: raise errors.Setup.MissingExecutableError( - 'Could not load ' + HADOOP_STABLE_URL) + 'Could not load ' + HADOOP_STABLE_URL + ) soup = bs4.BeautifulSoup(response.content, 'html.parser') link = soup.find('a', href=HADOOP_TAR_PATTERN) if link: @@ -94,7 +103,8 @@ def HadoopVersion() -> version.Version: if match: return version.Version(match.group(1)) raise errors.Setup.MissingExecutableError( - 'Could not find valid hadoop version at ' + HADOOP_STABLE_URL) + 'Could not find valid hadoop version at ' + HADOOP_STABLE_URL + ) def _GetHadoopURL(): @@ -106,8 +116,9 @@ def _GetHadoopURL(): The Hadoop download url. """ - return '{0}/hadoop-{1}/hadoop-{1}.tar.gz'.format(HADOOP_URL_BASE, - HadoopVersion()) + return '{0}/hadoop-{1}/hadoop-{1}.tar.gz'.format( + HADOOP_URL_BASE, HadoopVersion() + ) def CheckPrerequisites(): @@ -128,8 +139,10 @@ def _Install(vm): hadoop_url = FLAGS.hadoop_bin_url or _GetHadoopURL() vm.RemoteCommand( - ('mkdir {0} && curl -L {1} | ' - 'tar -C {0} --strip-components=1 -xzf -').format(HADOOP_DIR, hadoop_url)) + ( + 'mkdir {0} && curl -L {1} | tar -C {0} --strip-components=1 -xzf -' + ).format(HADOOP_DIR, hadoop_url) + ) def YumInstall(vm): @@ -150,9 +163,10 @@ def AptInstall(vm): def InstallGcsConnector(vm, install_dir=HADOOP_LIB_DIR): """Install the GCS connector for Hadoop, which allows I/O to GCS.""" - connector_url = ('https://storage.googleapis.com/hadoop-lib/gcs/' - 'gcs-connector-hadoop{}-latest.jar'.format( - HadoopVersion().major)) + connector_url = ( + 'https://storage.googleapis.com/hadoop-lib/gcs/' + 'gcs-connector-hadoop{}-latest.jar'.format(HadoopVersion().major) + ) vm.RemoteCommand('cd {0} && curl -O {1}'.format(install_dir, connector_url)) @@ -173,11 +187,13 @@ def InstallGcsConnector(vm, install_dir=HADOOP_LIB_DIR): REDUCE_SLOTS_PER_CORE = 4 / 3 -def _RenderConfig(vm, - master, - workers, - memory_fraction=YARN_MEMORY_FRACTION, - configure_s3=False): +def _RenderConfig( + vm, + master, + workers, + memory_fraction=YARN_MEMORY_FRACTION, + configure_s3=False, +): """Load Hadoop Condfiguration on VM.""" # Use first worker to get worker configuration worker = workers[0] @@ -239,13 +255,14 @@ def _RenderConfig(vm, 'aws_access_key': aws_access_key, 'aws_secret_key': aws_secret_key, 'optional_tools': optional_tools, - 'block_size': block_size + 'block_size': block_size, } for file_name in DATA_FILES: file_path = data.ResourcePath(file_name) - if (file_name == 'hadoop/workers.j2' and - HadoopVersion() < version.Version('3')): + if file_name == 'hadoop/workers.j2' and HadoopVersion() < version.Version( + '3' + ): file_name = 'hadoop/slaves.j2' remote_path = posixpath.join(HADOOP_CONF_DIR, os.path.basename(file_name)) if file_name.endswith('.j2'): @@ -274,7 +291,7 @@ def ConfigureAndStart(master, workers, start_yarn=True, configure_s3=False): master: VM. Master VM - will be the HDFS NameNode, YARN ResourceManager. workers: List of VMs. Each VM will run an HDFS DataNode, YARN node. start_yarn: bool. Start YARN and JobHistory server? Set to False if HDFS is - the only service required. Default: True. + the only service required. Default: True. configure_s3: Whether to configure Hadoop to access S3. """ vms = [master] + workers @@ -282,11 +299,15 @@ def ConfigureAndStart(master, workers, start_yarn=True, configure_s3=False): # node runs the worker daemons. workers = workers or [master] fn = functools.partial( - _RenderConfig, master=master, workers=workers, configure_s3=configure_s3) + _RenderConfig, master=master, workers=workers, configure_s3=configure_s3 + ) background_tasks.RunThreaded(fn, vms) - master.RemoteCommand("rm -f {0} && ssh-keygen -q -t rsa -N '' -f {0}".format( - HADOOP_PRIVATE_KEY)) + master.RemoteCommand( + "rm -f {0} && ssh-keygen -q -t rsa -N '' -f {0}".format( + HADOOP_PRIVATE_KEY + ) + ) public_key = master.RemoteCommand('cat {0}.pub'.format(HADOOP_PRIVATE_KEY))[0] @@ -298,13 +319,14 @@ def AddKey(vm): context = { 'hadoop_dir': HADOOP_DIR, 'vm_ips': [vm.internal_ip for vm in vms], - 'start_yarn': start_yarn + 'start_yarn': start_yarn, } # HDFS setup and formatting, YARN startup script_path = posixpath.join(HADOOP_DIR, 'start-hadoop.sh') master.RenderTemplate( - data.ResourcePath(START_HADOOP_SCRIPT), script_path, context=context) + data.ResourcePath(START_HADOOP_SCRIPT), script_path, context=context + ) master.RemoteCommand('bash {0}'.format(script_path)) logging.info('Sleeping 10s for Hadoop nodes to join.') @@ -313,8 +335,11 @@ def AddKey(vm): logging.info('Checking HDFS status.') hdfs_online_count = _GetHDFSOnlineNodeCount(master) if hdfs_online_count != len(workers): - raise ValueError('Not all nodes running HDFS: {0} < {1}'.format( - hdfs_online_count, len(workers))) + raise ValueError( + 'Not all nodes running HDFS: {0} < {1}'.format( + hdfs_online_count, len(workers) + ) + ) else: logging.info('HDFS running on all %d workers', len(workers)) @@ -322,7 +347,10 @@ def AddKey(vm): logging.info('Checking YARN status.') yarn_online_count = _GetYARNOnlineNodeCount(master) if yarn_online_count != len(workers): - raise ValueError('Not all nodes running YARN: {0} < {1}'.format( - yarn_online_count, len(workers))) + raise ValueError( + 'Not all nodes running YARN: {0} < {1}'.format( + yarn_online_count, len(workers) + ) + ) else: logging.info('YARN running on all %d workers', len(workers)) diff --git a/perfkitbenchmarker/linux_packages/hammerdb.py b/perfkitbenchmarker/linux_packages/hammerdb.py index 3dce85b589..a9293c2326 100644 --- a/perfkitbenchmarker/linux_packages/hammerdb.py +++ b/perfkitbenchmarker/linux_packages/hammerdb.py @@ -52,8 +52,14 @@ TAR_4_5 = f'HammerDB-{HAMMERDB_4_5}-Linux.tar.gz' TPCH_TABLES = [ - 'customer', 'lineitem', 'nation', 'orders', 'part', 'partsupp', - 'region', 'supplier' + 'customer', + 'lineitem', + 'nation', + 'orders', + 'part', + 'partsupp', + 'region', + 'supplier', ] PACKAGE_NAME = 'hammerdb' @@ -78,7 +84,7 @@ MAP_SCRIPT_TO_DATABASE_NAME = { HAMMERDB_SCRIPT_TPC_C: 'tpcc', - HAMMERDB_SCRIPT_TPC_H: 'tpch' + HAMMERDB_SCRIPT_TPC_H: 'tpch', } RUN_SCRIPT_TYPE = 'RUN' @@ -104,87 +110,119 @@ RESTORABLE = 'restorable' HAMMERDB_VERSION = flags.DEFINE_enum( - 'hammerdbcli_version', HAMMERDB_4_0, + 'hammerdbcli_version', + HAMMERDB_4_0, [HAMMERDB_4_0, HAMMERDB_4_3, HAMMERDB_4_5], 'Version of the hammerdb. Currently supported patched ' - 'version of 4.0 and the public versions of 4.3 and 4.5.') + 'version of 4.0 and the public versions of 4.3 and 4.5.', +) HAMMERDB_SCRIPT = flags.DEFINE_enum( - 'hammerdbcli_script', HAMMERDB_SCRIPT_TPC_C, + 'hammerdbcli_script', + HAMMERDB_SCRIPT_TPC_C, [HAMMERDB_SCRIPT_TPC_H, HAMMERDB_SCRIPT_TPC_C], - 'The script to run for hammerdb.') + 'The script to run for hammerdb.', +) _HAMMERDB_BUILD_TIMEOUT = flags.DEFINE_integer( - 'hammerdbcli_build_timeout', 15000, - 'Timeout hammerdb build phase when exceed build timeout.') + 'hammerdbcli_build_timeout', + 15000, + 'Timeout hammerdb build phase when exceed build timeout.', +) HAMMERDB_RUN_TIMEOUT = flags.DEFINE_integer( - 'hammerdbcli_run_timeout', None, - 'Timeout when hammerdb exceed run time. No timeout when None is specified') + 'hammerdbcli_run_timeout', + None, + 'Timeout when hammerdb exceed run time. No timeout when None is specified', +) _HAMMERDB_SET_LINUX_OPEN_FILE_LIMIT = flags.DEFINE_integer( - 'hammerdbcli_set_linux_open_file_limit', None, + 'hammerdbcli_set_linux_open_file_limit', + None, 'Override the default linux open file limit, ' 'hammerdb fails due when VU is over 1000 the open file limit is not ' - 'override.') + 'override.', +) HAMMERDB_TPCH_DEGREE_OF_PARALLEL = flags.DEFINE_integer( - 'hammerdbcli_tpch_degree_of_parallel', 2, - 'Number of parallel worker in tpch postgres.') + 'hammerdbcli_tpch_degree_of_parallel', + 2, + 'Number of parallel worker in tpch postgres.', +) HAMMERDB_TPCC_RAMPUP = flags.DEFINE_integer( - 'hammerdbcli_tpcc_rampup', 5, - 'Duration in minutes for the database to ramp up before ' - 'gathering sampels.') + 'hammerdbcli_tpcc_rampup', + 5, + 'Duration in minutes for the database to ramp up before gathering sampels.', +) HAMMERDB_TPCC_DURATION = flags.DEFINE_integer( - 'hammerdbcli_tpcc_duration', 10, - 'Duration in minutes of running the benchmark.') + 'hammerdbcli_tpcc_duration', + 10, + 'Duration in minutes of running the benchmark.', +) HAMMERDB_TPCC_NUM_WAREHOUSE = flags.DEFINE_integer( - 'hammerdbcli_tpcc_num_warehouse', 5, 'Number of warehouses set in TPCC') + 'hammerdbcli_tpcc_num_warehouse', 5, 'Number of warehouses set in TPCC' +) HAMMERDB_TPCC_ALL_WAREHOUSE = flags.DEFINE_bool( - 'hammerdbcli_tpcc_all_warehouse', False, 'Uses all ' - 'warehouse in TPCC benchmark') + 'hammerdbcli_tpcc_all_warehouse', + False, + 'Uses all warehouse in TPCC benchmark', +) NUM_RUN = flags.DEFINE_integer( - 'hammerdbcli_num_runs', 1, 'Run the benchmark ' - 'n times.') + 'hammerdbcli_num_runs', 1, 'Run the benchmark n times.' +) LOAD_TPCH_TABLES_TO_COLUMNAR_ENGINE = flags.DEFINE_bool( - 'hammerdbcli_load_tpch_tables_to_columnar_engine', False, - 'Load the table to columnar engine.') - -HAMMERDB_NUM_VU = flags.DEFINE_integer('hammerdbcli_num_vu', None, - 'Number of virtual users') + 'hammerdbcli_load_tpch_tables_to_columnar_engine', + False, + 'Load the table to columnar engine.', +) + +HAMMERDB_NUM_VU = flags.DEFINE_integer( + 'hammerdbcli_num_vu', None, 'Number of virtual users' +) HAMMERDB_BUILD_TPCC_NUM_VU = flags.DEFINE_integer( - 'hammerdbcli_build_tpcc_num_vu', None, 'Number of virtual ' + 'hammerdbcli_build_tpcc_num_vu', + None, + 'Number of virtual ' 'users to build the tpcc database. This is set to ' - 'hammerdb_num_vu if not set.') + 'hammerdb_num_vu if not set.', +) HAMMERDB_BUILD_TPCH_NUM_VU = flags.DEFINE_integer( - 'hammerdbcli_build_tpch_num_vu', 8, 'Number of virtual' - ' users when building tpch table') + 'hammerdbcli_build_tpch_num_vu', + 8, + 'Number of virtual users when building tpch table', +) HAMMERDB_TPCH_SCALE_FACTOR = flags.DEFINE_integer( - 'hammerdbcli_tpch_scale_factor', 1, 'Scaling factor for TPCH') + 'hammerdbcli_tpch_scale_factor', 1, 'Scaling factor for TPCH' +) HAMMERDB_OPTIMIZED_SERVER_CONFIGURATION = flags.DEFINE_enum( - 'hammerdbcli_optimized_server_configuration', NON_OPTIMIZED, + 'hammerdbcli_optimized_server_configuration', + NON_OPTIMIZED, [NON_OPTIMIZED, MINIMUM_RECOVERY, RESTORABLE], - 'Server configuration to use when ' - 'benchmarking IAAS DB using HammerDB tool') + 'Server configuration to use when benchmarking IAAS DB using HammerDB tool', +) TPCC_LOG_TRANSACTIONS = flags.DEFINE_bool( - 'hammerdbcli_tpcc_log_transactions', False, - 'Gather data for TPM every seconds. Only supported on hammerdb 4.3.') + 'hammerdbcli_tpcc_log_transactions', + False, + 'Gather data for TPM every seconds. Only supported on hammerdb 4.3.', +) HAMMERDB_TPCC_TIME_PROFILE = flags.DEFINE_bool( - 'hammerdbcli_tpcc_time_profile', False, + 'hammerdbcli_tpcc_time_profile', + False, 'Gather data for tpcc latency data by turning on time' ' profile flag in hammerdb. Might decrease the ' - 'the TPM/NOPM, gathered due to extra measurements.') + 'the TPM/NOPM, gathered due to extra measurements.', +) def SetDefaultConfig(): @@ -203,11 +241,13 @@ def CheckPrerequisites(_): # hammerdb 4.5 and later versions required glibc 2.29 or later, # which is not available on ubuntu1804 and earlier. if HAMMERDB_VERSION.value == HAMMERDB_4_5 and FLAGS.os_type in [ - os_types.UBUNTU1604, os_types.UBUNTU1804 + os_types.UBUNTU1604, + os_types.UBUNTU1804, ]: raise errors.Setup.InvalidFlagConfigurationError( 'Hammerdb version 4.5 is not supported on os type {}.' - 'Use a later version of the os or an earlier version of hammerdb.') + 'Use a later version of the os or an earlier version of hammerdb.' + ) # define Hammerdb exception @@ -218,20 +258,27 @@ class HammerdbBenchmarkError(Exception): class HammerDbTclScript(object): """Represents a TCL script that will be run inside of hammerdbcli.""" - def __init__(self, tcl_script_name: str, needed_parameters: FrozenSet[str], - path: str, script_type: str): + def __init__( + self, + tcl_script_name: str, + needed_parameters: FrozenSet[str], + path: str, + script_type: str, + ): self.tcl_script_name = tcl_script_name self.needed_parameters = needed_parameters self.path = path self.script_type = script_type - def Install(self, vm: virtual_machine.VirtualMachine, - tcl_script_parameters: Any): + def Install( + self, vm: virtual_machine.VirtualMachine, tcl_script_parameters: Any + ): PushCloudSqlTestFile(vm, self.tcl_script_name, self.path) for parameter in self.needed_parameters: - tcl_script_parameters.SearchAndReplaceInScript(vm, self.tcl_script_name, - parameter) + tcl_script_parameters.SearchAndReplaceInScript( + vm, self.tcl_script_name, parameter + ) @classmethod def CheckErrorFromHammerdb(cls, stdout: str): @@ -253,17 +300,24 @@ def CheckErrorFromHammerdb(cls, stdout: str): Raises: Exception: exception when hammerdb failed """ - if ('Error' in stdout or 'FAILED' in stdout or - 'Virtual Users remain running' in stdout): - raise HammerdbBenchmarkError('Script failed during the build phase ' - 'with stdout {0}'.format(stdout)) - - def Run(self, - vm: virtual_machine.BaseVirtualMachine, - timeout: Optional[int] = 60*60*6) -> str: + if ( + 'Error' in stdout + or 'FAILED' in stdout + or 'Virtual Users remain running' in stdout + ): + raise HammerdbBenchmarkError( + 'Script failed during the build phase with stdout {0}'.format(stdout) + ) + + def Run( + self, + vm: virtual_machine.BaseVirtualMachine, + timeout: Optional[int] = 60 * 60 * 6, + ) -> str: """Run hammerdbcli script.""" - script_location = '{0}/{1}'.format(LocalWorkingDirectory(), - self.tcl_script_name) + script_location = '{0}/{1}'.format( + LocalWorkingDirectory(), self.tcl_script_name + ) cmd = '' if TPCC_LOG_TRANSACTIONS.value: @@ -277,9 +331,13 @@ def Run(self, cmd = f'ulimit -n {_HAMMERDB_SET_LINUX_OPEN_FILE_LIMIT.value} &&' stdout, _ = vm.RemoteCommand( InDir( - HAMMERDB_RUN_LOCATION, 'PATH="$PATH:/opt/mssql-tools/bin" &&' - + cmd + 'sudo -E ./hammerdbcli auto {0}'.format(script_location)), - timeout=timeout) + HAMMERDB_RUN_LOCATION, + 'PATH="$PATH:/opt/mssql-tools/bin" &&' + + cmd + + 'sudo -E ./hammerdbcli auto {0}'.format(script_location), + ), + timeout=timeout, + ) self.CheckErrorFromHammerdb(stdout) return stdout @@ -303,38 +361,50 @@ def Run(self, SCRIPT_PARAMETER_TPCH_DEGREE_OF_PARALLEL = '{{DEGREE_OF_PARALLEL_TPC_H}}' SCRIPT_PARAMETER_TPCC_LOG_TRANSACTIONS = '{{LOG_TRANSACTIONS}}' SCRIPT_PARAMETER_WAIT_TO_COMPLETE = '{{WAIT_TO_COMPLETE}}' -TPCC_PARAMS = frozenset({SCRIPT_PARAMETER_IP, - SCRIPT_PARAMETER_PORT, - SCRIPT_PARAMETER_PASSWORD, - SCRIPT_PARAMETER_USER, - SCRIPT_PARAMETER_AZURE, - SCRIPT_PARAMETER_TPCC_USERS, - SCRIPT_PARAMETER_TPCC_NUM_WAREHOUSE, - SCRIPT_PARAMETER_TPCC_ALL_WAREHOUSE, - SCRIPT_PARAMETER_BUILD_TIMEOUT, - SCRIPT_PARAMETER_TPCC_RAMPUP, - SCRIPT_PARAMETER_TPCC_DURATION, - SCRIPT_PARAMETER_TPCC_BUILD_USERS, - SCRIPT_PARAMETER_TPCC_TIME_PROFILE, - SCRIPT_PARAMETER_TPCC_LOG_TRANSACTIONS, - SCRIPT_PARAMETER_WAIT_TO_COMPLETE}) - -TPCH_PARAMS = frozenset({SCRIPT_PARAMETER_IP, - SCRIPT_PARAMETER_PORT, - SCRIPT_PARAMETER_PASSWORD, - SCRIPT_PARAMETER_USER, - SCRIPT_PARAMETER_AZURE, - SCRIPT_PARAMETER_TPCH_USERS, - SCRIPT_PARAMETER_TPCH_SCALE_FACTOR, - SCRIPT_PARAMETER_TPCH_DEGREE_OF_PARALLEL, - SCRIPT_PARAMETER_BUILD_TIMEOUT}) +TPCC_PARAMS = frozenset({ + SCRIPT_PARAMETER_IP, + SCRIPT_PARAMETER_PORT, + SCRIPT_PARAMETER_PASSWORD, + SCRIPT_PARAMETER_USER, + SCRIPT_PARAMETER_AZURE, + SCRIPT_PARAMETER_TPCC_USERS, + SCRIPT_PARAMETER_TPCC_NUM_WAREHOUSE, + SCRIPT_PARAMETER_TPCC_ALL_WAREHOUSE, + SCRIPT_PARAMETER_BUILD_TIMEOUT, + SCRIPT_PARAMETER_TPCC_RAMPUP, + SCRIPT_PARAMETER_TPCC_DURATION, + SCRIPT_PARAMETER_TPCC_BUILD_USERS, + SCRIPT_PARAMETER_TPCC_TIME_PROFILE, + SCRIPT_PARAMETER_TPCC_LOG_TRANSACTIONS, + SCRIPT_PARAMETER_WAIT_TO_COMPLETE, +}) + +TPCH_PARAMS = frozenset({ + SCRIPT_PARAMETER_IP, + SCRIPT_PARAMETER_PORT, + SCRIPT_PARAMETER_PASSWORD, + SCRIPT_PARAMETER_USER, + SCRIPT_PARAMETER_AZURE, + SCRIPT_PARAMETER_TPCH_USERS, + SCRIPT_PARAMETER_TPCH_SCALE_FACTOR, + SCRIPT_PARAMETER_TPCH_DEGREE_OF_PARALLEL, + SCRIPT_PARAMETER_BUILD_TIMEOUT, +}) class TclScriptParameters(object): """Handle of the parameters that may be needed by a TCL script.""" - def __init__(self, ip, port, password, user, is_managed_azure, - hammerdb_script, script_type): + def __init__( + self, + ip, + port, + password, + user, + is_managed_azure, + hammerdb_script, + script_type, + ): if ':' in ip: ip = ip.replace(':', '\\:') self.map_search_to_replace = { @@ -343,7 +413,7 @@ def __init__(self, ip, port, password, user, is_managed_azure, SCRIPT_PARAMETER_PASSWORD: password, SCRIPT_PARAMETER_USER: user, SCRIPT_PARAMETER_AZURE: 'true' if is_managed_azure else 'false', - SCRIPT_PARAMETER_BUILD_TIMEOUT: _HAMMERDB_BUILD_TIMEOUT.value + SCRIPT_PARAMETER_BUILD_TIMEOUT: _HAMMERDB_BUILD_TIMEOUT.value, } if hammerdb_script == HAMMERDB_SCRIPT_TPC_H: @@ -355,126 +425,179 @@ def __init__(self, ip, port, password, user, is_managed_azure, tpch_user_param = HAMMERDB_NUM_VU.value self.map_search_to_replace.update({ - SCRIPT_PARAMETER_TPCH_DEGREE_OF_PARALLEL: - HAMMERDB_TPCH_DEGREE_OF_PARALLEL.value, - SCRIPT_PARAMETER_TPCH_USERS: - tpch_user_param, - SCRIPT_PARAMETER_TPCH_SCALE_FACTOR: - HAMMERDB_TPCH_SCALE_FACTOR.value, + SCRIPT_PARAMETER_TPCH_DEGREE_OF_PARALLEL: ( + HAMMERDB_TPCH_DEGREE_OF_PARALLEL.value + ), + SCRIPT_PARAMETER_TPCH_USERS: tpch_user_param, + SCRIPT_PARAMETER_TPCH_SCALE_FACTOR: HAMMERDB_TPCH_SCALE_FACTOR.value, }) elif hammerdb_script == HAMMERDB_SCRIPT_TPC_C: # Wait to complete forces the script to stop. # Set the wait time to tpcc duration plus rampup time and add extra 10 # minutes of buffer - wait_to_complete_seconds = (int(HAMMERDB_TPCC_DURATION.value) + - int(HAMMERDB_TPCC_RAMPUP.value)) * 60 + 600 + wait_to_complete_seconds = ( + int(HAMMERDB_TPCC_DURATION.value) + int(HAMMERDB_TPCC_RAMPUP.value) + ) * 60 + 600 self.map_search_to_replace.update({ - SCRIPT_PARAMETER_TPCC_DURATION: - HAMMERDB_TPCC_DURATION.value, - SCRIPT_PARAMETER_TPCC_RAMPUP: - HAMMERDB_TPCC_RAMPUP.value, - SCRIPT_PARAMETER_TPCC_BUILD_USERS: - HAMMERDB_BUILD_TPCC_NUM_VU.value, - SCRIPT_PARAMETER_TPCC_USERS: - HAMMERDB_NUM_VU.value, - SCRIPT_PARAMETER_TPCC_NUM_WAREHOUSE: - HAMMERDB_TPCC_NUM_WAREHOUSE.value, - SCRIPT_PARAMETER_TPCC_ALL_WAREHOUSE: - HAMMERDB_TPCC_ALL_WAREHOUSE.value, - SCRIPT_PARAMETER_TPCC_TIME_PROFILE: - 'true' if HAMMERDB_TPCC_TIME_PROFILE.value else 'false', - SCRIPT_PARAMETER_TPCC_LOG_TRANSACTIONS: - 'true' if TPCC_LOG_TRANSACTIONS.value else 'false', - SCRIPT_PARAMETER_WAIT_TO_COMPLETE: wait_to_complete_seconds + SCRIPT_PARAMETER_TPCC_DURATION: HAMMERDB_TPCC_DURATION.value, + SCRIPT_PARAMETER_TPCC_RAMPUP: HAMMERDB_TPCC_RAMPUP.value, + SCRIPT_PARAMETER_TPCC_BUILD_USERS: HAMMERDB_BUILD_TPCC_NUM_VU.value, + SCRIPT_PARAMETER_TPCC_USERS: HAMMERDB_NUM_VU.value, + SCRIPT_PARAMETER_TPCC_NUM_WAREHOUSE: ( + HAMMERDB_TPCC_NUM_WAREHOUSE.value + ), + SCRIPT_PARAMETER_TPCC_ALL_WAREHOUSE: ( + HAMMERDB_TPCC_ALL_WAREHOUSE.value + ), + SCRIPT_PARAMETER_TPCC_TIME_PROFILE: ( + 'true' if HAMMERDB_TPCC_TIME_PROFILE.value else 'false' + ), + SCRIPT_PARAMETER_TPCC_LOG_TRANSACTIONS: ( + 'true' if TPCC_LOG_TRANSACTIONS.value else 'false' + ), + SCRIPT_PARAMETER_WAIT_TO_COMPLETE: wait_to_complete_seconds, }) else: raise ValueError('Unknown hammerdb_script') - def SearchAndReplaceInScript(self, vm: virtual_machine.BaseVirtualMachine, - script_name: str, parameter: str): - SearchAndReplaceTclScript(vm, parameter, - self.map_search_to_replace[parameter], - script_name) + def SearchAndReplaceInScript( + self, + vm: virtual_machine.BaseVirtualMachine, + script_name: str, + parameter: str, + ): + SearchAndReplaceTclScript( + vm, parameter, self.map_search_to_replace[parameter], script_name + ) TPC_C_SQLSERVER_BUILD_SCRIPT = HammerDbTclScript( - 'hammerdb_sqlserver_tpc_c_build.tcl', TPCC_PARAMS, P3RF_CLOUD_SQL_TEST_DIR, - BUILD_SCRIPT_TYPE) + 'hammerdb_sqlserver_tpc_c_build.tcl', + TPCC_PARAMS, + P3RF_CLOUD_SQL_TEST_DIR, + BUILD_SCRIPT_TYPE, +) TPC_C_SQLSERVER_RUN_SCRIPT = HammerDbTclScript( - 'hammerdb_sqlserver_tpc_c_run.tcl', TPCC_PARAMS, P3RF_CLOUD_SQL_TEST_DIR, - RUN_SCRIPT_TYPE) + 'hammerdb_sqlserver_tpc_c_run.tcl', + TPCC_PARAMS, + P3RF_CLOUD_SQL_TEST_DIR, + RUN_SCRIPT_TYPE, +) TPC_H_SQLSERVER_BUILD_SCRIPT = HammerDbTclScript( - 'hammerdb_sqlserver_tpc_h_build.tcl', TPCH_PARAMS, P3RF_CLOUD_SQL_TEST_DIR, - BUILD_SCRIPT_TYPE) + 'hammerdb_sqlserver_tpc_h_build.tcl', + TPCH_PARAMS, + P3RF_CLOUD_SQL_TEST_DIR, + BUILD_SCRIPT_TYPE, +) TPC_H_SQLSERVER_RUN_SCRIPT = HammerDbTclScript( - 'hammerdb_sqlserver_tpc_h_run.tcl', TPCH_PARAMS, P3RF_CLOUD_SQL_TEST_DIR, - RUN_SCRIPT_TYPE) - -TPC_C_MYSQL_BUILD_SCRIPT = HammerDbTclScript('hammerdb_mysql_tpc_c_build.tcl', - TPCC_PARAMS, - P3RF_CLOUD_SQL_TEST_DIR, - BUILD_SCRIPT_TYPE) - -TPC_C_MYSQL_RUN_SCRIPT = HammerDbTclScript('hammerdb_mysql_tpc_c_run.tcl', - TPCC_PARAMS, P3RF_CLOUD_SQL_TEST_DIR, - RUN_SCRIPT_TYPE) - -TPC_H_MYSQL_BUILD_SCRIPT = HammerDbTclScript('hammerdb_mysql_tpc_h_build.tcl', - TPCH_PARAMS, - P3RF_CLOUD_SQL_TEST_DIR, - BUILD_SCRIPT_TYPE) - -TPC_H_MYSQL_RUN_SCRIPT = HammerDbTclScript('hammerdb_mysql_tpc_h_run.tcl', - TPCH_PARAMS, P3RF_CLOUD_SQL_TEST_DIR, - RUN_SCRIPT_TYPE) + 'hammerdb_sqlserver_tpc_h_run.tcl', + TPCH_PARAMS, + P3RF_CLOUD_SQL_TEST_DIR, + RUN_SCRIPT_TYPE, +) + +TPC_C_MYSQL_BUILD_SCRIPT = HammerDbTclScript( + 'hammerdb_mysql_tpc_c_build.tcl', + TPCC_PARAMS, + P3RF_CLOUD_SQL_TEST_DIR, + BUILD_SCRIPT_TYPE, +) + +TPC_C_MYSQL_RUN_SCRIPT = HammerDbTclScript( + 'hammerdb_mysql_tpc_c_run.tcl', + TPCC_PARAMS, + P3RF_CLOUD_SQL_TEST_DIR, + RUN_SCRIPT_TYPE, +) + +TPC_H_MYSQL_BUILD_SCRIPT = HammerDbTclScript( + 'hammerdb_mysql_tpc_h_build.tcl', + TPCH_PARAMS, + P3RF_CLOUD_SQL_TEST_DIR, + BUILD_SCRIPT_TYPE, +) + +TPC_H_MYSQL_RUN_SCRIPT = HammerDbTclScript( + 'hammerdb_mysql_tpc_h_run.tcl', + TPCH_PARAMS, + P3RF_CLOUD_SQL_TEST_DIR, + RUN_SCRIPT_TYPE, +) TPC_C_POSTGRES_BUILD_SCRIPT = HammerDbTclScript( 'hammerdb_postgres_tpc_c_build.tcl', - TPCC_PARAMS, P3RF_CLOUD_SQL_TEST_DIR, BUILD_SCRIPT_TYPE) + TPCC_PARAMS, + P3RF_CLOUD_SQL_TEST_DIR, + BUILD_SCRIPT_TYPE, +) TPC_C_POSTGRES_RUN_SCRIPT = HammerDbTclScript( 'hammerdb_postgres_tpc_c_run.tcl', - TPCC_PARAMS, P3RF_CLOUD_SQL_TEST_DIR, RUN_SCRIPT_TYPE) + TPCC_PARAMS, + P3RF_CLOUD_SQL_TEST_DIR, + RUN_SCRIPT_TYPE, +) TPC_H_POSTGRES_BUILD_SCRIPT = HammerDbTclScript( 'hammerdb_postgres_tpc_h_build.tcl', - TPCH_PARAMS, P3RF_CLOUD_SQL_TEST_DIR, BUILD_SCRIPT_TYPE) + TPCH_PARAMS, + P3RF_CLOUD_SQL_TEST_DIR, + BUILD_SCRIPT_TYPE, +) TPC_H_POSTGRES_RUN_SCRIPT = HammerDbTclScript( 'hammerdb_postgres_tpc_h_run.tcl', - TPCH_PARAMS, P3RF_CLOUD_SQL_TEST_DIR, RUN_SCRIPT_TYPE) + TPCH_PARAMS, + P3RF_CLOUD_SQL_TEST_DIR, + RUN_SCRIPT_TYPE, +) SCRIPT_MAPPING = { sql_engine_utils.MYSQL: { - HAMMERDB_SCRIPT_TPC_H: [TPC_H_MYSQL_BUILD_SCRIPT, - TPC_H_MYSQL_RUN_SCRIPT], - HAMMERDB_SCRIPT_TPC_C: [TPC_C_MYSQL_BUILD_SCRIPT, - TPC_C_MYSQL_RUN_SCRIPT] + HAMMERDB_SCRIPT_TPC_H: [ + TPC_H_MYSQL_BUILD_SCRIPT, + TPC_H_MYSQL_RUN_SCRIPT, + ], + HAMMERDB_SCRIPT_TPC_C: [ + TPC_C_MYSQL_BUILD_SCRIPT, + TPC_C_MYSQL_RUN_SCRIPT, + ], }, sql_engine_utils.SQLSERVER: { HAMMERDB_SCRIPT_TPC_H: [ - TPC_H_SQLSERVER_BUILD_SCRIPT, TPC_H_SQLSERVER_RUN_SCRIPT + TPC_H_SQLSERVER_BUILD_SCRIPT, + TPC_H_SQLSERVER_RUN_SCRIPT, ], HAMMERDB_SCRIPT_TPC_C: [ - TPC_C_SQLSERVER_BUILD_SCRIPT, TPC_C_SQLSERVER_RUN_SCRIPT - ] + TPC_C_SQLSERVER_BUILD_SCRIPT, + TPC_C_SQLSERVER_RUN_SCRIPT, + ], }, sql_engine_utils.POSTGRES: { - HAMMERDB_SCRIPT_TPC_H: [TPC_H_POSTGRES_BUILD_SCRIPT, - TPC_H_POSTGRES_RUN_SCRIPT], - HAMMERDB_SCRIPT_TPC_C: [TPC_C_POSTGRES_BUILD_SCRIPT, - TPC_C_POSTGRES_RUN_SCRIPT] + HAMMERDB_SCRIPT_TPC_H: [ + TPC_H_POSTGRES_BUILD_SCRIPT, + TPC_H_POSTGRES_RUN_SCRIPT, + ], + HAMMERDB_SCRIPT_TPC_C: [ + TPC_C_POSTGRES_BUILD_SCRIPT, + TPC_C_POSTGRES_RUN_SCRIPT, + ], }, } # TPCC queries are defined in the tpcc spec # http://www.tpc.org/tpc_documents_current_versions/pdf/tpc-c_v5.11.0.pdf TPCC_QUERY_TYPES = [ - 'neword', 'payment', 'delivery', 'slev', 'ostat', 'gettimestamp' + 'neword', + 'payment', + 'delivery', + 'slev', + 'ostat', + 'gettimestamp', ] # The ordering of the query metrics is used as an index to extract @@ -484,8 +607,9 @@ def SearchAndReplaceInScript(self, vm: virtual_machine.BaseVirtualMachine, TPCC_QUERY_METRICS_4_3 = ['CALLS', 'MIN', 'MAX', 'P99', 'P95', 'P50'] -def _GetFileContent(vm: virtual_machine.BaseVirtualMachine, - file_path: str) -> str: +def _GetFileContent( + vm: virtual_machine.BaseVirtualMachine, file_path: str +) -> str: stdout, _ = vm.RemoteCommand(f'cat {file_path}') return stdout @@ -502,14 +626,14 @@ def ParseTpcCTimeProfileResultsFromFile(stdout: str) -> List[sample.Sample]: stdout = stdout.split('>>>>')[-5:] for line in stdout: query_type = str.lower( - regex_util.ExtractGroup('PROC: ([A-Z]*)', line, flags=re.M)) + regex_util.ExtractGroup('PROC: ([A-Z]*)', line, flags=re.M) + ) for metric in TPCC_QUERY_METRICS_4_3: metric_regex = metric + ': ([0-9.]*)' metric_value = regex_util.ExtractGroup(metric_regex, line, flags=re.M) tpcc_metrics.append( - sample.Sample(query_type + '_' + metric, - metric_value, - MILLISECONDS)) + sample.Sample(query_type + '_' + metric, metric_value, MILLISECONDS) + ) return tpcc_metrics @@ -519,8 +643,10 @@ def ParseTpcCTimeProfileResults(stdout: str) -> List[sample.Sample]: # match a string like # Vuser 2:|neword|MIN-35|P50%-9970.5|P95%-148|P99%-153|MAX-16816|SAMPLES-87 percentile_dic = collections.defaultdict(list) - percentile_regex = (r'MIN-([0-9.]*)\|P50%-([0-9.]*)\|P95%-([0-9.]*)\|' - r'P99%-([0-9.]*)\|MAX-([0-9.]*)\|SAMPLES-([0-9.]*)') + percentile_regex = ( + r'MIN-([0-9.]*)\|P50%-([0-9.]*)\|P95%-([0-9.]*)\|' + r'P99%-([0-9.]*)\|MAX-([0-9.]*)\|SAMPLES-([0-9.]*)' + ) for line in stdout.split('\n'): for query_type in TPCC_QUERY_TYPES: if query_type in line: @@ -528,7 +654,8 @@ def ParseTpcCTimeProfileResults(stdout: str) -> List[sample.Sample]: if m: for idx, query_metric in enumerate(TPCC_QUERY_METRICS): percentile_dic[query_type + '_' + query_metric].append( - float(m.group(idx + 1))) + float(m.group(idx + 1)) + ) gathered_metrics = percentile_dic.keys() if len(gathered_metrics) != len(TPCC_QUERY_METRICS) * len(TPCC_QUERY_TYPES): @@ -539,9 +666,10 @@ def ParseTpcCTimeProfileResults(stdout: str) -> List[sample.Sample]: if 'SAMPLES' not in key: # Metrics is in micro seconds tpcc_metrics.append( - sample.Sample(key, - statistics.mean(percentile_dic[key]) / 1000, - MILLISECONDS)) + sample.Sample( + key, statistics.mean(percentile_dic[key]) / 1000, MILLISECONDS + ) + ) return tpcc_metrics @@ -561,16 +689,25 @@ def ParseTpcCTPMResultsFromFile(stdout: str) -> List[sample.Sample]: ramp_up_ends = time_series[0] + HAMMERDB_TPCC_RAMPUP.value * MINUTES_TO_MS - ramp_down_starts = ramp_up_ends + (HAMMERDB_TPCC_DURATION.value - * MINUTES_TO_MS) - tpm_sample = sample.CreateTimeSeriesSample(tpm_metrics, time_series, - sample.TPM_TIME_SERIES, TPM, 1, - ramp_up_ends, ramp_down_starts, {}) + ramp_down_starts = ramp_up_ends + ( + HAMMERDB_TPCC_DURATION.value * MINUTES_TO_MS + ) + tpm_sample = sample.CreateTimeSeriesSample( + tpm_metrics, + time_series, + sample.TPM_TIME_SERIES, + TPM, + 1, + ramp_up_ends, + ramp_down_starts, + {}, + ) return [tpm_sample] def ParseTpcCResults( - stdout: str, vm: virtual_machine.BaseVirtualMachine) -> List[sample.Sample]: + stdout: str, vm: virtual_machine.BaseVirtualMachine +) -> List[sample.Sample]: """Extract results from the TPC-C script.""" tpcc_metrics = ParseBasicTpcCResults(stdout) @@ -592,8 +729,10 @@ def ParseBasicTpcCResults(stdout: str) -> List[sample.Sample]: # match a string like: # "Vuser 1:TEST RESULT : System achieved 40213 NOPM from 92856 SQL Server TPM" - regex = (r'Vuser 1:TEST RESULT : System achieved ' - r'(\d*) NOPM from (\d*) (\w| )* TPM') + regex = ( + r'Vuser 1:TEST RESULT : System achieved ' + r'(\d*) NOPM from (\d*) (\w| )* TPM' + ) tpm = regex_util.ExtractInt(regex, stdout, group=2) nopm = regex_util.ExtractInt(regex, stdout, group=1) @@ -604,8 +743,9 @@ def ParseBasicTpcCResults(stdout: str) -> List[sample.Sample]: def GeoMean(float_array: List[float]) -> float: """Calculate the geomean of the numbers in float_array.""" - return functools.reduce(lambda x, y: x * y, - float_array)**(1.0 / len(float_array)) + return functools.reduce(lambda x, y: x * y, float_array) ** ( + 1.0 / len(float_array) + ) def ParseTpcHResults(stdout: str) -> List[sample.Sample]: @@ -628,7 +768,8 @@ def ParseTpcHResults(stdout: str) -> List[sample.Sample]: raise HammerdbBenchmarkError('Unexpected TPCH result count') results.append( - sample.Sample(QUERY_TIMES_GEOMEAN, GeoMean(query_times), SECONDS)) + sample.Sample(QUERY_TIMES_GEOMEAN, GeoMean(query_times), SECONDS) + ) return results @@ -650,24 +791,39 @@ def InLocalDir(command: str) -> str: return InDir(LocalWorkingDirectory(), command) -def PushCloudSqlTestFile(vm: virtual_machine.BaseVirtualMachine, data_file: str, - path: str): - vm.PushFile(data.ResourcePath(posixpath.join(path, data_file)), - LocalWorkingDirectory()) +def PushCloudSqlTestFile( + vm: virtual_machine.BaseVirtualMachine, data_file: str, path: str +): + vm.PushFile( + data.ResourcePath(posixpath.join(path, data_file)), + LocalWorkingDirectory(), + ) -def SearchAndReplaceGuestFile(vm: virtual_machine.BaseVirtualMachine, - directory: str, filename: str, search: str, - replace: str): +def SearchAndReplaceGuestFile( + vm: virtual_machine.BaseVirtualMachine, + directory: str, + filename: str, + search: str, + replace: str, +): vm.RemoteCommand( - InDir(directory, - 'sed -i.bak \'s:{0}:{1}:\' {2}'.format(search, replace, filename))) + InDir( + directory, + "sed -i.bak 's:{0}:{1}:' {2}".format(search, replace, filename), + ) + ) -def SearchAndReplaceTclScript(vm: virtual_machine.BaseVirtualMachine, - search: str, replace: str, script_name: str): - SearchAndReplaceGuestFile(vm, LocalWorkingDirectory(), - script_name, search, replace) +def SearchAndReplaceTclScript( + vm: virtual_machine.BaseVirtualMachine, + search: str, + replace: str, + script_name: str, +): + SearchAndReplaceGuestFile( + vm, LocalWorkingDirectory(), script_name, search, replace + ) def Install(vm: virtual_machine.BaseVirtualMachine): @@ -680,17 +836,23 @@ def Install(vm: virtual_machine.BaseVirtualMachine): tar_file = f'HammerDB-{HAMMERDB_VERSION.value}-Linux.tar.gz' vm.InstallPreprovisionedPackageData( - PACKAGE_NAME, [tar_file], HAMMERDB_RUN_LOCATION) + PACKAGE_NAME, [tar_file], HAMMERDB_RUN_LOCATION + ) vm.RemoteCommand( f'cd {HAMMERDB_RUN_LOCATION}; tar xzvf {tar_file}; ' - f'mv HammerDB-{HAMMERDB_VERSION.value}/* ./') + f'mv HammerDB-{HAMMERDB_VERSION.value}/* ./' + ) # Push Hammerdb install files if HAMMERDB_VERSION.value == HAMMERDB_4_0: install_file = 'install_hammerdb_4_0.sh' # Patches hammerdb 4.0 for Postgres on Azure and time profile frequency - files_required = ['pgolap.tcl.patch', 'pgoltp.tcl.patch', - 'postgresql.xml.patch', 'etprof-1.1.tm.patch', - install_file] + files_required = [ + 'pgolap.tcl.patch', + 'pgoltp.tcl.patch', + 'postgresql.xml.patch', + 'etprof-1.1.tm.patch', + install_file, + ] for file in files_required: PushCloudSqlTestFile(vm, file, P3RF_CLOUD_SQL_TEST_DIR) @@ -705,19 +867,29 @@ def Install(vm: virtual_machine.BaseVirtualMachine): vm.RemoteCommand('export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/') -def SetupConfig(vm: virtual_machine.BaseVirtualMachine, db_engine: str, - hammerdb_script: str, ip: str, port: str, password: str, - user: str, is_managed_azure: bool): +def SetupConfig( + vm: virtual_machine.BaseVirtualMachine, + db_engine: str, + hammerdb_script: str, + ip: str, + port: str, + password: str, + user: str, + is_managed_azure: bool, +): """Sets up the necessary scripts on the VM with the necessary parameters.""" db_engine = sql_engine_utils.GetDbEngineType(db_engine) if db_engine not in SCRIPT_MAPPING: - raise ValueError('{0} is currently not supported for running ' - 'hammerdb benchmarks.'.format(db_engine)) + raise ValueError( + '{0} is currently not supported for running ' + 'hammerdb benchmarks.'.format(db_engine) + ) if hammerdb_script not in SCRIPT_MAPPING[db_engine]: - raise ValueError('{0} is not a known hammerdb script.'.format( - hammerdb_script)) + raise ValueError( + '{0} is not a known hammerdb script.'.format(hammerdb_script) + ) scripts = SCRIPT_MAPPING[db_engine][hammerdb_script] @@ -729,7 +901,8 @@ def SetupConfig(vm: virtual_machine.BaseVirtualMachine, db_engine: str, user=user, is_managed_azure=is_managed_azure, hammerdb_script=hammerdb_script, - script_type=script.script_type) + script_type=script.script_type, + ) script.Install(vm, script_parameters) # Run all the build script or scripts before actual run phase @@ -737,10 +910,12 @@ def SetupConfig(vm: virtual_machine.BaseVirtualMachine, db_engine: str, scripts[i].Run(vm) -def Run(vm: virtual_machine.BaseVirtualMachine, - db_engine: str, - hammerdb_script: str, - timeout: Optional[int] = 60*60*8) -> List[sample.Sample]: +def Run( + vm: virtual_machine.BaseVirtualMachine, + db_engine: str, + hammerdb_script: str, + timeout: Optional[int] = 60 * 60 * 8, +) -> List[sample.Sample]: """Run the HammerDBCli Benchmark. Runs Hammerdb TPCC or TPCH script. @@ -752,12 +927,12 @@ def Run(vm: virtual_machine.BaseVirtualMachine, TPCH gathers the latency of the 22 TPCH queries. Args: - vm: The virtual machine to run on that has - Install and SetupConfig already invoked on it. + vm: The virtual machine to run on that has Install and SetupConfig already + invoked on it. db_engine: The type of database that the script is running on hammerdb_script: An enumeration from HAMMERDB_SCRIPT indicating which - script to run. Must have been prior setup with - SetupConfig method on the vm to work. + script to run. Must have been prior setup with SetupConfig method on the + vm to work. timeout: Timeout when running hammerdbcli Returns: @@ -789,17 +964,19 @@ def GetMetadata(db_engine: str): metadata['hammerdbcli_num_run'] = NUM_RUN.value if not FLAGS.use_managed_db and HAMMERDB_OPTIMIZED_SERVER_CONFIGURATION.value: metadata['hammerdbcli_optimized_server_configuration'] = ( - HAMMERDB_OPTIMIZED_SERVER_CONFIGURATION.value) + HAMMERDB_OPTIMIZED_SERVER_CONFIGURATION.value + ) if script == HAMMERDB_SCRIPT_TPC_H: metadata['hammerdbcli_scale_factor'] = HAMMERDB_TPCH_SCALE_FACTOR.value metadata['hammerdbcli_load_tpch_tables_to_columnar_engine'] = ( - LOAD_TPCH_TABLES_TO_COLUMNAR_ENGINE.value) - metadata['hammerdbcli_build_tpch_num_vu'] = ( - HAMMERDB_BUILD_TPCH_NUM_VU.value) + LOAD_TPCH_TABLES_TO_COLUMNAR_ENGINE.value + ) + metadata['hammerdbcli_build_tpch_num_vu'] = HAMMERDB_BUILD_TPCH_NUM_VU.value if db_engine == sql_engine_utils.POSTGRES: metadata['hammerdbcli_degree_of_parallel'] = ( - HAMMERDB_TPCH_DEGREE_OF_PARALLEL.value) + HAMMERDB_TPCH_DEGREE_OF_PARALLEL.value + ) elif script == HAMMERDB_SCRIPT_TPC_C: metadata['hammerdbcli_num_warehouse'] = HAMMERDB_TPCC_NUM_WAREHOUSE.value metadata['hammerdbcli_all_warehouse'] = HAMMERDB_TPCC_ALL_WAREHOUSE.value diff --git a/perfkitbenchmarker/linux_packages/hbase.py b/perfkitbenchmarker/linux_packages/hbase.py index 213753b19e..f8f67d46bd 100644 --- a/perfkitbenchmarker/linux_packages/hbase.py +++ b/perfkitbenchmarker/linux_packages/hbase.py @@ -32,15 +32,19 @@ FLAGS = flags.FLAGS flags.DEFINE_string('hbase_version', '1.3.5', 'HBase version.') -flags.DEFINE_string('hbase_bin_url', None, - 'Specify to override url from HBASE_URL_BASE.') +flags.DEFINE_string( + 'hbase_bin_url', None, 'Specify to override url from HBASE_URL_BASE.' +) HBASE_URL_BASE = 'https://www-us.apache.org/dist/hbase' HBASE_PATTERN = r'>(hbase-([\d\.]+)-bin.tar.gz)<' HBASE_VERSION_PATTERN = re.compile('HBase (.*)$', re.IGNORECASE | re.MULTILINE) -DATA_FILES = ['hbase/hbase-site.xml.j2', 'hbase/regionservers.j2', - 'hbase/hbase-env.sh.j2'] +DATA_FILES = [ + 'hbase/hbase-site.xml.j2', + 'hbase/regionservers.j2', + 'hbase/hbase-env.sh.j2', +] HBASE_DIR = posixpath.join(linux_packages.INSTALL_DIR, 'hbase') HBASE_BIN = posixpath.join(HBASE_DIR, 'bin') @@ -57,7 +61,8 @@ def _GetHBaseURL(): The HBase download url. """ return '{0}/{1}/hbase-{1}-bin.tar.gz'.format( - HBASE_URL_BASE, FLAGS.hbase_version) + HBASE_URL_BASE, FLAGS.hbase_version + ) def GetHBaseVersion(vm): @@ -86,8 +91,10 @@ def _Install(vm): vm.Install('curl') hbase_url = FLAGS.hbase_bin_url or _GetHBaseURL() vm.RemoteCommand( - ('mkdir -p {0} && curl -L {1} | ' - 'tar -C {0} --strip-components=1 -xzf -').format(HBASE_DIR, hbase_url)) + ( + 'mkdir -p {0} && curl -L {1} | tar -C {0} --strip-components=1 -xzf -' + ).format(HBASE_DIR, hbase_url) + ) def YumInstall(vm): @@ -103,8 +110,7 @@ def AptInstall(vm): def _RenderConfig(vm, master_ip, zk_ips, regionserver_ips): # Use the same heap configuration as Cassandra memory_mb = vm.total_memory_kb // 1024 - hbase_memory_mb = max(min(memory_mb // 2, 1024), - min(memory_mb // 4, 8192)) + hbase_memory_mb = max(min(memory_mb // 2, 1024), min(memory_mb // 4, 8192)) context = { 'master_ip': master_ip, 'worker_ips': regionserver_ips, @@ -116,8 +122,7 @@ def _RenderConfig(vm, master_ip, zk_ips, regionserver_ips): for file_name in DATA_FILES: file_path = data.ResourcePath(file_name) - remote_path = posixpath.join(HBASE_CONF_DIR, - os.path.basename(file_name)) + remote_path = posixpath.join(HBASE_CONF_DIR, os.path.basename(file_name)) if file_name.endswith('.j2'): vm.RenderTemplate(file_path, os.path.splitext(remote_path)[0], context) else: @@ -134,19 +139,26 @@ def ConfigureAndStart(master, regionservers, zk_nodes): vms = [master] + regionservers def LinkNativeLibraries(vm): - vm.RemoteCommand(('mkdir {0}/lib/native && ' - 'ln -s {1} {0}/lib/native/Linux-amd64-64').format( - HBASE_DIR, - posixpath.join(hadoop.HADOOP_DIR, 'lib', 'native'))) + vm.RemoteCommand( + ( + 'mkdir {0}/lib/native && ln -s {1} {0}/lib/native/Linux-amd64-64' + ).format(HBASE_DIR, posixpath.join(hadoop.HADOOP_DIR, 'lib', 'native')) + ) + background_tasks.RunThreaded(LinkNativeLibraries, vms) - fn = functools.partial(_RenderConfig, master_ip=master.internal_ip, - zk_ips=[vm.internal_ip for vm in zk_nodes], - regionserver_ips=[regionserver.internal_ip - for regionserver in regionservers]) + fn = functools.partial( + _RenderConfig, + master_ip=master.internal_ip, + zk_ips=[vm.internal_ip for vm in zk_nodes], + regionserver_ips=[ + regionserver.internal_ip for regionserver in regionservers + ], + ) background_tasks.RunThreaded(fn, vms) - master.RemoteCommand('{0} dfs -mkdir /hbase'.format( - posixpath.join(hadoop.HADOOP_BIN, 'hdfs'))) + master.RemoteCommand( + '{0} dfs -mkdir /hbase'.format(posixpath.join(hadoop.HADOOP_BIN, 'hdfs')) + ) master.RemoteCommand(posixpath.join(HBASE_BIN, 'start-hbase.sh')) diff --git a/perfkitbenchmarker/linux_packages/hpcc.py b/perfkitbenchmarker/linux_packages/hpcc.py index 388cb560d0..58508bee8a 100644 --- a/perfkitbenchmarker/linux_packages/hpcc.py +++ b/perfkitbenchmarker/linux_packages/hpcc.py @@ -36,9 +36,7 @@ PREPROVISIONED_DATA = { HPCC_TAR: '0a6fef7ab9f3347e549fed65ebb98234feea9ee18aea0c8f59baefbe3cf7ffb8' } -PACKAGE_DATA_URL = { - HPCC_TAR: HPCC_URL -} +PACKAGE_DATA_URL = {HPCC_TAR: HPCC_URL} HPCC_DIR = '%s/hpcc-1.5.0' % linux_packages.INSTALL_DIR HPCC_VERSION = '1.5.0' @@ -208,19 +206,30 @@ flags.DEFINE_enum( - 'hpcc_math_library', HPCC_MATH_LIBRARY_OPEN_BLAS, [ - HPCC_MATH_LIBRARY_OPEN_BLAS, HPCC_MATH_LIBRARY_MKL, - HPCC_MATH_LIBRARY_AMD_BLIS - ], 'The math library to use when compiling hpcc: openblas, mkl, or ' - 'amdblis. The default is openblas.') + 'hpcc_math_library', + HPCC_MATH_LIBRARY_OPEN_BLAS, + [ + HPCC_MATH_LIBRARY_OPEN_BLAS, + HPCC_MATH_LIBRARY_MKL, + HPCC_MATH_LIBRARY_AMD_BLIS, + ], + 'The math library to use when compiling hpcc: openblas, mkl, or ' + 'amdblis. The default is openblas.', +) flags.DEFINE_list( - 'hpcc_benchmarks', [], 'A list of benchmarks in HPCC to run. If none are ' + 'hpcc_benchmarks', + [], + 'A list of benchmarks in HPCC to run. If none are ' 'specified (the default), then all of the benchmarks are run. In 1.5.0, ' - 'the benchmarks may include the following: %s' % ', '.join(HPCC_BENCHMARKS)) + 'the benchmarks may include the following: %s' + % ', '.join(HPCC_BENCHMARKS), +) USE_INTEL_COMPILED_HPL = flags.DEFINE_boolean( - 'hpcc_use_intel_compiled_hpl', False, - 'Whether to use the intel compiled HPL') + 'hpcc_use_intel_compiled_hpl', + False, + 'Whether to use the intel compiled HPL', +) def CheckUseIntelCompiled(myflags: Dict[str, Any]) -> bool: @@ -240,11 +249,14 @@ def CheckUseIntelCompiled(myflags: Dict[str, Any]) -> bool: flags.register_validator( 'hpcc_benchmarks', - lambda hpcc_benchmarks: set(hpcc_benchmarks).issubset(set(HPCC_BENCHMARKS))) + lambda hpcc_benchmarks: set(hpcc_benchmarks).issubset(set(HPCC_BENCHMARKS)), +) flags.register_multi_flags_validator( - ['hpcc_math_library', 'hpcc_use_intel_compiled_hpl'], CheckUseIntelCompiled, + ['hpcc_math_library', 'hpcc_use_intel_compiled_hpl'], + CheckUseIntelCompiled, 'With --hpcc_use_intel_compiled_hpl must specify ' - f'--hpcc_math_library={HPCC_MATH_LIBRARY_MKL}') + f'--hpcc_math_library={HPCC_MATH_LIBRARY_MKL}', +) FLAGS = flags.FLAGS @@ -295,10 +307,12 @@ def Install(vm): # Using a pre-compiled HPL, no need to continue return vm.Install('openmpi') - vm.InstallPreprovisionedPackageData(PACKAGE_NAME, PREPROVISIONED_DATA.keys(), - linux_packages.INSTALL_DIR) - vm.RemoteCommand('cd %s && tar xvfz %s' % - (linux_packages.INSTALL_DIR, HPCC_TAR)) + vm.InstallPreprovisionedPackageData( + PACKAGE_NAME, PREPROVISIONED_DATA.keys(), linux_packages.INSTALL_DIR + ) + vm.RemoteCommand( + 'cd %s && tar xvfz %s' % (linux_packages.INSTALL_DIR, HPCC_TAR) + ) if FLAGS.hpcc_benchmarks: _LimitBenchmarksToRun(vm, set(FLAGS.hpcc_benchmarks)) @@ -311,21 +325,25 @@ def Install(vm): _CompileHpccAmdBlis(vm) else: raise errors.Setup.InvalidFlagConfigurationError( - 'Unexpected hpcc_math_library option encountered.') + 'Unexpected hpcc_math_library option encountered.' + ) def _CompileHpccOpenblas(vm): """Compile HPCC with OpenBlas.""" vm.Install('openblas') vm.RemoteCommand( - 'cp %s/hpl/setup/%s %s' % - (HPCC_DIR, HPCC_MAKEFILE_CBLAS, HPCC_MAKEFILE_PATH_OPEN_BLAS)) - sed_cmd = ('sed -i -e "/^MP/d" -e "s|gcc|mpicc|" -e "s|g77|mpicc|" ' - '-e "s|\\$(HOME)/netlib/ARCHIVES/Linux_PII|%s|" ' - '-e "s|libcblas.*|libopenblas.a|" ' - '-e "s|-funroll-loops|-funroll-loops -std=c99|" ' - '-e "s|\\-lm|\\-lgfortran \\-lm|" %s' % - (re.escape(openblas.OPENBLAS_DIR), HPCC_MAKEFILE_PATH_OPEN_BLAS)) + 'cp %s/hpl/setup/%s %s' + % (HPCC_DIR, HPCC_MAKEFILE_CBLAS, HPCC_MAKEFILE_PATH_OPEN_BLAS) + ) + sed_cmd = ( + 'sed -i -e "/^MP/d" -e "s|gcc|mpicc|" -e "s|g77|mpicc|" ' + '-e "s|\\$(HOME)/netlib/ARCHIVES/Linux_PII|%s|" ' + '-e "s|libcblas.*|libopenblas.a|" ' + '-e "s|-funroll-loops|-funroll-loops -std=c99|" ' + '-e "s|\\-lm|\\-lgfortran \\-lm|" %s' + % (re.escape(openblas.OPENBLAS_DIR), HPCC_MAKEFILE_PATH_OPEN_BLAS) + ) vm.RemoteCommand(sed_cmd) vm.RemoteCommand('cd %s; make arch=OPEN_BLAS' % HPCC_DIR) @@ -333,14 +351,18 @@ def _CompileHpccOpenblas(vm): def _CompileHpccAmdBlis(vm): """Compile HPCC with AMD BLIS.""" vm.Install('amdblis') - vm.RemoteCommand('cp %s/hpl/setup/%s %s' % - (HPCC_DIR, HPCC_MAKEFILE_CBLAS, HPCC_MAKEFILE_PATH_AMD_BLIS)) - sed_cmd = ('sed -i -e "/^MP/d" -e "s|gcc|mpicc|" -e "s|g77|mpicc|" ' - '-e "s|\\$(HOME)/netlib/ARCHIVES/Linux_PII|%s|" ' - '-e "s|libcblas.*|lib/zen/libblis.a|" ' - '-e "s|-funroll-loops|-funroll-loops -std=c99|" ' - '-e "s|\\-lm|\\-lgfortran \\-lm|" %s' % - (re.escape(amdblis.AMDBLIS_DIR), HPCC_MAKEFILE_PATH_AMD_BLIS)) + vm.RemoteCommand( + 'cp %s/hpl/setup/%s %s' + % (HPCC_DIR, HPCC_MAKEFILE_CBLAS, HPCC_MAKEFILE_PATH_AMD_BLIS) + ) + sed_cmd = ( + 'sed -i -e "/^MP/d" -e "s|gcc|mpicc|" -e "s|g77|mpicc|" ' + '-e "s|\\$(HOME)/netlib/ARCHIVES/Linux_PII|%s|" ' + '-e "s|libcblas.*|lib/zen/libblis.a|" ' + '-e "s|-funroll-loops|-funroll-loops -std=c99|" ' + '-e "s|\\-lm|\\-lgfortran \\-lm|" %s' + % (re.escape(amdblis.AMDBLIS_DIR), HPCC_MAKEFILE_PATH_AMD_BLIS) + ) vm.RemoteCommand(sed_cmd) vm.RemoteCommand('cd %s; make arch=AMD_BLIS' % HPCC_DIR) @@ -362,26 +384,35 @@ def _CompileHpccMKL(vm): _CompileHpccOpenblas(vm) vm.RemoteCommand('cd %s; rm hpcc' % HPCC_DIR) vm.Install('mkl') - vm.RemoteCommand('cp %s/hpl/setup/%s %s' % (HPCC_DIR, HPCC_MAKEFILE_CBLAS, - HPCC_MAKEFILE_PATH_MKL)) - mkl_lalib = ('-Wl,--start-group $(LAdir)/libfftw2xc_double_gnu.a ' - '$(LAdir)/libfftw2xf_double_gnu.a ' - '$(LAdir)/libmkl_intel_lp64.a ' - '$(LAdir)/libmkl_intel_thread.a ' - '$(LAdir)/libmkl_core.a ' - '$(LAdir)/libmkl_blas95_lp64.a ' - '-Wl,--end-group') - mkl_ccflags = (' -Wl,--no-as-needed -ldl -lmpi -liomp5 -lpthread -lm ' - '-DUSING_FFTW -DMKL_INT=long -DLONG_IS_64BITS') + vm.RemoteCommand( + 'cp %s/hpl/setup/%s %s' + % (HPCC_DIR, HPCC_MAKEFILE_CBLAS, HPCC_MAKEFILE_PATH_MKL) + ) + mkl_lalib = ( + '-Wl,--start-group $(LAdir)/libfftw2xc_double_gnu.a ' + '$(LAdir)/libfftw2xf_double_gnu.a ' + '$(LAdir)/libmkl_intel_lp64.a ' + '$(LAdir)/libmkl_intel_thread.a ' + '$(LAdir)/libmkl_core.a ' + '$(LAdir)/libmkl_blas95_lp64.a ' + '-Wl,--end-group' + ) + mkl_ccflags = ( + ' -Wl,--no-as-needed -ldl -lmpi -liomp5 -lpthread -lm ' + '-DUSING_FFTW -DMKL_INT=long -DLONG_IS_64BITS' + ) sed_cmd_mkl = ( 'sed -i -e "/^MP/d" -e "s|gcc|mpicc|" -e "s|g77|mpicc|" ' '-e "s|\\$(HOME)/netlib/ARCHIVES/Linux_PII|' '/opt/intel/mkl/lib/intel64|" ' '-e "s|\\$(LAdir)/libcblas.*|%s|" ' '-e "s|\\-lm|\\-lgfortran \\-lm|" ' - '-e "/CCFLAGS / s|$|%s|" %s' % - (re.escape(mkl_lalib), re.escape(mkl_ccflags), HPCC_MAKEFILE_PATH_MKL)) + '-e "/CCFLAGS / s|$|%s|" %s' + % (re.escape(mkl_lalib), re.escape(mkl_ccflags), HPCC_MAKEFILE_PATH_MKL) + ) vm.RemoteCommand(sed_cmd_mkl) - vm.RemoteCommand('source /opt/intel/compilers_and_libraries/linux/bin/' - 'compilervars.sh -arch intel64 -platform linux && ' - 'cd %s; make arch=intel64' % HPCC_DIR) + vm.RemoteCommand( + 'source /opt/intel/compilers_and_libraries/linux/bin/' + 'compilervars.sh -arch intel64 -platform linux && ' + 'cd %s; make arch=intel64' % HPCC_DIR + ) diff --git a/perfkitbenchmarker/linux_packages/hpcg.py b/perfkitbenchmarker/linux_packages/hpcg.py index a88c271c38..5535ac1503 100644 --- a/perfkitbenchmarker/linux_packages/hpcg.py +++ b/perfkitbenchmarker/linux_packages/hpcg.py @@ -32,34 +32,42 @@ HPCG_CUDA_11_BINARY = 'xhpcg-3.1_cuda-11_ompi-4.0_sm_60_sm70_sm80' HPCG_CUDA_10_TAR = 'hpcg-3.1_cuda-10_ompi-3.1_gcc485_sm_35_sm_50_sm_60_sm_70_sm75_ver_10_9_18.tgz' HPCG_CUDA_9_TAR = ( - 'hpcg-3.1_cuda9_ompi1.10.2_gcc485_sm_35_sm_50_sm_60_sm_70_ver_10_8_17.tgz') + 'hpcg-3.1_cuda9_ompi1.10.2_gcc485_sm_35_sm_50_sm_60_sm_70_ver_10_8_17.tgz' +) HPCG_CUDA_8_TAR = ( - 'hpcg-3.1_cuda8_ompi1.10.2_gcc485_sm_35_sm_50_sm_60_ver_3_28_17.tgz') + 'hpcg-3.1_cuda8_ompi1.10.2_gcc485_sm_35_sm_50_sm_60_ver_3_28_17.tgz' +) HPCG_CUDA_11 = posixpath.join(HPCG_BASE_URL, HPCG_CUDA_11_BINARY) HPCG_CUDA_10 = posixpath.join(HPCG_BASE_URL, HPCG_CUDA_10_TAR) HPCG_CUDA_9 = posixpath.join(HPCG_BASE_URL, HPCG_CUDA_9_TAR) HPCG_CUDA_8 = posixpath.join(HPCG_BASE_URL, HPCG_CUDA_8_TAR) HPCG_DIR = '%s/%s' % (linux_packages.INSTALL_DIR, 'hpcg') PREPROVISIONED_DATA = { - HPCG_CUDA_11_BINARY: - '580b609dd4ab866f17346babecbddd5fa2364eca05a802b3f42ac55522fd6be0', - HPCG_CUDA_10_TAR: - 'e0a7e6cbac9a4e1cfe0fe584de05f3740fface83ef3ba88313160b5da658fbb7', - HPCG_CUDA_9_TAR: - '384345954c20fbcd03b9d2dcfef9854a16ab942032763eab77c8f8b43a832204', - HPCG_CUDA_8_TAR: + HPCG_CUDA_11_BINARY: ( + '580b609dd4ab866f17346babecbddd5fa2364eca05a802b3f42ac55522fd6be0' + ), + HPCG_CUDA_10_TAR: ( + 'e0a7e6cbac9a4e1cfe0fe584de05f3740fface83ef3ba88313160b5da658fbb7' + ), + HPCG_CUDA_9_TAR: ( + '384345954c20fbcd03b9d2dcfef9854a16ab942032763eab77c8f8b43a832204' + ), + HPCG_CUDA_8_TAR: ( 'd63b8653ab3d04d09874c7bc5251d3083ee3d8cfb3105654a367d2aa619571ec' + ), } PACKAGE_DATA_URL = { HPCG_CUDA_11_BINARY: HPCG_CUDA_11, HPCG_CUDA_10_TAR: HPCG_CUDA_10, HPCG_CUDA_9_TAR: HPCG_CUDA_9, - HPCG_CUDA_8_TAR: HPCG_CUDA_8 + HPCG_CUDA_8_TAR: HPCG_CUDA_8, } HPCG_CUDA_11_BINARY = 'xhpcg-3.1_cuda-11_ompi-4.0_sm_60_sm70_sm80' HPCG_CUDA_10_BINARY = 'xhpcg-3.1_gcc_485_cuda-10.0.130_ompi-3.1.0_sm_35_sm_50_sm_60_sm_70_sm_75_ver_10_9_18' -HPCG_CUDA_8_BINARY = 'xhpcg-3.1_gcc_485_cuda8061_ompi_1_10_2_sm_35_sm_50_sm_60_ver_3_28_17' +HPCG_CUDA_8_BINARY = ( + 'xhpcg-3.1_gcc_485_cuda8061_ompi_1_10_2_sm_35_sm_50_sm_60_ver_3_28_17' +) HPCG_CUDA_9_BINARY = 'xhpcg-3.1_gcc_485_cuda90176_ompi_1_10_2_sm_35_sm_50_sm_60_sm_70_ver_10_8_17' CUDA_FILES = { @@ -88,15 +96,18 @@ def AptInstall(vm): if FLAGS.cuda_toolkit_version not in CUDA_FILES: raise cuda_toolkit.UnsupportedCudaVersionError( - f'HPCG only supports CUDA {sorted(CUDA_FILES)}') + f'HPCG only supports CUDA {sorted(CUDA_FILES)}' + ) hpcg_tar, hpcg_binary = CUDA_FILES[FLAGS.cuda_toolkit_version] - vm.InstallPreprovisionedPackageData(PACKAGE_NAME, [hpcg_tar], - linux_packages.INSTALL_DIR) + vm.InstallPreprovisionedPackageData( + PACKAGE_NAME, [hpcg_tar], linux_packages.INSTALL_DIR + ) vm.RemoteCommand('rm -rf %s' % HPCG_DIR) vm.RemoteCommand('mkdir %s' % HPCG_DIR) vm.RemoteCommand( - 'cd %s && tar xvf %s --directory=%s --strip-components=1' % - (linux_packages.INSTALL_DIR, hpcg_tar, HPCG_DIR)) + 'cd %s && tar xvf %s --directory=%s --strip-components=1' + % (linux_packages.INSTALL_DIR, hpcg_tar, HPCG_DIR) + ) # Create a symlink from the hpcg binary to 'hpcg' if FLAGS.cuda_toolkit_version == '11.0': # HPCG only release the binary that supports CUDA 11. Use the data from @@ -108,5 +119,4 @@ def AptInstall(vm): def YumInstall(_): """Install the HPCG package on the VM using yum.""" - raise NotImplementedError( - 'Installation of HPCG is only supported on Ubuntu') + raise NotImplementedError('Installation of HPCG is only supported on Ubuntu') diff --git a/perfkitbenchmarker/linux_packages/imb.py b/perfkitbenchmarker/linux_packages/imb.py index 75b9cdfb19..5238da88e7 100644 --- a/perfkitbenchmarker/linux_packages/imb.py +++ b/perfkitbenchmarker/linux_packages/imb.py @@ -1,4 +1,5 @@ r"""Installs MPI library (Intel or OpenMPI) and compiles Intel MPI benchmarks (IMB) from source.""" + import logging import posixpath from typing import List, Optional @@ -10,21 +11,26 @@ FLAGS = flags.FLAGS COMPILE_FROM_SOURCE = flags.DEFINE_bool( - 'imb_compile_from_source', True, - 'Whether to compile the Intel MPI benchmarks from source.') + 'imb_compile_from_source', + True, + 'Whether to compile the Intel MPI benchmarks from source.', +) _INTEL_DIR = '/opt/intel' -_INTEL_COMPILER_DIR = posixpath.join(_INTEL_DIR, - 'compilers_and_libraries/linux') -_INTEL_COMPILER_DIR_2020 = posixpath.join(_INTEL_DIR, - 'compilers_and_libraries_2020/linux') +_INTEL_COMPILER_DIR = posixpath.join( + _INTEL_DIR, 'compilers_and_libraries/linux' +) +_INTEL_COMPILER_DIR_2020 = posixpath.join( + _INTEL_DIR, 'compilers_and_libraries_2020/linux' +) # TBB: Intel's "Thread Building Blocks" for multithreaded programs # https://en.wikipedia.org/wiki/Threading_Building_Blocks _INTEL_FIX_TBBROOT_CMD = ( "sudo sed -i 's" "#TBBROOT=SUBSTITUTE_INSTALL_DIR_HERE#TBBROOT={compiler_dir}/tbb#' " - '{compiler_dir}/tbb/bin/tbbvars.sh') + '{compiler_dir}/tbb/bin/tbbvars.sh' +) # Source for the Intel MPI benchmarks _GITHUB_URL = 'https://github.com/intel/mpi-benchmarks.git' @@ -32,52 +38,58 @@ # Directory for the MPI benchmarks _MPI_BENCHMARK_DIR = 'mpi-benchmarks' # Checks out the Intel MPI benchmarks -_GIT_CHECKOUT_CMD = (f'git clone -n {_GITHUB_URL}; cd mpi-benchmarks; ' - f'git checkout {_GITHUB_COMMIT}') +_GIT_CHECKOUT_CMD = ( + f'git clone -n {_GITHUB_URL}; cd mpi-benchmarks; ' + f'git checkout {_GITHUB_COMMIT}' +) # Patch file and command to add latency histogram to Intel test code _PATCH_FILE = 'intelmpi.patch' _GIT_PATCH_CMD = f'patch -d {_MPI_BENCHMARK_DIR} -p3 < ~/{_PATCH_FILE}' # Enable verbose logging when mpirun fails due to a segfault -_ENABLE_VERBOSE_SEGFAULT_LOGS = ('echo 1 | sudo tee -a ' - '/proc/sys/kernel/print-fatal-signals') +_ENABLE_VERBOSE_SEGFAULT_LOGS = ( + 'echo 1 | sudo tee -a /proc/sys/kernel/print-fatal-signals' +) -def _InstallForIntelMpiLibrary( - vm) -> None: +def _InstallForIntelMpiLibrary(vm) -> None: """Compiles the Intel MPI benchmarks for Intel MPI library.""" if intel_repo.UseOneApi(): vm.InstallPackages('intel-oneapi-compiler-dpcpp-cpp') vm.InstallPackages('intel-oneapi-mpi-devel') # for mpi.h source_cmds = f'. {intel_repo.ONEAPI_VARS_FILE}' else: - source_cmds = (f'. {_INTEL_DIR}/mkl/bin/mklvars.sh intel64; ' - f'. {_INTEL_COMPILER_DIR}/bin/compilervars.sh intel64') + source_cmds = ( + f'. {_INTEL_DIR}/mkl/bin/mklvars.sh intel64; ' + f'. {_INTEL_COMPILER_DIR}/bin/compilervars.sh intel64' + ) for compiler_dir in (_INTEL_COMPILER_DIR, _INTEL_COMPILER_DIR_2020): vm.RemoteCommand( _INTEL_FIX_TBBROOT_CMD.format(compiler_dir=compiler_dir), - ignore_failure=True) + ignore_failure=True, + ) vm.RemoteCommand(_GIT_CHECKOUT_CMD) vm.PushDataFile(_PATCH_FILE) vm.RemoteCommand(_GIT_PATCH_CMD) # Default make uses the Intel compiler (mpiicc) not available in repos # {source_cmds} filled in at runtime due to differences in 2018/19 vs 2021 compile_benchmark_cmd = ( - f'cd {_MPI_BENCHMARK_DIR}; {source_cmds}; CC=mpicc CXX=mpicxx make') + f'cd {_MPI_BENCHMARK_DIR}; {source_cmds}; CC=mpicc CXX=mpicxx make' + ) vm.RemoteCommand(compile_benchmark_cmd) vm.RemoteCommand(_ENABLE_VERBOSE_SEGFAULT_LOGS) -def _InstallForOpenMpiLibrary( - vm) -> None: +def _InstallForOpenMpiLibrary(vm) -> None: """Compiles the Intel MPI benchmarks for OpenMPI library.""" vm.RemoteCommand(_GIT_CHECKOUT_CMD) vm.PushDataFile(_PATCH_FILE) vm.RemoteCommand(_GIT_PATCH_CMD) # When installing OpenMPI, openmpi.py runs ./configure.sh with --prefix=/usr. compile_benchmark_cmd = ( - f'cd {_MPI_BENCHMARK_DIR}; CC=/usr/bin/mpicc CXX=/usr/bin/mpicxx make') + f'cd {_MPI_BENCHMARK_DIR}; CC=/usr/bin/mpicc CXX=/usr/bin/mpicxx make' + ) vm.RemoteCommand(compile_benchmark_cmd) vm.RemoteCommand(_ENABLE_VERBOSE_SEGFAULT_LOGS) @@ -94,7 +106,8 @@ def Install(vm) -> None: elif FLAGS.mpi_vendor == 'openmpi': if not COMPILE_FROM_SOURCE.value: raise ValueError( - f'--mpi_vendor=openmpi requires --{COMPILE_FROM_SOURCE.name}') + f'--mpi_vendor=openmpi requires --{COMPILE_FROM_SOURCE.name}' + ) mpilib = 'openmpi' install_benchmarks = _InstallForOpenMpiLibrary @@ -107,9 +120,14 @@ def Install(vm) -> None: def _MpiRunCommandForIntelMpiLibrary( - vm, hosts: List[str], - total_processes: int, ppn: int, environment: List[str], - global_environment: List[str], tune: bool) -> str: + vm, + hosts: List[str], + total_processes: int, + ppn: int, + environment: List[str], + global_environment: List[str], + tune: bool, +) -> str: """String command to call mpirun using Intel MPI library. See Intel docs for details: @@ -143,7 +161,8 @@ def _MpiRunCommandForIntelMpiLibrary( if tune: cmd_elements.append('-tune') cmd_elements.extend( - f'-genv {variable}' for variable in sorted(global_environment)) + f'-genv {variable}' for variable in sorted(global_environment) + ) cmd_elements.append(f'-n {total_processes}') # hosts MUST remain in same order so that latency file created on first host hosts_str = ','.join(hosts) @@ -156,9 +175,12 @@ def _MpiRunCommandForIntelMpiLibrary( return ' '.join(cmd_elements) -def _MpiRunCommandForOpenMpiLibrary(hosts: List[str], total_processes: int, - npernode: int, - environment: List[str]) -> str: +def _MpiRunCommandForOpenMpiLibrary( + hosts: List[str], + total_processes: int, + npernode: int, + environment: List[str], +) -> str: """String command to call mpirun using OpenMPI library. Args: @@ -177,7 +199,8 @@ def _MpiRunCommandForOpenMpiLibrary(hosts: List[str], total_processes: int, cmd_elements = [f'{env_var}' for env_var in environment] cmd_elements.append('mpirun') cmd_elements.extend( - [f'-x {env_var.split("=", 1)[0]}' for env_var in environment]) + [f'-x {env_var.split("=", 1)[0]}' for env_var in environment] + ) # Useful for verifying process mapping. cmd_elements.append('-report-bindings') @@ -195,15 +218,21 @@ def _MpiRunCommandForOpenMpiLibrary(hosts: List[str], total_processes: int, return ' '.join(cmd_elements) -def MpiRunCommand(vm, - hosts: List[str], total_processes: int, ppn: int, - environment: List[str], global_environment: List[str], - tune: bool) -> Optional[str]: +def MpiRunCommand( + vm, + hosts: List[str], + total_processes: int, + ppn: int, + environment: List[str], + global_environment: List[str], + tune: bool, +) -> Optional[str]: """String command to call mpirun.""" if FLAGS.mpi_vendor == 'intel': - return _MpiRunCommandForIntelMpiLibrary(vm, hosts, total_processes, ppn, - environment, global_environment, - tune) + return _MpiRunCommandForIntelMpiLibrary( + vm, hosts, total_processes, ppn, environment, global_environment, tune + ) elif FLAGS.mpi_vendor == 'openmpi': - return _MpiRunCommandForOpenMpiLibrary(hosts, total_processes, ppn, - environment) + return _MpiRunCommandForOpenMpiLibrary( + hosts, total_processes, ppn, environment + ) diff --git a/perfkitbenchmarker/linux_packages/intel_oneapi_basekit.py b/perfkitbenchmarker/linux_packages/intel_oneapi_basekit.py index 47f4202181..ddc92391c2 100644 --- a/perfkitbenchmarker/linux_packages/intel_oneapi_basekit.py +++ b/perfkitbenchmarker/linux_packages/intel_oneapi_basekit.py @@ -16,41 +16,53 @@ # https://software.intel.com/content/www/us/en/develop/tools/oneapi/base-toolkit/download.html # Install requires 23.3GB disk space. -_HPC_VERSION = flags.DEFINE_enum('intel_oneapi_basekit_version', '2021.2.0', - ['2021.2.0'], - 'Version of Intel OneAPI base toolkit.') +_HPC_VERSION = flags.DEFINE_enum( + 'intel_oneapi_basekit_version', + '2021.2.0', + ['2021.2.0'], + 'Version of Intel OneAPI base toolkit.', +) PREPROVISIONED_DATA = { # original file name: l_BaseKit_p_2021.2.0.2883_offline.sh - '2021.2.0.sh': - '7e91f63614aa50cf931ca4264c941210fb829b4dc83ca00e94ca82efc7416a27', + '2021.2.0.sh': ( + '7e91f63614aa50cf931ca4264c941210fb829b4dc83ca00e94ca82efc7416a27' + ), } # command to run to get mpicc, etc files in PATH SOURCE_VARS_COMMAND = '. /opt/intel/oneapi/setvars.sh' # do not log these environment variables in _VerifyInstall -_IGNORE_VARIABLES_DEBUG = ('SSL_CLIENT', 'SSL_CONNECTION', 'XDG_SESSION_ID', - 'MANPATH') +_IGNORE_VARIABLES_DEBUG = ( + 'SSL_CLIENT', + 'SSL_CONNECTION', + 'XDG_SESSION_ID', + 'MANPATH', +) def Install(vm): """Installs Intel oneAPI.""" preprovisioned_filename = f'{_HPC_VERSION.value}.sh' - vm.InstallPreprovisionedPackageData('intel_oneapi_basekit', - [preprovisioned_filename], '.') + vm.InstallPreprovisionedPackageData( + 'intel_oneapi_basekit', [preprovisioned_filename], '.' + ) # needed for warning if disk size is too small vm.InstallPackages('bc') - vm.RemoteCommand(f'chmod +x {preprovisioned_filename}; ' - f'sudo ./{preprovisioned_filename} -a -s --eula accept; ' - f'rm {preprovisioned_filename}') + vm.RemoteCommand( + f'chmod +x {preprovisioned_filename}; ' + f'sudo ./{preprovisioned_filename} -a -s --eula accept; ' + f'rm {preprovisioned_filename}' + ) _VerifyInstall(vm) def _GetVariables(vm, use_source_vars: bool = False) -> Dict[str, List[str]]: """Returns all the environment variables on the VM.""" text, _ = vm.RemoteCommand( - f'{SOURCE_VARS_COMMAND} >/dev/null; env' if use_source_vars else 'env') + f'{SOURCE_VARS_COMMAND} >/dev/null; env' if use_source_vars else 'env' + ) ret: Dict[str, List[str]] = {} for line in text.splitlines(): variable, value = line.split('=', 1) @@ -59,8 +71,9 @@ def _GetVariables(vm, use_source_vars: bool = False) -> Dict[str, List[str]]: return ret -def _LogEnvDifferences(pre_variables: Dict[str, List[str]], - post_variables: Dict[str, List[str]]): +def _LogEnvDifferences( + pre_variables: Dict[str, List[str]], post_variables: Dict[str, List[str]] +): """Logs the differences between the two sets of environment variables.""" differences = [] for variable in sorted(set(list(pre_variables) + list(post_variables))): @@ -70,8 +83,11 @@ def _LogEnvDifferences(pre_variables: Dict[str, List[str]], if additions: differences.append(f'{variable}={":".join(additions)}') if differences: - logging.info('Changes in %s environment variables:\n%s', len(differences), - '\n'.join(differences)) + logging.info( + 'Changes in %s environment variables:\n%s', + len(differences), + '\n'.join(differences), + ) else: logging.info('No change in environment variables, most likely an error') @@ -84,5 +100,6 @@ def _VerifyInstall(vm): expected_mpi_root = f'/opt/intel/oneapi/mpi/{_HPC_VERSION.value}' mpi_root = post_variables.get('I_MPI_ROOT', []) if mpi_root != [expected_mpi_root]: - raise ValueError(f'Expected "I_MPI_ROOT={expected_mpi_root}" ' - f'have "{sorted(mpi_root)}"') + raise ValueError( + f'Expected "I_MPI_ROOT={expected_mpi_root}" have "{sorted(mpi_root)}"' + ) diff --git a/perfkitbenchmarker/linux_packages/intel_repo.py b/perfkitbenchmarker/linux_packages/intel_repo.py index 2f7ce968aa..21a3e829dc 100644 --- a/perfkitbenchmarker/linux_packages/intel_repo.py +++ b/perfkitbenchmarker/linux_packages/intel_repo.py @@ -32,17 +32,19 @@ _APT_REMOTE_REPO_FILE = posixpath.join(vm_util.VM_TMP_DIR, 'intel.list') # Command to add the GPG key and update the repo list. _APT_INSTALL_REPO_CMD = ';'.join([ - f'sudo apt-key add {_REMOTE_KEY_FILE}', f'rm {_REMOTE_KEY_FILE}', + f'sudo apt-key add {_REMOTE_KEY_FILE}', + f'rm {_REMOTE_KEY_FILE}', f'sudo mv {_APT_REMOTE_REPO_FILE} /etc/apt/sources.list.d/', - 'sudo apt-get update' + 'sudo apt-get update', ]) # APT constants for Intel oneAPI _ONEAPI_APT_URL = 'https://apt.repos.intel.com/oneapi' _ONEAPI_APT_FILE = '/etc/apt/sources.list.d/oneAPI.list' _ONEAPI_APT_INSTALL_REPO_CMD = ';'.join([ - f'sudo apt-key add {_REMOTE_KEY_FILE}', f'rm {_REMOTE_KEY_FILE}', + f'sudo apt-key add {_REMOTE_KEY_FILE}', + f'rm {_REMOTE_KEY_FILE}', f'echo "deb {_ONEAPI_APT_URL} all main" | sudo tee {_ONEAPI_APT_FILE}', - 'sudo apt-get update' + 'sudo apt-get update', ]) # YUM constants @@ -68,7 +70,8 @@ def UseOneApi(): return FLAGS.intelmpi_version.startswith( - '2021') or FLAGS.mkl_version.startswith('2021') + '2021' + ) or FLAGS.mkl_version.startswith('2021') def AptInstall(vm): @@ -90,7 +93,8 @@ def YumInstall(vm): tmp_remote_file = posixpath.basename(_ONEAPI_YUM_INSTALL_REPO_REMOTE_FILE) vm.PushDataFile(_ONEAPI_YUM_INSTALL_REPO_FILE, tmp_remote_file) vm.RemoteCommand( - f'sudo mv {tmp_remote_file} {_ONEAPI_YUM_INSTALL_REPO_REMOTE_FILE}') + f'sudo mv {tmp_remote_file} {_ONEAPI_YUM_INSTALL_REPO_REMOTE_FILE}' + ) else: vm.RemoteCommand(_YUM_INSTALL_REPO_CMD) # the /etc/yum.repos.d/intelproducts.repo file has the gpgkey listed as the @@ -99,7 +103,8 @@ def YumInstall(vm): diff, _, retcode = vm.RemoteCommandWithReturnCode(_YUM_DIFF_KEY_CMD) if retcode: raise errors.Setup.InvalidConfigurationError( - f'Intel GPG key does not match local key: {diff}') + f'Intel GPG key does not match local key: {diff}' + ) vm.RemoteCommand(f'rm {_YUM_DOWNLOAD_KEY}') # need to update with -y to force import of known GPG key vm.RemoteCommand('sudo yum update -y') @@ -116,8 +121,9 @@ def CopyIntelFiles(source_vm, target_vm) -> None: target_vm: Destination VM for the /opt/intel files """ root_dir = '/opt/intel' - mkdir_cmd = (f'sudo mkdir {root_dir}; ' - f'sudo chown {target_vm.user_name} {root_dir}') + mkdir_cmd = ( + f'sudo mkdir {root_dir}; sudo chown {target_vm.user_name} {root_dir}' + ) target_vm.RemoteCommand(mkdir_cmd) tar_cmd = f'cd {root_dir}; tar -cf - *' untar_cmd = f"ssh {target_vm.internal_ip} '(cd {root_dir} ; tar -xf -)'" diff --git a/perfkitbenchmarker/linux_packages/intelmpi.py b/perfkitbenchmarker/linux_packages/intelmpi.py index 63990d1396..241f32eaff 100644 --- a/perfkitbenchmarker/linux_packages/intelmpi.py +++ b/perfkitbenchmarker/linux_packages/intelmpi.py @@ -23,22 +23,27 @@ from perfkitbenchmarker import vm_util from perfkitbenchmarker.linux_packages import intel_repo -MPI_VERSION = flags.DEFINE_string('intelmpi_version', '2019.6-088', - 'MPI version.') +MPI_VERSION = flags.DEFINE_string( + 'intelmpi_version', '2019.6-088', 'MPI version.' +) FLAGS = flags.FLAGS _INTEL_ROOT = '/opt/intel' PREPROVISIONED_DATA = { - 'intel-mpi-2018.4-057.rpm.tar.gz': - 'fbcf98d246760330ef7dd661d2b75e7f90783943dfea1285e5aaeb4f0e1abf9e', - 'intel-mpi-2018.4-057.deb.tar.gz': - '9603b9241287874b5d4fe3ba806d1b672b1da576406a4c6dcd86e9d84e341bd1', - 'intel-mpi-2019.6-088.rpm.tar.gz': - 'a4683ff1bd66b57f9002ad63208a77aee172b5b1dc365d892af8d468d96ad8d5', - 'intel-mpi-2019.6-088.deb.tar.gz': - 'e5a8126dbc2d1aca889f987117be00dffe4344c58f8e622d3b8f5809cb5493e6', + 'intel-mpi-2018.4-057.rpm.tar.gz': ( + 'fbcf98d246760330ef7dd661d2b75e7f90783943dfea1285e5aaeb4f0e1abf9e' + ), + 'intel-mpi-2018.4-057.deb.tar.gz': ( + '9603b9241287874b5d4fe3ba806d1b672b1da576406a4c6dcd86e9d84e341bd1' + ), + 'intel-mpi-2019.6-088.rpm.tar.gz': ( + 'a4683ff1bd66b57f9002ad63208a77aee172b5b1dc365d892af8d468d96ad8d5' + ), + 'intel-mpi-2019.6-088.deb.tar.gz': ( + 'e5a8126dbc2d1aca889f987117be00dffe4344c58f8e622d3b8f5809cb5493e6' + ), } @@ -56,7 +61,8 @@ def MpiVars(vm) -> str: return intel_repo.ONEAPI_VARS_FILE txt, _ = vm.RemoteCommand( f'readlink -f {_INTEL_ROOT}/compilers_and_libraries*/' - 'linux/mpi/intel64/bin/mpivars.sh | sort | uniq') + 'linux/mpi/intel64/bin/mpivars.sh | sort | uniq' + ) files = txt.splitlines() if not files: raise ValueError('Could not find the mpivars.sh file') @@ -76,7 +82,6 @@ def SourceMpiVarsCommand(vm): def FixEnvironment(vm): """Changes system settings for optimal Intel MPI conditions. - Sets the ptrace_scope to 0, for details see: https://www.kernel.org/doc/Documentation/security/Yama.txt @@ -84,14 +89,17 @@ def FixEnvironment(vm): vm: The virtual machine to run on. """ if not vm.TryRemoteCommand('ulimit -l | grep unlimited'): - ulimit_fix_cmd = (f'echo "{vm.user_name} - memlock unlimited" | ' - 'sudo tee -a /etc/security/limits.conf') + ulimit_fix_cmd = ( + f'echo "{vm.user_name} - memlock unlimited" | ' + 'sudo tee -a /etc/security/limits.conf' + ) vm.RemoteCommand(ulimit_fix_cmd) logging.info('Rebooting to permamently set ulimit') vm.Reboot() vm.WaitForBootCompletion() _, stderr, exitcode = vm.RemoteCommandWithReturnCode( - 'sudo sysctl -w kernel.yama.ptrace_scope=0', ignore_failure=True) + 'sudo sysctl -w kernel.yama.ptrace_scope=0', ignore_failure=True + ) if exitcode: logging.info('Not setting yama ptrace as %s', stderr) @@ -107,9 +115,11 @@ def _InstallViaPreprovisionedData(vm, version: str) -> bool: return False vm.InstallPreprovisionedPackageData('intelmpi', [tarball], '.') extract_dir = posixpath.join(vm_util.VM_TMP_DIR, 'packages') - vm.RemoteCommand(f'mkdir -p {extract_dir}; ' - f'tar -C {extract_dir} -xvzf {tarball}; ' - f'rm {tarball}') + vm.RemoteCommand( + f'mkdir -p {extract_dir}; ' + f'tar -C {extract_dir} -xvzf {tarball}; ' + f'rm {tarball}' + ) vm.InstallPackages(f'{extract_dir}/*') return True @@ -149,7 +159,8 @@ def TestInstall(vms) -> None: # 'ip-..compute.internal so just check number of responses if len(hosts) != len(expected_hosts): raise ValueError( - f'Expected hosts {len(expected_hosts)} but have {len(hosts)}') + f'Expected hosts {len(expected_hosts)} but have {len(hosts)}' + ) logging.info('Hosts: %s', ','.join(hosts)) diff --git a/perfkitbenchmarker/linux_packages/ior.py b/perfkitbenchmarker/linux_packages/ior.py index 44d0502fc5..34c01972a1 100644 --- a/perfkitbenchmarker/linux_packages/ior.py +++ b/perfkitbenchmarker/linux_packages/ior.py @@ -28,10 +28,18 @@ GIT_REPO = 'https://github.com/hpc/ior' GIT_TAG = '945fba2aa2d571e8babc4f5f01e78e9f5e6e193e' _METADATA_KEYS = [ - 'Operation', '#Tasks', 'segcnt', 'blksiz', 'xsize', 'aggsize', 'API', 'fPP', + 'Operation', + '#Tasks', + 'segcnt', + 'blksiz', + 'xsize', + 'aggsize', + 'API', + 'fPP', ] -_MDTEST_RESULT_REGEX = (r'\s*(.*?)\s*:\s*(\d+\.\d+)\s*(\d+\.\d+)' - r'\s*(\d+\.\d+)\s*(\d+\.\d+)') +_MDTEST_RESULT_REGEX = ( + r'\s*(.*?)\s*:\s*(\d+\.\d+)\s*(\d+\.\d+)' r'\s*(\d+\.\d+)\s*(\d+\.\d+)' +) _MDTEST_SUMMARY_REGEX = r'(\d+) tasks, (\d+) files[^\n]*\n\s*\n(.*?)\n\s*\n' @@ -40,8 +48,10 @@ def Install(vm): vm.Install('openmpi') vm.RemoteCommand('git clone {0} {1}'.format(GIT_REPO, IOR_DIR)) vm.RemoteCommand('cd {0} && git checkout {1}'.format(IOR_DIR, GIT_TAG)) - vm.RemoteCommand('cd {0} && ./bootstrap && ./configure && make && ' - 'sudo make install'.format(IOR_DIR)) + vm.RemoteCommand( + 'cd {0} && ./bootstrap && ./configure && make && ' + 'sudo make install'.format(IOR_DIR) + ) def Uninstall(vm): @@ -63,11 +73,14 @@ def RunIOR(master_vm, num_tasks, script_path): def ParseIORResults(test_output): - """"Parses the test output and returns samples.""" - random_offsets = (ordering == 'random offsets' for ordering in - re.findall('ordering in a file = (.*)', test_output)) + """ "Parses the test output and returns samples.""" + random_offsets = ( + ordering == 'random offsets' + for ordering in re.findall('ordering in a file = (.*)', test_output) + ) match = re.search( - 'Summary of all tests:\n(.*?)Finished', test_output, re.DOTALL) + 'Summary of all tests:\n(.*?)Finished', test_output, re.DOTALL + ) if not match: raise errors.Benchmarks.RunError('Error parsing: "{test_output}"') fp = io.StringIO(re.sub(' +', ' ', match.group(1))) @@ -81,7 +94,7 @@ def ParseIORResults(test_output): iops = float(result_dict['Mean(OPs)']) results.extend([ sample.Sample('Bandwidth', bandwidth, 'MiB/s', metadata), - sample.Sample('IOPS', iops, 'OPs/s', metadata) + sample.Sample('IOPS', iops, 'OPs/s', metadata), ]) return results @@ -111,24 +124,30 @@ def ParseMdtestResults(test_output): summaries = re.findall(_MDTEST_SUMMARY_REGEX, test_output, re.DOTALL) for num_tasks, num_files, summary in summaries: metadata = { - 'command_line': command_line, 'num_tasks': num_tasks, + 'command_line': command_line, + 'num_tasks': num_tasks, 'drop_caches': FLAGS.mdtest_drop_caches, - 'num_files': num_files, 'dir_per_task': dir_per_task + 'num_files': num_files, + 'dir_per_task': dir_per_task, } result_lines = re.findall(_MDTEST_RESULT_REGEX, summary) for result_line in result_lines: op_type, max_ops, min_ops, mean_ops, std_dev = result_line if not float(mean_ops): continue - results.append(sample.Sample( - op_type + ' (Mean)', float(mean_ops), 'OPs/s', metadata)) + results.append( + sample.Sample(op_type + ' (Mean)', float(mean_ops), 'OPs/s', metadata) + ) if float(std_dev): results.extend([ sample.Sample( - op_type + ' (Max)', float(max_ops), 'OPs/s', metadata), + op_type + ' (Max)', float(max_ops), 'OPs/s', metadata + ), sample.Sample( - op_type + ' (Min)', float(min_ops), 'OPs/s', metadata), + op_type + ' (Min)', float(min_ops), 'OPs/s', metadata + ), sample.Sample( - op_type + ' (Std Dev)', float(std_dev), 'OPs/s', metadata) + op_type + ' (Std Dev)', float(std_dev), 'OPs/s', metadata + ), ]) return results diff --git a/perfkitbenchmarker/linux_packages/iperf.py b/perfkitbenchmarker/linux_packages/iperf.py index e2ef153e97..bb814a1e8b 100644 --- a/perfkitbenchmarker/linux_packages/iperf.py +++ b/perfkitbenchmarker/linux_packages/iperf.py @@ -31,12 +31,14 @@ def _Install(vm): vm.Install('build_tools') vm.Install('wget') - vm.RemoteCommand('wget -O %s/%s %s' % - (linux_packages.INSTALL_DIR, IPERF_TAR, IPERF_URL)) - - vm.RemoteCommand('cd %s; tar xvf %s; cd %s; ' - './configure; make; sudo make install' % - (linux_packages.INSTALL_DIR, IPERF_TAR, IPERF_DIR)) + vm.RemoteCommand( + 'wget -O %s/%s %s' % (linux_packages.INSTALL_DIR, IPERF_TAR, IPERF_URL) + ) + + vm.RemoteCommand( + 'cd %s; tar xvf %s; cd %s; ./configure; make; sudo make install' + % (linux_packages.INSTALL_DIR, IPERF_TAR, IPERF_DIR) + ) def YumInstall(vm): diff --git a/perfkitbenchmarker/linux_packages/iperf3.py b/perfkitbenchmarker/linux_packages/iperf3.py index 755448555b..a9141a67f4 100644 --- a/perfkitbenchmarker/linux_packages/iperf3.py +++ b/perfkitbenchmarker/linux_packages/iperf3.py @@ -22,7 +22,8 @@ def _Install(vm): vm.Install('build_tools') vm.InstallPackages('lib32z1') vm.RemoteCommand( - 'wget https://downloads.es.net/pub/iperf/iperf-3.9.tar.gz --no-check-certificate' + 'wget https://downloads.es.net/pub/iperf/iperf-3.9.tar.gz' + ' --no-check-certificate' ) vm.RemoteCommand('tar -xf iperf-3.9.tar.gz') vm.RemoteCommand('cd iperf-3.9 && ./configure') diff --git a/perfkitbenchmarker/linux_packages/kernel_compile.py b/perfkitbenchmarker/linux_packages/kernel_compile.py index 287b0ee3da..d2457e5bf1 100644 --- a/perfkitbenchmarker/linux_packages/kernel_compile.py +++ b/perfkitbenchmarker/linux_packages/kernel_compile.py @@ -26,8 +26,11 @@ def _Install(vm): vm.Install('build_tools') vm.Install('wget') vm.InstallPackages('bc') - vm.RemoteCommand('mkdir -p {0} && ' - 'cd {0} && wget {1}'.format(linux_packages.INSTALL_DIR, URL)) + vm.RemoteCommand( + 'mkdir -p {0} && cd {0} && wget {1}'.format( + linux_packages.INSTALL_DIR, URL + ) + ) def AptInstall(vm): @@ -39,5 +42,6 @@ def YumInstall(vm): def Cleanup(vm): - vm.RemoteCommand('cd {} && rm -f {}'.format(linux_packages.INSTALL_DIR, - TARBALL)) + vm.RemoteCommand( + 'cd {} && rm -f {}'.format(linux_packages.INSTALL_DIR, TARBALL) + ) diff --git a/perfkitbenchmarker/linux_packages/lapack.py b/perfkitbenchmarker/linux_packages/lapack.py index e50853bd26..67d32d3366 100644 --- a/perfkitbenchmarker/linux_packages/lapack.py +++ b/perfkitbenchmarker/linux_packages/lapack.py @@ -24,12 +24,11 @@ LAPACK_DIR = os.path.join(vm_util.VM_TMP_DIR, LAPACK_FOLDER) PACKAGE_NAME = 'lapack' PREPROVISIONED_DATA = { - LAPACK_TAR: + LAPACK_TAR: ( '888a50d787a9d828074db581c80b2d22bdb91435a673b1bf6cd6eb51aa50d1de' + ) } -PACKAGE_DATA_URL = { - LAPACK_TAR: LAPACK_URL -} +PACKAGE_DATA_URL = {LAPACK_TAR: LAPACK_URL} def _Install(vm): @@ -37,16 +36,17 @@ def _Install(vm): vm.Install('fortran') vm.Install('cmake') vm.InstallPreprovisionedPackageData( - PACKAGE_NAME, PREPROVISIONED_DATA.keys(), vm_util.VM_TMP_DIR) - vm.RemoteCommand( - 'cd %s; tar xf %s' % ( - vm_util.VM_TMP_DIR, LAPACK_TAR)) + PACKAGE_NAME, PREPROVISIONED_DATA.keys(), vm_util.VM_TMP_DIR + ) + vm.RemoteCommand('cd %s; tar xf %s' % (vm_util.VM_TMP_DIR, LAPACK_TAR)) vm.RemoteCommand( - 'cd %s; mv make.inc.example make.inc; cmake .; make -j %s' % ( - LAPACK_DIR, vm.num_cpus)) + 'cd %s; mv make.inc.example make.inc; cmake .; make -j %s' + % (LAPACK_DIR, vm.num_cpus) + ) vm.RemoteCommand( - 'cd %s; make -j %s' % (os.path.join(LAPACK_DIR, 'BLAS'), - vm.NumCpusForBenchmark())) + 'cd %s; make -j %s' + % (os.path.join(LAPACK_DIR, 'BLAS'), vm.NumCpusForBenchmark()) + ) def YumInstall(vm): diff --git a/perfkitbenchmarker/linux_packages/linux_boot.py b/perfkitbenchmarker/linux_packages/linux_boot.py index e7afadbc62..9b9a150490 100644 --- a/perfkitbenchmarker/linux_packages/linux_boot.py +++ b/perfkitbenchmarker/linux_packages/linux_boot.py @@ -7,7 +7,7 @@ import os import re import time -from typing import Optional, Tuple, List, Callable +from typing import Callable, List, Optional, Tuple from absl import flags import datetime_tz @@ -41,7 +41,8 @@ def PrepareBootScriptVM(aux_vm_ips: str, aux_vm_port: int) -> str: @vm_util.Retry(log_errors=False, poll_interval=1, timeout=300) def GetStartupScriptOutput( - vm: virtual_machine.BaseVirtualMachine, output_file: str) -> str: + vm: virtual_machine.BaseVirtualMachine, output_file: str +) -> str: """return fastboot script log.""" return vm.RemoteCommand(f'cat {output_file}')[0] @@ -67,15 +68,16 @@ def CollectBootSamples( vm: virtual_machine.VirtualMachine, runner_ip: Tuple[str, str], create_time: datetime.datetime, - include_networking_samples: bool = False) -> List[sample.Sample]: + include_networking_samples: bool = False, +) -> List[sample.Sample]: """Collect boot samples. Args: vm: The boot vm. runner_ip: Runner ip addresses to collect VM-to-VM metrics. create_time: VM creation time. - include_networking_samples: Boolean, whether to include samples such as - time to first egress/ingress packet. + include_networking_samples: Boolean, whether to include samples such as time + to first egress/ingress packet. Returns: A list of sample.Sample objects. @@ -152,7 +154,8 @@ class RegexMatcher: def ScrapeConsoleLogLines( log_lines: List[str], start_time: datetime.datetime, - matchers: List[RegexMatcher]) -> List[sample.Sample]: + matchers: List[RegexMatcher], +) -> List[sample.Sample]: """Extract timings from the guest console log lines. Args: @@ -180,16 +183,15 @@ def ScrapeConsoleLogLines( for key, val in times.items(): samples.append( - sample.Sample( - key, (val - start_time).total_seconds(), 'second', {} - ) + sample.Sample(key, (val - start_time).total_seconds(), 'second', {}) ) return samples -def CollectGuestSamples(vm: virtual_machine.VirtualMachine, - metric_start_time: float) -> List[sample.Sample]: +def CollectGuestSamples( + vm: virtual_machine.VirtualMachine, metric_start_time: float +) -> List[sample.Sample]: """Collect guest metrics. All metrics published are normalized against VM creation timestamp, including: @@ -252,7 +254,8 @@ def CollectGuestSamples(vm: virtual_machine.VirtualMachine, def CollectKernelSamples( - vm: virtual_machine.VirtualMachine, offset: float) -> List[sample.Sample]: + vm: virtual_machine.VirtualMachine, offset: float +) -> List[sample.Sample]: """Collect kernel metrics. Args: @@ -345,9 +348,8 @@ def ParseUserTotalTimes(system_d_string: str) -> Tuple[float, float]: Args: system_d_string: the string output from systemd-analyze string output. i.e. "Startup finished in 2.2s (kernel) + 1min 12.5s (userspace) = 1min - 14.774s" - "Startup finished in 448ms (firmware) + 1.913s (loader) + - 1.182s (kernel) + 52.438s (initrd) + 30.413s (userspace) = 1min 26.397s" + 14.774s" "Startup finished in 448ms (firmware) + 1.913s (loader) + 1.182s + (kernel) + 52.438s (initrd) + 30.413s (userspace) = 1min 26.397s" Returns: A tuple of user_start and total_time, both in seconds. @@ -485,7 +487,8 @@ def ParseSystemDCriticalChainServiceTime(systemd_out: str) -> Optional[float]: def WaitForReady( - vm: virtual_machine.VirtualMachine, timeout: int) -> Tuple[float, float]: + vm: virtual_machine.VirtualMachine, timeout: int +) -> Tuple[float, float]: """Ensure the system is ready for boot metrics inspection. Args: @@ -533,8 +536,8 @@ def GetGuestScriptsStart(vm: virtual_machine.VirtualMachine) -> Optional[float]: def GetSystemDCriticalChainMetric( - vm: virtual_machine.VirtualMachine, - metric: str) -> Optional[float]: + vm: virtual_machine.VirtualMachine, metric: str +) -> Optional[float]: stdout, _ = vm.RemoteCommand(f'sudo systemd-analyze critical-chain {metric}') return ParseSystemDCriticalChainOutput(stdout) @@ -602,7 +605,8 @@ def GetSystemDCriticalChainMetric( def CollectVmToVmSamples( vm: virtual_machine.VirtualMachine, runner_ip: Tuple[str, str], - create_time: datetime.datetime) -> List[sample.Sample]: + create_time: datetime.datetime, +) -> List[sample.Sample]: """Collect samples related to vm-to-vm networking.""" samples = [] vm_output = GetStartupScriptOutput(vm, BOOT_SCRIPT_OUTPUT) diff --git a/perfkitbenchmarker/linux_packages/lmbench.py b/perfkitbenchmarker/linux_packages/lmbench.py index d91a167582..9dabea68f6 100644 --- a/perfkitbenchmarker/linux_packages/lmbench.py +++ b/perfkitbenchmarker/linux_packages/lmbench.py @@ -26,8 +26,10 @@ def _Install(vm): """Installs the Lmbench package on the VM.""" vm.Install('build_tools') - vm.RemoteCommand('cd %s && git clone %s && cd %s && git checkout %s' % - (linux_packages.INSTALL_DIR, GIT, 'lmbench', COMMIT)) + vm.RemoteCommand( + 'cd %s && git clone %s && cd %s && git checkout %s' + % (linux_packages.INSTALL_DIR, GIT, 'lmbench', COMMIT) + ) def YumInstall(vm): diff --git a/perfkitbenchmarker/linux_packages/maven.py b/perfkitbenchmarker/linux_packages/maven.py index 3347380cd2..3432c2544d 100644 --- a/perfkitbenchmarker/linux_packages/maven.py +++ b/perfkitbenchmarker/linux_packages/maven.py @@ -20,10 +20,12 @@ from perfkitbenchmarker import linux_packages from six.moves.urllib.parse import urlparse -flags.DEFINE_string('maven_version', '3.6.3', - 'The version of maven') -flags.DEFINE_string('maven_mirror_url', None, - 'If specified, this URL will be used as a Maven mirror') +flags.DEFINE_string('maven_version', '3.6.3', 'The version of maven') +flags.DEFINE_string( + 'maven_mirror_url', + None, + 'If specified, this URL will be used as a Maven mirror', +) FLAGS = flags.FLAGS MVN_URL = 'https://archive.apache.org/dist/maven/maven-{0}/{1}/binaries/apache-maven-{1}-bin.tar.gz' MVN_DIR = posixpath.join(linux_packages.INSTALL_DIR, 'maven') @@ -38,14 +40,16 @@ PACKAGE_NAME = 'maven' PREPROVISIONED_DATA = { - 'apache-maven-{0}-bin.tar.gz'.format('3.6.1'): - '2528c35a99c30f8940cc599ba15d34359d58bec57af58c1075519b8cd33b69e7', - 'apache-maven-{0}-bin.tar.gz'.format('3.6.3'): - '26ad91d751b3a9a53087aefa743f4e16a17741d3915b219cf74112bf87a438c5' + 'apache-maven-{0}-bin.tar.gz'.format( + '3.6.1' + ): '2528c35a99c30f8940cc599ba15d34359d58bec57af58c1075519b8cd33b69e7', + 'apache-maven-{0}-bin.tar.gz'.format( + '3.6.3' + ): '26ad91d751b3a9a53087aefa743f4e16a17741d3915b219cf74112bf87a438c5', } PACKAGE_DATA_URL = { 'apache-maven-{0}-bin.tar.gz'.format('3.6.1'): MVN_URL.format('3', '3.6.1'), - 'apache-maven-{0}-bin.tar.gz'.format('3.6.3'): MVN_URL.format('3', '3.6.3') + 'apache-maven-{0}-bin.tar.gz'.format('3.6.3'): MVN_URL.format('3', '3.6.3'), } @@ -70,13 +74,15 @@ def GetRunCommand(arguments): parsed_url = urlparse(FLAGS.http_proxy) http_proxy_params = ' -Dhttp.proxyHost={host} -Dhttp.proxyPort={port}' command += http_proxy_params.format( - host=parsed_url.hostname, port=parsed_url.port) + host=parsed_url.hostname, port=parsed_url.port + ) if FLAGS['https_proxy'].present: parsed_url = urlparse(FLAGS.https_proxy) https_proxy_params = ' -Dhttps.proxyHost={host} -Dhttps.proxyPort={port}' command += https_proxy_params.format( - host=parsed_url.hostname, port=parsed_url.port) + host=parsed_url.hostname, port=parsed_url.port + ) maven_major_ver, maven_minor_ver, maven_patch_ver = _MavenVersionParts() # Don't output download progress. @@ -89,11 +95,13 @@ def GetRunCommand(arguments): def _GetJavaHome(vm): - out, _ = vm.RemoteCommand("java -XshowSettings:properties 2>&1 > /dev/null " - "| awk '/java.home/{print $3}'") + out, _ = vm.RemoteCommand( + 'java -XshowSettings:properties 2>&1 > /dev/null ' + "| awk '/java.home/{print $3}'" + ) out = out.strip() if '/jre' in out: - return out[:out.index('/jre')] + return out[: out.index('/jre')] else: return out @@ -121,11 +129,14 @@ def _Install(vm): PREPROVISIONED_DATA[maven_tar] = '' PACKAGE_DATA_URL[maven_tar] = maven_url maven_remote_path = posixpath.join(linux_packages.INSTALL_DIR, maven_tar) - vm.InstallPreprovisionedPackageData(PACKAGE_NAME, [maven_tar], - linux_packages.INSTALL_DIR) - vm.RemoteCommand(('mkdir -p {0} && ' - 'tar -C {0} --strip-components=1 -xzf {1}').format( - MVN_DIR, maven_remote_path)) + vm.InstallPreprovisionedPackageData( + PACKAGE_NAME, [maven_tar], linux_packages.INSTALL_DIR + ) + vm.RemoteCommand( + ('mkdir -p {0} && tar -C {0} --strip-components=1 -xzf {1}').format( + MVN_DIR, maven_remote_path + ) + ) java_home = _GetJavaHome(vm) @@ -135,12 +146,11 @@ def _Install(vm): vm.RemoteCommand(cmd) if FLAGS.maven_mirror_url: - settings_local_path = data.ResourcePath(os.path.join( - 'maven', 'settings.xml.j2')) + settings_local_path = data.ResourcePath( + os.path.join('maven', 'settings.xml.j2') + ) settings_remote_path = '~/.m2/settings.xml' - context = { - 'maven_mirror_url': FLAGS.maven_mirror_url - } + context = {'maven_mirror_url': FLAGS.maven_mirror_url} vm.RemoteCommand('mkdir -p ~/.m2') vm.RenderTemplate(settings_local_path, settings_remote_path, context) diff --git a/perfkitbenchmarker/linux_packages/mcperf.py b/perfkitbenchmarker/linux_packages/mcperf.py index ee94b32f22..b7c08286be 100644 --- a/perfkitbenchmarker/linux_packages/mcperf.py +++ b/perfkitbenchmarker/linux_packages/mcperf.py @@ -33,31 +33,33 @@ flags.DEFINE_enum( 'mcperf_protocol', - 'binary', ['binary', 'ascii'], - 'Protocol to use. Supported protocols are binary and ascii.') + 'binary', + ['binary', 'ascii'], + 'Protocol to use. Supported protocols are binary and ascii.', +) flags.DEFINE_list( - 'mcperf_qps', [], - 'Target aggregate QPS. If not set, target for peak qps.') -flags.DEFINE_integer( - 'mcperf_time', 300, - 'Maximum time to run (seconds).') + 'mcperf_qps', [], 'Target aggregate QPS. If not set, target for peak qps.' +) +flags.DEFINE_integer('mcperf_time', 300, 'Maximum time to run (seconds).') flags.DEFINE_string( - 'mcperf_keysize', '16', - 'Length of memcached keys (distribution).') + 'mcperf_keysize', '16', 'Length of memcached keys (distribution).' +) flags.DEFINE_string( - 'mcperf_valuesize', '128', - 'Length of memcached values (distribution).') + 'mcperf_valuesize', '128', 'Length of memcached values (distribution).' +) flags.DEFINE_integer( - 'mcperf_records', 10000, - 'Number of memcached records to use.') + 'mcperf_records', 10000, 'Number of memcached records to use.' +) flags.DEFINE_float( - 'mcperf_ratio', 0.0, - 'Ratio of set:get. By default, read only.') + 'mcperf_ratio', 0.0, 'Ratio of set:get. By default, read only.' +) flags.DEFINE_list( - 'mcperf_options', ['iadist=exponential:0.0'], + 'mcperf_options', + ['iadist=exponential:0.0'], 'Additional mcperf long-form options (--) in comma separated form. e.g.' '--mcperf_options=blocking,search=99:1000.' - 'See https://github.com/shaygalon/memcache-perf for all available options.') + 'See https://github.com/shaygalon/memcache-perf for all available options.', +) # If more than one value provided for threads, connections, depths, we will # enumerate all test configurations. e.g. @@ -65,30 +67,31 @@ # We will test following threads:connections:depths: # 1,3,5; 1,3,6; 1,4,5; 1,4,6; 2,3,5; 2,3,6; 2,4,5; 2,4,6; flags.DEFINE_list( - 'mcperf_threads', [1], - 'Number of total client threads to spawn per client VM.') -flags.DEFINE_list( - 'mcperf_connections', [1], - 'Number of connections to establish per client thread.') + 'mcperf_threads', + [1], + 'Number of total client threads to spawn per client VM.', +) flags.DEFINE_list( - 'mcperf_depths', [1], - 'Maximum depth to pipeline requests.') + 'mcperf_connections', + [1], + 'Number of connections to establish per client thread.', +) +flags.DEFINE_list('mcperf_depths', [1], 'Maximum depth to pipeline requests.') # Agent mode options. flags.DEFINE_integer( - 'mcperf_measure_connections', None, - 'Master client connections.') -flags.DEFINE_integer( - 'mcperf_measure_threads', None, - 'Master client thread count.') + 'mcperf_measure_connections', None, 'Master client connections.' +) flags.DEFINE_integer( - 'mcperf_measure_qps', None, - 'Master client QPS.') + 'mcperf_measure_threads', None, 'Master client thread count.' +) +flags.DEFINE_integer('mcperf_measure_qps', None, 'Master client QPS.') flags.DEFINE_integer( - 'mcperf_measure_depth', None, - 'Master client connection depth.') + 'mcperf_measure_depth', None, 'Master client connection depth.' +) _INCREMENTAL_LOAD = flags.DEFINE_float( - 'mcperf_incremental_load', None, 'Increments target qps until hits peak.') + 'mcperf_incremental_load', None, 'Increments target qps until hits peak.' +) # To use remote agent mode, we need at least 2 VMs. AGENT_MODE_MIN_CLIENT_VMS = 2 @@ -99,22 +102,27 @@ def CheckPrerequisites(): Raises: errors.Setup.InvalidFlagConfigurationError: On invalid flag configurations. """ - agent_mode_flags = [FLAGS['mcperf_measure_connections'].present, - FLAGS['mcperf_measure_threads'].present, - FLAGS['mcperf_measure_qps'].present, - FLAGS['mcperf_measure_depth'].present] + agent_mode_flags = [ + FLAGS['mcperf_measure_connections'].present, + FLAGS['mcperf_measure_threads'].present, + FLAGS['mcperf_measure_qps'].present, + FLAGS['mcperf_measure_depth'].present, + ] error_message = ( - 'To enable agent mode, set ' - 'memcached_mcperf_num_client_vms > 1.') + 'To enable agent mode, set memcached_mcperf_num_client_vms > 1.' + ) if any(agent_mode_flags) and ( - FLAGS.memcached_mcperf_num_client_vms < AGENT_MODE_MIN_CLIENT_VMS): + FLAGS.memcached_mcperf_num_client_vms < AGENT_MODE_MIN_CLIENT_VMS + ): raise errors.Setup.InvalidFlagConfigurationError(error_message) - if _INCREMENTAL_LOAD.value and (len(FLAGS.mcperf_qps) != 1 or - int(FLAGS.mcperf_qps[0]) == 0): + if _INCREMENTAL_LOAD.value and ( + len(FLAGS.mcperf_qps) != 1 or int(FLAGS.mcperf_qps[0]) == 0 + ): raise errors.Setup.InvalidFlagConfigurationError( 'To use dynamic load, set inital target qps with --mcperf_qps ' - 'and incremental with --mcperf_incremental_load.') + 'and incremental with --mcperf_incremental_load.' + ) def YumInstall(vm): @@ -139,7 +147,7 @@ def GetMetadata(): 'keysize': FLAGS.mcperf_keysize, 'valuesize': FLAGS.mcperf_valuesize, 'records': FLAGS.mcperf_records, - 'ratio': FLAGS.mcperf_ratio + 'ratio': FLAGS.mcperf_ratio, } if FLAGS.mcperf_options: metadata['options'] = FLAGS.mcperf_options @@ -152,13 +160,18 @@ def BuildCmd(server_ip, server_port, num_instances, options): server_ips = [] for idx in range(num_instances): server_ips.append(f'--server={server_ip}:{server_port + idx}') - cmd = [ - 'ulimit -n 32768; ', MCPERF_BIN, - '--keysize=%s' % FLAGS.mcperf_keysize, - '--valuesize=%s' % FLAGS.mcperf_valuesize, - '--records=%s' % FLAGS.mcperf_records, - '--roundrobin' if len(server_ips) > 1 else '' - ] + server_ips + options + cmd = ( + [ + 'ulimit -n 32768; ', + MCPERF_BIN, + '--keysize=%s' % FLAGS.mcperf_keysize, + '--valuesize=%s' % FLAGS.mcperf_valuesize, + '--records=%s' % FLAGS.mcperf_records, + '--roundrobin' if len(server_ips) > 1 else '', + ] + + server_ips + + options + ) if FLAGS.mcperf_protocol == 'binary': cmd.append('--binary') return cmd @@ -167,9 +180,7 @@ def BuildCmd(server_ip, server_port, num_instances, options): def Load(client_vm, server_ip, server_port): """Preload the server with data.""" logging.info('Loading memcached server.') - cmd = BuildCmd( - server_ip, server_port, 1, - ['--loadonly']) + cmd = BuildCmd(server_ip, server_port, 1, ['--loadonly']) client_vm.RemoteCommand(' '.join(cmd)) @@ -178,11 +189,17 @@ def RestartAgent(vm, threads): # Kill existing mcperf agent threads vm.RemoteCommand('pkill -9 mcperf', ignore_failure=True) # Make sure have enough file descriptor for the agent process. - vm.RemoteCommand(' '.join([ - 'ulimit -n 32768; ', - 'nohup', MCPERF_BIN, - '--threads=%s' % threads, '--agentmode', '&> log', '&' - ])) + vm.RemoteCommand( + ' '.join([ + 'ulimit -n 32768; ', + 'nohup', + MCPERF_BIN, + '--threads=%s' % threads, + '--agentmode', + '&> log', + '&', + ]) + ) def Run(vms, server_ip, server_port, num_instances): @@ -197,17 +214,16 @@ def Run(vms, server_ip, server_port, num_instances): if FLAGS.mcperf_measure_connections: runtime_options['measure_connections'] = FLAGS.mcperf_measure_connections measure_flags.append( - '--measure_connections=%s' % FLAGS.mcperf_measure_connections) + '--measure_connections=%s' % FLAGS.mcperf_measure_connections + ) if FLAGS.mcperf_measure_threads: runtime_options['measure_threads'] = FLAGS.mcperf_measure_threads if FLAGS.mcperf_measure_qps: runtime_options['measure_qps'] = FLAGS.mcperf_measure_qps - measure_flags.append( - '--measure_qps=%s' % FLAGS.mcperf_measure_qps) + measure_flags.append('--measure_qps=%s' % FLAGS.mcperf_measure_qps) if FLAGS.mcperf_measure_depth: runtime_options['measure_depth'] = FLAGS.mcperf_measure_depth - measure_flags.append( - '--measure_depth=%s' % FLAGS.mcperf_measure_depth) + measure_flags.append('--measure_depth=%s' % FLAGS.mcperf_measure_depth) for thread_count in FLAGS.mcperf_threads: runtime_options['threads'] = thread_count @@ -223,20 +239,31 @@ def Run(vms, server_ip, server_port, num_instances): target_qps = int(target_qps_list[0]) runtime_options['qps'] = target_qps or 'peak' remote_agents = ['--agent=%s' % vm.internal_ip for vm in vms[1:]] - cmd = BuildCmd(server_ip, server_port, num_instances, [ - '--noload', - '--qps=%s' % target_qps, - '--time=%s' % FLAGS.mcperf_time, - '--update=%s' % FLAGS.mcperf_ratio, - '--threads=%s' % (FLAGS.mcperf_measure_threads or thread_count), - '--connections=%s' % connection_count, - '--depth=%s' % depth, - ] + remote_agents + measure_flags + additional_flags) + cmd = BuildCmd( + server_ip, + server_port, + num_instances, + [ + '--noload', + '--qps=%s' % target_qps, + '--time=%s' % FLAGS.mcperf_time, + '--update=%s' % FLAGS.mcperf_ratio, + '--threads=%s' + % (FLAGS.mcperf_measure_threads or thread_count), + '--connections=%s' % connection_count, + '--depth=%s' % depth, + ] + + remote_agents + + measure_flags + + additional_flags, + ) try: stdout, _, retcode = master.RemoteHostCommandWithReturnCode( - ' '.join(cmd), timeout=FLAGS.mcperf_time * 2, - ignore_failure=True) + ' '.join(cmd), + timeout=FLAGS.mcperf_time * 2, + ignore_failure=True, + ) except errors.VmUtil.IssueCommandTimeoutError: break if retcode: @@ -246,10 +273,12 @@ def Run(vms, server_ip, server_port, num_instances): run_samples, actual_qps = ParseResults(stdout, metadata) samples.extend(run_samples) - if _INCREMENTAL_LOAD.value and (actual_qps / target_qps > - (1 - _INCREMENTAL_LOAD.value * 2)): + if _INCREMENTAL_LOAD.value and ( + actual_qps / target_qps > (1 - _INCREMENTAL_LOAD.value * 2) + ): target_qps_list.append( - int(target_qps) * (1 + _INCREMENTAL_LOAD.value)) + int(target_qps) * (1 + _INCREMENTAL_LOAD.value) + ) target_qps_list.pop(0) if not target_qps_list: break @@ -303,18 +332,19 @@ def ParseResults(result, metadata): for idx, stat in enumerate(latency_stats): if idx == len(latency_values): logging.warning( - 'Mutilate does not report %s latency for %s.', stat, metric) + 'Mutilate does not report %s latency for %s.', stat, metric + ) break samples.append( - sample.Sample(metric + '_' + stat, - float(latency_values[idx]), - 'usec', metadata)) + sample.Sample( + metric + '_' + stat, float(latency_values[idx]), 'usec', metadata + ) + ) # parse bandwidth for metric in ('TX', 'RX'): bw_regex = metric + BANDWIDTH_REGEX bw = regex_util.ExtractGroup(bw_regex, result) - samples.append( - sample.Sample(metric, float(bw), 'MB/s', metadata)) + samples.append(sample.Sample(metric, float(bw), 'MB/s', metadata)) qps = regex_util.ExtractFloat(QPS_REGEX, result) samples.append(sample.Sample('qps', qps, 'ops/s', metadata)) diff --git a/perfkitbenchmarker/linux_packages/memcached_server.py b/perfkitbenchmarker/linux_packages/memcached_server.py index aec34f3089..6993be4a54 100644 --- a/perfkitbenchmarker/linux_packages/memcached_server.py +++ b/perfkitbenchmarker/linux_packages/memcached_server.py @@ -25,13 +25,12 @@ MEMCACHED_PORT = 11211 -flags.DEFINE_integer('memcached_size_mb', 64, - 'Size of memcached cache in megabytes.') +flags.DEFINE_integer( + 'memcached_size_mb', 64, 'Size of memcached cache in megabytes.' +) -flags.DEFINE_integer('memcached_num_threads', 4, - 'Number of worker threads.') -flags.DEFINE_string('memcached_version', '1.6.9', - 'Memcached version to use.') +flags.DEFINE_integer('memcached_num_threads', 4, 'Number of worker threads.') +flags.DEFINE_string('memcached_version', '1.6.9', 'Memcached version to use.') DIR = linux_packages.INSTALL_DIR @@ -47,7 +46,8 @@ def _Install(vm): f'memcached-{FLAGS.memcached_version}.tar.gz --no-check-certificate; ' f'tar -zxvf memcached-{FLAGS.memcached_version}.tar.gz; ' f'cd memcached-{FLAGS.memcached_version}; ' - './configure && make && sudo make install') + './configure && make && sudo make install' + ) def YumInstall(vm): @@ -60,8 +60,11 @@ def AptInstall(vm): _Install(vm) -@vm_util.Retry(poll_interval=5, timeout=300, - retryable_exceptions=(errors.Resource.RetryableCreationError)) +@vm_util.Retry( + poll_interval=5, + timeout=300, + retryable_exceptions=(errors.Resource.RetryableCreationError), +) def _WaitForServerUp(vm, port=MEMCACHED_PORT): """Block until the memcached server is up and responsive. @@ -86,16 +89,19 @@ def _WaitForServerUp(vm, port=MEMCACHED_PORT): logging.info('Trying to connect to memcached at %s:%s', address, port) try: out, _ = vm.RemoteCommand( - '(echo -e "stats\n")| netcat -q 1 %s %s' % (address, port)) + '(echo -e "stats\n")| netcat -q 1 %s %s' % (address, port) + ) if out.startswith('STAT '): logging.info('memcached server stats received. Server up and running.') return except errors.VirtualMachine.RemoteCommandError as e: raise errors.Resource.RetryableCreationError( - 'memcached server not up yet: %s.' % str(e)) + 'memcached server not up yet: %s.' % str(e) + ) else: raise errors.Resource.RetryableCreationError( - 'memcached server not up yet. Expected "STAT" but got "%s".' % out) + 'memcached server not up yet. Expected "STAT" but got "%s".' % out + ) def ConfigureAndStart(vm, port=MEMCACHED_PORT, smp_affinity=False): @@ -124,7 +130,8 @@ def ConfigureAndStart(vm, port=MEMCACHED_PORT, smp_affinity=False): # update memory size f'-m {FLAGS.memcached_size_mb} ' # update security config to allow incoming network - '-l 0.0.0.0 -v &> log &') + '-l 0.0.0.0 -v &> log &' + ) _WaitForServerUp(vm, port) logging.info('memcached server configured and started.') @@ -132,8 +139,9 @@ def ConfigureAndStart(vm, port=MEMCACHED_PORT, smp_affinity=False): def GetVersion(vm): """Returns the version of the memcached server installed.""" - results, _ = vm.RemoteCommand('memcached -help |grep -m 1 "memcached"' - '| tr -d "\n"') + results, _ = vm.RemoteCommand( + 'memcached -help |grep -m 1 "memcached"| tr -d "\n"' + ) return results @@ -143,7 +151,8 @@ def StopMemcached(vm): def FlushMemcachedServer(ip, port): vm_util.IssueCommand( - ['(echo -e "flush_all\n" ; sleep 1)| netcat %s %s' % (ip, port)]) + ['(echo -e "flush_all\n" ; sleep 1)| netcat %s %s' % (ip, port)] + ) def AptUninstall(vm): diff --git a/perfkitbenchmarker/linux_packages/memtier.py b/perfkitbenchmarker/linux_packages/memtier.py index ef18b40542..0389f9d4f7 100644 --- a/perfkitbenchmarker/linux_packages/memtier.py +++ b/perfkitbenchmarker/linux_packages/memtier.py @@ -1114,9 +1114,7 @@ def _Run( pipeline, ) - file_name_suffix = '_'.join( - filter(None, [str(server_port), unique_id]) - ) + file_name_suffix = '_'.join(filter(None, [str(server_port), unique_id])) memtier_results_file_name = ( '_'.join([MEMTIER_RESULTS, file_name_suffix]) + '.log' ) diff --git a/perfkitbenchmarker/linux_packages/mkl.py b/perfkitbenchmarker/linux_packages/mkl.py index c775db545a..a01a90fd9c 100644 --- a/perfkitbenchmarker/linux_packages/mkl.py +++ b/perfkitbenchmarker/linux_packages/mkl.py @@ -31,16 +31,19 @@ # Default installs MKL as it was previously done via preprovisioned data _USE_MKL_REPO = flags.DEFINE_bool( - 'mkl_install_from_repo', False, + 'mkl_install_from_repo', + False, 'Whether to install MKL from the Intel repo. Default is to use ' 'preprovisioned data. Passing in True uses the Intel MPI repo or the ' - 'Intel oneAPI repo if --mkl_version is 2021 or later.') + 'Intel oneAPI repo if --mkl_version is 2021 or later.', +) # File contains MKL specific environment variables _MKL_VARS_FILE = '/opt/intel/mkl/bin/mklvars.sh' -MKL_VERSION = flags.DEFINE_string('mkl_version', _MKL_VERSION_REPO, - 'Version of Intel MKL to use') +MKL_VERSION = flags.DEFINE_string( + 'mkl_version', _MKL_VERSION_REPO, 'Version of Intel MKL to use' +) FLAGS = flags.FLAGS @@ -63,8 +66,10 @@ def UseMklRepo(): if FLAGS['mkl_install_from_repo'].present: if not _USE_MKL_REPO.value and MKL_VERSION.value != MKL_VERSION.default: # Need to check as caller expects to use the MKL version they specified - raise ValueError('To use preprovisioned data do not change ' - f'--mkl_version={MKL_VERSION.default}') + raise ValueError( + 'To use preprovisioned data do not change ' + f'--mkl_version={MKL_VERSION.default}' + ) return _USE_MKL_REPO.value return MKL_VERSION.value != MKL_VERSION.default @@ -93,10 +98,14 @@ def Install(vm): def _LogEnvVariables(vm): """Logs the MKL associated environment variables.""" env_vars = [] - for env_var in ('CPATH', 'LD_LIBRARY_PATH', 'MKLROOT', 'NLSPATH', - 'PKG_CONFIG_PATH'): - txt, _ = vm.RemoteCommand( - f'{SourceVarsCommand()}; echo ${env_var}') + for env_var in ( + 'CPATH', + 'LD_LIBRARY_PATH', + 'MKLROOT', + 'NLSPATH', + 'PKG_CONFIG_PATH', + ): + txt, _ = vm.RemoteCommand(f'{SourceVarsCommand()}; echo ${env_var}') env_vars.append(f'{env_var}={txt.strip()}') logging.info('MKL environment variables: %s', ' '.join(env_vars)) @@ -107,21 +116,27 @@ def _InstallFromPreprovisionedData(vm): vm.InstallPreprovisionedBenchmarkData(BENCHMARK_NAME, [MKL_TGZ], MKL_DIR) vm.RemoteCommand('cd {0} && tar zxvf {1}'.format(MKL_DIR, MKL_TGZ)) vm.RemoteCommand( - ('cd {0}/{1} && ' - 'sed -i "s/decline/accept/g" silent.cfg && ' - 'sudo ./install.sh --silent ./silent.cfg').format(MKL_DIR, MKL_TAG)) - vm.RemoteCommand('sudo chmod 755 /etc/bash.bashrc && ' - 'sudo chown ' + vm.user_name + ' /etc/bash.bashrc && ' - 'echo "source /opt/intel/mkl/bin/mklvars.sh intel64" ' - '>>/etc/bash.bashrc && ' - 'echo "export PATH=/opt/intel/bin:$PATH" ' - '>>/etc/bash.bashrc && ' - 'echo "export LD_LIBRARY_PATH=/opt/intel/lib/intel64:' - '/opt/intel/mkl/lib/intel64:$LD_LIBRARY_PATH" ' - '>>/etc/bash.bashrc && ' - 'echo "source /opt/intel/compilers_and_libraries/linux/bin/' - 'compilervars.sh -arch intel64 -platform linux" ' - '>>/etc/bash.bashrc') + ( + 'cd {0}/{1} && ' + 'sed -i "s/decline/accept/g" silent.cfg && ' + 'sudo ./install.sh --silent ./silent.cfg' + ).format(MKL_DIR, MKL_TAG) + ) + vm.RemoteCommand( + 'sudo chmod 755 /etc/bash.bashrc && sudo chown ' + + vm.user_name + + ' /etc/bash.bashrc && ' + 'echo "source /opt/intel/mkl/bin/mklvars.sh intel64" ' + '>>/etc/bash.bashrc && ' + 'echo "export PATH=/opt/intel/bin:$PATH" ' + '>>/etc/bash.bashrc && ' + 'echo "export LD_LIBRARY_PATH=/opt/intel/lib/intel64:' + '/opt/intel/mkl/lib/intel64:$LD_LIBRARY_PATH" ' + '>>/etc/bash.bashrc && ' + 'echo "source /opt/intel/compilers_and_libraries/linux/bin/' + 'compilervars.sh -arch intel64 -platform linux" ' + '>>/etc/bash.bashrc' + ) def _CompileInterfaces(vm): @@ -132,11 +147,12 @@ def _CompileInterfaces(vm): """ vm.Install('build_tools') mpi_lib = 'openmpi' - make_options = ('PRECISION=MKL_DOUBLE ' - 'interface=ilp64 ' - f'mpi={mpi_lib} ' - 'compiler=gnu') + make_options = ( + f'PRECISION=MKL_DOUBLE interface=ilp64 mpi={mpi_lib} compiler=gnu' + ) for interface in ('fftw2xc', 'fftw2xf', 'fftw3xc', 'fftw3xf'): - cmd = (f'cd /opt/intel/mkl/interfaces/{interface} && ' - f'sudo make libintel64 {make_options}') + cmd = ( + f'cd /opt/intel/mkl/interfaces/{interface} && ' + f'sudo make libintel64 {make_options}' + ) vm.RemoteCommand(cmd) diff --git a/perfkitbenchmarker/linux_packages/mofed.py b/perfkitbenchmarker/linux_packages/mofed.py index f6af4711ab..809cb3b0bd 100644 --- a/perfkitbenchmarker/linux_packages/mofed.py +++ b/perfkitbenchmarker/linux_packages/mofed.py @@ -34,30 +34,40 @@ } # Mellanox OpenFabrics drivers -MLNX_OFED_DOWNLOAD_URL = ('https://www.mellanox.com/downloads/ofed/MLNX_OFED-' - '{version}/MLNX_OFED_LINUX-{version}-{os}-x86_64.tgz') +MLNX_OFED_DOWNLOAD_URL = ( + 'https://www.mellanox.com/downloads/ofed/MLNX_OFED-' + '{version}/MLNX_OFED_LINUX-{version}-{os}-x86_64.tgz' +) def _Install(vm): """Installs the Mellanox OpenFabrics driver on the VM.""" if vm.OS_TYPE not in MOFED_OS_MAPPING: - raise ValueError('OS type {} not in {}'.format(vm.OS_TYPE, - sorted(MOFED_OS_MAPPING))) - driver = MLNX_OFED_DOWNLOAD_URL.format(version=FLAGS.mofed_version, - os=MOFED_OS_MAPPING[vm.OS_TYPE]) + raise ValueError( + 'OS type {} not in {}'.format(vm.OS_TYPE, sorted(MOFED_OS_MAPPING)) + ) + driver = MLNX_OFED_DOWNLOAD_URL.format( + version=FLAGS.mofed_version, os=MOFED_OS_MAPPING[vm.OS_TYPE] + ) vm.InstallPackages('libdapl2 libmlx4-1') vm.RemoteCommand(f'wget --retry-connrefused --tries=3 --waitretry=5 {driver}') vm.RemoteCommand('tar zxvf MLNX_OFED_LINUX-*-x86_64.tgz') - stdout, _ = vm.RemoteCommand('cd MLNX_OFED_LINUX-*-x86_64 && sudo ' - './mlnxofedinstall --force --skip-repo') + stdout, _ = vm.RemoteCommand( + 'cd MLNX_OFED_LINUX-*-x86_64 && sudo ' + './mlnxofedinstall --force --skip-repo' + ) if 'Installation passed successfully' not in stdout: raise errors.Benchmarks.PrepareException( - 'Mellanox OpenFabrics driver isn\'t installed successfully.') + "Mellanox OpenFabrics driver isn't installed successfully." + ) vm.RemoteCommand('sudo /etc/init.d/openibd restart') - vm.RemoteCommand("sudo sed -i -e 's/# OS.EnableRDMA=y/" - "OS.EnableRDMA=y/g' /etc/waagent.conf") - vm.RemoteCommand("sudo sed -i -e 's/# OS.UpdateRdmaDriver=y/" - "OS.UpdateRdmaDriver=y/g' /etc/waagent.conf") + vm.RemoteCommand( + "sudo sed -i -e 's/# OS.EnableRDMA=y/OS.EnableRDMA=y/g' /etc/waagent.conf" + ) + vm.RemoteCommand( + "sudo sed -i -e 's/# OS.UpdateRdmaDriver=y/" + "OS.UpdateRdmaDriver=y/g' /etc/waagent.conf" + ) vm.Reboot() # Check IB status. stdout, _ = vm.RemoteCommand('sudo ibdev2netdev -v') diff --git a/perfkitbenchmarker/linux_packages/mongodb_server.py b/perfkitbenchmarker/linux_packages/mongodb_server.py index a3dcdf74d6..48fcaf4a19 100644 --- a/perfkitbenchmarker/linux_packages/mongodb_server.py +++ b/perfkitbenchmarker/linux_packages/mongodb_server.py @@ -29,22 +29,26 @@ def _GetConfigPath(): def _Setup(vm): """Setup mongodb.""" vm.RemoteCommand( - 'sudo sed -i "s|bindIp|# bindIp|" {}'.format(_GetConfigPath())) + 'sudo sed -i "s|bindIp|# bindIp|" {}'.format(_GetConfigPath()) + ) def YumInstall(vm): """Installs the mongodb package on the VM.""" vm.RemoteCommand('sudo setenforce 0') releasever, _ = vm.RemoteCommand( - 'distro=$(sed -n \'s/^distroverpkg=//p\' /etc/yum.conf);' - 'echo $(rpm -q --qf "%{version}" -f /etc/$distro)') + "distro=$(sed -n 's/^distroverpkg=//p' /etc/yum.conf);" + 'echo $(rpm -q --qf "%{version}" -f /etc/$distro)' + ) mongodb_repo = ( '[mongodb-org-3.0]\nname=MongoDB Repository\nbaseurl=' 'https://repo.mongodb.org/yum/redhat/{0}/mongodb-org/3.0/x86_64/' - '\ngpgcheck=0\nenabled=1').format(releasever.strip()) + '\ngpgcheck=0\nenabled=1' + ).format(releasever.strip()) vm.RemoteCommand( - 'echo "%s" | sudo tee /etc/yum.repos.d/mongodb-org-3.0.repo' % - mongodb_repo) + 'echo "%s" | sudo tee /etc/yum.repos.d/mongodb-org-3.0.repo' + % mongodb_repo + ) vm.InstallPackages('mongodb-org') _Setup(vm) @@ -53,11 +57,13 @@ def AptInstall(vm): """Installs the mongodb package on the VM.""" vm.RemoteCommand( 'wget -qO - https://www.mongodb.org/static/pgp/server-3.0.asc' - ' | sudo apt-key add -') + ' | sudo apt-key add -' + ) vm.RemoteCommand( 'echo "deb https://repo.mongodb.org/apt/ubuntu ' '$(lsb_release -c -s)/mongodb-org/3.0 multiverse" | ' - 'sudo tee /etc/apt/sources.list.d/mongodb-org-3.0.list') + 'sudo tee /etc/apt/sources.list.d/mongodb-org-3.0.list' + ) vm.AptUpdate() vm.RemoteCommand('sudo apt-get install mongodb-org -y --force-yes') _Setup(vm) diff --git a/perfkitbenchmarker/linux_packages/mpi.py b/perfkitbenchmarker/linux_packages/mpi.py index 79677ff360..0d5b9d78d7 100644 --- a/perfkitbenchmarker/linux_packages/mpi.py +++ b/perfkitbenchmarker/linux_packages/mpi.py @@ -41,6 +41,7 @@ class MpiData: "time_min":186.85 } """ + bytes: Optional[int] = None repetitions: Optional[int] = None data: Optional[Dict[str, Union[int, float]]] = None @@ -54,6 +55,7 @@ class MpiResult: For example this could be the PingPong run results. """ + benchmark: str data: List[MpiData] groups: Optional[int] = None @@ -65,6 +67,7 @@ class MpiResult: @dataclasses.dataclass class MpiResponse: """Response to the RunMpiStats call.""" + mpi_run: str args: str vendor: str @@ -80,6 +83,7 @@ class MpiRequest: See the FLAGS.mpi_XXX definitions in mpi_benchmark.py for the definitions. """ + # TODO(andytzhu): Get rid of Any, and handle importing linux_virtual_machine # without encountering circular dependencies vms: List[Any] # virtual machine @@ -105,15 +109,25 @@ class MpiRequest: # The same order as the output in the print_tail function in the patched code LATENCY_HEADERS: List[str] = [ - 'latency_min', 'latency_p10', 'latency_p25', 'latency_p50', 'latency_p75', - 'latency_p90', 'latency_p95', 'latency_p99', 'latency_p99.5', - 'latency_p99.9', 'latency_p99.99', 'latency_max' + 'latency_min', + 'latency_p10', + 'latency_p25', + 'latency_p50', + 'latency_p75', + 'latency_p90', + 'latency_p95', + 'latency_p99', + 'latency_p99.5', + 'latency_p99.9', + 'latency_p99.99', + 'latency_max', ] # Regexs for parsing I_MPI_DEBUG=4 and higher output _MPI_STARTUP_PREFIX = r'^\[(?P\d+)\] MPI startup\(\):\s+' -_MPI_ENV_RE = re.compile(_MPI_STARTUP_PREFIX + - r'(?PI_MPI.*?)=(?P.*)') +_MPI_ENV_RE = re.compile( + _MPI_STARTUP_PREFIX + r'(?PI_MPI.*?)=(?P.*)' +) def Install(vm) -> None: @@ -146,7 +160,8 @@ def VerifyInstall(vms) -> None: include_zero_byte=False, compile_from_source=True, record_latencies=False, - multi=True) + multi=True, + ) RunMpiStats(vms[0], request) @@ -174,16 +189,24 @@ def RunMpiStats(vm, request: MpiRequest) -> MpiResponse: """ hosts = [vm.internal_ip for vm in request.vms] - mpirun = imb.MpiRunCommand(vm, hosts, request.total_processes, request.ppn, - request.environment, request.global_environment, - request.tune) + mpirun = imb.MpiRunCommand( + vm, + hosts, + request.total_processes, + request.ppn, + request.environment, + request.global_environment, + request.tune, + ) if request.record_latencies: - latency_file = '/tmp/latency-{}-{}.txt'.format(request.tests[0], - uuid.uuid4().hex[:8]) + latency_file = '/tmp/latency-{}-{}.txt'.format( + request.tests[0], uuid.uuid4().hex[:8] + ) else: latency_file = None common = ' '.join( - BuildMpiBenchmarkArgs(request, latency_file, bool(request.ppn))) + BuildMpiBenchmarkArgs(request, latency_file, bool(request.ppn)) + ) try: stdout, stderr = vm.RobustRemoteCommand(mpirun + ' ' + common) except errors.VirtualMachine.RemoteCommandError: @@ -191,7 +214,8 @@ def RunMpiStats(vm, request: MpiRequest) -> MpiResponse: for client_vm in request.vms: logging.info('VM syslog for %s', client_vm.name) client_vm.RemoteCommand( - 'sudo tail -n 100 /var/log/syslog /var/log/messages || exit') + 'sudo tail -n 100 /var/log/syslog /var/log/messages || exit' + ) raise if stderr: # SSH displays a warning but this could also contain mpirun errors @@ -209,7 +233,8 @@ def RunMpiStats(vm, request: MpiRequest) -> MpiResponse: version=GetMpiVersion(vm), results=results, mpi_pinning=omb.ParseMpiPinning(lines), - mpi_env=ParseMpiEnv(lines)) + mpi_env=ParseMpiEnv(lines), + ) class MpiResultParser(Iterable[MpiResult]): @@ -217,9 +242,11 @@ class MpiResultParser(Iterable[MpiResult]): This is an iterator where each next item is an MpiResult. """ + _NAME = re.compile('^# Benchmarking (.*)') _GROUP1 = re.compile( - r'.*?(?P\d+) groups of (?P\d+) processes') + r'.*?(?P\d+) groups of (?P\d+) processes' + ) _GROUP2 = re.compile(r'# #processes = (?P\d+)') _GROUP_LAYOUT = re.compile(r'# Group\s+(\d+):\s+(.*)') _GROUP_LAYOUT_FOLLOWON = re.compile(r'#\s+(\d+[\s\d]*)') @@ -375,8 +402,9 @@ def _Data(self, headers: Sequence[str]) -> Iterator[MpiData]: ] percentiles: Dict[str, float] = dict(zip(LATENCY_HEADERS, values)) if not on_deck: - logging.warning('Have percentiles but no previous mpidata %s', - percentiles) + logging.warning( + 'Have percentiles but no previous mpidata %s', percentiles + ) continue if sum(values) == 0.0: # only tests that have been patched have the percentile metrics @@ -393,13 +421,15 @@ def _Data(self, headers: Sequence[str]) -> Iterator[MpiData]: number_bytes = data.pop('bytes', 0) repetitions = data.pop('repetitions', -1) on_deck = MpiData( - bytes=number_bytes, repetitions=repetitions, data=data) + bytes=number_bytes, repetitions=repetitions, data=data + ) if on_deck: # Last record in this stanza was a normal MPI row. yield on_deck - def _DataIntoMap(self, headers: Sequence[str], - data: Sequence[str]) -> Dict[str, Union[int, float]]: + def _DataIntoMap( + self, headers: Sequence[str], data: Sequence[str] + ) -> Dict[str, Union[int, float]]: """Converts the a row of data from the MPI results into a dict. Args: @@ -419,8 +449,9 @@ def _ConvertValue(self, header: str, value: str) -> Union[int, float]: return int(value) if header in self._INT_COLUMNS else float(value) -def BuildMpiBenchmarkArgs(request: MpiRequest, latency_file: Optional[str], - ppn_set: bool) -> List[str]: +def BuildMpiBenchmarkArgs( + request: MpiRequest, latency_file: Optional[str], ppn_set: bool +) -> List[str]: """Creates the common arguments to pass to mpirun. See https://software.intel.com/en-us/imb-user-guide-command-line-control @@ -440,8 +471,10 @@ def BuildMpiBenchmarkArgs(request: MpiRequest, latency_file: Optional[str], else: args.append(request.suite) # only add -msglog if told to do so - if request.suite in ('IMB-MPI1', 'IMB-RMA', - 'IMB-NBC') and request.msglog_max is not None: + if ( + request.suite in ('IMB-MPI1', 'IMB-RMA', 'IMB-NBC') + and request.msglog_max is not None + ): if request.msglog_min is None: arg = request.msglog_max else: @@ -478,8 +511,9 @@ def BuildMpiBenchmarkArgs(request: MpiRequest, latency_file: Optional[str], return args -def _CreateMpiDataForHistogram(grouped_lines: List[List[str]], - results: List[MpiResult]) -> None: +def _CreateMpiDataForHistogram( + grouped_lines: List[List[str]], results: List[MpiResult] +) -> None: """Adds histogram data from the histogram file to existing data. The MPI parsed results are passed in as some benchmarks runs can do many @@ -494,7 +528,8 @@ def _CreateMpiDataForHistogram(grouped_lines: List[List[str]], # MPI runs that time out should not have histogram data associated with it. for result in results: acceptable_mpi_data.extend( - mpi_data for mpi_data in result.data if not mpi_data.is_error) + mpi_data for mpi_data in result.data if not mpi_data.is_error + ) histograms: List[MpiData] = [] for lines in grouped_lines: histograms.extend(_CombineHistogramEntries(lines)) @@ -503,8 +538,9 @@ def _CreateMpiDataForHistogram(grouped_lines: List[List[str]], mpi_data.histogram = histogram.histogram -def _MpiHistogramAcceptable(mpi_data: List[MpiData], - histograms: List[MpiData]) -> bool: +def _MpiHistogramAcceptable( + mpi_data: List[MpiData], histograms: List[MpiData] +) -> bool: """Returns whether the parsed MpiResults MpiData matches with the histograms. Criteria: @@ -517,15 +553,21 @@ def _MpiHistogramAcceptable(mpi_data: List[MpiData], histograms: List of MpiData histograms parsed for this run. """ if len(mpi_data) != len(histograms): - logging.warning('Have %s parsed MPI data but only %s histograms', - len(mpi_data), len(histograms)) + logging.warning( + 'Have %s parsed MPI data but only %s histograms', + len(mpi_data), + len(histograms), + ) return False for mpi_data, histogram in zip(mpi_data, histograms): bytes_same = mpi_data.bytes == histogram.bytes repetitions_same = mpi_data.repetitions == histogram.repetitions if not bytes_same or not repetitions_same: - logging.warning('Parsed MPI data %s does not match with histogram %s', - mpi_data, histogram) + logging.warning( + 'Parsed MPI data %s does not match with histogram %s', + mpi_data, + histogram, + ) return False return True @@ -580,11 +622,13 @@ def _CombineHistogramEntries(lines: Iterable[str]) -> Iterator[MpiData]: yield MpiData( bytes=number_bytes, histogram=dict(histogram), - repetitions=sum(histogram.values())) + repetitions=sum(histogram.values()), + ) -def _GroupLatencyLines(vm, latency_file: str, - packets_per_run: int) -> List[List[str]]: +def _GroupLatencyLines( + vm, latency_file: str, packets_per_run: int +) -> List[List[str]]: r"""Parses the histogram latency file copied from the remote VM. The latency file contains multiple sub-runs concatenated together. Each of @@ -602,23 +646,29 @@ def _GroupLatencyLines(vm, latency_file: str, List of lists of strings of length packets_per_run or an empty list if there is a problem dividing up the lines into groups. """ - local_file: str = os.path.join(temp_dir.GetRunDirPath(), - os.path.basename(latency_file)) + local_file: str = os.path.join( + temp_dir.GetRunDirPath(), os.path.basename(latency_file) + ) if vm.TryRemoteCommand(f'test -f {latency_file}'): vm.PullFile(local_file, latency_file) else: - logging.warning('Skipping gathering latency as %s file missing', - latency_file) + logging.warning( + 'Skipping gathering latency as %s file missing', latency_file + ) return [] with open(local_file) as reader: lines = [line.strip() for line in reader.readlines()] number_groups = len(lines) // packets_per_run if packets_per_run * number_groups != len(lines): - logging.warning('File %s has %s lines, cannot be divided into size %s', - local_file, len(lines), packets_per_run) + logging.warning( + 'File %s has %s lines, cannot be divided into size %s', + local_file, + len(lines), + packets_per_run, + ) return [] return [ - lines[i:i + packets_per_run] + lines[i : i + packets_per_run] for i in range(0, len(lines), packets_per_run) ] diff --git a/perfkitbenchmarker/linux_packages/mssql2019.py b/perfkitbenchmarker/linux_packages/mssql2019.py index 5bd17deb99..faa7c0c65d 100644 --- a/perfkitbenchmarker/linux_packages/mssql2019.py +++ b/perfkitbenchmarker/linux_packages/mssql2019.py @@ -21,19 +21,24 @@ def YumInstall(vm): """Installs the ms sql package on the RedHat VM.""" - vm.RemoteCommand('sudo curl -o /etc/yum.repos.d/mssql-server.repo ' - 'https://packages.microsoft.com/config/rhel/8/' - 'mssql-server-2019.repo') - vm.RemoteCommand('sudo curl -o /etc/yum.repos.d/msprod.repo ' - 'https://packages.microsoft.com/config/rhel/8/prod.repo') + vm.RemoteCommand( + 'sudo curl -o /etc/yum.repos.d/mssql-server.repo ' + 'https://packages.microsoft.com/config/rhel/8/' + 'mssql-server-2019.repo' + ) + vm.RemoteCommand( + 'sudo curl -o /etc/yum.repos.d/msprod.repo ' + 'https://packages.microsoft.com/config/rhel/8/prod.repo' + ) vm.InstallPackages('mssql-server') vm.RemoteCommand('sudo yum remove unixODBC-utf16 unixODBC-utf16-devel') - vm.RemoteCommand('sudo ACCEPT_EULA=Y yum install -y ' - 'mssql-tools unixODBC-devel') vm.RemoteCommand( - r'echo PATH="$PATH:/opt/mssql-tools/bin" >> ~/.bash_profile') + 'sudo ACCEPT_EULA=Y yum install -y mssql-tools unixODBC-devel' + ) + vm.RemoteCommand(r'echo PATH="$PATH:/opt/mssql-tools/bin" >> ~/.bash_profile') vm.RemoteCommand( - r'echo export PATH="$PATH:/opt/mssql-tools/bin" >> ~/.bashrc') + r'echo export PATH="$PATH:/opt/mssql-tools/bin" >> ~/.bashrc' + ) vm.RemoteCommand('source ~/.bashrc') diff --git a/perfkitbenchmarker/linux_packages/mssql2022.py b/perfkitbenchmarker/linux_packages/mssql2022.py index 708863adc0..cfd407087b 100644 --- a/perfkitbenchmarker/linux_packages/mssql2022.py +++ b/perfkitbenchmarker/linux_packages/mssql2022.py @@ -21,32 +21,42 @@ def YumInstall(vm): """Installs the ms sql package on the RedHat VM.""" - vm.RemoteCommand('sudo curl -o /etc/yum.repos.d/mssql-server.repo ' - 'https://packages.microsoft.com/config/rhel/8/' - 'mssql-server-2022.repo') - vm.RemoteCommand('sudo curl -o /etc/yum.repos.d/msprod.repo ' - 'https://packages.microsoft.com/config/rhel/8/prod.repo') + vm.RemoteCommand( + 'sudo curl -o /etc/yum.repos.d/mssql-server.repo ' + 'https://packages.microsoft.com/config/rhel/8/' + 'mssql-server-2022.repo' + ) + vm.RemoteCommand( + 'sudo curl -o /etc/yum.repos.d/msprod.repo ' + 'https://packages.microsoft.com/config/rhel/8/prod.repo' + ) vm.InstallPackages('mssql-server') - vm.RemoteCommand('sudo ACCEPT_EULA=Y yum install -y ' - 'mssql-tools unixODBC-devel') vm.RemoteCommand( - r'echo PATH="$PATH:/opt/mssql-tools/bin" >> ~/.bash_profile') + 'sudo ACCEPT_EULA=Y yum install -y mssql-tools unixODBC-devel' + ) + vm.RemoteCommand(r'echo PATH="$PATH:/opt/mssql-tools/bin" >> ~/.bash_profile') vm.RemoteCommand( - r'echo export PATH="$PATH:/opt/mssql-tools/bin" >> ~/.bashrc') + r'echo export PATH="$PATH:/opt/mssql-tools/bin" >> ~/.bashrc' + ) vm.RemoteCommand('source ~/.bashrc') - vm.RemoteCommand('sudo firewall-cmd --zone=public ' - '--add-port=1433/tcp --permanent') + vm.RemoteCommand( + 'sudo firewall-cmd --zone=public --add-port=1433/tcp --permanent' + ) vm.RemoteCommand('sudo firewall-cmd --reload') def AptInstall(vm): - vm.RemoteCommand('wget -qO- https://packages.microsoft.com/keys/microsoft.asc' - ' | sudo tee /etc/apt/trusted.gpg.d/microsoft.asc') - vm.RemoteCommand('sudo add-apt-repository "$(wget -qO- ' - 'https://packages.microsoft.com/config/ubuntu/20.04/' - 'mssql-server-2022.list)"') + vm.RemoteCommand( + 'wget -qO- https://packages.microsoft.com/keys/microsoft.asc' + ' | sudo tee /etc/apt/trusted.gpg.d/microsoft.asc' + ) + vm.RemoteCommand( + 'sudo add-apt-repository "$(wget -qO- ' + 'https://packages.microsoft.com/config/ubuntu/20.04/' + 'mssql-server-2022.list)"' + ) vm.RemoteCommand('sudo apt-get update') vm.InstallPackages('mssql-server') diff --git a/perfkitbenchmarker/linux_packages/mssql_tools.py b/perfkitbenchmarker/linux_packages/mssql_tools.py index b45393ea00..4ee0b6b176 100644 --- a/perfkitbenchmarker/linux_packages/mssql_tools.py +++ b/perfkitbenchmarker/linux_packages/mssql_tools.py @@ -34,24 +34,29 @@ os_types.UBUNTU1604: '16.04', os_types.UBUNTU1804: '18.04', os_types.UBUNTU2004: '20.04', - os_types.UBUNTU2204: '22.04' + os_types.UBUNTU2204: '22.04', } def YumInstall(vm): """Installs the ms sql package on the RedHat VM.""" - vm.RemoteCommand('sudo curl -o /etc/yum.repos.d/mssql-server.repo ' - 'https://packages.microsoft.com/config/rhel/8/' - 'mssql-server-2022.repo') - vm.RemoteCommand('sudo curl -o /etc/yum.repos.d/msprod.repo ' - 'https://packages.microsoft.com/config/rhel/8/prod.repo') + vm.RemoteCommand( + 'sudo curl -o /etc/yum.repos.d/mssql-server.repo ' + 'https://packages.microsoft.com/config/rhel/8/' + 'mssql-server-2022.repo' + ) + vm.RemoteCommand( + 'sudo curl -o /etc/yum.repos.d/msprod.repo ' + 'https://packages.microsoft.com/config/rhel/8/prod.repo' + ) - vm.RemoteCommand('sudo ACCEPT_EULA=Y yum install -y mssql-tools ' - 'unixODBC-devel') vm.RemoteCommand( - r'echo PATH="$PATH:/opt/mssql-tools/bin" >> ~/.bash_profile') + 'sudo ACCEPT_EULA=Y yum install -y mssql-tools unixODBC-devel' + ) + vm.RemoteCommand(r'echo PATH="$PATH:/opt/mssql-tools/bin" >> ~/.bash_profile') vm.RemoteCommand( - r'echo export PATH="$PATH:/opt/mssql-tools/bin" >> ~/.bashrc') + r'echo export PATH="$PATH:/opt/mssql-tools/bin" >> ~/.bashrc' + ) vm.RemoteCommand('source ~/.bashrc') @@ -62,16 +67,19 @@ def AptInstall(vm): vm: Virtual Machine to install on. """ vm.Install('unixodbc_dev') - vm.RemoteCommand( - 'curl {key} | sudo apt-key add -'.format(key=_DEB_REPO_KEY)) + vm.RemoteCommand('curl {key} | sudo apt-key add -'.format(key=_DEB_REPO_KEY)) deb_repro_file = _DEB_REPO_FILE.format(os=OS_TYPE_MAPPING[vm.OS_TYPE]) vm.RemoteCommand( - 'curl {file} | sudo tee {location}'.format(file=deb_repro_file, - location=_DEB_FILE_LOCATION)) + 'curl {file} | sudo tee {location}'.format( + file=deb_repro_file, location=_DEB_FILE_LOCATION + ) + ) vm.RemoteCommand('sudo apt-get update') vm.RemoteCommand('sudo ACCEPT_EULA=Y /usr/bin/apt-get -y install mssql-tools') vm.RemoteCommand( - r'sudo sed -i "1 i\export PATH=$PATH:/opt/mssql-tools/bin/" ~/.bashrc') + r'sudo sed -i "1 i\export PATH=$PATH:/opt/mssql-tools/bin/" ~/.bashrc' + ) vm.RemoteCommand( r'sudo sed -i "1 i\export PATH=$PATH:/opt/mssql-tools/bin/" ' - '/etc/bash.bashrc') + '/etc/bash.bashrc' + ) diff --git a/perfkitbenchmarker/linux_packages/multichase.py b/perfkitbenchmarker/linux_packages/multichase.py index 05bc9db024..278c97cf85 100644 --- a/perfkitbenchmarker/linux_packages/multichase.py +++ b/perfkitbenchmarker/linux_packages/multichase.py @@ -23,12 +23,17 @@ def _Install(vm): """Installs the multichase package on the VM.""" vm.Install('build_tools') - vm.RemoteCommand('rm -rf {path} && mkdir -p {path}'.format( - path=INSTALL_PATH)) - vm.RemoteCommand('git clone --recursive {git_path} {dir}'.format( - dir=INSTALL_PATH, git_path=GIT_PATH)) - vm.RemoteCommand('cd {dir} && git checkout {version} && make'.format( - dir=INSTALL_PATH, version=GIT_VERSION)) + vm.RemoteCommand('rm -rf {path} && mkdir -p {path}'.format(path=INSTALL_PATH)) + vm.RemoteCommand( + 'git clone --recursive {git_path} {dir}'.format( + dir=INSTALL_PATH, git_path=GIT_PATH + ) + ) + vm.RemoteCommand( + 'cd {dir} && git checkout {version} && make'.format( + dir=INSTALL_PATH, version=GIT_VERSION + ) + ) def YumInstall(vm): diff --git a/perfkitbenchmarker/linux_packages/mutilate.py b/perfkitbenchmarker/linux_packages/mutilate.py index 6c71df1397..46094fc440 100644 --- a/perfkitbenchmarker/linux_packages/mutilate.py +++ b/perfkitbenchmarker/linux_packages/mutilate.py @@ -33,31 +33,33 @@ flags.DEFINE_enum( 'mutilate_protocol', - 'binary', ['binary', 'ascii'], - 'Protocol to use. Supported protocols are binary and ascii.') + 'binary', + ['binary', 'ascii'], + 'Protocol to use. Supported protocols are binary and ascii.', +) flags.DEFINE_list( - 'mutilate_qps', [], - 'Target aggregate QPS. If not set, target for peak qps.') -flags.DEFINE_integer( - 'mutilate_time', 300, - 'Maximum time to run (seconds).') + 'mutilate_qps', [], 'Target aggregate QPS. If not set, target for peak qps.' +) +flags.DEFINE_integer('mutilate_time', 300, 'Maximum time to run (seconds).') flags.DEFINE_string( - 'mutilate_keysize', '16', - 'Length of memcached keys (distribution).') + 'mutilate_keysize', '16', 'Length of memcached keys (distribution).' +) flags.DEFINE_string( - 'mutilate_valuesize', '128', - 'Length of memcached values (distribution).') + 'mutilate_valuesize', '128', 'Length of memcached values (distribution).' +) flags.DEFINE_integer( - 'mutilate_records', 10000, - 'Number of memcached records to use.') + 'mutilate_records', 10000, 'Number of memcached records to use.' +) flags.DEFINE_float( - 'mutilate_ratio', 0.0, - 'Ratio of set:get. By default, read only.') + 'mutilate_ratio', 0.0, 'Ratio of set:get. By default, read only.' +) flags.DEFINE_list( - 'mutilate_options', ['iadist=exponential:0.0'], + 'mutilate_options', + ['iadist=exponential:0.0'], 'Additional mutilate long-form options (--) in comma separated form. e.g.' '--mutilate_options=blocking,search=99:1000.' - 'See https://github.com/leverich/mutilate for all available options.') + 'See https://github.com/leverich/mutilate for all available options.', +) # If more than one value provided for threads, connections, depths, we will # enumerate all test configurations. e.g. @@ -65,30 +67,33 @@ # We will test following threads:connections:depths: # 1,3,5; 1,3,6; 1,4,5; 1,4,6; 2,3,5; 2,3,6; 2,4,5; 2,4,6; flags.DEFINE_list( - 'mutilate_threads', ['1'], - 'Number of total client threads to spawn per client VM.') + 'mutilate_threads', + ['1'], + 'Number of total client threads to spawn per client VM.', +) flags.DEFINE_list( - 'mutilate_connections', ['1'], - 'Number of connections to establish per client thread.') + 'mutilate_connections', + ['1'], + 'Number of connections to establish per client thread.', +) flags.DEFINE_list( - 'mutilate_depths', ['1'], - 'Maximum depth to pipeline requests.') + 'mutilate_depths', ['1'], 'Maximum depth to pipeline requests.' +) # Agent mode options. flags.DEFINE_integer( - 'mutilate_measure_connections', None, - 'Master client connections.') -flags.DEFINE_integer( - 'mutilate_measure_threads', None, - 'Master client thread count.') + 'mutilate_measure_connections', None, 'Master client connections.' +) flags.DEFINE_integer( - 'mutilate_measure_qps', None, - 'Master client QPS.') + 'mutilate_measure_threads', None, 'Master client thread count.' +) +flags.DEFINE_integer('mutilate_measure_qps', None, 'Master client QPS.') flags.DEFINE_integer( - 'mutilate_measure_depth', None, - 'Master client connection depth.') + 'mutilate_measure_depth', None, 'Master client connection depth.' +) _INCREMENTAL_LOAD = flags.DEFINE_float( - 'mutilate_incremental_load', None, 'Increments target qps until hits peak.') + 'mutilate_incremental_load', None, 'Increments target qps until hits peak.' +) # To use remote agent mode, we need at least 2 VMs. AGENT_MODE_MIN_CLIENT_VMS = 2 @@ -99,22 +104,27 @@ def CheckPrerequisites(): Raises: errors.Setup.InvalidFlagConfigurationError: On invalid flag configurations. """ - agent_mode_flags = [FLAGS['mutilate_measure_connections'].present, - FLAGS['mutilate_measure_threads'].present, - FLAGS['mutilate_measure_qps'].present, - FLAGS['mutilate_measure_depth'].present] + agent_mode_flags = [ + FLAGS['mutilate_measure_connections'].present, + FLAGS['mutilate_measure_threads'].present, + FLAGS['mutilate_measure_qps'].present, + FLAGS['mutilate_measure_depth'].present, + ] error_message = ( - 'To enable agent mode, set ' - 'memcached_mutilate_num_client_vms > 1.') + 'To enable agent mode, set memcached_mutilate_num_client_vms > 1.' + ) if any(agent_mode_flags) and ( - FLAGS.memcached_mutilate_num_client_vms < AGENT_MODE_MIN_CLIENT_VMS): + FLAGS.memcached_mutilate_num_client_vms < AGENT_MODE_MIN_CLIENT_VMS + ): raise errors.Setup.InvalidFlagConfigurationError(error_message) - if _INCREMENTAL_LOAD.value and (len(FLAGS.mutilate_qps) != 1 or - int(FLAGS.mutilate_qps[0]) == 0): + if _INCREMENTAL_LOAD.value and ( + len(FLAGS.mutilate_qps) != 1 or int(FLAGS.mutilate_qps[0]) == 0 + ): raise errors.Setup.InvalidFlagConfigurationError( 'To use dynamic load, set inital target qps with --mutilate_qps ' - 'and incremental with --mutilate_incremental_load.') + 'and incremental with --mutilate_incremental_load.' + ) def YumInstall(vm): @@ -128,7 +138,8 @@ def AptInstall(vm): vm.InstallPackages(APT_PACKAGES) vm.RemoteCommand('git clone {0} {1}'.format(GIT_REPO, MUTILATE_DIR)) vm.RemoteCommand( - f'sed -i "s|int total|long total|g" {MUTILATE_DIR}/mutilate.cc') + f'sed -i "s|int total|long total|g" {MUTILATE_DIR}/mutilate.cc' + ) vm.RemoteCommand('cd {0} && sudo scons'.format(MUTILATE_DIR)) @@ -141,7 +152,7 @@ def GetMetadata(): 'keysize': FLAGS.mutilate_keysize, 'valuesize': FLAGS.mutilate_valuesize, 'records': FLAGS.mutilate_records, - 'ratio': FLAGS.mutilate_ratio + 'ratio': FLAGS.mutilate_ratio, } if FLAGS.mutilate_options: metadata['options'] = FLAGS.mutilate_options @@ -154,13 +165,18 @@ def BuildCmd(server_ip, server_port, num_instances, options): server_ips = [] for idx in range(num_instances): server_ips.append(f'--server={server_ip}:{server_port + idx}') - cmd = [ - 'ulimit -n 32768; ', MUTILATE_BIN, - '--keysize=%s' % FLAGS.mutilate_keysize, - '--valuesize=%s' % FLAGS.mutilate_valuesize, - '--records=%s' % FLAGS.mutilate_records, - '--roundrobin' if len(server_ips) > 1 else '' - ] + server_ips + options + cmd = ( + [ + 'ulimit -n 32768; ', + MUTILATE_BIN, + '--keysize=%s' % FLAGS.mutilate_keysize, + '--valuesize=%s' % FLAGS.mutilate_valuesize, + '--records=%s' % FLAGS.mutilate_records, + '--roundrobin' if len(server_ips) > 1 else '', + ] + + server_ips + + options + ) if FLAGS.mutilate_protocol == 'binary': cmd.append('--binary') return cmd @@ -169,9 +185,7 @@ def BuildCmd(server_ip, server_port, num_instances, options): def Load(client_vm, server_ip, server_port): """Preload the server with data.""" logging.info('Loading memcached server.') - cmd = BuildCmd( - server_ip, server_port, 1, - ['--loadonly']) + cmd = BuildCmd(server_ip, server_port, 1, ['--loadonly']) client_vm.RemoteCommand(' '.join(cmd)) @@ -180,11 +194,16 @@ def RestartAgent(vm, threads): # Kill existing mutilate agent threads vm.RemoteCommand('pkill -9 mutilate', ignore_failure=True) # Make sure have enough file descriptor for the agent process. - vm.RemoteCommand(' '.join([ - 'ulimit -n 32768; ' - 'nohup', MUTILATE_BIN, - '--threads=%s' % threads, '--agentmode', '&> log', '&' - ])) + vm.RemoteCommand( + ' '.join([ + 'ulimit -n 32768; nohup', + MUTILATE_BIN, + '--threads=%s' % threads, + '--agentmode', + '&> log', + '&', + ]) + ) def Run(vms, server_ip, server_port, num_instances): @@ -199,17 +218,16 @@ def Run(vms, server_ip, server_port, num_instances): if FLAGS.mutilate_measure_connections: runtime_options['measure_connections'] = FLAGS.mutilate_measure_connections measure_flags.append( - '--measure_connections=%s' % FLAGS.mutilate_measure_connections) + '--measure_connections=%s' % FLAGS.mutilate_measure_connections + ) if FLAGS.mutilate_measure_threads: runtime_options['measure_threads'] = FLAGS.mutilate_measure_threads if FLAGS.mutilate_measure_qps: runtime_options['measure_qps'] = FLAGS.mutilate_measure_qps - measure_flags.append( - '--measure_qps=%s' % FLAGS.mutilate_measure_qps) + measure_flags.append('--measure_qps=%s' % FLAGS.mutilate_measure_qps) if FLAGS.mutilate_measure_depth: runtime_options['measure_depth'] = FLAGS.mutilate_measure_depth - measure_flags.append( - '--measure_depth=%s' % FLAGS.mutilate_measure_depth) + measure_flags.append('--measure_depth=%s' % FLAGS.mutilate_measure_depth) for thread_count in FLAGS.mutilate_threads: runtime_options['threads'] = thread_count @@ -225,20 +243,31 @@ def Run(vms, server_ip, server_port, num_instances): target_qps = int(target_qps_list[0]) runtime_options['qps'] = target_qps or 'peak' remote_agents = ['--agent=%s' % vm.internal_ip for vm in vms[1:]] - cmd = BuildCmd(server_ip, server_port, num_instances, [ - '--noload', - '--qps=%s' % target_qps, - '--time=%s' % FLAGS.mutilate_time, - '--update=%s' % FLAGS.mutilate_ratio, - '--threads=%s' % (FLAGS.mutilate_measure_threads or thread_count), - '--connections=%s' % connection_count, - '--depth=%s' % depth, - ] + remote_agents + measure_flags + additional_flags) + cmd = BuildCmd( + server_ip, + server_port, + num_instances, + [ + '--noload', + '--qps=%s' % target_qps, + '--time=%s' % FLAGS.mutilate_time, + '--update=%s' % FLAGS.mutilate_ratio, + '--threads=%s' + % (FLAGS.mutilate_measure_threads or thread_count), + '--connections=%s' % connection_count, + '--depth=%s' % depth, + ] + + remote_agents + + measure_flags + + additional_flags, + ) try: stdout, _, retcode = master.RemoteHostCommandWithReturnCode( - ' '.join(cmd), timeout=FLAGS.mutilate_time * 2, - ignore_failure=True) + ' '.join(cmd), + timeout=FLAGS.mutilate_time * 2, + ignore_failure=True, + ) except errors.VmUtil.IssueCommandTimeoutError: break if retcode: @@ -248,10 +277,12 @@ def Run(vms, server_ip, server_port, num_instances): run_samples, actual_qps = ParseResults(stdout, metadata) samples.extend(run_samples) - if _INCREMENTAL_LOAD.value and (actual_qps / target_qps > - (1 - _INCREMENTAL_LOAD.value * 2)): + if _INCREMENTAL_LOAD.value and ( + actual_qps / target_qps > (1 - _INCREMENTAL_LOAD.value * 2) + ): target_qps_list.append( - int(target_qps) * (1 + _INCREMENTAL_LOAD.value)) + int(target_qps) * (1 + _INCREMENTAL_LOAD.value) + ) target_qps_list.pop(0) if not target_qps_list: break @@ -302,18 +333,19 @@ def ParseResults(result, metadata): for idx, stat in enumerate(latency_stats): if idx == len(latency_values): logging.warning( - 'Mutilate does not report %s latency for %s.', stat, metric) + 'Mutilate does not report %s latency for %s.', stat, metric + ) break samples.append( - sample.Sample(metric + '_' + stat, - float(latency_values[idx]), - 'usec', metadata)) + sample.Sample( + metric + '_' + stat, float(latency_values[idx]), 'usec', metadata + ) + ) # parse bandwidth for metric in ('TX', 'RX'): bw_regex = metric + BANDWIDTH_REGEX bw = regex_util.ExtractGroup(bw_regex, result) - samples.append( - sample.Sample(metric, float(bw), 'MB/s', metadata)) + samples.append(sample.Sample(metric, float(bw), 'MB/s', metadata)) qps = regex_util.ExtractFloat(QPS_REGEX, result) samples.append(sample.Sample('qps', qps, 'ops/s', metadata)) diff --git a/perfkitbenchmarker/linux_packages/mxnet.py b/perfkitbenchmarker/linux_packages/mxnet.py index 97e3f75744..bec0e65ec2 100644 --- a/perfkitbenchmarker/linux_packages/mxnet.py +++ b/perfkitbenchmarker/linux_packages/mxnet.py @@ -34,11 +34,11 @@ def GetEnvironmentVars(vm): long_bit = output.strip() lib_name = 'lib' if long_bit == '32' else 'lib64' return ' '.join([ - 'PATH=%s${PATH:+:${PATH}}' % - posixpath.join(cuda_toolkit.CUDA_HOME, 'bin'), + 'PATH=%s${PATH:+:${PATH}}' + % posixpath.join(cuda_toolkit.CUDA_HOME, 'bin'), 'CUDA_HOME=%s' % cuda_toolkit.CUDA_HOME, - 'LD_LIBRARY_PATH=%s${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}' % - posixpath.join(cuda_toolkit.CUDA_HOME, lib_name), + 'LD_LIBRARY_PATH=%s${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}' + % posixpath.join(cuda_toolkit.CUDA_HOME, lib_name), ]) @@ -51,10 +51,11 @@ def GetMXNetVersion(vm): Returns: installed python MXNet version as a string """ - stdout, _ = vm.RemoteCommand( - ('echo -e "import mxnet\nprint(mxnet.__version__)" | {0} python' - .format(GetEnvironmentVars(vm))) - ) + stdout, _ = vm.RemoteCommand(( + 'echo -e "import mxnet\nprint(mxnet.__version__)" | {0} python'.format( + GetEnvironmentVars(vm) + ) + )) return stdout.strip() @@ -66,15 +67,17 @@ def Install(vm): vm.Install('cuda_toolkit') if float(FLAGS.cuda_toolkit_version) < 11: cuda_version = FLAGS.cuda_toolkit_version.replace('.', '') - vm.RemoteCommand('sudo pip install mxnet-cu{}=={}'.format( - cuda_version, FLAGS.mx_version)) + vm.RemoteCommand( + 'sudo pip install mxnet-cu{}=={}'.format( + cuda_version, FLAGS.mx_version + ) + ) else: # mxnet-cu110 starts in version 1.8, which requires Python 3. # TODO(tohaowu). Migrate mxnet to version 1.8 and Python 3. raise cuda_toolkit.UnsupportedCudaVersionError() elif FLAGS.mx_device == 'cpu': - vm.RemoteCommand('sudo pip install mxnet=={}'.format( - FLAGS.mx_version)) + vm.RemoteCommand('sudo pip install mxnet=={}'.format(FLAGS.mx_version)) def Uninstall(vm): diff --git a/perfkitbenchmarker/linux_packages/mxnet_cnn.py b/perfkitbenchmarker/linux_packages/mxnet_cnn.py index 0944e39a52..0e441784e4 100644 --- a/perfkitbenchmarker/linux_packages/mxnet_cnn.py +++ b/perfkitbenchmarker/linux_packages/mxnet_cnn.py @@ -13,20 +13,24 @@ # limitations under the License. """Module containing MXNet CNN installation and cleanup functions.""" from absl import flags + FLAGS = flags.FLAGS MXNET_GIT = 'https://github.com/apache/incubator-mxnet.git' -flags.DEFINE_string('mxnet_commit_hash', - '2700ddbbeef212879802f7f0c0812192ec5c2b77', - 'git commit hash of desired mxnet commit.') +flags.DEFINE_string( + 'mxnet_commit_hash', + '2700ddbbeef212879802f7f0c0812192ec5c2b77', + 'git commit hash of desired mxnet commit.', +) def Install(vm): """Installs MXNet on the VM.""" vm.InstallPackages('git') vm.RemoteCommand('git clone %s' % MXNET_GIT) - vm.RemoteCommand('cd incubator-mxnet && git checkout %s' % - FLAGS.mxnet_commit_hash) + vm.RemoteCommand( + 'cd incubator-mxnet && git checkout %s' % FLAGS.mxnet_commit_hash + ) def Uninstall(vm): diff --git a/perfkitbenchmarker/linux_packages/mysql56.py b/perfkitbenchmarker/linux_packages/mysql56.py index d1531955f1..96bdd1c987 100644 --- a/perfkitbenchmarker/linux_packages/mysql56.py +++ b/perfkitbenchmarker/linux_packages/mysql56.py @@ -23,21 +23,23 @@ MYSQL_URL = 'https://dev.mysql.com/get/' + MYSQL_RPM PACKAGE_NAME = 'mysql' PREPROVISIONED_DATA = { - MYSQL_RPM: + MYSQL_RPM: ( '81b2256f778bb3972054257edda2c2a82fcec455cae3d45ba9c8778a46aa8eb3' + ) } -PACKAGE_DATA_URL = { - MYSQL_RPM: MYSQL_URL -} +PACKAGE_DATA_URL = {MYSQL_RPM: MYSQL_URL} def YumInstall(vm): """Installs the mysql package on the VM.""" vm.RemoteCommand('sudo setenforce 0') - vm.InstallPreprovisionedPackageData(PACKAGE_NAME, PREPROVISIONED_DATA.keys(), - linux_packages.INSTALL_DIR) - vm.RemoteCommand('sudo rpm -ivh --force %s' % - posixpath.join(linux_packages.INSTALL_DIR, MYSQL_RPM)) + vm.InstallPreprovisionedPackageData( + PACKAGE_NAME, PREPROVISIONED_DATA.keys(), linux_packages.INSTALL_DIR + ) + vm.RemoteCommand( + 'sudo rpm -ivh --force %s' + % posixpath.join(linux_packages.INSTALL_DIR, MYSQL_RPM) + ) vm.InstallPackages('mysql-server') vm.RemoteCommand('sudo service mysqld start') vm.RemoteCommand('/usr/bin/mysqladmin -u root password "%s"' % MYSQL_PSWD) @@ -45,10 +47,14 @@ def YumInstall(vm): def AptInstall(vm): """Installs the mysql package on the VM.""" - vm.RemoteCommand('echo "mysql-server-5.6 mysql-server/root_password password ' - '%s" | sudo debconf-set-selections' % MYSQL_PSWD) - vm.RemoteCommand('echo "mysql-server-5.6 mysql-server/root_password_again ' - 'password %s" | sudo debconf-set-selections' % MYSQL_PSWD) + vm.RemoteCommand( + 'echo "mysql-server-5.6 mysql-server/root_password password ' + '%s" | sudo debconf-set-selections' % MYSQL_PSWD + ) + vm.RemoteCommand( + 'echo "mysql-server-5.6 mysql-server/root_password_again ' + 'password %s" | sudo debconf-set-selections' % MYSQL_PSWD + ) vm.InstallPackages('mysql-server') diff --git a/perfkitbenchmarker/linux_packages/mysql57.py b/perfkitbenchmarker/linux_packages/mysql57.py index ac0195472b..25814630eb 100644 --- a/perfkitbenchmarker/linux_packages/mysql57.py +++ b/perfkitbenchmarker/linux_packages/mysql57.py @@ -23,21 +23,23 @@ MYSQL_URL = 'https://dev.mysql.com/get/' + MYSQL_RPM PACKAGE_NAME = 'mysql' PREPROVISIONED_DATA = { - MYSQL_RPM: + MYSQL_RPM: ( '81b2256f778bb3972054257edda2c2a82fcec455cae3d45ba9c8778a46aa8eb3' + ) } -PACKAGE_DATA_URL = { - MYSQL_RPM: MYSQL_URL -} +PACKAGE_DATA_URL = {MYSQL_RPM: MYSQL_URL} def YumInstall(vm): """Installs the mysql package on the VM.""" vm.RemoteCommand('sudo setenforce 0') - vm.InstallPreprovisionedPackageData(PACKAGE_NAME, PREPROVISIONED_DATA.keys(), - linux_packages.INSTALL_DIR) - vm.RemoteCommand('sudo rpm -ivh --force %s' % - posixpath.join(linux_packages.INSTALL_DIR, MYSQL_RPM)) + vm.InstallPreprovisionedPackageData( + PACKAGE_NAME, PREPROVISIONED_DATA.keys(), linux_packages.INSTALL_DIR + ) + vm.RemoteCommand( + 'sudo rpm -ivh --force %s' + % posixpath.join(linux_packages.INSTALL_DIR, MYSQL_RPM) + ) vm.InstallPackages('mysql-server') vm.RemoteCommand('sudo service mysqld start') vm.RemoteCommand('/usr/bin/mysqladmin -u root password "%s"' % MYSQL_PSWD) @@ -45,10 +47,14 @@ def YumInstall(vm): def AptInstall(vm): """Installs the mysql package on the VM.""" - vm.RemoteCommand('echo "mysql-server-5.7 mysql-server/root_password password ' - '%s" | sudo debconf-set-selections' % MYSQL_PSWD) - vm.RemoteCommand('echo "mysql-server-5.7 mysql-server/root_password_again ' - 'password %s" | sudo debconf-set-selections' % MYSQL_PSWD) + vm.RemoteCommand( + 'echo "mysql-server-5.7 mysql-server/root_password password ' + '%s" | sudo debconf-set-selections' % MYSQL_PSWD + ) + vm.RemoteCommand( + 'echo "mysql-server-5.7 mysql-server/root_password_again ' + 'password %s" | sudo debconf-set-selections' % MYSQL_PSWD + ) vm.InstallPackages('mysql-server') diff --git a/perfkitbenchmarker/linux_packages/mysql80.py b/perfkitbenchmarker/linux_packages/mysql80.py index 1c4741d36a..5e37ff2fb4 100644 --- a/perfkitbenchmarker/linux_packages/mysql80.py +++ b/perfkitbenchmarker/linux_packages/mysql80.py @@ -28,14 +28,21 @@ def YumInstall(vm): def AptInstall(vm): """Installs the mysql package on the VM.""" - vm.RemoteCommand('wget -c ' - 'https://repo.mysql.com//mysql-apt-config_0.8.17-1_all.deb') - vm.RemoteCommand('echo mysql-apt-config mysql-apt-config/select-server' - ' select mysql-8.0 | sudo debconf-set-selections') - vm.RemoteCommand('echo mysql-apt-config mysql-apt-config/select-product' - ' select Ok | sudo debconf-set-selections') - vm.RemoteCommand('sudo -E DEBIAN_FRONTEND=noninteractive dpkg -i' - ' mysql-apt-config_0.8.17-1_all.deb') + vm.RemoteCommand( + 'wget -c https://repo.mysql.com//mysql-apt-config_0.8.17-1_all.deb' + ) + vm.RemoteCommand( + 'echo mysql-apt-config mysql-apt-config/select-server' + ' select mysql-8.0 | sudo debconf-set-selections' + ) + vm.RemoteCommand( + 'echo mysql-apt-config mysql-apt-config/select-product' + ' select Ok | sudo debconf-set-selections' + ) + vm.RemoteCommand( + 'sudo -E DEBIAN_FRONTEND=noninteractive dpkg -i' + ' mysql-apt-config_0.8.17-1_all.deb' + ) _, stderr = vm.RemoteCommand('sudo apt-get update', ignore_failure=True) @@ -47,17 +54,23 @@ def AptInstall(vm): match = re.match('.*NO_PUBKEY ([A-Z0-9]*)', stderr) if match: key = match.group(1) - vm.RemoteCommand('sudo apt-key adv ' - f'--keyserver keyserver.ubuntu.com --recv-keys {key}') + vm.RemoteCommand( + 'sudo apt-key adv ' + f'--keyserver keyserver.ubuntu.com --recv-keys {key}' + ) else: raise RuntimeError('No public key found by regex.') else: raise RuntimeError(stderr) - vm.RemoteCommand('echo "mysql-server-8.0 mysql-server/root_password password ' - f'{MYSQL_PSWD}" | sudo debconf-set-selections') - vm.RemoteCommand('echo "mysql-server-8.0 mysql-server/root_password_again ' - f'password {MYSQL_PSWD}" | sudo debconf-set-selections') + vm.RemoteCommand( + 'echo "mysql-server-8.0 mysql-server/root_password password ' + f'{MYSQL_PSWD}" | sudo debconf-set-selections' + ) + vm.RemoteCommand( + 'echo "mysql-server-8.0 mysql-server/root_password_again ' + f'password {MYSQL_PSWD}" | sudo debconf-set-selections' + ) vm.InstallPackages('mysql-server') diff --git a/perfkitbenchmarker/linux_packages/mysqlclient56.py b/perfkitbenchmarker/linux_packages/mysqlclient56.py index 7dde237af6..3a715a6c65 100644 --- a/perfkitbenchmarker/linux_packages/mysqlclient56.py +++ b/perfkitbenchmarker/linux_packages/mysqlclient56.py @@ -26,7 +26,8 @@ def AptInstall(vm): """Installs the mysql package on the VM.""" vm.RemoteCommand( 'sudo add-apt-repository ' - "'deb http://archive.ubuntu.com/ubuntu trusty universe'") + "'deb http://archive.ubuntu.com/ubuntu trusty universe'" + ) vm.RemoteCommand('sudo apt-get update') vm.RemoteCommand('sudo apt-get -y install mysql-client-5.6') diff --git a/perfkitbenchmarker/linux_packages/nccl.py b/perfkitbenchmarker/linux_packages/nccl.py index 1c4ee5710e..9c49319594 100644 --- a/perfkitbenchmarker/linux_packages/nccl.py +++ b/perfkitbenchmarker/linux_packages/nccl.py @@ -26,8 +26,9 @@ ) flags.DEFINE_string('nccl_net_plugin', None, 'NCCL network plugin name') flags.DEFINE_string('nccl_mpi', '/usr/bin/mpirun', 'MPI binary path') -flags.DEFINE_string('nccl_mpi_home', '/usr/lib/x86_64-linux-gnu/openmpi', - 'MPI home') +flags.DEFINE_string( + 'nccl_mpi_home', '/usr/lib/x86_64-linux-gnu/openmpi', 'MPI home' +) flags.DEFINE_string('nccl_home', '$HOME/nccl/build', 'NCCL home') FLAGS = flags.FLAGS @@ -37,21 +38,28 @@ def _Build(vm): """Installs the NCCL package on the VM.""" - vm.RemoteCommand('[ -d "nccl" ] || git clone {git_repo} --branch {version}' - .format(git_repo=GIT_REPO, version=FLAGS.nccl_version)) + vm.RemoteCommand( + '[ -d "nccl" ] || git clone {git_repo} --branch {version}'.format( + git_repo=GIT_REPO, version=FLAGS.nccl_version + ) + ) cuda_home = cuda_toolkit.CUDA_HOME vm.InstallPackages('build-essential devscripts debhelper fakeroot') env_vars = {} - env_vars['PATH'] = (r'{cuda_bin_path}:$PATH' - .format(cuda_bin_path=posixpath.join(cuda_home, 'bin'))) - env_vars['CUDA_HOME'] = (r'{cuda_home}'.format(cuda_home=cuda_home)) - env_vars['LD_LIBRARY_PATH'] = (r'{lib_path}:$LD_LIBRARY_PATH' - .format(lib_path=posixpath.join( - cuda_home, 'lib64'))) + env_vars['PATH'] = r'{cuda_bin_path}:$PATH'.format( + cuda_bin_path=posixpath.join(cuda_home, 'bin') + ) + env_vars['CUDA_HOME'] = r'{cuda_home}'.format(cuda_home=cuda_home) + env_vars['LD_LIBRARY_PATH'] = r'{lib_path}:$LD_LIBRARY_PATH'.format( + lib_path=posixpath.join(cuda_home, 'lib64') + ) - vm.RemoteCommand('cd nccl && {env} make -j 20 pkg.debian.build' - .format(env=vm_util.DictionaryToEnvString(env_vars))) + vm.RemoteCommand( + 'cd nccl && {env} make -j 20 pkg.debian.build'.format( + env=vm_util.DictionaryToEnvString(env_vars) + ) + ) def AptInstall(vm): @@ -61,17 +69,18 @@ def AptInstall(vm): vm.Install('cuda_toolkit') _Build(vm) - vm.InstallPackages('--allow-downgrades --allow-change-held-packages ' - '{build}libnccl2_*+cuda{cuda}_amd64.deb ' - '{build}libnccl-dev_*+cuda{cuda}_amd64.deb' - .format( - build='./nccl/build/pkg/deb/', - cuda=FLAGS.cuda_toolkit_version)) + vm.InstallPackages( + '--allow-downgrades --allow-change-held-packages ' + '{build}libnccl2_*+cuda{cuda}_amd64.deb ' + '{build}libnccl-dev_*+cuda{cuda}_amd64.deb'.format( + build='./nccl/build/pkg/deb/', cuda=FLAGS.cuda_toolkit_version + ) + ) if FLAGS.nccl_net_plugin: vm.RemoteCommand( - 'echo "deb https://packages.cloud.google.com/apt google-fast-socket main" ' - '| sudo tee /etc/apt/sources.list.d/google-fast-socket.list' + 'echo "deb https://packages.cloud.google.com/apt google-fast-socket' + ' main" | sudo tee /etc/apt/sources.list.d/google-fast-socket.list' ) vm.RemoteCommand( 'curl -s -L https://packages.cloud.google.com/apt/doc/apt-key.gpg ' @@ -86,13 +95,15 @@ def AptInstall(vm): if FLAGS.aws_efa: vm.InstallPackages('libudev-dev libtool autoconf') vm.RemoteCommand('git clone https://github.com/aws/aws-ofi-nccl.git -b aws') - vm.RemoteCommand('cd aws-ofi-nccl && ./autogen.sh && ./configure ' - '--with-mpi={mpi} ' - '--with-libfabric=/opt/amazon/efa ' - '--with-nccl={nccl} ' - '--with-cuda={cuda} && sudo make && ' - 'sudo make install'.format( - mpi=FLAGS.nccl_mpi_home, - nccl=FLAGS.nccl_home, - cuda='/usr/local/cuda-{}'.format( - FLAGS.cuda_toolkit_version))) + vm.RemoteCommand( + 'cd aws-ofi-nccl && ./autogen.sh && ./configure ' + '--with-mpi={mpi} ' + '--with-libfabric=/opt/amazon/efa ' + '--with-nccl={nccl} ' + '--with-cuda={cuda} && sudo make && ' + 'sudo make install'.format( + mpi=FLAGS.nccl_mpi_home, + nccl=FLAGS.nccl_home, + cuda='/usr/local/cuda-{}'.format(FLAGS.cuda_toolkit_version), + ) + ) diff --git a/perfkitbenchmarker/linux_packages/netperf.py b/perfkitbenchmarker/linux_packages/netperf.py index 5d2b33f9d3..9bc7628057 100644 --- a/perfkitbenchmarker/linux_packages/netperf.py +++ b/perfkitbenchmarker/linux_packages/netperf.py @@ -24,13 +24,15 @@ from perfkitbenchmarker import regex_util flags.DEFINE_integer( - 'netperf_histogram_buckets', 100, + 'netperf_histogram_buckets', + 100, 'The number of buckets per bucket array in a netperf histogram. Netperf ' 'keeps one array for latencies in the single usec range, one for the ' '10-usec range, one for the 100-usec range, and so on until the 10-sec ' 'range. The default value that netperf uses is 100. Using more will ' 'increase the precision of the histogram samples that the netperf ' - 'benchmark produces.') + 'benchmark produces.', +) FLAGS = flags.FLAGS NETPERF_TAR = 'netperf-2.7.0.tar.gz' NETPERF_URL = f'https://github.com/HewlettPackard/netperf/archive/{NETPERF_TAR}' @@ -63,10 +65,13 @@ def _Install(vm): f'CFLAGS=-DHIST_NUM_OF_BUCKET={FLAGS.netperf_histogram_buckets} ' './configure --enable-burst ' '--enable-demo --enable-histogram ' - '&& make && sudo make install') + '&& make && sudo make install' + ) - vm.RemoteCommand(f'cd {NETPERF_EXAMPLE_DIR} && chmod +x runemomniaggdemo.sh' - '&& chmod +x find_max_burst.sh') + vm.RemoteCommand( + f'cd {NETPERF_EXAMPLE_DIR} && chmod +x runemomniaggdemo.sh' + '&& chmod +x find_max_burst.sh' + ) # Set keepalive to a low value to ensure that the control connection # is not closed by the cloud networking infrastructure. @@ -99,13 +104,15 @@ def _CopyTar(vm): if vm.CLOUD != provider_info.KUBERNETES: try: vm.PushDataFile( - NETPERF_TAR, remote_path=(linux_packages.INSTALL_DIR + '/')) + NETPERF_TAR, remote_path=(linux_packages.INSTALL_DIR + '/') + ) return except data.ResourceNotFound: pass vm.Install('curl') vm.RemoteCommand( - f'curl {NETPERF_URL} -L -o {linux_packages.INSTALL_DIR}/{NETPERF_TAR}') + f'curl {NETPERF_URL} -L -o {linux_packages.INSTALL_DIR}/{NETPERF_TAR}' + ) def YumInstall(vm): @@ -142,7 +149,8 @@ def ParseHistogram(netperf_stdout): # >100_SECS: 0 # HIST_TOTAL: 444658 histogram_text = regex_util.ExtractGroup( - '(UNIT_USEC.*?)>100_SECS', netperf_stdout, flags=re.S) + '(UNIT_USEC.*?)>100_SECS', netperf_stdout, flags=re.S + ) # The total number of usecs that this row of the histogram represents. row_size = 10.0 @@ -151,8 +159,9 @@ def ParseHistogram(netperf_stdout): for l in histogram_text.splitlines(): buckets = [int(b) for b in l.split(':')[1:]] bucket_size = row_size / len(buckets) - hist.update({(i * bucket_size): count - for i, count in enumerate(buckets) if count}) + hist.update( + {(i * bucket_size): count for i, count in enumerate(buckets) if count} + ) # Each row is 10x larger than the previous row. row_size *= 10 diff --git a/perfkitbenchmarker/linux_packages/neuron.py b/perfkitbenchmarker/linux_packages/neuron.py index 688f38c76e..9672b9861e 100644 --- a/perfkitbenchmarker/linux_packages/neuron.py +++ b/perfkitbenchmarker/linux_packages/neuron.py @@ -18,8 +18,10 @@ from perfkitbenchmarker import virtual_machine ENV = flags.DEFINE_string( - 'neuron_env', 'aws_neuron_venv_pytorch', - 'The Python virtual environment to install Neuron pip package.') + 'neuron_env', + 'aws_neuron_venv_pytorch', + 'The Python virtual environment to install Neuron pip package.', +) FLAGS = flags.FLAGS @@ -38,7 +40,8 @@ def YumInstall(vm: virtual_machine.BaseVirtualMachine) -> None: metadata_expire=0 EOF""") vm.RemoteCommand( - 'sudo rpm --import https://yum.repos.neuron.amazonaws.com/GPG-PUB-KEY-AMAZON-AWS-NEURON.PUB' + 'sudo rpm --import' + ' https://yum.repos.neuron.amazonaws.com/GPG-PUB-KEY-AMAZON-AWS-NEURON.PUB' ) # Update OS packages @@ -78,7 +81,8 @@ def YumInstall(vm: virtual_machine.BaseVirtualMachine) -> None: # Install packages from repos vm.RemoteCommand( - f'{path} pip3 config set global.extra-index-url "https://pip.repos.neuron.amazonaws.com"' + f'{path} pip3 config set global.extra-index-url' + ' "https://pip.repos.neuron.amazonaws.com"' ) # Install Neuron packages diff --git a/perfkitbenchmarker/linux_packages/nginx.py b/perfkitbenchmarker/linux_packages/nginx.py index 6161aed4e8..b6c708558f 100644 --- a/perfkitbenchmarker/linux_packages/nginx.py +++ b/perfkitbenchmarker/linux_packages/nginx.py @@ -16,23 +16,27 @@ from perfkitbenchmarker import errors -RHEL_REPO = ('[nginx]\n' - 'name=nginx repo\n' - 'baseurl=https://nginx.org/packages/rhel/$releasever/$basearch/\n' - 'gpgcheck=0\n' - 'enabled=1') +RHEL_REPO = ( + '[nginx]\n' + 'name=nginx repo\n' + 'baseurl=https://nginx.org/packages/rhel/$releasever/$basearch/\n' + 'gpgcheck=0\n' + 'enabled=1' +) def YumInstall(vm): """Installs nginx on the VM.""" - vm.RemoteCommand('echo \'%s\' | ' - 'sudo tee /etc/yum.repos.d/nginx.repo' % RHEL_REPO) + vm.RemoteCommand( + "echo '%s' | sudo tee /etc/yum.repos.d/nginx.repo" % RHEL_REPO + ) try: vm.InstallPackages('nginx') except errors.VmUtil.IssueCommandError: # Amazon Linux does not have a releasever configured. - vm.RemoteCommand('sudo sed -i -e "s/\\$releasever/6/" ' - '/etc/yum.repos.d/nginx.repo') + vm.RemoteCommand( + 'sudo sed -i -e "s/\\$releasever/6/" /etc/yum.repos.d/nginx.repo' + ) vm.InstallPackages('nginx') diff --git a/perfkitbenchmarker/linux_packages/nvidia_docker.py b/perfkitbenchmarker/linux_packages/nvidia_docker.py index 5832dadebd..e4535ea2be 100644 --- a/perfkitbenchmarker/linux_packages/nvidia_docker.py +++ b/perfkitbenchmarker/linux_packages/nvidia_docker.py @@ -18,8 +18,10 @@ Installation: https://github.com/NVIDIA/nvidia-docker """ from absl import flags -_VERSION = flags.DEFINE_string('nvidia_docker_version', None, - 'The version of nvidia docker to install.') + +_VERSION = flags.DEFINE_string( + 'nvidia_docker_version', None, 'The version of nvidia docker to install.' +) def CheckNvidiaDockerExists(vm): @@ -34,12 +36,16 @@ def AptInstall(vm): if CheckNvidiaDockerExists(vm): return vm.Install('docker') - vm.RemoteCommand('curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey ' - '| sudo apt-key add -') - vm.RemoteCommand('curl -s -L https://nvidia.github.io/nvidia-docker/' - '$(. /etc/os-release;echo $ID$VERSION_ID)' - '/nvidia-docker.list | sudo tee ' - '/etc/apt/sources.list.d/nvidia-container-toolkit.list') + vm.RemoteCommand( + 'curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey ' + '| sudo apt-key add -' + ) + vm.RemoteCommand( + 'curl -s -L https://nvidia.github.io/nvidia-docker/' + '$(. /etc/os-release;echo $ID$VERSION_ID)' + '/nvidia-docker.list | sudo tee ' + '/etc/apt/sources.list.d/nvidia-container-toolkit.list' + ) vm.RemoteCommand('sudo apt-get update') version = f'={_VERSION.value}' if _VERSION.value else '' vm.InstallPackages(f'nvidia-docker2{version}') @@ -52,10 +58,12 @@ def YumInstall(vm): if CheckNvidiaDockerExists(vm): return vm.Install('docker') - vm.RemoteCommand('curl -s -L https://nvidia.github.io/' - 'nvidia-container-runtime/' - '$(. /etc/os-release;echo $ID$VERSION_ID)/' - 'nvidia-container-runtime.repo | sudo tee /etc/yum.repos.d/' - 'nvidia-container-runtime.repo') + vm.RemoteCommand( + 'curl -s -L https://nvidia.github.io/' + 'nvidia-container-runtime/' + '$(. /etc/os-release;echo $ID$VERSION_ID)/' + 'nvidia-container-runtime.repo | sudo tee /etc/yum.repos.d/' + 'nvidia-container-runtime.repo' + ) vm.RemoteCommand('sudo tee /etc/yum.repos.d/nvidia-container-runtime.repo') vm.InstallPackages('nvidia-container-runtime-hook') diff --git a/perfkitbenchmarker/linux_packages/nvidia_driver.py b/perfkitbenchmarker/linux_packages/nvidia_driver.py index e11d543964..52dbde4518 100644 --- a/perfkitbenchmarker/linux_packages/nvidia_driver.py +++ b/perfkitbenchmarker/linux_packages/nvidia_driver.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Module containing NVIDIA Driver installation. -""" +"""Module containing NVIDIA Driver installation.""" import re from absl import flags @@ -39,13 +38,15 @@ EXTRACT_CLOCK_SPEEDS_REGEX = r'(\S*).*,\s*(\S*)' -flag_util.DEFINE_integerlist('gpu_clock_speeds', - None, - 'desired gpu clock speeds in the form ' - '[memory clock, graphics clock]') +flag_util.DEFINE_integerlist( + 'gpu_clock_speeds', + None, + 'desired gpu clock speeds in the form [memory clock, graphics clock]', +) -flags.DEFINE_boolean('gpu_autoboost_enabled', None, - 'whether gpu autoboost is enabled') +flags.DEFINE_boolean( + 'gpu_autoboost_enabled', None, 'whether gpu autoboost is enabled' +) flags.DEFINE_string( 'nvidia_driver_version', @@ -55,18 +56,29 @@ 'For example, "418.67" or "418.87.01."' ), ) -flags.DEFINE_boolean('nvidia_driver_force_install', False, - 'Whether to install NVIDIA driver, even if it is already ' - 'installed.') +flags.DEFINE_boolean( + 'nvidia_driver_force_install', + False, + 'Whether to install NVIDIA driver, even if it is already installed.', +) -flags.DEFINE_string('nvidia_driver_x_library_path', '/usr/lib', - 'X library path for nvidia driver installation') +flags.DEFINE_string( + 'nvidia_driver_x_library_path', + '/usr/lib', + 'X library path for nvidia driver installation', +) -flags.DEFINE_string('nvidia_driver_x_module_path', '/usr/lib/xorg/modules', - 'X module path for nvidia driver installation') +flags.DEFINE_string( + 'nvidia_driver_x_module_path', + '/usr/lib/xorg/modules', + 'X module path for nvidia driver installation', +) -flags.DEFINE_boolean('nvidia_driver_persistence_mode', None, - 'whether to enable persistence mode on the NVIDIA GPU') +flags.DEFINE_boolean( + 'nvidia_driver_persistence_mode', + None, + 'whether to enable persistence mode on the NVIDIA GPU', +) FLAGS = flags.FLAGS @@ -138,8 +150,9 @@ def GetDriverVersion(vm): match = re.search(regex, stdout) if match: return str(match.group(1)) - raise NvidiaSmiParseOutputError('Unable to parse driver version from {}' - .format(stdout)) + raise NvidiaSmiParseOutputError( + 'Unable to parse driver version from {}'.format(stdout) + ) def GetGpuType(vm): @@ -185,11 +198,11 @@ def GetGpuType(vm): else: gpu_types.append(splitted[2]) except: - raise NvidiaSmiParseOutputError('Unable to parse gpu type from {}' - .format(stdout)) + raise NvidiaSmiParseOutputError( + 'Unable to parse gpu type from {}'.format(stdout) + ) if any(gpu_type != gpu_types[0] for gpu_type in gpu_types): - raise HeterogeneousGpuTypesError( - 'PKB only supports one type of gpu per VM') + raise HeterogeneousGpuTypesError('PKB only supports one type of gpu per VM') if 'K80' in gpu_types[0]: return NVIDIA_TESLA_K80 @@ -211,7 +224,8 @@ def GetGpuType(vm): return NVIDIA_H100 else: raise UnsupportedClockSpeedError( - 'Gpu type {0} is not supported by PKB'.format(gpu_types[0])) + 'Gpu type {0} is not supported by PKB'.format(gpu_types[0]) + ) def GetGpuMem(vm: virtual_machine.BaseVirtualMachine) -> int: @@ -224,7 +238,8 @@ def GetGpuMem(vm: virtual_machine.BaseVirtualMachine) -> int: Integer indicating the NVIDIA GPU memory on the vm. """ stdout, _ = vm.RemoteCommand( - 'sudo nvidia-smi --query-gpu=memory.total --id=0 --format=csv') + 'sudo nvidia-smi --query-gpu=memory.total --id=0 --format=csv' + ) return regex_util.ExtractInt(r'(\d+) MiB', stdout.split('\n')[1]) @@ -237,8 +252,9 @@ def QueryNumberOfGpus(vm): Returns: Integer indicating the number of NVIDIA GPUs present on the vm. """ - stdout, _ = vm.RemoteCommand('sudo nvidia-smi --query-gpu=count --id=0 ' - '--format=csv') + stdout, _ = vm.RemoteCommand( + 'sudo nvidia-smi --query-gpu=count --id=0 --format=csv' + ) return int(stdout.split()[1]) @@ -271,10 +287,12 @@ def GetPeerToPeerTopology(vm): # Delimit each GPU result with semicolons, # and simplify the result character set to 'Y' and 'N'. - return (';'.join(results) - .replace('X', 'Y') # replace X (self) with Y - .replace('OK', 'Y') # replace OK with Y - .replace('NS', 'N')) # replace NS (not supported) with N + return ( + ';'.join(results) + .replace('X', 'Y') # replace X (self) with Y + .replace('OK', 'Y') # replace OK with Y + .replace('NS', 'N') + ) # replace NS (not supported) with N def SetAndConfirmGpuClocks(vm): @@ -302,11 +320,15 @@ def SetAndConfirmGpuClocks(vm): SetAutoboostDefaultPolicy(vm, autoboost_enabled) num_gpus = QueryNumberOfGpus(vm) for i in range(num_gpus): - if QueryGpuClockSpeed(vm, i) != (desired_memory_clock, - desired_graphics_clock): + if QueryGpuClockSpeed(vm, i) != ( + desired_memory_clock, + desired_graphics_clock, + ): raise UnsupportedClockSpeedError( 'Unrecoverable error setting GPU #{} clock speed to {},{}'.format( - i, desired_memory_clock, desired_graphics_clock)) + i, desired_memory_clock, desired_graphics_clock + ) + ) def SetGpuClockSpeed(vm, memory_clock_speed, graphics_clock_speed): @@ -321,11 +343,11 @@ def SetGpuClockSpeed(vm, memory_clock_speed, graphics_clock_speed): for device_id in range(num_gpus): current_clock_speeds = QueryGpuClockSpeed(vm, device_id) if current_clock_speeds != (memory_clock_speed, graphics_clock_speed): - vm.RemoteCommand('sudo nvidia-smi -ac {},{} --id={}'.format( - memory_clock_speed, - graphics_clock_speed, - device_id - )) + vm.RemoteCommand( + 'sudo nvidia-smi -ac {},{} --id={}'.format( + memory_clock_speed, graphics_clock_speed, device_id + ) + ) def QueryGpuClockSpeed(vm, device_id): @@ -340,13 +362,15 @@ def QueryGpuClockSpeed(vm, device_id): Returns: Tuple of clock speeds in MHz in the form (memory clock, graphics clock). """ - query = ('sudo nvidia-smi --query-gpu=clocks.applications.memory,' - 'clocks.applications.graphics --format=csv --id={0}' - .format(device_id)) + query = ( + 'sudo nvidia-smi --query-gpu=clocks.applications.memory,' + 'clocks.applications.graphics --format=csv --id={0}'.format(device_id) + ) stdout, _ = vm.RemoteCommand(query) clock_speeds = stdout.splitlines()[1] - matches = regex_util.ExtractAllMatches(EXTRACT_CLOCK_SPEEDS_REGEX, - clock_speeds)[0] + matches = regex_util.ExtractAllMatches( + EXTRACT_CLOCK_SPEEDS_REGEX, clock_speeds + )[0] return (matches[0], matches[1]) @@ -375,8 +399,11 @@ def SetAutoboostDefaultPolicy(vm, autoboost_enabled): for device_id in range(num_gpus): current_state = QueryAutoboostPolicy(vm, device_id) if current_state['autoboost_default'] != autoboost_enabled: - vm.RemoteCommand('sudo nvidia-smi --auto-boost-default={0} --id={1}' - .format(1 if autoboost_enabled else 0, device_id)) + vm.RemoteCommand( + 'sudo nvidia-smi --auto-boost-default={0} --id={1}'.format( + 1 if autoboost_enabled else 0, device_id + ) + ) def QueryAutoboostPolicy(vm, device_id): @@ -408,13 +435,14 @@ def QueryAutoboostPolicy(vm, device_id): } if (autoboost_match is None) or (autoboost_default_match is None): - raise NvidiaSmiParseOutputError('Unable to parse Auto Boost policy from {}' - .format(stdout)) + raise NvidiaSmiParseOutputError( + 'Unable to parse Auto Boost policy from {}'.format(stdout) + ) return { - 'autoboost': nvidia_smi_output_string_to_value[ - autoboost_match.group(1)], + 'autoboost': nvidia_smi_output_string_to_value[autoboost_match.group(1)], 'autoboost_default': nvidia_smi_output_string_to_value[ - autoboost_default_match.group(1)] + autoboost_default_match.group(1) + ], } diff --git a/perfkitbenchmarker/linux_packages/oldisim_dependencies.py b/perfkitbenchmarker/linux_packages/oldisim_dependencies.py index 5cf71d3f1c..8f5d52b59d 100644 --- a/perfkitbenchmarker/linux_packages/oldisim_dependencies.py +++ b/perfkitbenchmarker/linux_packages/oldisim_dependencies.py @@ -16,10 +16,8 @@ import os -YUM_PACKAGES = ('bc gengetopt libevent-devel ' - 'google-perftools-devel scons') -APT_PACKAGES = ('bc gengetopt libevent-dev ' - 'libgoogle-perftools-dev scons') +YUM_PACKAGES = 'bc gengetopt libevent-devel google-perftools-devel scons' +APT_PACKAGES = 'bc gengetopt libevent-dev libgoogle-perftools-dev scons' OLDISIM_GIT = 'https://github.com/GoogleCloudPlatform/oldisim.git' OLDISIM_DIR = 'oldisim' @@ -31,9 +29,11 @@ def _Install(vm, packages): vm.Install('build_tools') vm.InstallPackages(packages) vm.RemoteCommand('git clone --recursive %s' % OLDISIM_GIT) - vm.RemoteCommand('cd %s && git checkout %s && ' - 'scons -j$(cat /proc/cpuinfo | grep processor | wc -l)' % - (OLDISIM_DIR, OLDISIM_VERSION)) + vm.RemoteCommand( + 'cd %s && git checkout %s && ' + 'scons -j$(cat /proc/cpuinfo | grep processor | wc -l)' + % (OLDISIM_DIR, OLDISIM_VERSION) + ) def YumInstall(vm): diff --git a/perfkitbenchmarker/linux_packages/omb.py b/perfkitbenchmarker/linux_packages/omb.py index 53f1767af7..82c8c863c5 100644 --- a/perfkitbenchmarker/linux_packages/omb.py +++ b/perfkitbenchmarker/linux_packages/omb.py @@ -18,39 +18,54 @@ flags.DEFINE_enum('mpi_vendor', 'intel', ['intel', 'openmpi'], 'MPI provider.') flags.DEFINE_list( - 'omb_mpi_env', [], 'Comma separated list of environment variables, e.g. ' - '--omb_mpi_env=FI_PROVIDER=tcp,FI_LOG_LEVEL=info') + 'omb_mpi_env', + [], + 'Comma separated list of environment variables, e.g. ' + '--omb_mpi_env=FI_PROVIDER=tcp,FI_LOG_LEVEL=info', +) flags.DEFINE_list( - 'omb_mpi_genv', [], 'Comma separated list of global environment variables, ' + 'omb_mpi_genv', + [], + 'Comma separated list of global environment variables, ' 'i.e. environment variables to be applied to all nodes, e.g. ' '--omb_mpi_genv=I_MPI_PIN_PROCESSOR_LIST=0,I_MPI_PIN=1 ' 'When running with Intel MPI, these translate to -genv mpirun options. ' 'When running with OpenMPI, both --omb_mpi_env and --omb_mpi_genv are ' - 'treated the same via the -x mpirun option') + 'treated the same via the -x mpirun option', +) flags.register_validator( 'omb_mpi_env', lambda env_params: all('=' in param for param in env_params), - message='--omb_mpi_env values must be in format "key=value" or "key="') + message='--omb_mpi_env values must be in format "key=value" or "key="', +) flags.register_validator( 'omb_mpi_genv', lambda genv_params: all('=' in param for param in genv_params), - message='--omb_mpi_genv values must be in format "key=value" or "key="') + message='--omb_mpi_genv values must be in format "key=value" or "key="', +) VERSION = '5.7.1' _PKG_NAME = 'osu-micro-benchmarks' -_DATA_URL = ('http://mvapich.cse.ohio-state.edu/download/mvapich/' - f'{_PKG_NAME}-{VERSION}.tgz') +_DATA_URL = ( + 'http://mvapich.cse.ohio-state.edu/download/mvapich/' + f'{_PKG_NAME}-{VERSION}.tgz' +) _SRC_DIR = f'{_PKG_NAME}-{VERSION}' _TARBALL = f'{_PKG_NAME}-{VERSION}.tgz' _RUN_DIR = f'/usr/local/libexec/{_PKG_NAME}/mpi' # Benchmarks that can only be run with a single thread per host _SINGLE_THREADED_BENCHMARKS = frozenset({ - 'acc_latency', 'bibw', 'bw', 'cas_latency', 'fop_latency', 'get_bw', - 'get_latency' + 'acc_latency', + 'bibw', + 'bw', + 'cas_latency', + 'fop_latency', + 'get_bw', + 'get_latency', }) PREPROVISIONED_DATA = { @@ -64,41 +79,57 @@ re.compile(r'^# Synchronization: (?P.*)'), re.compile(r'^# Number of Sender threads: (?P\d+)'), re.compile(r'^# Number of Receiver threads: (?P\d+)'), - re.compile(r'# \[ pairs: (?P\d+) \] ' - r'\[ window size: (?P\d+) \]'), + re.compile( + r'# \[ pairs: (?P\d+) \] ' + r'\[ window size: (?P\d+) \]' + ), ] -_LATENCY_RE = re.compile(r'^(?P\d+)\s+' - r'(?P[\d\.]+)\s*$', re.X | re.MULTILINE) +_LATENCY_RE = re.compile( + r'^(?P\d+)\s+' r'(?P[\d\.]+)\s*$', re.X | re.MULTILINE +) # the "overall" value is the metric recorded. Others are put into metadata _COMPUTE_RE = re.compile( r'^(?P^\d+)\s+' r'(?P[\d\.]+)\s+' r'(?P[\d\.]+)\s+' r'(?P[\d\.]+)\s+' - r'(?P[\d\.]+)\s*$', re.X | re.MULTILINE) + r'(?P[\d\.]+)\s*$', + re.X | re.MULTILINE, +) # parse MPI pinning _MPI_STARTUP_PREFIX = r'^\[(?P\d+)\] MPI startup\(\):\s+' -_MPI_PIN_RE = re.compile(_MPI_STARTUP_PREFIX + (r'(?P\d+)\s+' - r'(?P\d+)\s+' - r'(?P\S+)\s+' - r'.*?(?P[\d,-]+)')) +_MPI_PIN_RE = re.compile( + _MPI_STARTUP_PREFIX + + ( + r'(?P\d+)\s+' + r'(?P\d+)\s+' + r'(?P\S+)\s+' + r'.*?(?P[\d,-]+)' + ) +) _PKB_NODE_RE = re.compile(r'pkb-(?P.*?)-(?P\d+)') # parameters to pass into the benchmark _NUMBER_ITERATIONS = flags.DEFINE_integer( - 'omb_iterations', None, 'Number of iterations to run in a test.') -_SYNC_OPTION = flags.DEFINE_string('omb_sync_option', None, - '--sync-option value to pass in') - -_NUM_SERVER_THREADS = flags.DEFINE_integer('omb_server_threads', None, - 'Number of server threads to use.') + 'omb_iterations', None, 'Number of iterations to run in a test.' +) +_SYNC_OPTION = flags.DEFINE_string( + 'omb_sync_option', None, '--sync-option value to pass in' +) + +_NUM_SERVER_THREADS = flags.DEFINE_integer( + 'omb_server_threads', None, 'Number of server threads to use.' +) _NUM_RECEIVER_THREADS = flags.DEFINE_integer( - 'omb_receiver_threads', None, 'Number of server threads to use.') + 'omb_receiver_threads', None, 'Number of server threads to use.' +) flag_util.DEFINE_integerlist( - 'omb_mpi_processes', flag_util.IntegerList([1, 0]), - 'MPI processes to use per host. 1=One process, 0=only real cores') + 'omb_mpi_processes', + flag_util.IntegerList([1, 0]), + 'MPI processes to use per host. 1=One process, 0=only real cores', +) _MPI_PERHOST = flags.DEFINE_integer('omb_perhost', 1, 'MPI option -perhost.') @@ -114,6 +145,7 @@ class _RunType: supports_full: Whether this benchmark supports --full. long_running: Whether this benchmark takes a long time to run. """ + columns: Tuple[str] value_column: str units: str = 'usec' @@ -123,24 +155,51 @@ class _RunType: # Benchmarks that support --full _LATENCY = _RunType( - ('size', 'latency', 'min_latency', 'max_latency', 'iterations'), 'latency') + ('size', 'latency', 'min_latency', 'max_latency', 'iterations'), 'latency' +) _LATENCY_NOSIZE = _RunType( - ('latency', 'min_latency', 'max_latency', 'iterations'), 'latency') + ('latency', 'min_latency', 'max_latency', 'iterations'), 'latency' +) _COMPUTE = _RunType( - ('size', 'overall', 'compute', 'collection_init', 'mpi_test', 'mpi_wait', - 'min_comm', 'max_comm', 'pure_comm', 'overlap'), 'mpi_wait') + ( + 'size', + 'overall', + 'compute', + 'collection_init', + 'mpi_test', + 'mpi_wait', + 'min_comm', + 'max_comm', + 'pure_comm', + 'overlap', + ), + 'mpi_wait', +) _COMPUTE_NOSIZE = _RunType( - ('overall', 'compute', 'collection_init', 'mpi_test', 'mpi_wait', - 'pure_comm', 'min_comm', 'max_comm', 'overlap'), 'mpi_wait') + ( + 'overall', + 'compute', + 'collection_init', + 'mpi_test', + 'mpi_wait', + 'pure_comm', + 'min_comm', + 'max_comm', + 'overlap', + ), + 'mpi_wait', +) # Benchmarks that do not support --full _LATENCY_SIZE_ONLY = _RunType(('size', 'latency'), 'latency', 'usec', False) _BANDWIDTH = _RunType(('size', 'bandwidth'), 'bandwidth', 'MB/s', False) -_BANDWIDTH_MESSAGES = _RunType(('size', 'bandwidth', 'messages_per_second'), - 'bandwidth', 'MB/s', False) +_BANDWIDTH_MESSAGES = _RunType( + ('size', 'bandwidth', 'messages_per_second'), 'bandwidth', 'MB/s', False +) # The get_acc_latency and latency_mt take a really long time to finish -_LATENCY_LONG_RUNNING = _RunType(('size', 'latency'), 'latency', 'usec', False, - True) +_LATENCY_LONG_RUNNING = _RunType( + ('size', 'latency'), 'latency', 'usec', False, True +) BENCHMARKS: Dict[str, _RunType] = { 'acc_latency': _LATENCY_SIZE_ONLY, @@ -184,7 +243,7 @@ class _RunType: 'reduce': _LATENCY, 'reduce_scatter': _LATENCY, 'scatter': _LATENCY, - 'scatterv': _LATENCY + 'scatterv': _LATENCY, } @@ -209,6 +268,7 @@ class RunResult: perhost: MPI option -perhost. mpi_env: environment variables to set for mpirun command. """ + name: str metadata: Dict[str, Any] data: List[Dict[str, float]] @@ -229,8 +289,9 @@ class RunResult: class RunRequest: test_name: str vms: List[Any] # virtual machine - message_size: Optional[ - Union[str, int]] = None # default: run all message sizes + message_size: Optional[Union[str, int]] = ( + None # default: run all message sizes + ) FLAGS = flags.FLAGS @@ -239,15 +300,18 @@ class RunRequest: def _InstallForIntelMpi(vm) -> None: """Installs the omb package with IntelMPI lib on the VM.""" vm.Install('intelmpi') - txt, _ = vm.RemoteCommand(f'{intelmpi.SourceMpiVarsCommand(vm)}; ' - 'which mpicc mpicxx') + txt, _ = vm.RemoteCommand( + f'{intelmpi.SourceMpiVarsCommand(vm)}; which mpicc mpicxx' + ) mpicc_path, mpicxx_path = txt.splitlines() vm.Install('build_tools') vm.InstallPreprovisionedPackageData('omb', [_TARBALL], '.') vm.RemoteCommand(f'tar -xvf {_TARBALL}') - vm.RemoteCommand(f'cd {_SRC_DIR}; {intelmpi.SourceMpiVarsCommand(vm)}; ' - f'./configure CC={mpicc_path} CXX={mpicxx_path}; ' - 'make; sudo make install') + vm.RemoteCommand( + f'cd {_SRC_DIR}; {intelmpi.SourceMpiVarsCommand(vm)}; ' + f'./configure CC={mpicc_path} CXX={mpicxx_path}; ' + 'make; sudo make install' + ) _TestInstall([vm]) @@ -259,8 +323,10 @@ def _InstallForOpenMpi(vm) -> None: vm.Install('build_tools') vm.InstallPreprovisionedPackageData('omb', [_TARBALL], '.') vm.RemoteCommand(f'tar -xvf {_TARBALL}') - vm.RemoteCommand(f'cd {_SRC_DIR}; ./configure CC={mpicc_path} ' - f'CXX={mpicxx_path}; make; sudo make install') + vm.RemoteCommand( + f'cd {_SRC_DIR}; ./configure CC={mpicc_path} ' + f'CXX={mpicxx_path}; make; sudo make install' + ) _TestInstall([vm]) @@ -319,12 +385,16 @@ def RunBenchmark(request: RunRequest) -> Iterator[RunResult]: number_processes = processes_per_host * len(vms) try: start_time = time.time() - txt, full_cmd = _RunBenchmark(vms[0], name, _MPI_PERHOST.value, - number_processes, vms, params) + txt, full_cmd = _RunBenchmark( + vms[0], name, _MPI_PERHOST.value, number_processes, vms, params + ) run_time = time.time() - start_time except errors.VirtualMachine.RemoteCommandError: - logging.exception('Error running %s benchmark with %s MPI proccesses', - name, number_processes) + logging.exception( + 'Error running %s benchmark with %s MPI proccesses', + name, + number_processes, + ) continue yield RunResult( @@ -342,11 +412,13 @@ def RunBenchmark(request: RunRequest) -> Iterator[RunResult]: pinning=ParseMpiPinning(txt.splitlines()), perhost=_MPI_PERHOST.value, mpi_env={ - k: v for k, v in [ + k: v + for k, v in [ envvar.split('=', 1) for envvar in FLAGS.omb_mpi_env + FLAGS.omb_mpi_genv ] - }) + }, + ) def _GetMpiVersion(vm) -> Optional[str]: @@ -365,7 +437,8 @@ def _RunBenchmarkWithIntelMpi( global_environment: List[str], number_processes: int = None, hosts: List[Any] = None, - options: Dict[str, Any] = None) -> Tuple[str, str]: + options: Dict[str, Any] = None, +) -> Tuple[str, str]: """Runs the microbenchmark using Intel MPI library.""" # Create the mpirun command full_benchmark_path = _PathToBenchmark(vm, name) @@ -373,7 +446,8 @@ def _RunBenchmarkWithIntelMpi( mpirun_cmd.extend(sorted(environment)) mpirun_cmd.append('mpirun') mpirun_cmd.extend( - f'-genv {variable}' for variable in sorted(global_environment)) + f'-genv {variable}' for variable in sorted(global_environment) + ) if perhost: mpirun_cmd.append(f'-perhost {perhost}') if number_processes: @@ -394,14 +468,16 @@ def _RunBenchmarkWithIntelMpi( return txt, full_cmd -def _RunBenchmarkWithOpenMpi(vm, - name: str, - perhost: int, - environment: List[str], - global_environment: List[str], - number_processes: int = None, - hosts: List[Any] = None, - options: Dict[str, Any] = None) -> Tuple[str, str]: +def _RunBenchmarkWithOpenMpi( + vm, + name: str, + perhost: int, + environment: List[str], + global_environment: List[str], + number_processes: int = None, + hosts: List[Any] = None, + options: Dict[str, Any] = None, +) -> Tuple[str, str]: """Runs the microbenchmark using OpenMPI library.""" # Create the mpirun command full_env = sorted(environment + global_environment) @@ -421,7 +497,8 @@ def _RunBenchmarkWithOpenMpi(vm, mpirun_cmd.append('--use-hwthread-cpus') if hosts: host_ips = ','.join( - [f'{vm.internal_ip}:slots={number_processes}' for vm in hosts]) + [f'{vm.internal_ip}:slots={number_processes}' for vm in hosts] + ) mpirun_cmd.append(f'-host {host_ips}') mpirun_cmd.append(full_benchmark_path) if options: @@ -436,12 +513,14 @@ def _RunBenchmarkWithOpenMpi(vm, return txt, full_cmd -def _RunBenchmark(vm, - name: str, - perhost: int, - number_processes: int = None, - hosts: List[Any] = None, - options: Dict[str, Any] = None) -> Tuple[str, str]: +def _RunBenchmark( + vm, + name: str, + perhost: int, + number_processes: int = None, + hosts: List[Any] = None, + options: Dict[str, Any] = None, +) -> Tuple[str, str]: """Runs the microbenchmark. Args: @@ -458,8 +537,16 @@ def _RunBenchmark(vm, run_impl = _RunBenchmarkWithIntelMpi if FLAGS.mpi_vendor == 'openmpi': run_impl = _RunBenchmarkWithOpenMpi - return run_impl(vm, name, perhost, FLAGS.omb_mpi_env, FLAGS.omb_mpi_genv, - number_processes, hosts, options) + return run_impl( + vm, + name, + perhost, + FLAGS.omb_mpi_env, + FLAGS.omb_mpi_genv, + number_processes, + hosts, + options, + ) def _PathToBenchmark(vm, name: str) -> str: @@ -516,8 +603,9 @@ def StartBlock(line: str) -> bool: return lines[1:] if lines else [] -def _ParseBenchmarkData(benchmark_name: str, - txt: str) -> List[Dict[str, float]]: +def _ParseBenchmarkData( + benchmark_name: str, txt: str +) -> List[Dict[str, float]]: """Returns the parsed metrics from the benchmark stdout. Text for benchmark_name='bw': @@ -542,7 +630,8 @@ def _ParseBenchmarkData(benchmark_name: str, if len(columns) != len(row_data): raise ValueError( f'Expected {len(columns)} columns ({columns}) in the ' - f'{benchmark_name} benchmark, received {len(row_data)} ({row_data})') + f'{benchmark_name} benchmark, received {len(row_data)} ({row_data})' + ) row_with_headers = dict(zip(columns, row_data)) for int_column in ('size', 'iterations'): if int_column in row_with_headers: diff --git a/perfkitbenchmarker/linux_packages/openfoam.py b/perfkitbenchmarker/linux_packages/openfoam.py index 8828e71bff..4831c45835 100644 --- a/perfkitbenchmarker/linux_packages/openfoam.py +++ b/perfkitbenchmarker/linux_packages/openfoam.py @@ -55,5 +55,6 @@ def AptInstall(vm): # Separate commands since $WM_PROJECT_DIR comes from the OpenFOAM bashrc. vm.RemoteCommand(f'cat {openfoam_bash_path} | tee $HOME/.bashrc') - vm.RemoteCommand('cat $WM_PROJECT_DIR/bin/tools/RunFunctions | ' - 'tee -a $HOME/.bashrc') + vm.RemoteCommand( + 'cat $WM_PROJECT_DIR/bin/tools/RunFunctions | tee -a $HOME/.bashrc' + ) diff --git a/perfkitbenchmarker/linux_packages/openjdk.py b/perfkitbenchmarker/linux_packages/openjdk.py index 43e205828a..59bdc5d989 100644 --- a/perfkitbenchmarker/linux_packages/openjdk.py +++ b/perfkitbenchmarker/linux_packages/openjdk.py @@ -24,9 +24,11 @@ JAVA_HOME = '/usr' OPENJDK_VERSION = flags.DEFINE_integer( - 'openjdk_version', None, + 'openjdk_version', + None, 'Version of openjdk to use. By default, the oldest non-end-of-life LTS ' - 'version of openjdk is automatically detected.') + 'version of openjdk is automatically detected.', +) # Earlier elements of list are preferred. @@ -38,6 +40,7 @@ def _Install(vm, get_package_name_for_version: Callable[[int], str]): """Installs the OpenJDK package on the VM.""" + def DetectJava(): for version in KNOWN_JAVA_VERSIONS: if vm.HasPackage(get_package_name_for_version(version)): @@ -46,12 +49,14 @@ def DetectJava(): version = OPENJDK_VERSION.value or DetectJava() if not version: raise errors.VirtualMachine.VirtualMachineError( - f'No OpenJDK candidate found for {vm.name}.') + f'No OpenJDK candidate found for {vm.name}.' + ) vm.InstallPackages(get_package_name_for_version(version)) def YumInstall(vm): """Installs the OpenJDK package on the VM.""" + def OpenJdkPackage(version: int) -> str: numeric_version = version > 8 and version or f'1.{version}.0' if vm.OS_TYPE == os_types.AMAZONLINUX2023: @@ -62,6 +67,7 @@ def OpenJdkPackage(version: int) -> str: build_name = 'openjdk' return f'java-{numeric_version}-{build_name}-devel' + _Install(vm, OpenJdkPackage) diff --git a/perfkitbenchmarker/linux_packages/openjdk_neoverse.py b/perfkitbenchmarker/linux_packages/openjdk_neoverse.py index b88569f77f..ce225a802f 100644 --- a/perfkitbenchmarker/linux_packages/openjdk_neoverse.py +++ b/perfkitbenchmarker/linux_packages/openjdk_neoverse.py @@ -6,8 +6,9 @@ FLAGS = flags.FLAGS GCC_VERSION = 10 -NEOVERSE_CFLAGS = ('-mcpu=neoverse-n1 -march=armv8.2-a -mtune=neoverse-n1 ' - '-lgcc_s -fPIC ') +NEOVERSE_CFLAGS = ( + '-mcpu=neoverse-n1 -march=armv8.2-a -mtune=neoverse-n1 -lgcc_s -fPIC ' +) NEOVERSE_LDFLAGS = '-Wl,--allow-multiple-definition -Wl,-lgcc_s -lgcc_s' @@ -24,21 +25,28 @@ def InstallNeoverseCompiledOpenJDK(vm, jdk_version: int): os_type = FLAGS.os_type if os_type != os_types.UBUNTU1804: - raise errors.Config.InvalidValue(f'OS Type must be {os_types.UBUNTU1804},' - f'current os type is {FLAGS.os_type}.') + raise errors.Config.InvalidValue( + f'OS Type must be {os_types.UBUNTU1804},' + f'current os type is {FLAGS.os_type}.' + ) if jdk_version != 11: - raise errors.Config.InvalidValue('OpenJDK Version must equal 11, ' - f'current version is {jdk_version}.') + raise errors.Config.InvalidValue( + f'OpenJDK Version must equal 11, current version is {jdk_version}.' + ) vm.Install('ubuntu_toolchain') # This command fails without Ubuntu 1804 - gcc-10 isn't available. vm.InstallPackages(f'gcc-{GCC_VERSION} g++-{GCC_VERSION}') - vm.InstallPackages('mercurial autoconf build-essential ' - 'unzip zip libx11-dev libxext-dev libxrender-dev ' - 'libxrandr-dev libxtst-dev libxt-dev libcups2-dev ' - 'libfontconfig1-dev libasound2-dev') - vm.RemoteCommand('cd /scratch/ && hg clone ' - f'http://hg.openjdk.java.net/jdk-updates/jdk{jdk_version}u/') + vm.InstallPackages( + 'mercurial autoconf build-essential ' + 'unzip zip libx11-dev libxext-dev libxrender-dev ' + 'libxrandr-dev libxtst-dev libxt-dev libcups2-dev ' + 'libfontconfig1-dev libasound2-dev' + ) + vm.RemoteCommand( + 'cd /scratch/ && hg clone ' + f'http://hg.openjdk.java.net/jdk-updates/jdk{jdk_version}u/' + ) vm.RemoteCommand( f'cd /scratch/jdk{jdk_version}u/ && ' @@ -47,15 +55,19 @@ def InstallNeoverseCompiledOpenJDK(vm, jdk_version: int): f'--with-extra-cflags="{NEOVERSE_CFLAGS}" ' f'--with-extra-cxxflags="{NEOVERSE_CFLAGS}" ' f'--with-extra-ldflags="{NEOVERSE_LDFLAGS}" ' - f'--with-boot-jdk=/usr/lib/jvm/java-{jdk_version}-openjdk-arm64') + f'--with-boot-jdk=/usr/lib/jvm/java-{jdk_version}-openjdk-arm64' + ) vm.RemoteCommand(f'cd /scratch/jdk{jdk_version}u/ && make JOBS=50') # The build output folder is different for other OpenJDK versions e.g. 13 build_dir = '/scratch/jdk11u/build/linux-aarch64-normal-server-release' for binary in ['java', 'javac']: - vm.RemoteCommand(f'sudo update-alternatives --install /usr/bin/{binary} ' - f'{binary} {build_dir}/jdk/bin/{binary} 1') - vm.RemoteCommand(f'sudo update-alternatives --set {binary} ' - f'{build_dir}/jdk/bin/{binary}') + vm.RemoteCommand( + f'sudo update-alternatives --install /usr/bin/{binary} ' + f'{binary} {build_dir}/jdk/bin/{binary} 1' + ) + vm.RemoteCommand( + f'sudo update-alternatives --set {binary} {build_dir}/jdk/bin/{binary}' + ) vm.RemoteCommand(f'{binary} --version') diff --git a/perfkitbenchmarker/linux_packages/openmpi.py b/perfkitbenchmarker/linux_packages/openmpi.py index 4f4cef4921..e67adb0f91 100644 --- a/perfkitbenchmarker/linux_packages/openmpi.py +++ b/perfkitbenchmarker/linux_packages/openmpi.py @@ -24,17 +24,25 @@ FLAGS = flags.FLAGS -flags.DEFINE_string('openmpi_version', '3.1.2', - 'OpenMPI version to install, such as 3.1.2 and 4.0.2.' - 'Set to empty to ignore the intallation of OpenMPI.') -flags.DEFINE_bool('openmpi_enable_shared', False, - 'Whether openmpi should build shared libraries ' - 'in addition to static ones.') -flags.DEFINE_bool('openmpi_with_cuda_support', False, - 'Compile with CUDA support') -flags.DEFINE_string('openmpi_configs', None, - 'command line options to be provided to ./configure for' - 'OpenMPI compilation') +flags.DEFINE_string( + 'openmpi_version', + '3.1.2', + 'OpenMPI version to install, such as 3.1.2 and 4.0.2.' + 'Set to empty to ignore the intallation of OpenMPI.', +) +flags.DEFINE_bool( + 'openmpi_enable_shared', + False, + 'Whether openmpi should build shared libraries in addition to static ones.', +) +flags.DEFINE_bool( + 'openmpi_with_cuda_support', False, 'Compile with CUDA support' +) +flags.DEFINE_string( + 'openmpi_configs', + None, + 'command line options to be provided to ./configure forOpenMPI compilation', +) MPI_URL_BASE = 'https://download.open-mpi.org/release/open-mpi' REMOVE_MPI_CMD = 'autoremove -y libopenmpi-dev openmpi-bin openmpi-common' @@ -80,12 +88,14 @@ def _Install(vm): first_dot_pos = version_to_install.find('.') second_dot_pos = version_to_install.find('.', first_dot_pos + 1) major_version = version_to_install[0:second_dot_pos] - mpi_tar = ('openmpi-{version}.tar.gz'.format(version=version_to_install)) - mpi_url = ('{mpi_url_base}/v{major_version}/{mpi_tar}'.format( - mpi_url_base=MPI_URL_BASE, major_version=major_version, mpi_tar=mpi_tar)) + mpi_tar = 'openmpi-{version}.tar.gz'.format(version=version_to_install) + mpi_url = '{mpi_url_base}/v{major_version}/{mpi_tar}'.format( + mpi_url_base=MPI_URL_BASE, major_version=major_version, mpi_tar=mpi_tar + ) install_dir = posixpath.join( linux_packages.INSTALL_DIR, - 'openmpi-{version}'.format(version=version_to_install)) + 'openmpi-{version}'.format(version=version_to_install), + ) vm.Install('build_tools') vm.Install('wget') @@ -96,27 +106,36 @@ def _Install(vm): config_options = [] config_options.append('--enable-static') config_options.append('--prefix=/usr') - config_options.append('--enable-shared' if FLAGS.openmpi_enable_shared - else '--disable-shared') + config_options.append( + '--enable-shared' if FLAGS.openmpi_enable_shared else '--disable-shared' + ) if FLAGS.openmpi_with_cuda_support: - config_options.append('--with-cuda=/usr/local/cuda-{version}/' - .format(version=FLAGS.cuda_toolkit_version)) - config_options.append('--with-cuda-libdir=/usr/local/cuda-{version}/lib64/' - .format(version=FLAGS.cuda_toolkit_version)) + config_options.append( + '--with-cuda=/usr/local/cuda-{version}/'.format( + version=FLAGS.cuda_toolkit_version + ) + ) + config_options.append( + '--with-cuda-libdir=/usr/local/cuda-{version}/lib64/'.format( + version=FLAGS.cuda_toolkit_version + ) + ) if FLAGS.openmpi_configs: config_options.append(FLAGS.openmpi_configs) config_cmd = './configure {}'.format(' '.join(config_options)) vm.RobustRemoteCommand( - 'cd %s/openmpi-%s && %s && make -j %s && sudo make install' % - (install_dir, version_to_install, config_cmd, make_jobs)) + 'cd %s/openmpi-%s && %s && make -j %s && sudo make install' + % (install_dir, version_to_install, config_cmd, make_jobs) + ) def GetMpiDir(): """Returns the installation directory of OpenMPI.""" mpi_dir = posixpath.join( linux_packages.INSTALL_DIR, - 'openmpi-{version}'.format(version=FLAGS.openmpi_version)) + 'openmpi-{version}'.format(version=FLAGS.openmpi_version), + ) return mpi_dir @@ -125,7 +144,8 @@ def YumInstall(vm): if not FLAGS.openmpi_version: return vm.RobustRemoteCommand( - 'sudo yum {}'.format(REMOVE_MPI_CMD), ignore_failure=True) + 'sudo yum {}'.format(REMOVE_MPI_CMD), ignore_failure=True + ) _Install(vm) @@ -134,7 +154,8 @@ def AptInstall(vm): if not FLAGS.openmpi_version: return vm.RobustRemoteCommand( - 'sudo apt-get {}'.format(REMOVE_MPI_CMD), ignore_failure=True) + 'sudo apt-get {}'.format(REMOVE_MPI_CMD), ignore_failure=True + ) _Install(vm) diff --git a/perfkitbenchmarker/linux_packages/pgadapter.py b/perfkitbenchmarker/linux_packages/pgadapter.py index 1a5f197c1b..abc3e516b8 100644 --- a/perfkitbenchmarker/linux_packages/pgadapter.py +++ b/perfkitbenchmarker/linux_packages/pgadapter.py @@ -17,7 +17,9 @@ database. See https://cloud.google.com/spanner/docs/pgadapter. """ -PGADAPTER_URL = 'https://storage.googleapis.com/pgadapter-jar-releases/pgadapter.tar.gz' +PGADAPTER_URL = ( + 'https://storage.googleapis.com/pgadapter-jar-releases/pgadapter.tar.gz' +) def AptInstall(vm): diff --git a/perfkitbenchmarker/linux_packages/pgbench.py b/perfkitbenchmarker/linux_packages/pgbench.py index 3dc79a7063..5d0dbbc2c0 100644 --- a/perfkitbenchmarker/linux_packages/pgbench.py +++ b/perfkitbenchmarker/linux_packages/pgbench.py @@ -35,7 +35,8 @@ def YumInstall(vm): """Raises exception when trying to install on yum-based VMs.""" raise NotImplementedError( 'PKB currently only supports the installation of pgbench on ' - 'Debian-based VMs') + 'Debian-based VMs' + ) def AptUninstall(vm): @@ -45,8 +46,9 @@ def AptUninstall(vm): vm.RemoteCommand(remove_str + package) -def MakeSamplesFromOutput(pgbench_stderr, num_clients, num_jobs, - additional_metadata): +def MakeSamplesFromOutput( + pgbench_stderr, num_clients, num_jobs, additional_metadata +): """Creates sample objects from the given pgbench output and metadata. Two samples will be returned, one containing a latency list and @@ -80,17 +82,19 @@ def MakeSamplesFromOutput(pgbench_stderr, num_clients, num_jobs, return [tps_sample, latency_sample] -def RunPgBench(benchmark_spec, - relational_db, - vm, - test_db_name, - client_counts, - job_counts, - seconds_to_pause, - seconds_per_test, - metadata, - file=None, - path=None): +def RunPgBench( + benchmark_spec, + relational_db, + vm, + test_db_name, + client_counts, + job_counts, + seconds_to_pause, + seconds_per_test, + metadata, + file=None, + path=None, +): """Run Pgbench on the client VM. Args: diff --git a/perfkitbenchmarker/linux_packages/pip.py b/perfkitbenchmarker/linux_packages/pip.py index b0927556a7..f89f465831 100644 --- a/perfkitbenchmarker/linux_packages/pip.py +++ b/perfkitbenchmarker/linux_packages/pip.py @@ -29,13 +29,14 @@ from perfkitbenchmarker import linux_packages from perfkitbenchmarker import vm_util from perfkitbenchmarker.linux_packages import python - import requests # NOTE: versionless (latest) URL is in root directory and versions have their # own subdirectories. GET_PIP_URL = 'https://bootstrap.pypa.io/pip/get-pip.py' -GET_PIP_VERSIONED_URL = 'https://bootstrap.pypa.io/pip/{python_version}/get-pip.py' +GET_PIP_VERSIONED_URL = ( + 'https://bootstrap.pypa.io/pip/{python_version}/get-pip.py' +) def Install(vm, pip_cmd='pip', python_cmd='python'): @@ -51,8 +52,9 @@ def Install(vm, pip_cmd='pip', python_cmd='python'): pip_path = '/usr/bin/' + pip_cmd # Create an sh shim that redirects to python -m pip vm.RemoteCommand( - f"echo 'exec {python_cmd} -m pip \"$@\"'| sudo tee {pip_path} " - f'&& sudo chmod 755 {pip_path}') + f'echo \'exec {python_cmd} -m pip "$@"\'| sudo tee {pip_path} ' + f'&& sudo chmod 755 {pip_path}' + ) else: # get-pip.py has the appropriate latest version of pip for all Python # versions. Prefer it over linux packages or easy_install @@ -68,13 +70,16 @@ def Install(vm, pip_cmd='pip', python_cmd='python'): get_pip_url = versioned_url else: get_pip_url = GET_PIP_URL + # get_pip can suffer from various network issues. @vm_util.Retry( max_retries=5, - retryable_exceptions=(errors.VirtualMachine.RemoteCommandError,)) + retryable_exceptions=(errors.VirtualMachine.RemoteCommandError,), + ) def GetPipWithRetries(): vm.RemoteCommand( - f'curl {get_pip_url} -o get_pip.py && sudo {python_cmd} get_pip.py') + f'curl {get_pip_url} -o get_pip.py && sudo {python_cmd} get_pip.py' + ) GetPipWithRetries() @@ -83,13 +88,17 @@ def GetPipWithRetries(): # Record installed Python packages install_dir = linux_packages.INSTALL_DIR - vm.RemoteCommand(f'mkdir -p {install_dir} ' - f'&& {pip_cmd} freeze | tee {install_dir}/requirements.txt') + vm.RemoteCommand( + f'mkdir -p {install_dir} ' + f'&& {pip_cmd} freeze | tee {install_dir}/requirements.txt' + ) def Uninstall(vm, pip_cmd='pip'): """Uninstalls the pip package on the VM.""" install_dir = linux_packages.INSTALL_DIR - vm.RemoteCommand(f'{pip_cmd} freeze | grep --fixed-strings --line-regexp ' - f'--invert-match --file {install_dir}/requirements.txt | ' - f'xargs --no-run-if-empty sudo {pip_cmd} uninstall -y') + vm.RemoteCommand( + f'{pip_cmd} freeze | grep --fixed-strings --line-regexp ' + f'--invert-match --file {install_dir}/requirements.txt | ' + f'xargs --no-run-if-empty sudo {pip_cmd} uninstall -y' + ) diff --git a/perfkitbenchmarker/linux_packages/postgres13.py b/perfkitbenchmarker/linux_packages/postgres13.py index 6079979a81..4146adf19d 100644 --- a/perfkitbenchmarker/linux_packages/postgres13.py +++ b/perfkitbenchmarker/linux_packages/postgres13.py @@ -13,20 +13,21 @@ # limitations under the License. -"""Module containing install postgresql server. -""" +"""Module containing install postgresql server.""" def AptInstall(vm): """Installs the postgres package on the VM.""" vm.RemoteCommand( - 'sudo sh -c \'echo ' + "sudo sh -c 'echo " '"deb https://apt.postgresql.org/pub/repos/apt ' '$(lsb_release -cs)-pgdg main" ' - '> /etc/apt/sources.list.d/pgdg.list\'') + "> /etc/apt/sources.list.d/pgdg.list'" + ) vm.InstallPackages('wget') vm.RemoteCommand( 'wget --quiet -O - ' - 'https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -') + 'https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -' + ) vm.RemoteCommand('sudo apt-get update') vm.RemoteCommand('sudo apt-get -y install postgresql-13') diff --git a/perfkitbenchmarker/linux_packages/postgres_client.py b/perfkitbenchmarker/linux_packages/postgres_client.py index c4bf5d8175..abfde6e46f 100644 --- a/perfkitbenchmarker/linux_packages/postgres_client.py +++ b/perfkitbenchmarker/linux_packages/postgres_client.py @@ -19,14 +19,18 @@ def _Install(vm): """Installs the postgres client package on the VM.""" # Install Postgres client - vm.RemoteCommand('sudo sh -c ' - '\'echo "deb https://apt.postgresql.org/pub/repos/apt' - ' $(lsb_release -cs)-pgdg main" >' - ' /etc/apt/sources.list.d/pgdg.list\'') + vm.RemoteCommand( + 'sudo sh -c ' + '\'echo "deb https://apt.postgresql.org/pub/repos/apt' + ' $(lsb_release -cs)-pgdg main" >' + " /etc/apt/sources.list.d/pgdg.list'" + ) vm.InstallPackages('wget') - vm.RemoteCommand('wget --quiet -O - ' - 'https://www.postgresql.org/media/keys/ACCC4CF8.asc ' - '| sudo apt-key add -') + vm.RemoteCommand( + 'wget --quiet -O - ' + 'https://www.postgresql.org/media/keys/ACCC4CF8.asc ' + '| sudo apt-key add -' + ) vm.InstallPackages('postgresql-client') diff --git a/perfkitbenchmarker/linux_packages/python.py b/perfkitbenchmarker/linux_packages/python.py index b5caaf618d..4f94456822 100644 --- a/perfkitbenchmarker/linux_packages/python.py +++ b/perfkitbenchmarker/linux_packages/python.py @@ -20,8 +20,9 @@ from perfkitbenchmarker import vm_util # Gets major.minor version of python -GET_VERSION = ('import sys; ' - 'print(".".join(str(v) for v in sys.version_info[:2]))') +GET_VERSION = ( + 'import sys; print(".".join(str(v) for v in sys.version_info[:2]))' +) def Install(vm): @@ -44,7 +45,8 @@ def _SetDefaultPythonIfNeeded(vm): """ @vm_util.Retry( - retryable_exceptions=(errors.VirtualMachine.RemoteCommandError,)) + retryable_exceptions=(errors.VirtualMachine.RemoteCommandError,) + ) def _RunCommand(command): return vm.RemoteCommandWithReturnCode(command, ignore_failure=True) @@ -54,25 +56,30 @@ def _RunCommand(command): python_found = not return_code if python_found: logging.info( - 'Default version of python: %s', (stdout or stderr).strip().split()[-1]) + 'Default version of python: %s', (stdout or stderr).strip().split()[-1] + ) return logging.info('Trying to set the default python version') _, _, python2_not_found = _RunCommand('ls /usr/bin/python2') if python2_not_found: raise errors.Setup.PythonPackageRequirementUnfulfilled( - 'No default version of python set and /usr/bin/python2 does not exist') + 'No default version of python set and /usr/bin/python2 does not exist' + ) _, _, update_alternatives_failed = _RunCommand( - 'sudo update-alternatives --set python /usr/bin/python2') + 'sudo update-alternatives --set python /usr/bin/python2' + ) if update_alternatives_failed: # Ubuntu 20 lets you install python2, but not update-alternatives to point # python to it. Also some distros might not include update-alternatives. # In any case fall back to a symlink vm.RemoteCommandWithReturnCode( - 'sudo ln -s /usr/bin/python2 /usr/bin/python') + 'sudo ln -s /usr/bin/python2 /usr/bin/python' + ) _, txt, python_not_found = _RunCommand(python_version_cmd) if python_not_found: raise errors.Setup.PythonPackageRequirementUnfulfilled( - "'python' command not working after setting alias.") + "'python' command not working after setting alias." + ) logging.info('Set default python version to %s', txt.strip().split()[-1]) diff --git a/perfkitbenchmarker/linux_packages/pytorch.py b/perfkitbenchmarker/linux_packages/pytorch.py index f282f724af..25b3d9eb1c 100644 --- a/perfkitbenchmarker/linux_packages/pytorch.py +++ b/perfkitbenchmarker/linux_packages/pytorch.py @@ -25,8 +25,9 @@ flags.DEFINE_string('torch_version', '1.7.1', 'The torch version.') flags.DEFINE_string('torchvision_version', '0.8.2', 'The torchvision version.') flags.DEFINE_string('torchaudio_version', '0.7.2', 'The torchaudio version.') -flags.DEFINE_string('torch_env', 'PATH=/opt/conda/bin:$PATH', - 'The torch install environment.') +flags.DEFINE_string( + 'torch_env', 'PATH=/opt/conda/bin:$PATH', 'The torch install environment.' +) _PYTORCH_WHL = 'https://download.pytorch.org/whl/{compute_platform}' @@ -66,5 +67,6 @@ def Install(vm): def Uninstall(vm): """Uninstalls TensorFlow on the VM.""" - vm.RemoteCommand(f'{FLAGS.torch_env} pip uninstall ' - 'torch torchvision torchaudio') + vm.RemoteCommand( + f'{FLAGS.torch_env} pip uninstall torch torchvision torchaudio' + ) diff --git a/perfkitbenchmarker/linux_packages/redis_enterprise.py b/perfkitbenchmarker/linux_packages/redis_enterprise.py index 72aa166335..347ec7a31b 100644 --- a/perfkitbenchmarker/linux_packages/redis_enterprise.py +++ b/perfkitbenchmarker/linux_packages/redis_enterprise.py @@ -37,71 +37,102 @@ FLAGS = flags.FLAGS _LICENSE = flags.DEFINE_string( - 'enterprise_redis_license_file', 'enterprise_redis_license', - 'Name of the redis enterprise license file to use.') + 'enterprise_redis_license_file', + 'enterprise_redis_license', + 'Name of the redis enterprise license file to use.', +) _LICENSE_PATH = flags.DEFINE_string( - 'enterprise_redis_license_path', None, - 'If none, defaults to local data directory joined with _LICENSE.') + 'enterprise_redis_license_path', + None, + 'If none, defaults to local data directory joined with _LICENSE.', +) _TUNE_ON_STARTUP = flags.DEFINE_boolean( - 'enterprise_redis_tune_on_startup', True, - 'Whether to tune core config during startup.') + 'enterprise_redis_tune_on_startup', + True, + 'Whether to tune core config during startup.', +) _PROXY_THREADS = flags.DEFINE_integer( - 'enterprise_redis_proxy_threads', None, - 'Number of redis proxy threads to use.') + 'enterprise_redis_proxy_threads', + None, + 'Number of redis proxy threads to use.', +) _SHARDS = flags.DEFINE_integer( - 'enterprise_redis_shard_count', None, - 'Number of redis shards per database. Each shard is a redis thread.') + 'enterprise_redis_shard_count', + None, + 'Number of redis shards per database. Each shard is a redis thread.', +) _LOAD_RECORDS = flags.DEFINE_integer( - 'enterprise_redis_load_records', 1000000, - 'Number of keys to pre-load into Redis. Use ' - '--enterprise_redis_data_size_bytes to calculate how much space will be ' - 'used. Due to overhead, a ballpark estimate is 1KB record takes 1.5KB of ' - 'utilization. See https://docs.redis.com/latest/rs/concepts/memory-performance/' - 'for more info.') + 'enterprise_redis_load_records', + 1000000, + 'Number of keys to pre-load into Redis. Use' + ' --enterprise_redis_data_size_bytes to calculate how much space will be' + ' used. Due to overhead, a ballpark estimate is 1KB record takes 1.5KB of' + ' utilization. See' + ' https://docs.redis.com/latest/rs/concepts/memory-performance/for more' + ' info.', +) _RUN_RECORDS = flags.DEFINE_integer( - 'enterprise_redis_run_records', 1000000, - 'Number of requests per loadgen client to send to the ' - 'Redis server.') + 'enterprise_redis_run_records', + 1000000, + 'Number of requests per loadgen client to send to the Redis server.', +) _PIPELINES = flags.DEFINE_integer( - 'enterprise_redis_pipeline', 9, - 'Number of pipelines to use.') + 'enterprise_redis_pipeline', 9, 'Number of pipelines to use.' +) _LOADGEN_CLIENTS = flags.DEFINE_integer( - 'enterprise_redis_loadgen_clients', 24, - 'Number of clients per loadgen vm.') + 'enterprise_redis_loadgen_clients', 24, 'Number of clients per loadgen vm.' +) _MAX_THREADS = flags.DEFINE_integer( - 'enterprise_redis_max_threads', 40, - 'Maximum number of memtier threads to use.') + 'enterprise_redis_max_threads', + 40, + 'Maximum number of memtier threads to use.', +) _MIN_THREADS = flags.DEFINE_integer( - 'enterprise_redis_min_threads', 18, - 'Minimum number of memtier threads to use.') + 'enterprise_redis_min_threads', + 18, + 'Minimum number of memtier threads to use.', +) _THREAD_INCREMENT = flags.DEFINE_integer( - 'enterprise_redis_thread_increment', 1, - 'Number of memtier threads to increment by.') + 'enterprise_redis_thread_increment', + 1, + 'Number of memtier threads to increment by.', +) _LATENCY_THRESHOLD = flags.DEFINE_integer( - 'enterprise_redis_latency_threshold', 1100, - 'The latency threshold in microseconds ' - 'until the test stops.') + 'enterprise_redis_latency_threshold', + 1100, + 'The latency threshold in microseconds until the test stops.', +) _PIN_WORKERS = flags.DEFINE_boolean( - 'enterprise_redis_pin_workers', False, - 'Whether to pin the proxy threads after startup.') + 'enterprise_redis_pin_workers', + False, + 'Whether to pin the proxy threads after startup.', +) _DISABLE_CPU_IDS = flags.DEFINE_list( - 'enterprise_redis_disable_cpu_ids', None, - 'List of cpus to disable by id.') + 'enterprise_redis_disable_cpu_ids', None, 'List of cpus to disable by id.' +) _DATA_SIZE = flags.DEFINE_integer( - 'enterprise_redis_data_size_bytes', 100, - 'The size of the data to write to redis enterprise.') + 'enterprise_redis_data_size_bytes', + 100, + 'The size of the data to write to redis enterprise.', +) _NUM_DATABASES = flags.DEFINE_integer( - 'enterprise_redis_db_count', 1, - 'The number of databases to create on the cluster.') + 'enterprise_redis_db_count', + 1, + 'The number of databases to create on the cluster.', +) _REPLICATION = flags.DEFINE_bool( - 'enterprise_redis_db_replication', False, + 'enterprise_redis_db_replication', + False, 'If true, replicates each database to another node. Doubles the amount of ' - 'memory used by the database records.') + 'memory used by the database records.', +) _MEMORY_SIZE_PERCENTAGE = flags.DEFINE_float( - 'enterprise_redis_memory_size_percentage', 0.80, + 'enterprise_redis_memory_size_percentage', + 0.80, 'The percentage amount of memory to use out of all the available memory ' 'reported by rladmin for provisioning databases. 1 means use all available ' - 'memory, which in practice tends to be error-prone.') + 'memory, which in practice tends to be error-prone.', +) _VM = virtual_machine.VirtualMachine _ThroughputSampleTuple = Tuple[float, List[sample.Sample]] @@ -123,12 +154,15 @@ # These checksums correspond to version 6.2.4-54. To update, run # 'sha256sum ' and replace the values # below. - _RHEL_TAR: - 'fb0b7aa5f115eb0bc2ac4fb958aaa7ad92bb260f2251a221a15b01fbdf4d2d14', - _XENIAL_TAR: - 'f78a6bb486f3dfb3e5ba9b5be86b1c880d0c76a08eb0dc4bd3aaaf9cc210406d', - _BIONIC_TAR: - 'dfe568958b243368c1f1c08c9cce9f660fa06e1bce38fa88f90503e344466927', + _RHEL_TAR: ( + 'fb0b7aa5f115eb0bc2ac4fb958aaa7ad92bb260f2251a221a15b01fbdf4d2d14' + ), + _XENIAL_TAR: ( + 'f78a6bb486f3dfb3e5ba9b5be86b1c880d0c76a08eb0dc4bd3aaaf9cc210406d' + ), + _BIONIC_TAR: ( + 'dfe568958b243368c1f1c08c9cce9f660fa06e1bce38fa88f90503e344466927' + ), } _THREAD_OPTIMIZATION_RATIO = 0.75 @@ -160,50 +194,62 @@ def Install(vm: _VM) -> None: vm.PushFile(license_path, posixpath.join(_WORKING_DIR, _LICENSE.value)) # Check for the tarfile in the data directory first. - vm.InstallPreprovisionedPackageData(_PACKAGE_NAME, [_GetTarName()], - _WORKING_DIR) - vm.RemoteCommand('cd {dir} && sudo tar xvf {tar}'.format( - dir=_WORKING_DIR, tar=_GetTarName())) + vm.InstallPreprovisionedPackageData( + _PACKAGE_NAME, [_GetTarName()], _WORKING_DIR + ) + vm.RemoteCommand( + 'cd {dir} && sudo tar xvf {tar}'.format( + dir=_WORKING_DIR, tar=_GetTarName() + ) + ) if FLAGS.os_type == os_types.UBUNTU1804: # Fix Ubuntu 18.04 DNS conflict vm.RemoteCommand( - 'echo "DNSStubListener=no" | sudo tee -a /etc/systemd/resolved.conf') + 'echo "DNSStubListener=no" | sudo tee -a /etc/systemd/resolved.conf' + ) vm.RemoteCommand('sudo mv /etc/resolv.conf /etc/resolv.conf.orig') vm.RemoteCommand( - 'sudo ln -s /run/systemd/resolve/resolv.conf /etc/resolv.conf') + 'sudo ln -s /run/systemd/resolve/resolv.conf /etc/resolv.conf' + ) vm.RemoteCommand('sudo service systemd-resolved restart') install_cmd = './install.sh -y' if not _TUNE_ON_STARTUP.value: install_cmd = 'CONFIG_systune=no ./install.sh -y -n' - vm.RemoteCommand('cd {dir} && sudo {install}'.format( - dir=_WORKING_DIR, install=install_cmd)) + vm.RemoteCommand( + 'cd {dir} && sudo {install}'.format(dir=_WORKING_DIR, install=install_cmd) + ) def _JoinCluster(server_vm: _VM, vm: _VM) -> None: """Joins a Redis Enterprise cluster.""" logging.info('Joining redis enterprise cluster.') - vm.RemoteCommand('sudo /opt/redislabs/bin/rladmin cluster join ' - 'nodes {server_vm_ip} ' - 'username {username} ' - 'password {password} '.format( - server_vm_ip=server_vm.internal_ip, - username=_USERNAME, - password=FLAGS.run_uri)) + vm.RemoteCommand( + 'sudo /opt/redislabs/bin/rladmin cluster join ' + 'nodes {server_vm_ip} ' + 'username {username} ' + 'password {password} '.format( + server_vm_ip=server_vm.internal_ip, + username=_USERNAME, + password=FLAGS.run_uri, + ) + ) def CreateCluster(vms: List[_VM]) -> None: """Creates a Redis Enterprise cluster on the VM.""" logging.info('Creating redis enterprise cluster.') - vms[0].RemoteCommand('sudo /opt/redislabs/bin/rladmin cluster create ' - 'license_file {license_file} ' - 'name redis-cluster ' - 'username {username} ' - 'password {password} '.format( - license_file=posixpath.join(_WORKING_DIR, - _LICENSE.value), - username=_USERNAME, - password=FLAGS.run_uri)) + vms[0].RemoteCommand( + 'sudo /opt/redislabs/bin/rladmin cluster create ' + 'license_file {license_file} ' + 'name redis-cluster ' + 'username {username} ' + 'password {password} '.format( + license_file=posixpath.join(_WORKING_DIR, _LICENSE.value), + username=_USERNAME, + password=FLAGS.run_uri, + ) + ) for vm in vms[1:]: _JoinCluster(vms[0], vm) @@ -213,9 +259,10 @@ def OfflineCores(vms: List[_VM]) -> None: def _Offline(vm): for cpu_id in _DISABLE_CPU_IDS.value or []: - vm.RemoteCommand('sudo bash -c ' - '"echo 0 > /sys/devices/system/cpu/cpu%s/online"' % - cpu_id) + vm.RemoteCommand( + 'sudo bash -c "echo 0 > /sys/devices/system/cpu/cpu%s/online"' + % cpu_id + ) background_tasks.RunThreaded(_Offline, vms) @@ -223,10 +270,12 @@ def _Offline(vm): def TuneProxy(vm: _VM, proxy_threads: Optional[int] = None) -> None: """Tunes the number of Redis proxies on the cluster.""" proxy_threads = proxy_threads or _PROXY_THREADS.value - vm.RemoteCommand('sudo /opt/redislabs/bin/rladmin tune ' - 'proxy all ' - f'max_threads {proxy_threads} ' - f'threads {proxy_threads} ') + vm.RemoteCommand( + 'sudo /opt/redislabs/bin/rladmin tune ' + 'proxy all ' + f'max_threads {proxy_threads} ' + f'threads {proxy_threads} ' + ) vm.RemoteCommand('sudo /opt/redislabs/bin/dmc_ctl restart') @@ -249,7 +298,8 @@ def _Pin(vm): proxies_per_node = proxy_threads // numa_nodes for node in range(numa_nodes): node_cpu_list = vm.RemoteCommand( - 'cat /sys/devices/system/node/node%d/cpulist' % node)[0].strip() + 'cat /sys/devices/system/node/node%d/cpulist' % node + )[0].strip() # List the PIDs of the Redis worker processes and pin a sliding window of # `proxies_per_node` workers to the NUMA nodes in increasing order. vm.RemoteCommand( @@ -263,7 +313,9 @@ def _Pin(vm): r'xargs -i sudo taskset -pc {node_cpu_list} {{}} '.format( proxies_already_partitioned=proxies_per_node * node, proxies_per_node=proxies_per_node, - node_cpu_list=node_cpu_list)) + node_cpu_list=node_cpu_list, + ) + ) background_tasks.RunThreaded(_Pin, vms) @@ -275,12 +327,13 @@ def GetDatabaseMemorySize(vm: _VM) -> int: node_output = output.splitlines()[2] provisional_ram = node_output.split()[7] size_gb = float(provisional_ram.split('/')[0].strip('GB')) * ( - _MEMORY_SIZE_PERCENTAGE.value) + _MEMORY_SIZE_PERCENTAGE.value + ) return int(size_gb * _ONE_GIGABYTE) @dataclasses.dataclass(frozen=True) -class LoadRequest(): +class LoadRequest: key_minimum: int key_maximum: int redis_port: int @@ -303,7 +356,8 @@ def _BuildLoadCommand(request: LoadRequest) -> str: '--key-pattern S:S ' f'--key-minimum {request.key_minimum} ' f'--key-maximum {request.key_maximum} ' - '-n allkeys ') + '-n allkeys ' + ) if request.cluster_mode: command += '--cluster-mode' return command @@ -316,10 +370,12 @@ def _LoadDatabaseSingleVM(load_vm: _VM, request: LoadRequest) -> None: load_vm.RemoteCommand(command) -def LoadDatabases(redis_vms: List[_VM], - load_vms: List[_VM], - endpoints: List[Tuple[str, int]], - shards: Optional[int] = None) -> None: +def LoadDatabases( + redis_vms: List[_VM], + load_vms: List[_VM], + endpoints: List[Tuple[str, int]], + shards: Optional[int] = None, +) -> None: """Loads the databases before performing tests.""" vms = load_vms + redis_vms load_requests = [] @@ -335,14 +391,16 @@ def LoadDatabases(redis_vms: List[_VM], key_maximum=(i + 1) * load_records_per_vm, redis_port=port, cluster_mode=cluster_mode, - server_ip=endpoint))) + server_ip=endpoint, + ), + )) background_tasks.RunThreaded( _LoadDatabaseSingleVM, [(arg, {}) for arg in load_requests] ) -class HttpClient(): +class HttpClient: """HTTP Client for interacting with Redis REST API.""" def __init__(self, server_vms: List[_VM]): @@ -355,8 +413,7 @@ def __init__(self, server_vms: List[_VM]): def _LogCurlifiedCommand(self, response: requests.Response) -> None: """Logs the version of the request that can be run from curl.""" request = response.request - command = (f'curl -v -k -u {_USERNAME}:{FLAGS.run_uri} ' - f'-X {request.method} ') + command = f'curl -v -k -u {_USERNAME}:{FLAGS.run_uri} -X {request.method} ' if request.body: body = request.body.decode('UTF-8') command += f'-d "{body}" -H "Content-type: application/json" ' @@ -398,13 +455,15 @@ def GetDatabase(self, uid: int) -> Optional[_Json]: @vm_util.Retry( poll_interval=5, - retryable_exceptions=(errors.Resource.RetryableCreationError,)) + retryable_exceptions=(errors.Resource.RetryableCreationError,), + ) def WaitForDatabaseUp(self, uid: int) -> None: """Waits for the Redis Enterprise database to become active.""" db = self.GetDatabase(uid) if not db: raise errors.Benchmarks.RunError( - 'Database %s does not exist, expected to be waiting for startup.') + 'Database %s does not exist, expected to be waiting for startup.' + ) if db['status'] != 'active': raise errors.Resource.RetryableCreationError() @@ -424,7 +483,8 @@ def CreateDatabase(self, shards: Optional[int] = None) -> _Json: if not self.provisional_memory: self.provisional_memory = GetDatabaseMemorySize(self.vms[0]) per_db_memory_size = int( - self.provisional_memory * len(self.vms) / _NUM_DATABASES.value) + self.provisional_memory * len(self.vms) / _NUM_DATABASES.value + ) content = { 'name': 'redisdb', 'type': 'redis', @@ -447,12 +507,14 @@ def CreateDatabase(self, shards: Optional[int] = None) -> _Json: logging.info('Creating Redis Enterprise database with %s.', content) r = self.session.post( - f'{self.api_base_url}/v1/bdbs', json=content, verify=False) + f'{self.api_base_url}/v1/bdbs', json=content, verify=False + ) self._LogCurlifiedCommand(r) if r.status_code != 200: raise errors.Benchmarks.RunError( f'Unable to create database: status code: {r.status_code}, ' - f'reason {r.reason}.') + f'reason {r.reason}.' + ) self.WaitForDatabaseUp(r.json()['uid']) logging.info('Finished creating Redis Enterprise database %s.', r.json()) return r.json() @@ -464,7 +526,8 @@ def CreateDatabases(self, shards: Optional[int] = None) -> None: @vm_util.Retry( poll_interval=5, - retryable_exceptions=(errors.Resource.RetryableDeletionError,)) + retryable_exceptions=(errors.Resource.RetryableDeletionError,), + ) def DeleteDatabase(self, uid: int) -> None: """Deletes the database from the cluster.""" logging.info('Deleting Redis Enterprise database (uid: %s).', uid) @@ -480,8 +543,9 @@ def DeleteDatabases(self) -> None: self.DeleteDatabase(uid) -def _BuildRunCommand(host: str, threads: int, port: int, - shards: Optional[int] = None) -> str: +def _BuildRunCommand( + host: str, threads: int, port: int, shards: Optional[int] = None +) -> str: """Spawns a memtier_benchmark on the load_vm against the redis_vm:port. Args: @@ -496,18 +560,20 @@ def _BuildRunCommand(host: str, threads: int, port: int, if threads == 0: return None - result = ('sudo /opt/redislabs/bin/memtier_benchmark ' - f'-s {host} ' - f'-a {FLAGS.run_uri} ' - f'-p {str(port)} ' - f'-t {str(threads)} ' - '--ratio 1:1 ' - f'--pipeline {str(_PIPELINES.value)} ' - f'-c {str(_LOADGEN_CLIENTS.value)} ' - f'-d {str(_DATA_SIZE.value)} ' - '--test-time 30 ' - '--key-minimum 1 ' - f'--key-maximum {str(_LOAD_RECORDS.value)} ') + result = ( + 'sudo /opt/redislabs/bin/memtier_benchmark ' + f'-s {host} ' + f'-a {FLAGS.run_uri} ' + f'-p {str(port)} ' + f'-t {str(threads)} ' + '--ratio 1:1 ' + f'--pipeline {str(_PIPELINES.value)} ' + f'-c {str(_LOADGEN_CLIENTS.value)} ' + f'-d {str(_DATA_SIZE.value)} ' + '--test-time 30 ' + '--key-minimum 1 ' + f'--key-maximum {str(_LOAD_RECORDS.value)} ' + ) shards = shards or _SHARDS.value if shards > 1: result += '--cluster-mode ' @@ -515,8 +581,9 @@ def _BuildRunCommand(host: str, threads: int, port: int, @dataclasses.dataclass(frozen=True) -class Result(): +class Result: """Individual throughput and latency result.""" + throughput: int latency_usec: int metadata: Dict[str, Any] @@ -533,16 +600,18 @@ def ParseResults(output: str) -> List[Result]: results = [] for interval in output_json.get('intervals'): results.append( - Result( - interval.get('total_req'), interval.get('avg_latency'), interval)) + Result(interval.get('total_req'), interval.get('avg_latency'), interval) + ) return results -def Run(redis_vms: List[_VM], - load_vms: List[_VM], - shards: Optional[int] = None, - proxy_threads: Optional[int] = None, - memtier_threads: Optional[int] = None) -> _ThroughputSampleTuple: +def Run( + redis_vms: List[_VM], + load_vms: List[_VM], + shards: Optional[int] = None, + proxy_threads: Optional[int] = None, + memtier_threads: Optional[int] = None, +) -> _ThroughputSampleTuple: """Run memtier against enterprise redis and measure latency and throughput. This function runs memtier against the redis server vm with increasing memtier @@ -576,13 +645,15 @@ def Run(redis_vms: List[_VM], if len(set(endpoints)) < _NUM_DATABASES.value: raise errors.Benchmarks.RunError( f'Wrong number of unique endpoints {endpoints}, ' - f'expected {_NUM_DATABASES.value}.') + f'expected {_NUM_DATABASES.value}.' + ) if threads > max_threads: raise errors.Benchmarks.RunError( 'min threads %s higher than max threads %s, ' - 'raise --enterprise_redis_max_threads') + 'raise --enterprise_redis_max_threads' + ) - while (cur_max_latency < latency_threshold and threads <= max_threads): + while cur_max_latency < latency_threshold and threads <= max_threads: # Set up run commands run_cmds = [ _BuildRunCommand(endpoint, threads, port, shards) @@ -596,7 +667,8 @@ def Run(redis_vms: List[_VM], .format( user=_USERNAME, password=FLAGS.run_uri, - )) + ) + ) args += [((redis_vm, measurement_command), {})] # Run background_tasks.RunThreaded( @@ -613,26 +685,39 @@ def Run(redis_vms: List[_VM], cur_max_latency = max(cur_max_latency, latency) if latency < 1000: max_throughput_for_completion_latency_under_1ms = max( - max_throughput_for_completion_latency_under_1ms, result.throughput) + max_throughput_for_completion_latency_under_1ms, result.throughput + ) - logging.info('Threads : %d (%f ops/sec, %f ms latency) < %f ms latency', - threads, result.throughput, latency, latency_threshold) + logging.info( + 'Threads : %d (%f ops/sec, %f ms latency) < %f ms latency', + threads, + result.throughput, + latency, + latency_threshold, + ) threads += _THREAD_INCREMENT.value if cur_max_latency >= 1000: results.append( - sample.Sample('max_throughput_for_completion_latency_under_1ms', - max_throughput_for_completion_latency_under_1ms, 'ops/s', - metadata)) + sample.Sample( + 'max_throughput_for_completion_latency_under_1ms', + max_throughput_for_completion_latency_under_1ms, + 'ops/s', + metadata, + ) + ) - logging.info('Max throughput under 1ms: %s ops/sec.', - max_throughput_for_completion_latency_under_1ms) + logging.info( + 'Max throughput under 1ms: %s ops/sec.', + max_throughput_for_completion_latency_under_1ms, + ) return max_throughput_for_completion_latency_under_1ms, results -def GetMetadata(shards: int, threads: int, - proxy_threads: int) -> Dict[str, Any]: +def GetMetadata( + shards: int, threads: int, proxy_threads: int +) -> Dict[str, Any]: """Returns metadata associated with the run. Args: @@ -661,7 +746,7 @@ def GetMetadata(shards: int, threads: int, } -class ThroughputOptimizer(): +class ThroughputOptimizer: """Class that searches for the shard/proxy_thread count for best throughput. Attributes: @@ -680,26 +765,34 @@ def __init__(self, server_vms: List[_VM], client_vms: List[_VM]): # Determines the search space for the optimization algorithm. We multiply # the size by 2 which should be a large enough search space. - matrix_size = max(server_vms[0].num_cpus, - FLAGS.enterprise_redis_proxy_threads or 0, - FLAGS.enterprise_redis_shard_count or 0) * 2 - self.results: _ThroughputSampleMatrix = ( - [[() for i in range(matrix_size)] for i in range(matrix_size)]) + matrix_size = ( + max( + server_vms[0].num_cpus, + FLAGS.enterprise_redis_proxy_threads or 0, + FLAGS.enterprise_redis_shard_count or 0, + ) + * 2 + ) + self.results: _ThroughputSampleMatrix = [ + [() for i in range(matrix_size)] for i in range(matrix_size) + ] def _CreateAndLoadDatabases(self, shards: int) -> None: """Creates and loads all the databases needed for the run.""" self.client.CreateDatabases(shards) LoadDatabases( - self.server_vms, - self.client_vms, - self.client.GetEndpoints(), - shards) + self.server_vms, self.client_vms, self.client.GetEndpoints(), shards + ) - def _FullRun(self, shard_count: int, - proxy_thread_count: int) -> _ThroughputSampleTuple: + def _FullRun( + self, shard_count: int, proxy_thread_count: int + ) -> _ThroughputSampleTuple: """Recreates databases if needed, then runs the test.""" - logging.info('Starting new run with %s shards, %s proxy threads', - shard_count, proxy_thread_count) + logging.info( + 'Starting new run with %s shards, %s proxy threads', + shard_count, + proxy_thread_count, + ) server_vm = self.server_vms[0] # Recreate the DB if needed @@ -720,35 +813,47 @@ def _FullRun(self, shard_count: int, self.min_threads) # pyformat: disable self.min_threads = max( self.min_threads, - int(results[1][-1].metadata['threads'] * _THREAD_OPTIMIZATION_RATIO)) + int(results[1][-1].metadata['threads'] * _THREAD_OPTIMIZATION_RATIO), + ) return results - def _GetResult(self, shards: int, - proxy_threads: int) -> _ThroughputSampleTuple: + def _GetResult( + self, shards: int, proxy_threads: int + ) -> _ThroughputSampleTuple: if not self.results[shards - 1][proxy_threads - 1]: self.results[shards - 1][proxy_threads - 1] = self._FullRun( - shards, proxy_threads) + shards, proxy_threads + ) return self.results[shards - 1][proxy_threads - 1] - def _GetNeighborsToCheck(self, shards: int, - proxy_threads: int) -> List[Tuple[int, int]]: + def _GetNeighborsToCheck( + self, shards: int, proxy_threads: int + ) -> List[Tuple[int, int]]: """Returns the shards/proxy_threads neighbor to check.""" - vary_proxy_threads = [(shards, proxy_threads - 1), - (shards, proxy_threads + 1)] + vary_proxy_threads = [ + (shards, proxy_threads - 1), + (shards, proxy_threads + 1), + ] vary_shards = [(shards - 1, proxy_threads), (shards + 1, proxy_threads)] return vary_proxy_threads + vary_shards - def _GetOptimalNeighbor(self, shards: int, - proxy_threads: int) -> Tuple[int, int]: + def _GetOptimalNeighbor( + self, shards: int, proxy_threads: int + ) -> Tuple[int, int]: """Returns the shards/proxy_threads neighbor with the best throughput.""" optimal_shards = shards optimal_proxy_threads = proxy_threads optimal_throughput, _ = self._GetResult(shards, proxy_threads) for shards_count, proxy_threads_count in self._GetNeighborsToCheck( - shards, proxy_threads): - if (shards_count < 1 or shards_count > len(self.results) or - proxy_threads_count < 1 or proxy_threads_count > len(self.results)): + shards, proxy_threads + ): + if ( + shards_count < 1 + or shards_count > len(self.results) + or proxy_threads_count < 1 + or proxy_threads_count > len(self.results) + ): continue throughput, _ = self._GetResult(shards_count, proxy_threads_count) @@ -779,10 +884,14 @@ def DoGraphSearch(self) -> _ThroughputSampleTuple: proxy_thread_count = num_cpus - shard_count while True: - logging.info('Checking shards: %s, proxy_threads: %s', shard_count, - proxy_thread_count) + logging.info( + 'Checking shards: %s, proxy_threads: %s', + shard_count, + proxy_thread_count, + ) optimal_shards, optimal_proxies = self._GetOptimalNeighbor( - shard_count, proxy_thread_count) + shard_count, proxy_thread_count + ) if (shard_count, proxy_thread_count) == (optimal_shards, optimal_proxies): break shard_count = optimal_shards @@ -812,6 +921,7 @@ def GetOptimalThroughput(self) -> _ThroughputSampleTuple: """ # If only optimizing proxy threads, do a linear search. if (_SHARDS.value and not _PROXY_THREADS.value) or ( - _PROXY_THREADS.value and not _SHARDS.value): + _PROXY_THREADS.value and not _SHARDS.value + ): return self.DoLinearSearch() return self.DoGraphSearch() diff --git a/perfkitbenchmarker/linux_packages/redis_server.py b/perfkitbenchmarker/linux_packages/redis_server.py index c4b11eaa8b..5c27f607b7 100644 --- a/perfkitbenchmarker/linux_packages/redis_server.py +++ b/perfkitbenchmarker/linux_packages/redis_server.py @@ -21,8 +21,9 @@ from perfkitbenchmarker import os_types -class RedisEvictionPolicy(): +class RedisEvictionPolicy: """Enum of options for --redis_eviction_policy.""" + NOEVICTION = 'noeviction' ALLKEYS_LRU = 'allkeys-lru' VOLATILE_LRU = 'volatile-lru' @@ -31,37 +32,57 @@ class RedisEvictionPolicy(): VOLATILE_TTL = 'volatile-ttl' -_VERSION = flags.DEFINE_string('redis_server_version', '6.2.1', - 'Version of redis server to use.') +_VERSION = flags.DEFINE_string( + 'redis_server_version', '6.2.1', 'Version of redis server to use.' +) _IO_THREADS = flags.DEFINE_integer( - 'redis_server_io_threads', 4, 'Only supported for redis version >= 6, the ' - 'number of redis server IO threads to use.') + 'redis_server_io_threads', + 4, + 'Only supported for redis version >= 6, the ' + 'number of redis server IO threads to use.', +) _IO_THREADS_DO_READS = flags.DEFINE_bool( - 'redis_server_io_threads_do_reads', False, + 'redis_server_io_threads_do_reads', + False, 'If true, makes both reads and writes use IO threads instead of just ' - 'writes.') + 'writes.', +) _IO_THREAD_AFFINITY = flags.DEFINE_bool( - 'redis_server_io_threads_cpu_affinity', False, - 'If true, attempts to pin IO threads to CPUs.') + 'redis_server_io_threads_cpu_affinity', + False, + 'If true, attempts to pin IO threads to CPUs.', +) _ENABLE_SNAPSHOTS = flags.DEFINE_bool( - 'redis_server_enable_snapshots', False, - 'If true, uses the default redis snapshot policy.') + 'redis_server_enable_snapshots', + False, + 'If true, uses the default redis snapshot policy.', +) _NUM_PROCESSES = flags.DEFINE_integer( 'redis_total_num_processes', 1, 'Total number of redis server processes. Useful when running with a redis ' 'version lower than 6.', - lower_bound=1) + lower_bound=1, +) _EVICTION_POLICY = flags.DEFINE_enum( - 'redis_eviction_policy', RedisEvictionPolicy.NOEVICTION, [ - RedisEvictionPolicy.NOEVICTION, RedisEvictionPolicy.ALLKEYS_LRU, - RedisEvictionPolicy.VOLATILE_LRU, RedisEvictionPolicy.ALLKEYS_RANDOM, - RedisEvictionPolicy.VOLATILE_RANDOM, RedisEvictionPolicy.VOLATILE_TTL - ], 'Redis eviction policy when maxmemory limit is reached. This requires ' - 'running clients with larger amounts of data than Redis can hold.') + 'redis_eviction_policy', + RedisEvictionPolicy.NOEVICTION, + [ + RedisEvictionPolicy.NOEVICTION, + RedisEvictionPolicy.ALLKEYS_LRU, + RedisEvictionPolicy.VOLATILE_LRU, + RedisEvictionPolicy.ALLKEYS_RANDOM, + RedisEvictionPolicy.VOLATILE_RANDOM, + RedisEvictionPolicy.VOLATILE_TTL, + ], + 'Redis eviction policy when maxmemory limit is reached. This requires ' + 'running clients with larger amounts of data than Redis can hold.', +) REDIS_SIMULATE_AOF = flags.DEFINE_bool( - 'redis_simulate_aof', False, 'If true, simulate usage of ' - 'disks on the server for aof backups. ') + 'redis_simulate_aof', + False, + 'If true, simulate usage of disks on the server for aof backups. ', +) # Default port for Redis DEFAULT_PORT = 6379 @@ -85,7 +106,8 @@ def _Install(vm) -> None: vm.Install('wget') vm.RemoteCommand(f'cd {linux_packages.INSTALL_DIR}; git clone {REDIS_GIT}') vm.RemoteCommand( - f'cd {GetRedisDir()} && git checkout {_VERSION.value} && make') + f'cd {GetRedisDir()} && git checkout {_VERSION.value} && make' + ) def YumInstall(vm) -> None: @@ -174,18 +196,22 @@ def Start(vm) -> None: """Start redis server process.""" # 10 is an arbituary multiplier that ensures this value is high enough. mux_sessions = 10 * _NUM_PROCESSES.value - vm.RemoteCommand(f'echo "\nMaxSessions {mux_sessions}" | ' - 'sudo tee -a /etc/ssh/sshd_config') + vm.RemoteCommand( + f'echo "\nMaxSessions {mux_sessions}" | sudo tee -a /etc/ssh/sshd_config' + ) # Redis tuning parameters, see # https://www.techandme.se/performance-tips-for-redis-cache-server/. # This command works on 2nd generation of VMs only. - update_sysvtl = vm.TryRemoteCommand('echo "' - 'vm.overcommit_memory = 1\n' - 'net.core.somaxconn = 65535\n' - '" | sudo tee -a /etc/sysctl.conf') + update_sysvtl = vm.TryRemoteCommand( + 'echo "' + 'vm.overcommit_memory = 1\n' + 'net.core.somaxconn = 65535\n' + '" | sudo tee -a /etc/sysctl.conf' + ) # /usr/sbin/sysctl is not applicable on certain distros. commit_sysvtl = vm.TryRemoteCommand( - 'sudo /usr/sbin/sysctl -p || sudo sysctl -p') + 'sudo /usr/sbin/sysctl -p || sudo sysctl -p' + ) if not (update_sysvtl and commit_sysvtl): logging.info('Fail to optimize overcommit_memory and socket connections.') for port in GetRedisPorts(): diff --git a/perfkitbenchmarker/linux_packages/s64da.py b/perfkitbenchmarker/linux_packages/s64da.py index 7dedd4047f..aeed938154 100644 --- a/perfkitbenchmarker/linux_packages/s64da.py +++ b/perfkitbenchmarker/linux_packages/s64da.py @@ -15,7 +15,6 @@ import collections import dataclasses import datetime - from typing import List, Tuple from perfkitbenchmarker import relational_db as r_db @@ -63,19 +62,26 @@ def AptInstall(vm: virtual_machine.BaseVirtualMachine) -> None: """Installs the s64da package on the VM.""" vm.Install('pip3') cmds = [ - 'wget ' - f'https://github.com/swarm64/s64da-benchmark-toolkit/archive/refs/tags/{S64DA_VERSION}.tar.gz', + ( + 'wget ' + f'https://github.com/swarm64/s64da-benchmark-toolkit/archive/refs/tags/{S64DA_VERSION}.tar.gz' + ), f'tar -xf v{S64DA_VERSION}.tar.gz', f'mv s64da-benchmark-toolkit-{S64DA_VERSION} {WORKING_DIRECTORY_NAME}', - f'python3 -m pip install -r {WORKING_DIRECTORY_NAME}/requirements.txt' + f'python3 -m pip install -r {WORKING_DIRECTORY_NAME}/requirements.txt', ] for cmd in cmds: vm.RemoteCommand(cmd) -def PrepareBenchmark(vm: virtual_machine.BaseVirtualMachine, - db: r_db.BaseRelationalDb, benchmark: str, schema: str, - scale_factor: int, max_jobs: int) -> None: +def PrepareBenchmark( + vm: virtual_machine.BaseVirtualMachine, + db: r_db.BaseRelationalDb, + benchmark: str, + schema: str, + scale_factor: int, + max_jobs: int, +) -> None: """Load a database with a dataset.""" vm.RemoteCommand( InLocalDir( @@ -83,36 +89,47 @@ def PrepareBenchmark(vm: virtual_machine.BaseVirtualMachine, f'--dsn={db.client_vm_query_tools.GetDSNConnectionString(DB_NAME)} ' f'--benchmark={benchmark} --schema={schema} ' f'--scale-factor={scale_factor} ' - f'--max-jobs {max_jobs}')) - - -def RunBenchmark(vm: virtual_machine.BaseVirtualMachine, - db: r_db.BaseRelationalDb, benchmark: str, oltp_workers: int, - olap_workers: int, duration: int, olap_timeout: str, - ramp_up_duration: int, - run_on_replica: bool) -> List[sample.Sample]: + f'--max-jobs {max_jobs}' + ) + ) + + +def RunBenchmark( + vm: virtual_machine.BaseVirtualMachine, + db: r_db.BaseRelationalDb, + benchmark: str, + oltp_workers: int, + olap_workers: int, + duration: int, + olap_timeout: str, + ramp_up_duration: int, + run_on_replica: bool, +) -> List[sample.Sample]: """Runs the benchmark and gathers the data.""" olap_query_tools = db.client_vm_query_tools if run_on_replica: olap_query_tools = db.client_vm_query_tools_for_replica vm.RemoteCommand( InLocalDir( - f'./run_benchmark ' + './run_benchmark ' f'--dsn={db.client_vm_query_tools.GetDSNConnectionString(DB_NAME)} ' f'{benchmark} ' - f'--dont-wait-until-enough-data ' + '--dont-wait-until-enough-data ' f'--oltp-workers={oltp_workers} ' f'--olap-workers={olap_workers} ' f'--duration={duration} ' - f'--output=csv ' - f'--csv-interval=1 ' + '--output=csv ' + '--csv-interval=1 ' f'--olap-dsns={olap_query_tools.GetDSNConnectionString(DB_NAME)} ' - f'--olap-timeout={olap_timeout} ')) + f'--olap-timeout={olap_timeout} ' + ) + ) olap_results, _ = vm.RemoteCommand(InLocalDir(f'cat {OLAP_RESULT_PATH}')) oltp_results, _ = vm.RemoteCommand(InLocalDir(f'cat {OLTP_RESULT_PATH}')) return ParseOLAPResults(olap_results) + ParseOLTPResults( - oltp_results, ramp_up_duration) + oltp_results, ramp_up_duration + ) def ParseTime(time: str) -> datetime.datetime: @@ -125,8 +142,8 @@ def ParseSamples(stdout: str) -> List[List[str]]: def SplitRampUpSamples( - rows: List[OLTPRow], - ramp_up_duration: int) -> Tuple[List[OLTPRow], List[OLTPRow]]: + rows: List[OLTPRow], ramp_up_duration: int +) -> Tuple[List[OLTPRow], List[OLTPRow]]: """Splits samples before ramp up and after ramp up. The samples returned from the s64da benchmarks are in chronological order. @@ -188,8 +205,10 @@ def ParseOLAPResults(stdout: str) -> List[sample.Sample]: ] results.append( - sample.Sample('query_times_geomean', - sample.GeoMean(averaged_result.values()), 's')) + sample.Sample( + 'query_times_geomean', sample.GeoMean(averaged_result.values()), 's' + ) + ) results.append(sample.Sample(TPCH_FAILURE_RATE, failure_rate, '%')) return results @@ -211,31 +230,39 @@ def ParseOLTPResults(stdout: str, ramp_up_duration: int) -> List[sample.Sample]: transaction_type=row[1], time=ParseTime(row[0]), total_transactions=float(row[2]), - successful_transactions=float(row[3]))) + successful_transactions=float(row[3]), + ) + ) oltp_rows = [row for row in oltp_rows if row.transaction_type == ALL_TYPES] before_ramp_up_rows, after_ramp_up_rows = SplitRampUpSamples( - oltp_rows, ramp_up_duration) + oltp_rows, ramp_up_duration + ) successful_transactions_before_ramp_up = 0 if before_ramp_up_rows: # Get the number of total transaction before ramp up. last_metrics_before_ramp_up = before_ramp_up_rows[-1] successful_transactions_before_ramp_up = ( - last_metrics_before_ramp_up.successful_transactions) + last_metrics_before_ramp_up.successful_transactions + ) # Get total the transaction number last_metrics = after_ramp_up_rows[-1] - failure_rate = (last_metrics.total_transactions - - last_metrics.successful_transactions - ) / last_metrics.total_transactions * 100 + failure_rate = ( + (last_metrics.total_transactions - last_metrics.successful_transactions) + / last_metrics.total_transactions + * 100 + ) successful_transactions_after_ramp_up = ( - last_metrics.successful_transactions - - successful_transactions_before_ramp_up) + last_metrics.successful_transactions + - successful_transactions_before_ramp_up + ) tpm = int( - (successful_transactions_after_ramp_up * 60.0) / len(after_ramp_up_rows)) + (successful_transactions_after_ramp_up * 60.0) / len(after_ramp_up_rows) + ) results.append(sample.Sample(TPM, tpm, TPM)) results.append(sample.Sample(TPCC_FAILURE_RATE, failure_rate, '%')) return results diff --git a/perfkitbenchmarker/linux_packages/scimark2.py b/perfkitbenchmarker/linux_packages/scimark2.py index f9bf351be3..782e321ea3 100644 --- a/perfkitbenchmarker/linux_packages/scimark2.py +++ b/perfkitbenchmarker/linux_packages/scimark2.py @@ -40,13 +40,14 @@ PACKAGE_NAME = 'scimark2' PREPROVISIONED_DATA = { - JAVA_JAR: - '6f84f949c3167b385da1a9957ecd53fe0111b42e981e0c481be53dba0504305f', - C_ZIP: '223464cd7e90b4c22e2af08dbae6f6faa33e65b01e1c58e5a176837bc67958be' + JAVA_JAR: ( + '6f84f949c3167b385da1a9957ecd53fe0111b42e981e0c481be53dba0504305f' + ), + C_ZIP: '223464cd7e90b4c22e2af08dbae6f6faa33e65b01e1c58e5a176837bc67958be', } PACKAGE_DATA_URL = { JAVA_JAR: posixpath.join(BASE_URL, JAVA_JAR), - C_ZIP: posixpath.join(BASE_URL, C_ZIP) + C_ZIP: posixpath.join(BASE_URL, C_ZIP), } @@ -58,7 +59,8 @@ def Install(vm): vm.Install('unzip') vm.RemoteCommand('rm -rf {0} && mkdir {0}'.format(PATH)) vm.InstallPreprovisionedPackageData( - PACKAGE_NAME, PREPROVISIONED_DATA.keys(), PATH) + PACKAGE_NAME, PREPROVISIONED_DATA.keys(), PATH + ) cmds = [ '(mkdir {0} && cd {0} && unzip {1}/{2})'.format(C_SRC, PATH, C_ZIP), '(cd {0} && make CFLAGS="{1}")'.format(C_SRC, C_CFLAGS), diff --git a/perfkitbenchmarker/linux_packages/shoc_benchmark_suite.py b/perfkitbenchmarker/linux_packages/shoc_benchmark_suite.py index 2937dd8870..7924494fdc 100644 --- a/perfkitbenchmarker/linux_packages/shoc_benchmark_suite.py +++ b/perfkitbenchmarker/linux_packages/shoc_benchmark_suite.py @@ -14,7 +14,8 @@ """Module containing SHOC Benchmark Suite installation and - cleanup functions. + +cleanup functions. """ @@ -33,7 +34,7 @@ def _IsShocInstalled(vm): """Returns whether shoc is installed or not""" command = os.path.join(SHOC_BIN_DIR, 'shocdriver') resp, _ = vm.RemoteHostCommand('command -v %s' % command, ignore_failure=True) - if resp.rstrip() == "": + if resp.rstrip() == '': return False return True @@ -46,17 +47,23 @@ def AptInstall(vm): vm.InstallPackages(APT_PACKAGES) vm.Install('cuda_toolkit') - vm.RemoteCommand('cd %s && git clone %s' % - (linux_packages.INSTALL_DIR, SHOC_GIT_URL)) - vm.RemoteCommand(('cd %s && ./configure ' - 'CUDA_CPPFLAGS=-gencode=arch=compute_37,code=compute_37 ' - 'NVCC=/usr/local/cuda/bin/nvcc') - % SHOC_DIR) + vm.RemoteCommand( + 'cd %s && git clone %s' % (linux_packages.INSTALL_DIR, SHOC_GIT_URL) + ) + vm.RemoteCommand( + ( + 'cd %s && ./configure ' + 'CUDA_CPPFLAGS=-gencode=arch=compute_37,code=compute_37 ' + 'NVCC=/usr/local/cuda/bin/nvcc' + ) + % SHOC_DIR + ) vm.RemoteCommand('cd %s && make -j8 && make install' % SHOC_DIR) def YumInstall(vm): """TODO: PKB currently only supports the installation of SHOC - on Ubuntu. + + on Ubuntu. """ raise NotImplementedError() diff --git a/perfkitbenchmarker/linux_packages/silo.py b/perfkitbenchmarker/linux_packages/silo.py index 5d61a20c10..f79eb333cb 100644 --- a/perfkitbenchmarker/linux_packages/silo.py +++ b/perfkitbenchmarker/linux_packages/silo.py @@ -20,10 +20,14 @@ GIT_REPO = 'https://github.com/stephentu/silo.git' GIT_TAG = '62d2d498984bf69d3b46a74e310e1fd12fd1f692' SILO_DIR = '%s/silo' % linux_packages.INSTALL_DIR -APT_PACKAGES = ('libjemalloc-dev libnuma-dev libdb++-dev ' - 'libmysqld-dev libaio-dev libssl-dev') -YUM_PACKAGES = ('jemalloc-devel numactl-devel libdb-cxx-devel mysql-devel ' - 'libaio-devel openssl-devel') +APT_PACKAGES = ( + 'libjemalloc-dev libnuma-dev libdb++-dev ' + 'libmysqld-dev libaio-dev libssl-dev' +) +YUM_PACKAGES = ( + 'jemalloc-devel numactl-devel libdb-cxx-devel mysql-devel ' + 'libaio-devel openssl-devel' +) def _Install(vm): @@ -31,19 +35,20 @@ def _Install(vm): nthreads = vm.NumCpusForBenchmark() * 2 vm.Install('build_tools') vm.RemoteCommand('git clone {0} {1}'.format(GIT_REPO, SILO_DIR)) - vm.RemoteCommand('cd {0} && git checkout {1}'.format(SILO_DIR, - GIT_TAG)) + vm.RemoteCommand('cd {0} && git checkout {1}'.format(SILO_DIR, GIT_TAG)) # This is due to a failing clone command when executing behind a proxy. # Replacing the protocol to https instead of git fixes the issue. vm.RemoteCommand('git config --global url."https://".insteadOf git://') # Disable -Wmaybe-uninitialized errors when GCC has the option to workaround # a spurious error in masstree. - cxx = '"g++ -std=gnu++0x \ - $(echo | gcc -Wmaybe-uninitialized -E - >/dev/null 2>&1 && \ - echo -Wno-error=maybe-uninitialized)"' + cxx = ( + '"g++ -std=gnu++0x $(echo | gcc -Wmaybe-uninitialized -E -' + ' >/dev/null 2>&1 && echo -Wno-error=maybe-uninitialized)"' + ) vm.RemoteCommand( 'cd {0} && CXX={2} MODE=perf DEBUG=0 CHECK_INVARIANTS=0 make -j{1} dbtest' - .format(SILO_DIR, nthreads, cxx)) + .format(SILO_DIR, nthreads, cxx) + ) def YumInstall(vm): diff --git a/perfkitbenchmarker/linux_packages/snowsql.py b/perfkitbenchmarker/linux_packages/snowsql.py index 7192dc2cde..9e06db3057 100644 --- a/perfkitbenchmarker/linux_packages/snowsql.py +++ b/perfkitbenchmarker/linux_packages/snowsql.py @@ -15,7 +15,10 @@ SNOWSQL_INSTALL_LOCATION = '~/bin' SNOWSQL_VERSION = '1.2.5' -SNOWSQL_DOWNLOAD_URL = 'https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-%s-linux_x86_64.bash' % SNOWSQL_VERSION +SNOWSQL_DOWNLOAD_URL = ( + 'https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-%s-linux_x86_64.bash' + % SNOWSQL_VERSION +) def AptInstall(vm): @@ -26,7 +29,8 @@ def AptInstall(vm): vm.RemoteCommand( 'SNOWSQL_DEST=%s SNOWSQL_LOGIN_SHELL=~/.profile ' 'bash snowsql-%s-linux_x86_64.bash' - % (SNOWSQL_INSTALL_LOCATION, SNOWSQL_VERSION)) + % (SNOWSQL_INSTALL_LOCATION, SNOWSQL_VERSION) + ) vm.RemoteCommand('chmod +x %s/snowsql' % SNOWSQL_INSTALL_LOCATION) @@ -35,7 +39,8 @@ def YumInstall(vm): del vm raise NotImplementedError( 'PKB currently only supports the installation of snowsql on ' - 'Debian-based VMs') + 'Debian-based VMs' + ) def AptUninstall(vm): diff --git a/perfkitbenchmarker/linux_packages/spark.py b/perfkitbenchmarker/linux_packages/spark.py index 608b46a7ef..9ce8acbc0d 100644 --- a/perfkitbenchmarker/linux_packages/spark.py +++ b/perfkitbenchmarker/linux_packages/spark.py @@ -39,11 +39,13 @@ FLAGS = flags.FLAGS _SPARK_VERSION_FLAG = flags.DEFINE_string( - 'spark_version', None, - 'Version of spark. Defaults to latest.') + 'spark_version', None, 'Version of spark. Defaults to latest.' +) DATA_FILES = [ - 'spark/spark-defaults.conf.j2', 'spark/spark-env.sh.j2', 'spark/workers.j2' + 'spark/spark-defaults.conf.j2', + 'spark/spark-env.sh.j2', + 'spark/workers.j2', ] SPARK_DOWNLOADS = 'https://downloads.apache.org/spark' @@ -68,7 +70,8 @@ def SparkVersion() -> version.Version: response = requests.get(SPARK_DOWNLOADS) if not response.ok: raise errors.Setup.MissingExecutableError( - 'Could not load ' + SPARK_DOWNLOADS) + 'Could not load ' + SPARK_DOWNLOADS + ) soup = bs4.BeautifulSoup(response.content, 'html.parser') found_versions = [] for link in soup.find_all('a', href=SPARK_VERSION_DIR_PATTERN): @@ -77,7 +80,8 @@ def SparkVersion() -> version.Version: found_versions.append(version.Version(match.group(1))) if not found_versions: raise errors.Setup.MissingExecutableError( - 'Could not find valid spark versions at ' + SPARK_DOWNLOADS) + 'Could not find valid spark versions at ' + SPARK_DOWNLOADS + ) return max(found_versions) @@ -92,8 +96,10 @@ def _ScalaVersion() -> version.Version: def SparkExamplesJarPath() -> str: return posixpath.join( - SPARK_DIR, 'examples/jars/', - f'spark-examples_{_ScalaVersion()}-{SparkVersion()}.jar') + SPARK_DIR, + 'examples/jars/', + f'spark-examples_{_ScalaVersion()}-{SparkVersion()}.jar', + ) def CheckPrerequisites(): @@ -115,11 +121,15 @@ def Install(vm): # Also used on Spark's classpath to support s3a client. vm.Install('hadoop') spark_version = SparkVersion() - spark_url = (f'{SPARK_DOWNLOADS}/spark-{spark_version}/' - f'spark-{spark_version}-bin-without-hadoop.tgz') + spark_url = ( + f'{SPARK_DOWNLOADS}/spark-{spark_version}/' + f'spark-{spark_version}-bin-without-hadoop.tgz' + ) vm.RemoteCommand( - ('mkdir {0} && curl -L {1} | ' - 'tar -C {0} --strip-components=1 -xzf -').format(SPARK_DIR, spark_url)) + ( + 'mkdir {0} && curl -L {1} | tar -C {0} --strip-components=1 -xzf -' + ).format(SPARK_DIR, spark_url) + ) # Scheduling constants. @@ -131,11 +141,13 @@ def Install(vm): SPARK_WORKER_VCPUS = 'spark.executor.cores' -def GetConfiguration(driver_memory_mb: int, - worker_memory_mb: int, - worker_cores: int, - num_workers: int, - configure_s3: bool = False) -> Dict[str, str]: +def GetConfiguration( + driver_memory_mb: int, + worker_memory_mb: int, + worker_cores: int, + num_workers: int, + configure_s3: bool = False, +) -> Dict[str, str]: """Calculate Spark configuration. Shared between VMs and k8s.""" conf = { SPARK_DRIVER_MEMORY: f'{driver_memory_mb}m', @@ -144,7 +156,7 @@ def GetConfiguration(driver_memory_mb: int, 'spark.executor.instances': str(num_workers), # Tell spark not to run job if it can't schedule all workers. This would # silently degrade performance. - 'spark.scheduler.minRegisteredResourcesRatio': '1' + 'spark.scheduler.minRegisteredResourcesRatio': '1', } if configure_s3: # Configure S3A Hadoop's S3 filesystem @@ -158,11 +170,13 @@ def GetConfiguration(driver_memory_mb: int, return conf -def _RenderConfig(vm, - leader, - workers, - memory_fraction=SPARK_MEMORY_FRACTION, - configure_s3=False): +def _RenderConfig( + vm, + leader, + workers, + memory_fraction=SPARK_MEMORY_FRACTION, + configure_s3=False, +): """Load Spark Condfiguration on VM.""" # Use first worker to get worker configuration worker = workers[0] @@ -175,7 +189,8 @@ def _RenderConfig(vm, worker_memory_mb=worker_memory_mb, worker_cores=worker_cores, num_workers=len(workers), - configure_s3=configure_s3) + configure_s3=configure_s3, + ) if vm.scratch_disks: # TODO(pclay): support multiple scratch disks. A current suboptimal @@ -198,7 +213,7 @@ def _RenderConfig(vm, 'worker_memory': spark_conf[SPARK_WORKER_MEMORY], 'hadoop_cmd': hadoop.HADOOP_CMD, 'python_cmd': 'python3', - 'optional_tools': optional_tools + 'optional_tools': optional_tools, } for file_name in DATA_FILES: @@ -215,9 +230,7 @@ def _RenderConfig(vm, def _GetOnlineWorkerCount(leader): """Curl Spark Master Web UI for worker status.""" - cmd = ('curl http://localhost:8080 ' - "| grep 'Alive Workers' " - "| grep -o '[0-9]\\+'") + cmd = "curl http://localhost:8080 | grep 'Alive Workers' | grep -o '[0-9]\\+'" stdout = leader.RemoteCommand(cmd)[0] return int(stdout) @@ -238,11 +251,13 @@ def ConfigureAndStart(leader, workers, configure_s3=False): # node runs the worker daemons. workers = workers or [leader] fn = functools.partial( - _RenderConfig, leader=leader, workers=workers, configure_s3=configure_s3) + _RenderConfig, leader=leader, workers=workers, configure_s3=configure_s3 + ) background_tasks.RunThreaded(fn, vms) - leader.RemoteCommand("rm -f {0} && ssh-keygen -q -t rsa -N '' -f {0}".format( - SPARK_PRIVATE_KEY)) + leader.RemoteCommand( + "rm -f {0} && ssh-keygen -q -t rsa -N '' -f {0}".format(SPARK_PRIVATE_KEY) + ) public_key = leader.RemoteCommand('cat {0}.pub'.format(SPARK_PRIVATE_KEY))[0] @@ -252,8 +267,7 @@ def AddKey(vm): background_tasks.RunThreaded(AddKey, vms) # HDFS setup and formatting, Spark startup - leader.RemoteCommand( - 'bash {0}/start-all.sh'.format(SPARK_SBIN)) + leader.RemoteCommand('bash {0}/start-all.sh'.format(SPARK_SBIN)) logging.info('Sleeping 10s for Spark nodes to join.') time.sleep(10) @@ -261,7 +275,10 @@ def AddKey(vm): logging.info('Checking Spark status.') worker_online_count = _GetOnlineWorkerCount(leader) if worker_online_count != len(workers): - raise ValueError('Not all nodes running Spark: {0} < {1}'.format( - worker_online_count, len(workers))) + raise ValueError( + 'Not all nodes running Spark: {0} < {1}'.format( + worker_online_count, len(workers) + ) + ) else: logging.info('Spark running on all %d workers', len(workers)) diff --git a/perfkitbenchmarker/linux_packages/speccpu.py b/perfkitbenchmarker/linux_packages/speccpu.py index 757318b642..b6a668ca4e 100644 --- a/perfkitbenchmarker/linux_packages/speccpu.py +++ b/perfkitbenchmarker/linux_packages/speccpu.py @@ -33,51 +33,66 @@ FLAGS = flags.FLAGS flags.DEFINE_string( - 'runspec_tar', None, + 'runspec_tar', + None, 'Used by the PKB speccpu benchmarks. Name of the .tgz file to use. ' - 'Defaults to None. ') + 'Defaults to None. ', +) flags.DEFINE_string( - 'runspec_config', None, + 'runspec_config', + None, 'Used by the PKB speccpu benchmarks. Name of the cfg file to use as the ' 'SPEC CPU config file provided to the runspec binary via its --config ' 'flag. If the benchmark is run using an .iso file, then the ' 'cfg file must be placed in the local PKB data directory and will be ' 'copied to the remote machine prior to executing runspec/runcpu. Defaults ' 'to None. ' - 'See README.md for instructions if running with a repackaged .tgz file.') + 'See README.md for instructions if running with a repackaged .tgz file.', +) flags.DEFINE_string( - 'runspec_build_tool_version', None, + 'runspec_build_tool_version', + None, 'Version of gcc/g++/gfortran. This should match runspec_config. Note, if ' 'neither runspec_config and runspec_build_tool_version is set, the test ' 'install gcc/g++/gfortran-4.7, since that matches default config version. ' 'If runspec_config is set, but not runspec_build_tool_version, default ' 'version of build tools will be installed. Also this flag only works with ' - 'debian.') + 'debian.', +) flags.DEFINE_integer( - 'runspec_iterations', 3, + 'runspec_iterations', + 3, 'Used by the PKB speccpu benchmarks. The number of benchmark iterations ' - 'to execute, provided to the runspec binary via its --iterations flag.') + 'to execute, provided to the runspec binary via its --iterations flag.', +) flags.DEFINE_string( - 'runspec_define', '', + 'runspec_define', + '', 'Used by the PKB speccpu benchmarks. Optional comma-separated list of ' 'SYMBOL[=VALUE] preprocessor macros provided to the runspec binary via ' - 'repeated --define flags. Example: numa,smt,sse=SSE4.2') + 'repeated --define flags. Example: numa,smt,sse=SSE4.2', +) flags.DEFINE_boolean( - 'runspec_enable_32bit', False, + 'runspec_enable_32bit', + False, 'Used by the PKB speccpu benchmarks. If set, multilib packages will be ' 'installed on the remote machine to enable use of 32-bit SPEC CPU ' 'binaries. This may be useful when running on memory-constrained instance ' 'types (i.e. less than 2 GiB memory/core), where 64-bit execution may be ' - 'problematic.') + 'problematic.', +) flags.DEFINE_boolean( - 'runspec_keep_partial_results', False, + 'runspec_keep_partial_results', + False, 'Used by the PKB speccpu benchmarks. If set, the benchmark will report ' 'an aggregate score even if some of the SPEC CPU component tests ' 'failed with status "NR". Available results will be saved, and PKB samples ' 'will be marked with a metadata value of partial=true. If unset, partial ' - 'failures are treated as errors.') + 'failures are treated as errors.', +) flags.DEFINE_boolean( - 'runspec_estimate_spec', False, + 'runspec_estimate_spec', + False, 'Used by the PKB speccpu benchmarks. If set, the benchmark will report ' 'an estimated aggregate score even if SPEC CPU did not compute one. ' 'This usually occurs when --runspec_iterations is less than 3. ' @@ -86,13 +101,20 @@ 'estimated_SPECfp(R)_rate_base. Available results will be saved, ' 'and PKB samples will be marked with a metadata value of partial=true. If ' 'unset, SPECint(R)_rate_base20** and SPECfp(R)_rate_base20** are listed ' - 'in the metadata under missing_results.') -flags.DEFINE_enum('spec_runmode', BASE_MODE, [BASE_MODE, PEAK_MODE, ALL_MODE], - 'Run mode to use. Defaults to base. ') + 'in the metadata under missing_results.', +) +flags.DEFINE_enum( + 'spec_runmode', + BASE_MODE, + [BASE_MODE, PEAK_MODE, ALL_MODE], + 'Run mode to use. Defaults to base. ', +) flags.DEFINE_string( - 'runspec_script', None, + 'runspec_script', + None, 'Used by the PKB speccpu benchmarks. If set, the benchmark will execute ' - 'this script instead of invoking runspec binary directly.') + 'this script instead of invoking runspec binary directly.', +) VM_STATE_ATTR = 'speccpu_vm_state' @@ -119,7 +141,8 @@ def _CheckTarFile(vm, runspec_config, examine_members, speccpu_vm_state): 'Invalid runspec_config value: {0}{1}When running speccpu with a ' 'tar file, runspec_config cannot specify a file in a sub-directory. ' 'See README.md for information about running speccpu with a tar ' - 'file.'.format(runspec_config, os.linesep)) + 'file.'.format(runspec_config, os.linesep) + ) if not examine_members: return @@ -128,15 +151,18 @@ def _CheckTarFile(vm, runspec_config, examine_members, speccpu_vm_state): vm.PushFile(local_cfg_file_path, speccpu_vm_state.cfg_file_path) scratch_dir = vm.GetScratchDir() - cfg_member = '{0}/config/{1}'.format(speccpu_vm_state.base_spec_dir, - runspec_config) - required_members = itertools.chain(speccpu_vm_state.required_members, - [cfg_member]) + cfg_member = '{0}/config/{1}'.format( + speccpu_vm_state.base_spec_dir, runspec_config + ) + required_members = itertools.chain( + speccpu_vm_state.required_members, [cfg_member] + ) missing_members = [] for member in required_members: stdout, _ = vm.RemoteCommand( 'cd {scratch_dir} && (test -f {member} || test -d {member}) ; echo $?' - .format(scratch_dir=scratch_dir, member=member)) + .format(scratch_dir=scratch_dir, member=member) + ) if stdout.strip() != '0': missing_members.append(member) @@ -146,8 +172,9 @@ def _CheckTarFile(vm, runspec_config, examine_members, speccpu_vm_state): '{linesep}This is an indication that the tar file is formatted ' 'incorrectly. See README.md for information about the expected format ' 'of the tar file.'.format( - linesep=os.linesep, - members=os.linesep.join(sorted(missing_members)))) + linesep=os.linesep, members=os.linesep.join(sorted(missing_members)) + ) + ) def _CheckIsoAndCfgFile(runspec_config, spec_iso, clang_flag): @@ -170,7 +197,8 @@ def _CheckIsoAndCfgFile(runspec_config, spec_iso, clang_flag): 'in the perfkitbenchmarker/data directory (or one of the specified ' 'data directories if the --data_search_paths flag is used). Visit ' 'https://www.spec.org/ to learn more about purchasing %(iso)s.', - {'iso': spec_iso}) + {'iso': spec_iso}, + ) raise # Search for the cfg. @@ -183,7 +211,9 @@ def _CheckIsoAndCfgFile(runspec_config, spec_iso, clang_flag): 'perfkitbenchmarker/data directory (or one of the specified data ' 'directories if the --data_search_paths flag is used). Visit ' 'https://www.spec.org/cpu2006/docs/runspec.html#about_config to learn ' - 'more about config files.', runspec_config) + 'more about config files.', + runspec_config, + ) raise if not clang_flag: # 2017 ISO does not contain clang.xml @@ -198,7 +228,9 @@ def _CheckIsoAndCfgFile(runspec_config, spec_iso, clang_flag): 'must be in the perfkitbenchmarker/data directory (or one of the ' 'specified data directories if the --data_search_paths flag is ' 'used). Visit https://www.spec.org/cpu2017/docs/flag-description.html ' - 'to learn more about flag files.', clang_flag) + 'to learn more about flag files.', + clang_flag, + ) raise @@ -269,16 +301,20 @@ def UpdateConfig(self, scratch_dir): scratch_dir: The scratch directory on the VM that SPEC is installed on. """ self.spec_dir = posixpath.join(scratch_dir, self.base_spec_dir) - self.cfg_file_path = posixpath.join(self.spec_dir, 'config', - os.path.basename(self.runspec_config)) + self.cfg_file_path = posixpath.join( + self.spec_dir, 'config', os.path.basename(self.runspec_config) + ) if self.base_iso_file_path: self.iso_file_path = posixpath.join(scratch_dir, self.base_iso_file_path) if self.base_mount_dir: self.mount_dir = posixpath.join(scratch_dir, self.base_mount_dir) if self.base_clang_flag_file_path: self.clang_flag_file_path = posixpath.join( - self.spec_dir, 'config', 'flags', - os.path.basename(self.base_clang_flag_file_path)) + self.spec_dir, + 'config', + 'flags', + os.path.basename(self.base_clang_flag_file_path), + ) def InstallSPECCPU(vm, speccpu_vm_state): @@ -294,12 +330,18 @@ def InstallSPECCPU(vm, speccpu_vm_state): # Since this will override 'build_tools' installation, install this # before we install 'build_tools' package _PrepareWithPreprovisionedTarFile(vm, speccpu_vm_state) - _CheckTarFile(vm, speccpu_vm_state.runspec_config, stages.PROVISION - in FLAGS.run_stage, speccpu_vm_state) + _CheckTarFile( + vm, + speccpu_vm_state.runspec_config, + stages.PROVISION in FLAGS.run_stage, + speccpu_vm_state, + ) except errors.Setup.BadPreprovisionedDataError: - _CheckIsoAndCfgFile(speccpu_vm_state.runspec_config, - speccpu_vm_state.base_iso_file_path, - speccpu_vm_state.base_clang_flag_file_path) + _CheckIsoAndCfgFile( + speccpu_vm_state.runspec_config, + speccpu_vm_state.base_iso_file_path, + speccpu_vm_state.base_clang_flag_file_path, + ) _PrepareWithIsoFile(vm, speccpu_vm_state) vm.Install('speccpu') @@ -332,11 +374,16 @@ def _PrepareWithPreprovisionedTarFile(vm, speccpu_vm_state): speccpu_vm_state: SpecInstallConfigurations. Install configuration for spec. """ scratch_dir = vm.GetScratchDir() - vm.InstallPreprovisionedPackageData(speccpu_vm_state.package_name, - [speccpu_vm_state.base_tar_file_path], - scratch_dir) - vm.RemoteCommand('cd {dir} && tar xvfz {tar}'.format( - dir=scratch_dir, tar=speccpu_vm_state.base_tar_file_path)) + vm.InstallPreprovisionedPackageData( + speccpu_vm_state.package_name, + [speccpu_vm_state.base_tar_file_path], + scratch_dir, + ) + vm.RemoteCommand( + 'cd {dir} && tar xvfz {tar}'.format( + dir=scratch_dir, tar=speccpu_vm_state.base_tar_file_path + ) + ) def _PrepareWithIsoFile(vm, speccpu_vm_state): @@ -361,18 +408,28 @@ def _PrepareWithIsoFile(vm, speccpu_vm_state): # Extract files from the iso to the cpu2006 or cpu2017 directory. vm.RemoteCommand('mkdir {0}'.format(speccpu_vm_state.mount_dir)) - vm.RemoteCommand('sudo mount -t iso9660 -o loop {0} {1}'.format( - speccpu_vm_state.iso_file_path, speccpu_vm_state.mount_dir)) - vm.RemoteCommand('cp -r {0}/* {1}'.format(speccpu_vm_state.mount_dir, - speccpu_vm_state.spec_dir)) + vm.RemoteCommand( + 'sudo mount -t iso9660 -o loop {0} {1}'.format( + speccpu_vm_state.iso_file_path, speccpu_vm_state.mount_dir + ) + ) + vm.RemoteCommand( + 'cp -r {0}/* {1}'.format( + speccpu_vm_state.mount_dir, speccpu_vm_state.spec_dir + ) + ) # cpu2017 iso does not come with config directory nor clang.xml if speccpu_vm_state.clang_flag_file_path: - vm.RemoteCommand('mkdir -p {0}'.format( - os.path.dirname(speccpu_vm_state.clang_flag_file_path))) + vm.RemoteCommand( + 'mkdir -p {0}'.format( + os.path.dirname(speccpu_vm_state.clang_flag_file_path) + ) + ) vm.PushFile( data.ResourcePath(speccpu_vm_state.base_clang_flag_file_path), - speccpu_vm_state.clang_flag_file_path) + speccpu_vm_state.clang_flag_file_path, + ) vm.RemoteCommand('chmod -R 755 {0}'.format(speccpu_vm_state.spec_dir)) @@ -516,8 +573,9 @@ def _ExtractScore(stdout, vm, keep_partial_results, runspec_metric): # Skip over failed runs, but count them since they make the overall # result invalid. not_reported = benchmark.count('NR') - if not_reported > 1 or (not_reported == 1 and - FLAGS.spec_runmode != PEAK_MODE): + if not_reported > 1 or ( + not_reported == 1 and FLAGS.spec_runmode != PEAK_MODE + ): logging.warning('SPEC CPU missing result: %s', benchmark) missing_results.append(str(benchmark.split()[0])) continue @@ -542,7 +600,8 @@ def _ExtractScore(stdout, vm, keep_partial_results, runspec_metric): if peak_score_str: peak_score_float = float(peak_score_str) results.append( - sample.Sample(str(name) + ':peak', peak_score_float, '', metadata)) + sample.Sample(str(name) + ':peak', peak_score_float, '', metadata) + ) if spec_score is None and FLAGS.spec_runmode != PEAK_MODE: missing_results.append(spec_name) @@ -552,16 +611,19 @@ def _ExtractScore(stdout, vm, keep_partial_results, runspec_metric): metadata['partial'] = 'true' metadata['missing_results'] = ','.join(missing_results) else: - raise errors.Benchmarks.RunError('speccpu: results missing, see log: ' + - ','.join(missing_results)) + raise errors.Benchmarks.RunError( + 'speccpu: results missing, see log: ' + ','.join(missing_results) + ) if spec_score: results.append(sample.Sample(spec_name, spec_score, '', metadata)) elif FLAGS.runspec_estimate_spec: estimated_spec_score = _GeometricMean(scores) results.append( - sample.Sample('estimated_' + spec_name, estimated_spec_score, '', - metadata)) + sample.Sample( + 'estimated_' + spec_name, estimated_spec_score, '', metadata + ) + ) if peak_score: results.append(sample.Sample(peak_name, float(peak_score), '', metadata)) @@ -573,14 +635,12 @@ def _GeometricMean(arr): product = 1 for val in arr: product *= val - return product**(1.0 / len(arr)) + return product ** (1.0 / len(arr)) -def ParseOutput(vm, - log_files, - is_partial_results, - runspec_metric, - results_directory=None): +def ParseOutput( + vm, log_files, is_partial_results, runspec_metric, results_directory=None +): """Retrieves the SPEC CPU output from the VM and parses it. Args: @@ -600,11 +660,15 @@ def ParseOutput(vm, for log in log_files: results_dir = results_directory or '%s/result' % speccpu_vm_state.spec_dir - stdout, _ = vm.RemoteCommand( - 'cat %s/%s' % (results_dir, log)) + stdout, _ = vm.RemoteCommand('cat %s/%s' % (results_dir, log)) results.extend( - _ExtractScore(stdout, vm, FLAGS.runspec_keep_partial_results or - is_partial_results, runspec_metric)) + _ExtractScore( + stdout, + vm, + FLAGS.runspec_keep_partial_results or is_partial_results, + runspec_metric, + ) + ) return results @@ -622,10 +686,12 @@ def Run(vm, cmd, benchmark_subset, version_specific_parameters=None): A Tuple of (stdout, stderr) the run output. """ speccpu_vm_state = getattr(vm, VM_STATE_ATTR, None) - runspec_flags = [('config', - posixpath.basename(speccpu_vm_state.cfg_file_path)), - ('tune', FLAGS.spec_runmode), ('size', 'ref'), - ('iterations', FLAGS.runspec_iterations)] + runspec_flags = [ + ('config', posixpath.basename(speccpu_vm_state.cfg_file_path)), + ('tune', FLAGS.spec_runmode), + ('size', 'ref'), + ('iterations', FLAGS.runspec_iterations), + ] if FLAGS.runspec_define: for runspec_define in FLAGS.runspec_define.split(','): runspec_flags.append(('define', runspec_define)) @@ -639,10 +705,16 @@ def Run(vm, cmd, benchmark_subset, version_specific_parameters=None): runspec_cmd = f'sudo bash {FLAGS.runspec_script}' else: runspec_cmd = '{cmd} --noreportable {flags} {subset}'.format( - cmd=cmd, flags=fl, subset=benchmark_subset) - - cmd = ' && '.join(('cd {0}'.format(speccpu_vm_state.spec_dir), - 'rm -rf result', '. ./shrc', '. ./shrc', runspec_cmd)) + cmd=cmd, flags=fl, subset=benchmark_subset + ) + + cmd = ' && '.join(( + 'cd {0}'.format(speccpu_vm_state.spec_dir), + 'rm -rf result', + '. ./shrc', + '. ./shrc', + runspec_cmd, + )) return vm.RobustRemoteCommand(cmd) diff --git a/perfkitbenchmarker/linux_packages/speccpu2006.py b/perfkitbenchmarker/linux_packages/speccpu2006.py index 52c235d4bc..05cb1072e4 100644 --- a/perfkitbenchmarker/linux_packages/speccpu2006.py +++ b/perfkitbenchmarker/linux_packages/speccpu2006.py @@ -48,8 +48,9 @@ def GetSpecInstallConfig(scratch_dir): install_config.base_tar_file_path = _SPECCPU2006_TAR install_config.required_members = _TAR_REQUIRED_MEMBERS install_config.log_format = _LOG_FORMAT - install_config.runspec_config = (FLAGS.runspec_config or - _DEFAULT_RUNSPEC_CONFIG) + install_config.runspec_config = ( + FLAGS.runspec_config or _DEFAULT_RUNSPEC_CONFIG + ) install_config.UpdateConfig(scratch_dir) return install_config diff --git a/perfkitbenchmarker/linux_packages/speccpu2017.py b/perfkitbenchmarker/linux_packages/speccpu2017.py index 8b4868ba16..7d6e8d49e0 100644 --- a/perfkitbenchmarker/linux_packages/speccpu2017.py +++ b/perfkitbenchmarker/linux_packages/speccpu2017.py @@ -19,19 +19,30 @@ FLAGS = flags.FLAGS -flags.DEFINE_list('spec17_subset', ['intspeed', 'fpspeed', 'intrate', 'fprate'], - 'Specify which speccpu2017 tests to run. Accepts a list of ' - 'benchmark suites (intspeed, fpspeed, intrate, fprate) ' - 'or individual benchmark names. Defaults to all suites.') -flags.DEFINE_integer('spec17_copies', None, - 'Number of copies to run for rate tests. If not set ' - 'default to number of cpu cores using lscpu.') -flags.DEFINE_integer('spec17_threads', None, - 'Number of threads to run for speed tests. If not set ' - 'default to number of cpu threads using lscpu.') -flags.DEFINE_boolean('spec17_fdo', False, - 'Run with feedback directed optimization on peak. ' - 'Default to False.') +flags.DEFINE_list( + 'spec17_subset', + ['intspeed', 'fpspeed', 'intrate', 'fprate'], + 'Specify which speccpu2017 tests to run. Accepts a list of ' + 'benchmark suites (intspeed, fpspeed, intrate, fprate) ' + 'or individual benchmark names. Defaults to all suites.', +) +flags.DEFINE_integer( + 'spec17_copies', + None, + 'Number of copies to run for rate tests. If not set ' + 'default to number of cpu cores using lscpu.', +) +flags.DEFINE_integer( + 'spec17_threads', + None, + 'Number of threads to run for speed tests. If not set ' + 'default to number of cpu threads using lscpu.', +) +flags.DEFINE_boolean( + 'spec17_fdo', + False, + 'Run with feedback directed optimization on peak. Default to False.', +) _PACKAGE_NAME = 'speccpu2017' _MOUNT_DIR = 'cpu2017_mnt' @@ -64,13 +75,14 @@ def GetSpecInstallConfig(scratch_dir): install_config.package_name = _PACKAGE_NAME install_config.base_mount_dir = _MOUNT_DIR install_config.base_spec_dir = _SPECCPU2017_DIR - install_config.base_tar_file_path = (FLAGS.runspec_tar or _SPECCPU2017_TAR) + install_config.base_tar_file_path = FLAGS.runspec_tar or _SPECCPU2017_TAR install_config.base_iso_file_path = _SPECCPU2017_ISO install_config.base_clang_flag_file_path = _DEFAULT_CLANG_FLAG install_config.required_members = _TAR_REQUIRED_MEMBERS install_config.log_format = _LOG_FORMAT - install_config.runspec_config = (FLAGS.runspec_config or - _DEFAULT_RUNSPEC_CONFIG) + install_config.runspec_config = ( + FLAGS.runspec_config or _DEFAULT_RUNSPEC_CONFIG + ) install_config.UpdateConfig(scratch_dir) return install_config @@ -82,13 +94,15 @@ def Install(vm): # But because we may have x86 or arm architecture machines, just rerun the # install script to regenerate the runner scripts based on what spec detects # to be the vm architecture. - vm.RemoteCommand('echo yes | {0}/cpu2017/install.sh'.format( - vm.GetScratchDir())) + vm.RemoteCommand( + 'echo yes | {0}/cpu2017/install.sh'.format(vm.GetScratchDir()) + ) # Updates SPECCPU 2017. # spec17 had several updates, hence let's ensure to run the latest version vm.RemoteCommand( - 'echo yes | {0}/cpu2017/bin/runcpu --update'.format(vm.GetScratchDir())) + 'echo yes | {0}/cpu2017/bin/runcpu --update'.format(vm.GetScratchDir()) + ) def AptInstall(vm): diff --git a/perfkitbenchmarker/linux_packages/specjbb.py b/perfkitbenchmarker/linux_packages/specjbb.py index b5a05208db..a987d566bf 100644 --- a/perfkitbenchmarker/linux_packages/specjbb.py +++ b/perfkitbenchmarker/linux_packages/specjbb.py @@ -22,25 +22,27 @@ SPEC_DIR = 'spec' _MOUNT_SPECJBB_ISO = flags.DEFINE_bool( - 'mount_specjbb_iso', True, 'Whether specjbb mounts iso or not') + 'mount_specjbb_iso', True, 'Whether specjbb mounts iso or not' +) def Install(vm) -> None: """Prepares a SPEC client by copying SPEC to the VM.""" mount_dir = 'spec_mnt' vm.RemoteCommand(f'mkdir -p {mount_dir} {SPEC_DIR}') - vm.InstallPreprovisionedBenchmarkData(_BENCHMARK_NAME, [SPEC_JBB_2015_ISO], - '~/') + vm.InstallPreprovisionedBenchmarkData( + _BENCHMARK_NAME, [SPEC_JBB_2015_ISO], '~/' + ) if _MOUNT_SPECJBB_ISO.value: vm.RemoteCommand( - f'sudo mount -t iso9660 -o loop {SPEC_JBB_2015_ISO} {mount_dir}') + f'sudo mount -t iso9660 -o loop {SPEC_JBB_2015_ISO} {mount_dir}' + ) vm.RemoteCommand(f'cp -r {mount_dir}/* {SPEC_DIR}') vm.RemoteCommand(f'sudo umount {mount_dir} && sudo rm -rf {mount_dir}') else: vm.InstallPackages('p7zip-full') vm.InstallPackages('p7zip-rar') - vm.RemoteCommand( - f'7z x -o{mount_dir} {SPEC_JBB_2015_ISO}') + vm.RemoteCommand(f'7z x -o{mount_dir} {SPEC_JBB_2015_ISO}') vm.RemoteCommand(f'cp -r {mount_dir}/* {SPEC_DIR}') vm.RemoteCommand(f'rm -rf {mount_dir}') @@ -50,7 +52,8 @@ def Uninstall(vm) -> None: if _MOUNT_SPECJBB_ISO.value: vm.RemoteCommand(f'sudo umount {SPEC_DIR}', ignore_failure=True) vm.RemoteCommand( - f'rm -rf {SPEC_DIR} {SPEC_JBB_2015_ISO}', ignore_failure=True) + f'rm -rf {SPEC_DIR} {SPEC_JBB_2015_ISO}', ignore_failure=True + ) def AptInstall(vm) -> None: diff --git a/perfkitbenchmarker/linux_packages/sysbench.py b/perfkitbenchmarker/linux_packages/sysbench.py index ab6127c8dc..17977f1740 100644 --- a/perfkitbenchmarker/linux_packages/sysbench.py +++ b/perfkitbenchmarker/linux_packages/sysbench.py @@ -20,9 +20,11 @@ FLAGS = flags.FLAGS _IGNORE_CONCURRENT = flags.DEFINE_bool( - 'sysbench_ignore_concurrent_modification', False, + 'sysbench_ignore_concurrent_modification', + False, 'If true, ignores concurrent modification P0001 exceptions thrown by ' - 'some databases.') + 'some databases.', +) GIT_REPO = 'https://github.com/akopytov/sysbench' @@ -32,19 +34,22 @@ SYSBENCH_DIR = '~/sysbench' # Inserts this error code on line 534. -CONCURRENT_MODS = ('534 i !strcmp(con->sql_state, "P0001")/* concurrent ' - 'modification */ ||') +CONCURRENT_MODS = ( + '534 i !strcmp(con->sql_state, "P0001")/* concurrent modification */ ||' +) def _Install(vm): """Installs the sysbench package on the VM.""" vm.RemoteCommand( - f'git clone {GIT_REPO} {SYSBENCH_DIR} --branch {RELEASE_TAG}') + f'git clone {GIT_REPO} {SYSBENCH_DIR} --branch {RELEASE_TAG}' + ) if _IGNORE_CONCURRENT.value: driver_file = f'{SYSBENCH_DIR}/src/drivers/pgsql/drv_pgsql.c' vm.RemoteCommand(f"sed -i '{CONCURRENT_MODS}' {driver_file}") - vm.RemoteCommand(f'cd {SYSBENCH_DIR} && ./autogen.sh ' - '&& ./configure --with-pgsql') + vm.RemoteCommand( + f'cd {SYSBENCH_DIR} && ./autogen.sh && ./configure --with-pgsql' + ) vm.RemoteCommand(f'cd {SYSBENCH_DIR} && make -j && sudo make install') diff --git a/perfkitbenchmarker/linux_packages/sysbench05plus.py b/perfkitbenchmarker/linux_packages/sysbench05plus.py index cc6f6051e8..8f126c21df 100644 --- a/perfkitbenchmarker/linux_packages/sysbench05plus.py +++ b/perfkitbenchmarker/linux_packages/sysbench05plus.py @@ -23,9 +23,13 @@ from perfkitbenchmarker import linux_packages SYSBENCH05PLUS_PATH = '%s/bin/sysbench' % linux_packages.INSTALL_DIR -PREPARE_SCRIPT_PATH = ('%s/share/doc/sysbench/tests/db/parallel_prepare.lua' % - linux_packages.INSTALL_DIR) -OLTP_SCRIPT_PATH = '%s/share/doc/sysbench/tests/db/oltp.lua' % linux_packages.INSTALL_DIR +PREPARE_SCRIPT_PATH = ( + '%s/share/doc/sysbench/tests/db/parallel_prepare.lua' + % linux_packages.INSTALL_DIR +) +OLTP_SCRIPT_PATH = ( + '%s/share/doc/sysbench/tests/db/oltp.lua' % linux_packages.INSTALL_DIR +) def _Install(vm): @@ -34,27 +38,36 @@ def _Install(vm): vm.InstallPackages('bzr') vm.RemoteCommand('cd ~ && bzr branch lp:sysbench') vm.RemoteCommand( - ('cd ~/sysbench && ./autogen.sh &&' - ' ./configure --prefix=%s --mandir=%s/share/man &&' - ' make') % (linux_packages.INSTALL_DIR, linux_packages.INSTALL_DIR)) + ( + 'cd ~/sysbench && ./autogen.sh &&' + ' ./configure --prefix=%s --mandir=%s/share/man &&' + ' make' + ) + % (linux_packages.INSTALL_DIR, linux_packages.INSTALL_DIR) + ) vm.RemoteCommand('cd ~/sysbench && sudo make install') - vm.RemoteCommand('sudo mkdir %s/share/doc/sysbench/tests/db -p' % - linux_packages.INSTALL_DIR) - vm.RemoteCommand('sudo cp ~/sysbench/sysbench/tests/db/*' - ' %s/share/doc/sysbench/tests/db/' % - linux_packages.INSTALL_DIR) + vm.RemoteCommand( + 'sudo mkdir %s/share/doc/sysbench/tests/db -p' + % linux_packages.INSTALL_DIR + ) + vm.RemoteCommand( + 'sudo cp ~/sysbench/sysbench/tests/db/* %s/share/doc/sysbench/tests/db/' + % linux_packages.INSTALL_DIR + ) def YumInstall(vm): - """ Installs SysBench 0.5 for Rhel/CentOS. We have to build from source!""" + """Installs SysBench 0.5 for Rhel/CentOS. We have to build from source!""" vm.InstallPackages('mysql mysql-server mysql-devel') _Install(vm) def AptInstall(vm): """Installs the sysbench 0.5 on the VM.""" - vm.RemoteCommand('sudo ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so ' - '/usr/lib/x86_64-linux-gnu/libmysqlclient_r.so') + vm.RemoteCommand( + 'sudo ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so ' + '/usr/lib/x86_64-linux-gnu/libmysqlclient_r.so' + ) vm.InstallPackages('mysql-client mysql-server libmysqlclient-dev') _Install(vm) diff --git a/perfkitbenchmarker/linux_packages/sysbench1.py b/perfkitbenchmarker/linux_packages/sysbench1.py index 0bdb9b49fd..8267ee2382 100644 --- a/perfkitbenchmarker/linux_packages/sysbench1.py +++ b/perfkitbenchmarker/linux_packages/sysbench1.py @@ -24,5 +24,6 @@ def Install(vm): 'curl -s ' 'https://packagecloud.io/install/repositories/akopytov/' 'sysbench/script.deb.sh ' - '| sudo bash') + '| sudo bash' + ) vm.InstallPackages('sysbench') diff --git a/perfkitbenchmarker/linux_packages/tailbench.py b/perfkitbenchmarker/linux_packages/tailbench.py index 5c552ec03e..3f52a1538b 100644 --- a/perfkitbenchmarker/linux_packages/tailbench.py +++ b/perfkitbenchmarker/linux_packages/tailbench.py @@ -17,6 +17,7 @@ from typing import List from perfkitbenchmarker import sample + PACKAGE_NAME = 'tailbench' TAILBENCH = 'tailbench-v0.9' @@ -28,14 +29,16 @@ TAILBENCH_INPUT_TAR = 'tailbench.inputs.tgz' TAILBENCH_INPUT_URL = 'http://tailbench.csail.mit.edu/' + TAILBENCH_INPUT_TAR PREPROVISIONED_DATA = { - TAILBENCH_TAR: - 'b26ead61a857e4f6cb904d0b0d1af07b3b509ea0c62685d696f8a26883ee94a5', - TAILBENCH_INPUT_TAR: + TAILBENCH_TAR: ( + 'b26ead61a857e4f6cb904d0b0d1af07b3b509ea0c62685d696f8a26883ee94a5' + ), + TAILBENCH_INPUT_TAR: ( '783b743e3d0d0b162bf92b93e219f67baa7c99666cb528a353d95721019817dd' + ), } PACKAGE_DATA_URL = { TAILBENCH_TAR: TAILBENCH_URL, - TAILBENCH_INPUT_TAR: TAILBENCH_INPUT_URL + TAILBENCH_INPUT_TAR: TAILBENCH_INPUT_URL, } BENCHMARK_NAME = 'tailbench' @@ -151,10 +154,9 @@ def _ParseResultsFile(input_file, name='') -> List[_TestResult]: return test_results -def BuildHistogramSamples(input_file, - name='', - metric='', - additional_metadata=None) -> List[sample.Sample]: +def BuildHistogramSamples( + input_file, name='', metric='', additional_metadata=None +) -> List[sample.Sample]: """Builds a list of samples for a the results of a test. Args: @@ -170,9 +172,15 @@ def BuildHistogramSamples(input_file, test_results = _ParseResultsFile(input_file, name) return [ - sample.CreateHistogramSample(result.histogram, result.name, - result.subname, 'ms', additional_metadata, - metric) for result in test_results + sample.CreateHistogramSample( + result.histogram, + result.name, + result.subname, + 'ms', + additional_metadata, + metric, + ) + for result in test_results ] @@ -191,6 +199,8 @@ def RunTailbench(vm, tailbench_tests): vm.RemoteCommand( f'cd {INSTALL_DIR} && ' f'sudo python3 {INSTALL_DIR}/{TAILBENCH}/utilities/parselats.py ' - f'{INSTALL_DIR}/{TAILBENCH}/{test}/lats.bin') - vm.RemoteCommand(f'sudo mv {INSTALL_DIR}/lats.txt ' - f'{INSTALL_DIR}/results/{test}.txt') + f'{INSTALL_DIR}/{TAILBENCH}/{test}/lats.bin' + ) + vm.RemoteCommand( + f'sudo mv {INSTALL_DIR}/lats.txt {INSTALL_DIR}/results/{test}.txt' + ) diff --git a/perfkitbenchmarker/linux_packages/tcmalloc.py b/perfkitbenchmarker/linux_packages/tcmalloc.py index 96287c05d8..fefd888419 100644 --- a/perfkitbenchmarker/linux_packages/tcmalloc.py +++ b/perfkitbenchmarker/linux_packages/tcmalloc.py @@ -25,12 +25,17 @@ from absl import flags from perfkitbenchmarker import errors -flags.DEFINE_enum('tcmalloc_version', 'off', - ['off', 'gperftools', 'experimental'], - 'the tcmalloc version to be preloaded') +flags.DEFINE_enum( + 'tcmalloc_version', + 'off', + ['off', 'gperftools', 'experimental'], + 'the tcmalloc version to be preloaded', +) flags.DEFINE_string( - 'tcmalloc_experimental_url', '', - 'the GCS URL for downloading the tcmalloc experimental lib') + 'tcmalloc_experimental_url', + '', + 'the GCS URL for downloading the tcmalloc experimental lib', +) flags.DEFINE_string( 'tcmalloc_settings', '', @@ -54,10 +59,12 @@ def AptInstall(vm): settings = FLAGS.tcmalloc_settings.split(',') for setting in settings: if setting: - vm.RemoteCommand('echo "export {setting}" | sudo tee -a {tmp}'.format( - setting=setting, # e.g. 'TCMALLOC_RELEASE_RATE=0.5' - tmp=TEMP_BASHRC, - )) + vm.RemoteCommand( + 'echo "export {setting}" | sudo tee -a {tmp}'.format( + setting=setting, # e.g. 'TCMALLOC_RELEASE_RATE=0.5' + tmp=TEMP_BASHRC, + ) + ) if FLAGS.tcmalloc_version == 'gperftools': vm.InstallPackages('libgoogle-perftools-dev') @@ -72,7 +79,8 @@ def AptInstall(vm): path1=libtcmalloc_paths[0], path2=libtcmalloc_paths[1], tmp=TEMP_BASHRC, - )) + ) + ) if FLAGS.tcmalloc_version == 'experimental': vm.Install('google_cloud_sdk') @@ -82,17 +90,22 @@ def AptInstall(vm): '&& echo "export LD_PRELOAD={path}" | sudo tee -a {tmp}'.format( url=FLAGS.tcmalloc_experimental_url, path=local_path, - tmp=TEMP_BASHRC)) + tmp=TEMP_BASHRC, + ) + ) # The environment variables must be exported before a potential termination # of bashrc when the shell is not interactive - vm.RemoteCommand('sudo cat {tmp} {bashrc} | sudo tee {bashrc}'.format( - tmp=TEMP_BASHRC, - bashrc=BASHRC, - )) + vm.RemoteCommand( + 'sudo cat {tmp} {bashrc} | sudo tee {bashrc}'.format( + tmp=TEMP_BASHRC, + bashrc=BASHRC, + ) + ) # Verify that libtcmalloc is preloaded in new process stdout, unused_stderr = vm.RemoteCommand('echo $LD_PRELOAD') if 'libtcmalloc.so' not in stdout: raise errors.Setup.InvalidSetupError( - 'Fail to install tcmalloc. LD_PRELOAD="{}"'.format(stdout)) + 'Fail to install tcmalloc. LD_PRELOAD="{}"'.format(stdout) + ) diff --git a/perfkitbenchmarker/linux_packages/tensorflow.py b/perfkitbenchmarker/linux_packages/tensorflow.py index 820b9bf646..1909c35159 100644 --- a/perfkitbenchmarker/linux_packages/tensorflow.py +++ b/perfkitbenchmarker/linux_packages/tensorflow.py @@ -21,23 +21,32 @@ FLAGS = flags.FLAGS -flags.DEFINE_string('tf_cpu_pip_package', - 'https://anaconda.org/intel/tensorflow/1.12.0/download/' - 'tensorflow-1.12.0-cp27-cp27mu-linux_x86_64.whl', - 'TensorFlow CPU pip package to install. By default, PKB ' - 'will install an Intel-optimized CPU build when using ' - 'CPUs.') -flags.DEFINE_string('tf_gpu_pip_package', 'tensorflow-gpu==1.12.0', - 'TensorFlow GPU pip package to install. By default, PKB ' - 'will install tensorflow-gpu==1.12 when using GPUs.') flags.DEFINE_string( - 't2t_pip_package', 'tensor2tensor==1.7', + 'tf_cpu_pip_package', + 'https://anaconda.org/intel/tensorflow/1.12.0/download/' + 'tensorflow-1.12.0-cp27-cp27mu-linux_x86_64.whl', + 'TensorFlow CPU pip package to install. By default, PKB ' + 'will install an Intel-optimized CPU build when using ' + 'CPUs.', +) +flags.DEFINE_string( + 'tf_gpu_pip_package', + 'tensorflow-gpu==1.12.0', + 'TensorFlow GPU pip package to install. By default, PKB ' + 'will install tensorflow-gpu==1.12 when using GPUs.', +) +flags.DEFINE_string( + 't2t_pip_package', + 'tensor2tensor==1.7', 'Tensor2Tensor pip package to install. By default, PKB ' - 'will install tensor2tensor==1.7 .') -flags.DEFINE_string('tf_cnn_benchmarks_branch', - 'cnn_tf_v1.12_compatible', - 'TensorFlow CNN branchmarks branch that is compatible with ' - 'A TensorFlow version.') + 'will install tensor2tensor==1.7 .', +) +flags.DEFINE_string( + 'tf_cnn_benchmarks_branch', + 'cnn_tf_v1.12_compatible', + 'TensorFlow CNN branchmarks branch that is compatible with ' + 'A TensorFlow version.', +) NCCL_URL = 'https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64/nvidia-machine-learning-repo-ubuntu1604_1.0.0-1_amd64.deb' NCCL_PACKAGE = 'nvidia-machine-learning-repo-ubuntu1604_1.0.0-1_amd64.deb' @@ -58,11 +67,12 @@ def GetEnvironmentVars(vm): long_bit = output.strip() lib_name = 'lib' if long_bit == '32' else 'lib64' env_vars.extend([ - 'PATH=%s${PATH:+:${PATH}}' % - posixpath.join(cuda_toolkit.CUDA_HOME, 'bin'), + 'PATH=%s${PATH:+:${PATH}}' + % posixpath.join(cuda_toolkit.CUDA_HOME, 'bin'), 'CUDA_HOME=%s' % cuda_toolkit.CUDA_HOME, - 'LD_LIBRARY_PATH=%s${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}' % - posixpath.join(cuda_toolkit.CUDA_HOME, lib_name)]) + 'LD_LIBRARY_PATH=%s${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}' + % posixpath.join(cuda_toolkit.CUDA_HOME, lib_name), + ]) if FLAGS.aws_s3_region: env_vars.append('AWS_REGION={}'.format(FLAGS.aws_s3_region)) return ' '.join(env_vars) @@ -77,18 +87,19 @@ def GetTensorFlowVersion(vm): Returns: installed python tensorflow version as a string """ - stdout, _ = vm.RemoteCommand( - ('echo -e "import tensorflow\nprint(tensorflow.__version__)" | {0} python' - .format(GetEnvironmentVars(vm))) - ) + stdout, _ = vm.RemoteCommand(( + 'echo -e "import tensorflow\nprint(tensorflow.__version__)" | {0}' + ' python'.format(GetEnvironmentVars(vm)) + )) return stdout.strip() def Install(vm): """Installs TensorFlow on the VM.""" has_gpu = nvidia_driver.CheckNvidiaGpuExists(vm) - tf_pip_package = (FLAGS.tf_gpu_pip_package if has_gpu else - FLAGS.tf_cpu_pip_package) + tf_pip_package = ( + FLAGS.tf_gpu_pip_package if has_gpu else FLAGS.tf_cpu_pip_package + ) if has_gpu: vm.Install('cuda_toolkit') @@ -99,19 +110,21 @@ def Install(vm): vm.RemoteCommand('sudo pip install requests') vm.RemoteCommand('sudo pip install --upgrade absl-py') vm.RemoteCommand('sudo pip install --upgrade %s' % tf_pip_package) - vm.RemoteCommand( - 'sudo pip install --upgrade %s' % FLAGS.t2t_pip_package) + vm.RemoteCommand('sudo pip install --upgrade %s' % FLAGS.t2t_pip_package) vm.InstallPackages('git') _, _, retcode = vm.RemoteHostCommandWithReturnCode( - 'test -d benchmarks', ignore_failure=True) + 'test -d benchmarks', ignore_failure=True + ) if retcode != 0: - vm.RemoteCommand( - 'git clone https://github.com/tensorflow/benchmarks.git') + vm.RemoteCommand('git clone https://github.com/tensorflow/benchmarks.git') vm.RemoteCommand( 'cd benchmarks && git checkout {}'.format(FLAGS.tf_cnn_benchmarks_branch) ) - if FLAGS.cloud == 'AWS' and FLAGS.tf_data_dir and ( - not FLAGS.tf_use_local_data): + if ( + FLAGS.cloud == 'AWS' + and FLAGS.tf_data_dir + and (not FLAGS.tf_use_local_data) + ): vm.Install('aws_credentials') diff --git a/perfkitbenchmarker/linux_packages/tensorflow_models.py b/perfkitbenchmarker/linux_packages/tensorflow_models.py index 6bba16a33e..7ad57512d9 100644 --- a/perfkitbenchmarker/linux_packages/tensorflow_models.py +++ b/perfkitbenchmarker/linux_packages/tensorflow_models.py @@ -13,20 +13,24 @@ # limitations under the License. """Module containing TensorFlow models installation and cleanup functions.""" from absl import flags + FLAGS = flags.FLAGS TF_MODELS_GIT = 'https://github.com/tensorflow/models.git' -flags.DEFINE_string('tensorflow_models_commit_hash', - '4fa82ae1cb08c374a44e2713e731f57d44bf7e61', - 'git commit hash of desired TensorFlow models commit.') +flags.DEFINE_string( + 'tensorflow_models_commit_hash', + '4fa82ae1cb08c374a44e2713e731f57d44bf7e61', + 'git commit hash of desired TensorFlow models commit.', +) def Install(vm): """Installs TensorFlow models on the VM.""" vm.InstallPackages('git') vm.RemoteCommand('git clone {}'.format(TF_MODELS_GIT)) - vm.RemoteCommand('cd models && git checkout {}'.format( - FLAGS.tensorflow_models_commit_hash)) + vm.RemoteCommand( + 'cd models && git checkout {}'.format(FLAGS.tensorflow_models_commit_hash) + ) def Uninstall(vm): diff --git a/perfkitbenchmarker/linux_packages/tensorflow_serving.py b/perfkitbenchmarker/linux_packages/tensorflow_serving.py index 5aee15e1fa..ef56e0536d 100644 --- a/perfkitbenchmarker/linux_packages/tensorflow_serving.py +++ b/perfkitbenchmarker/linux_packages/tensorflow_serving.py @@ -11,17 +11,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Module containing TensorFlow Serving installation functions. - -""" +"""Module containing TensorFlow Serving installation functions.""" import posixpath from absl import flags from perfkitbenchmarker import linux_packages from perfkitbenchmarker import vm_util VM_TMP_DIR = vm_util.VM_TMP_DIR -TF_SERVING_BASE_DIRECTORY = posixpath.join(linux_packages.INSTALL_DIR, - 'serving') +TF_SERVING_BASE_DIRECTORY = posixpath.join( + linux_packages.INSTALL_DIR, 'serving' +) FLAGS = flags.FLAGS @@ -41,19 +40,22 @@ def InstallTensorFlowServingAPI(vm): """ pip_package_output_dir = posixpath.join(VM_TMP_DIR, 'tf_serving_pip_package') - pip_package = posixpath.join(pip_package_output_dir, - 'tensorflow_serving_api*.whl') + pip_package = posixpath.join( + pip_package_output_dir, 'tensorflow_serving_api*.whl' + ) vm.Install('pip3') vm.RemoteCommand('sudo pip3 install --upgrade pip') # Build the pip package from the same source as the serving binary - vm.RemoteCommand('sudo docker run --rm -v {0}:{0} ' - 'benchmarks/tensorflow-serving-devel ' - 'bash -c "bazel build --config=nativeopt ' - 'tensorflow_serving/tools/pip_package:build_pip_package && ' - 'bazel-bin/tensorflow_serving/tools/pip_package/' - 'build_pip_package {0}"'.format(pip_package_output_dir)) + vm.RemoteCommand( + 'sudo docker run --rm -v {0}:{0} ' + 'benchmarks/tensorflow-serving-devel ' + 'bash -c "bazel build --config=nativeopt ' + 'tensorflow_serving/tools/pip_package:build_pip_package && ' + 'bazel-bin/tensorflow_serving/tools/pip_package/' + 'build_pip_package {0}"'.format(pip_package_output_dir) + ) vm.RemoteCommand('sudo pip3 install {0}'.format(pip_package)) @@ -62,24 +64,31 @@ def BuildDockerImages(vm): """Builds the Docker images from source Dockerfiles for a pre-built env.""" vm.InstallPackages('git') - vm.RemoteHostCommand('cd {0} && git clone -b {1} ' - 'https://github.com/tensorflow/serving'.format( - linux_packages.INSTALL_DIR, FLAGS.tf_serving_branch)) + vm.RemoteHostCommand( + 'cd {0} && git clone -b {1} https://github.com/tensorflow/serving'.format( + linux_packages.INSTALL_DIR, FLAGS.tf_serving_branch + ) + ) setup_script = posixpath.join( linux_packages.INSTALL_DIR, - 'serving/tensorflow_serving/tools/docker/Dockerfile.devel') + 'serving/tensorflow_serving/tools/docker/Dockerfile.devel', + ) # Changes the TensorFlow git branch to tf_serving_branch - vm_util.ReplaceText(vm, 'ARG TF_SERVING_VERSION_GIT_BRANCH=master', - 'ARG TF_SERVING_VERSION_GIT_BRANCH={}' - .format(FLAGS.tf_serving_branch), setup_script) + vm_util.ReplaceText( + vm, + 'ARG TF_SERVING_VERSION_GIT_BRANCH=master', + 'ARG TF_SERVING_VERSION_GIT_BRANCH={}'.format(FLAGS.tf_serving_branch), + setup_script, + ) # Build an optimized binary for TF Serving, and keep all the build artifacts vm.RemoteHostCommand( 'sudo docker build --target binary_build ' '-t benchmarks/tensorflow-serving-devel ' '-f {0}/tensorflow_serving/tools/docker/Dockerfile.devel ' - '{0}/tensorflow_serving/tools/docker/'.format(TF_SERVING_BASE_DIRECTORY)) + '{0}/tensorflow_serving/tools/docker/'.format(TF_SERVING_BASE_DIRECTORY) + ) # Create a serving image with the optimized model_server binary vm.RemoteHostCommand( @@ -88,7 +97,8 @@ def BuildDockerImages(vm): '--build-arg ' 'TF_SERVING_BUILD_IMAGE=benchmarks/tensorflow-serving-devel ' '-f {0}/tensorflow_serving/tools/docker/Dockerfile ' - '{0}/tensorflow_serving/tools/docker/'.format(TF_SERVING_BASE_DIRECTORY)) + '{0}/tensorflow_serving/tools/docker/'.format(TF_SERVING_BASE_DIRECTORY) + ) def InstallFromDocker(vm): diff --git a/perfkitbenchmarker/linux_packages/tomcat.py b/perfkitbenchmarker/linux_packages/tomcat.py index 1958a67e61..59581dd19d 100644 --- a/perfkitbenchmarker/linux_packages/tomcat.py +++ b/perfkitbenchmarker/linux_packages/tomcat.py @@ -27,8 +27,10 @@ from perfkitbenchmarker import linux_packages -TOMCAT_URL = ('https://archive.apache.org/dist/tomcat/tomcat-8/v8.0.28/bin/' - 'apache-tomcat-8.0.28.tar.gz') +TOMCAT_URL = ( + 'https://archive.apache.org/dist/tomcat/tomcat-8/v8.0.28/bin/' + 'apache-tomcat-8.0.28.tar.gz' +) TOMCAT_DIR = posixpath.join(linux_packages.INSTALL_DIR, 'tomcat') TOMCAT_HTTP_PORT = 8080 @@ -50,29 +52,38 @@ def _Install(vm): vm.Install('openjdk') vm.Install('curl') vm.RemoteCommand( - ('mkdir -p {0} && curl -L {1} | ' - 'tar -C {0} --strip-components 1 -xzf -').format(TOMCAT_DIR, - FLAGS.tomcat_url)) + ( + 'mkdir -p {0} && curl -L {1} | tar -C {0} --strip-components 1 -xzf -' + ).format(TOMCAT_DIR, FLAGS.tomcat_url) + ) # Use a non-blocking protocool, and disable access logging (which isn't very # helpful during load tests). vm.RemoteCommand( - ("""sed -i.bak -e '/Connector port="8080"/ """ - 's/protocol="[^"]\\+"/protocol="{0}"/\' ' - '-e "/org.apache.catalina.valves.AccessLogValve/,+3d" ' - '{1}').format( - _TOMCAT_PROTOCOL, _TOMCAT_SERVER_CONF)) + ( + """sed -i.bak -e '/Connector port="8080"/ """ + 's/protocol="[^"]\\+"/protocol="{0}"/\' ' + '-e "/org.apache.catalina.valves.AccessLogValve/,+3d" ' + '{1}' + ).format(_TOMCAT_PROTOCOL, _TOMCAT_SERVER_CONF) + ) # Quiet down localhost logs. vm.RemoteCommand( - ("sed -i.bak " - r"-e 's/\(2localhost.org.apache.*.level\)\s\+=.*$/\1 = WARN/' " - ' {0}').format(_TOMCAT_LOGGING_CONF)) + ( + 'sed -i.bak ' + r"-e 's/\(2localhost.org.apache.*.level\)\s\+=.*$/\1 = WARN/' " + ' {0}' + ).format(_TOMCAT_LOGGING_CONF) + ) # Expire sessions quickly. vm.RemoteCommand( - ("sed -i.bak " - r"-e 's,\(\)30\(\),\11\2,' " - " {0}").format(_TOMCAT_WEB_CONF)) + ( + 'sed -i.bak ' + r"-e 's,\(\)30\(\),\11\2,' " + ' {0}' + ).format(_TOMCAT_WEB_CONF) + ) def YumInstall(vm): diff --git a/perfkitbenchmarker/linux_packages/ubuntu_toolchain.py b/perfkitbenchmarker/linux_packages/ubuntu_toolchain.py index 227acb9127..4e36ff8dac 100644 --- a/perfkitbenchmarker/linux_packages/ubuntu_toolchain.py +++ b/perfkitbenchmarker/linux_packages/ubuntu_toolchain.py @@ -19,5 +19,6 @@ def AptInstall(vm): vm.RemoteCommand( 'curl -sL ' f'"http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x{key}"' - ' | sudo apt-key add') + ' | sudo apt-key add' + ) vm.RemoteCommand('sudo apt update') diff --git a/perfkitbenchmarker/linux_packages/unixbench.py b/perfkitbenchmarker/linux_packages/unixbench.py index 61c62f6cea..d3573eb81b 100644 --- a/perfkitbenchmarker/linux_packages/unixbench.py +++ b/perfkitbenchmarker/linux_packages/unixbench.py @@ -18,8 +18,9 @@ from perfkitbenchmarker import linux_packages UNIXBENCH_TAR = 'v5.1.3.tar.gz' -UNIXBENCH_URL = ('https://github.com/kdlucas/byte-unixbench/archive/' - + UNIXBENCH_TAR) +UNIXBENCH_URL = ( + 'https://github.com/kdlucas/byte-unixbench/archive/' + UNIXBENCH_TAR +) UNIXBENCH_DIR = '%s/byte-unixbench-5.1.3/UnixBench' % linux_packages.INSTALL_DIR @@ -27,16 +28,19 @@ def _Install(vm): """Installs the UnixBench package on the VM.""" vm.Install('build_tools') vm.Install('wget') - vm.RemoteCommand('wget {0} -P {1}'.format(UNIXBENCH_URL, - linux_packages.INSTALL_DIR)) - vm.RemoteCommand('cd {0} && tar xvzf {1}'.format(linux_packages.INSTALL_DIR, - UNIXBENCH_TAR)) + vm.RemoteCommand( + 'wget {0} -P {1}'.format(UNIXBENCH_URL, linux_packages.INSTALL_DIR) + ) + vm.RemoteCommand( + 'cd {0} && tar xvzf {1}'.format(linux_packages.INSTALL_DIR, UNIXBENCH_TAR) + ) def YumInstall(vm): """Installs the UnixBench package on the VM.""" - vm.InstallPackages('libX11-devel mesa-libGL-devel ' - 'perl-Time-HiRes libXext-devel') + vm.InstallPackages( + 'libX11-devel mesa-libGL-devel perl-Time-HiRes libXext-devel' + ) _Install(vm) diff --git a/perfkitbenchmarker/linux_packages/wrk.py b/perfkitbenchmarker/linux_packages/wrk.py index 990d274568..0d61c2bf81 100644 --- a/perfkitbenchmarker/linux_packages/wrk.py +++ b/perfkitbenchmarker/linux_packages/wrk.py @@ -46,10 +46,11 @@ def _Install(vm): vm.Install('curl') vm.Install('openssl') - vm.RemoteCommand(('mkdir -p {0} && curl -L {1} ' - '| tar --strip-components=1 -C {0} -xzf -').format( - WRK_DIR, - WRK_URL)) + vm.RemoteCommand( + ( + 'mkdir -p {0} && curl -L {1} | tar --strip-components=1 -C {0} -xzf -' + ).format(WRK_DIR, WRK_URL) + ) vm.RemoteCommand('cd {} && make'.format(WRK_DIR)) vm.PushDataFile(_LUA_SCRIPT_NAME, _LUA_SCRIPT_PATH) @@ -74,8 +75,7 @@ def _ParseOutput(output_text): raise ValueError('{0} not found in\n{1}'.format(_CSV_PREFIX, output_text)) csv_fp = six.StringIO(str(output_text).rsplit(_CSV_PREFIX, 1)[-1]) reader = csv.DictReader(csv_fp) - if (frozenset(reader.fieldnames) != - frozenset(['variable', 'value', 'unit'])): + if frozenset(reader.fieldnames) != frozenset(['variable', 'value', 'unit']): raise ValueError('Unexpected fields: {}'.format(reader.fieldnames)) for row in reader: yield row['variable'], float(row['value']), row['unit'] @@ -89,20 +89,34 @@ def Run(vm, target, connections=1, duration=60): target: URL to fetch. connections: Number of concurrent connections. duration: Duration of the test, in seconds. + Yields: sample.Sample objects with results. """ threads = min(connections, vm.NumCpusForBenchmark()) - cmd = ('{wrk} --connections={connections} --threads={threads} ' - '--duration={duration} ' - '--timeout={timeout} ' - '--script={script} {target}').format( - wrk=WRK_PATH, connections=connections, threads=threads, - script=_LUA_SCRIPT_PATH, target=target, - duration=duration, timeout=_TIMEOUT) + cmd = ( + '{wrk} --connections={connections} --threads={threads} ' + '--duration={duration} ' + '--timeout={timeout} ' + '--script={script} {target}' + ).format( + wrk=WRK_PATH, + connections=connections, + threads=threads, + script=_LUA_SCRIPT_PATH, + target=target, + duration=duration, + timeout=_TIMEOUT, + ) stdout, _ = vm.RemoteCommand(cmd) for variable, value, unit in _ParseOutput(stdout): - yield sample.Sample(variable, value, unit, - metadata={'connections': connections, - 'threads': threads, - 'duration': duration}) + yield sample.Sample( + variable, + value, + unit, + metadata={ + 'connections': connections, + 'threads': threads, + 'duration': duration, + }, + ) diff --git a/perfkitbenchmarker/linux_packages/wrk2.py b/perfkitbenchmarker/linux_packages/wrk2.py index 1be4f2d8b4..ca0757cca2 100644 --- a/perfkitbenchmarker/linux_packages/wrk2.py +++ b/perfkitbenchmarker/linux_packages/wrk2.py @@ -23,19 +23,23 @@ from perfkitbenchmarker import sample from perfkitbenchmarker import vm_util -WRK2_URL = ('https://github.com/giltene/wrk2/archive/' - 'c4250acb6921c13f8dccfc162d894bd7135a2979.tar.gz') +WRK2_URL = ( + 'https://github.com/giltene/wrk2/archive/' + 'c4250acb6921c13f8dccfc162d894bd7135a2979.tar.gz' +) WRK2_DIR = posixpath.join(vm_util.VM_TMP_DIR, 'wrk2') WRK2_PATH = posixpath.join(WRK2_DIR, 'wrk') _CORRECTED = flags.DEFINE_bool( - 'wrk2_corrected_latency', False, + 'wrk2_corrected_latency', + False, 'Whether or not response latency is corrected.\n' 'If True, wrk2 measure response latency from the time the ' 'transmission should have occurred according to the constant ' 'throughput configured for the run.\n' 'If False, response latency is the time that actual ' - 'transmission of a request occured.') + 'transmission of a request occured.', +) def _Install(vm): @@ -43,9 +47,10 @@ def _Install(vm): vm.Install('build_tools') vm.Install('openssl') vm.RemoteCommand( - ('mkdir -p {0} && ' - 'curl -L {1} | tar -xzf - -C {0} --strip-components 1').format( - WRK2_DIR, WRK2_URL)) + ( + 'mkdir -p {0} && curl -L {1} | tar -xzf - -C {0} --strip-components 1' + ).format(WRK2_DIR, WRK2_URL) + ) vm.RemoteCommand('make -C {}'.format(WRK2_DIR)) @@ -74,7 +79,9 @@ def _ParseOutput(output_text): inner_pat = r'^\s*(\d+\.\d+)%\s+(\d+\.\d+)(us|ms|s|m)\s*\n' regex = re.compile( r'^\s*Latency Distribution \(HdrHistogram - Recorded Latency\)\n' - r'((?:^' + inner_pat + ')+)', re.MULTILINE) + r'((?:^' + inner_pat + ')+)', + re.MULTILINE, + ) m = regex.search(output_text) if not m: raise ValueError('No match for {} in\n{}'.format(regex, output_text)) @@ -85,13 +92,13 @@ def _ParseOutput(output_text): value_in_ms = float(value) elif unit == 'us': unit = 'ms' - value_in_ms = float(value) / 1000. + value_in_ms = float(value) / 1000.0 elif unit == 's': unit = 'ms' - value_in_ms = float(value) * 1000. + value_in_ms = float(value) * 1000.0 elif unit == 'm': unit = 'ms' - value_in_ms = float(value) * 60. * 1000. + value_in_ms = float(value) * 60.0 * 1000.0 else: raise ValueError('Unknown unit {} for {}'.format(unit, m.group(1))) yield variable, value_in_ms, unit @@ -116,13 +123,9 @@ def _ParseOutput(output_text): raise ValueError('More than 10% of requests failed.') -def Run(vm, - target, - rate, - connections=1, - duration=60, - script_path=None, - threads=None): +def Run( + vm, target, rate, connections=1, duration=60, script_path=None, threads=None +): """Runs wrk against a given target. Args: @@ -139,18 +142,21 @@ def Run(vm, """ if threads is None: threads = min(connections, vm.NumCpusForBenchmark()) - cmd = ('{wrk} ' - '--rate={rate} ' - '--connections={connections} ' - '--threads={threads} ' - '--duration={duration} ' - '--{corrected}').format( - wrk=WRK2_PATH, - connections=connections, - threads=threads, - rate=rate, - duration=duration, - corrected=('latency' if _CORRECTED.value else 'u_latency')) + cmd = ( + '{wrk} ' + '--rate={rate} ' + '--connections={connections} ' + '--threads={threads} ' + '--duration={duration} ' + '--{corrected}' + ).format( + wrk=WRK2_PATH, + connections=connections, + threads=threads, + rate=rate, + duration=duration, + corrected=('latency' if _CORRECTED.value else 'u_latency'), + ) if script_path: cmd += ' --script ' + script_path cmd += ' ' + target @@ -165,5 +171,6 @@ def Run(vm, 'threads': threads, 'duration': duration, 'target_rate': rate, - 'corrected': _CORRECTED.value - }) + 'corrected': _CORRECTED.value, + }, + ) diff --git a/perfkitbenchmarker/linux_packages/xgboost.py b/perfkitbenchmarker/linux_packages/xgboost.py index 336fe6b0ff..f0a4b79731 100644 --- a/perfkitbenchmarker/linux_packages/xgboost.py +++ b/perfkitbenchmarker/linux_packages/xgboost.py @@ -19,10 +19,14 @@ from perfkitbenchmarker import linux_packages from perfkitbenchmarker.linux_packages import nvidia_driver -_ENV = flags.DEFINE_string('xgboost_env', 'PATH=/opt/conda/bin:$PATH', - 'The xboost install environment.') -_VERSION = flags.DEFINE_string('xgboost_version', '1.4.2', - 'The XGBoost version.') +_ENV = flags.DEFINE_string( + 'xgboost_env', + 'PATH=/opt/conda/bin:$PATH', + 'The xboost install environment.', +) +_VERSION = flags.DEFINE_string( + 'xgboost_version', '1.4.2', 'The XGBoost version.' +) FLAGS = flags.FLAGS @@ -46,8 +50,10 @@ def Install(vm): """Installs XGBoost on the VM.""" vm.Install('build_tools') install_dir = posixpath.join(linux_packages.INSTALL_DIR, 'xgboost') - vm.RemoteCommand('git clone --recursive https://github.com/dmlc/xgboost ' - f'--branch v{_VERSION.value} {install_dir}') + vm.RemoteCommand( + 'git clone --recursive https://github.com/dmlc/xgboost ' + f'--branch v{_VERSION.value} {install_dir}' + ) nccl_make_option = '' nccl_install_option = '' if nvidia_driver.QueryNumberOfGpus(vm) > 1: @@ -63,9 +69,13 @@ def Install(vm): build_dir = posixpath.join(install_dir, 'build') package_dir = posixpath.join(install_dir, 'python-package') vm.RemoteCommand(f'mkdir -p {build_dir}') - vm.RemoteCommand(f'cd {build_dir} && ' - f'{cuda_env} cmake .. {cuda_make_option} {nccl_make_option}') + vm.RemoteCommand( + f'cd {build_dir} && ' + f'{cuda_env} cmake .. {cuda_make_option} {nccl_make_option}' + ) vm.RemoteCommand(f'cd {build_dir} && make -j4') - vm.RemoteCommand(f'cd {package_dir} && ' - f'{_ENV.value} python3 setup.py install ' - f'{cuda_install_option} {nccl_install_option}') + vm.RemoteCommand( + f'cd {package_dir} && ' + f'{_ENV.value} python3 setup.py install ' + f'{cuda_install_option} {nccl_install_option}' + ) diff --git a/perfkitbenchmarker/linux_packages/ycsb.py b/perfkitbenchmarker/linux_packages/ycsb.py index c1e03d7297..2693132778 100644 --- a/perfkitbenchmarker/linux_packages/ycsb.py +++ b/perfkitbenchmarker/linux_packages/ycsb.py @@ -1038,6 +1038,7 @@ def RunStaircaseLoads(self, vms, workloads, **kwargs): for client_count, target_qps_per_vm in _GetThreadsQpsPerLoaderList( len(vms) ): + @vm_util.Retry( retryable_exceptions=ycsb_stats.CombineHdrLogError, timeout=-1, @@ -1207,10 +1208,7 @@ def Run( samples = self._RunLowestLatencyMode(vms, workloads, run_kwargs, database) else: samples = list(self.RunStaircaseLoads(vms, workloads, **run_kwargs)) - if ( - FLAGS.ycsb_sleep_after_load_in_sec > 0 - and not SKIP_LOAD_STAGE.value - ): + if FLAGS.ycsb_sleep_after_load_in_sec > 0 and not SKIP_LOAD_STAGE.value: for s in samples: s.metadata['sleep_after_load_in_sec'] = ( FLAGS.ycsb_sleep_after_load_in_sec diff --git a/perfkitbenchmarker/linux_packages/ycsb_stats.py b/perfkitbenchmarker/linux_packages/ycsb_stats.py index 29d5b62ef8..96ade648d2 100644 --- a/perfkitbenchmarker/linux_packages/ycsb_stats.py +++ b/perfkitbenchmarker/linux_packages/ycsb_stats.py @@ -276,13 +276,11 @@ def _GetStatsToWrite(self) -> list[str]: stats_to_write = set() for _, status_result in sorted(self.status_time_series.items()): for op_result in status_result.op_results: - stats_to_write.update( - [ - stat - for stat in op_result.statistics.keys() - if _IsStatusLatencyStatistic(stat) or stat == 'Count' - ] - ) + stats_to_write.update([ + stat + for stat in op_result.statistics.keys() + if _IsStatusLatencyStatistic(stat) or stat == 'Count' + ]) return list(stats_to_write) def WriteStatusTimeSeriesToFile(self) -> None: diff --git a/perfkitbenchmarker/linux_virtual_machine.py b/perfkitbenchmarker/linux_virtual_machine.py index 42786251f2..b9fc9c0b64 100644 --- a/perfkitbenchmarker/linux_virtual_machine.py +++ b/perfkitbenchmarker/linux_virtual_machine.py @@ -57,7 +57,9 @@ OS_PRETTY_NAME_REGEXP = r'PRETTY_NAME="(.*)"' -_EPEL_URL = 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-{}.noarch.rpm' +_EPEL_URL = ( + 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-{}.noarch.rpm' +) CLEAR_BUILD_REGEXP = r'Installed version:\s*(.*)\s*' UPDATE_RETRIES = 5 DEFAULT_SSH_PORT = 22 @@ -87,21 +89,30 @@ # TODO(user): update these to use a flag holder as recommended # in go/python-tips/051 -flags.DEFINE_bool('setup_remote_firewall', False, - 'Whether PKB should configure the firewall of each remote' - 'VM to make sure it accepts all internal connections.') - -flags.DEFINE_list('sysctl', [], - 'Sysctl values to set. This flag should be a comma-separated ' - 'list of path=value pairs. Each pair will be appended to' - '/etc/sysctl.conf. ' - 'For example, if you pass ' - '--sysctls=vm.dirty_background_ratio=10,vm.dirty_ratio=25, ' - 'PKB will append "vm.dirty_background_ratio=10" and' - '"vm.dirty_ratio=25" on separate lines to /etc/sysctrl.conf') - -flags.DEFINE_bool('reboot_after_changing_sysctl', False, - 'Whether PKB should reboot after applying sysctl changes') +flags.DEFINE_bool( + 'setup_remote_firewall', + False, + 'Whether PKB should configure the firewall of each remote' + 'VM to make sure it accepts all internal connections.', +) + +flags.DEFINE_list( + 'sysctl', + [], + 'Sysctl values to set. This flag should be a comma-separated ' + 'list of path=value pairs. Each pair will be appended to' + '/etc/sysctl.conf. ' + 'For example, if you pass ' + '--sysctls=vm.dirty_background_ratio=10,vm.dirty_ratio=25, ' + 'PKB will append "vm.dirty_background_ratio=10" and' + '"vm.dirty_ratio=25" on separate lines to /etc/sysctrl.conf', +) + +flags.DEFINE_bool( + 'reboot_after_changing_sysctl', + False, + 'Whether PKB should reboot after applying sysctl changes', +) flags.DEFINE_list( 'set_files', @@ -112,82 +123,112 @@ 'pass --set_files=/sys/kernel/mm/transparent_hugepage/enabled=always, ' 'then PKB will write "always" to ' '/sys/kernel/mm/transparent_hugepage/enabled before starting ' - 'the benchmark.') - -flags.DEFINE_bool('network_enable_BBR', False, - 'A shortcut to enable BBR congestion control on the network. ' - 'equivalent to appending to --sysctls the following values ' - '"net.core.default_qdisc=fq, ' - '"net.ipv4.tcp_congestion_control=bbr" ') - -flags.DEFINE_integer('num_disable_cpus', None, - 'Number of CPUs to disable on the virtual machine.' - 'If the VM has n CPUs, you can disable at most n-1.', - lower_bound=1) -flags.DEFINE_integer('disk_fill_size', 0, - 'Size of file to create in GBs.') -flags.DEFINE_enum('disk_fs_type', _DEFAULT_DISK_FS_TYPE, - [_DEFAULT_DISK_FS_TYPE, 'xfs'], - 'File system type used to format disk.') + 'the benchmark.', +) + +flags.DEFINE_bool( + 'network_enable_BBR', + False, + 'A shortcut to enable BBR congestion control on the network. ' + 'equivalent to appending to --sysctls the following values ' + '"net.core.default_qdisc=fq, ' + '"net.ipv4.tcp_congestion_control=bbr" ', +) + flags.DEFINE_integer( - 'disk_block_size', None, 'Block size to format disk with.' - 'Defaults to 4096 for ext4.') + 'num_disable_cpus', + None, + 'Number of CPUs to disable on the virtual machine.' + 'If the VM has n CPUs, you can disable at most n-1.', + lower_bound=1, +) +flags.DEFINE_integer('disk_fill_size', 0, 'Size of file to create in GBs.') +flags.DEFINE_enum( + 'disk_fs_type', + _DEFAULT_DISK_FS_TYPE, + [_DEFAULT_DISK_FS_TYPE, 'xfs'], + 'File system type used to format disk.', +) +flags.DEFINE_integer( + 'disk_block_size', + None, + 'Block size to format disk with.Defaults to 4096 for ext4.', +) flags.DEFINE_bool( - 'enable_transparent_hugepages', None, 'Whether to enable or ' + 'enable_transparent_hugepages', + None, + 'Whether to enable or ' 'disable transparent hugepages. If unspecified, the setting ' - 'is unchanged from the default in the OS.') + 'is unchanged from the default in the OS.', +) flags.DEFINE_integer( - 'ssh_retries', 10, 'Default number of times to retry SSH.', lower_bound=0) + 'ssh_retries', 10, 'Default number of times to retry SSH.', lower_bound=0 +) flags.DEFINE_integer( - 'scp_connect_timeout', 30, 'timeout for SCP connection.', lower_bound=0) + 'scp_connect_timeout', 30, 'timeout for SCP connection.', lower_bound=0 +) flags.DEFINE_string( - 'append_kernel_command_line', None, + 'append_kernel_command_line', + None, 'String to append to the kernel command line. The presence of any ' 'non-empty string will cause a reboot to occur after VM prepare. ' - 'If unspecified, the kernel command line will be unmodified.') + 'If unspecified, the kernel command line will be unmodified.', +) flags.DEFINE_integer( - 'tcp_max_receive_buffer', None, + 'tcp_max_receive_buffer', + None, 'Changes the third component of the sysctl value net.ipv4.tcp_rmem. ' 'This sets the maximum receive buffer for TCP socket connections in bytes. ' 'Increasing this value may increase single stream TCP throughput ' - 'for high latency connections') + 'for high latency connections', +) flags.DEFINE_integer( - 'tcp_max_send_buffer', None, + 'tcp_max_send_buffer', + None, 'Changes the third component of the sysctl value net.ipv4.tcp_wmem. ' 'This sets the maximum send buffer for TCP socket connections in bytes. ' 'Increasing this value may increase single stream TCP throughput ' - 'for high latency connections') + 'for high latency connections', +) _TCP_MAX_NOTSENT_BYTES = flags.DEFINE_integer( - 'tcp_max_notsent_bytes', None, + 'tcp_max_notsent_bytes', + None, 'Changes the third component of the sysctl value ' 'net.ipv4.tcp_notsent_lowat. This sets the maximum number of unsent bytes ' 'for TCP socket connections. Decreasing this value may to reduce usage ' - 'of kernel memory.' + 'of kernel memory.', ) flags.DEFINE_integer( - 'rmem_max', None, + 'rmem_max', + None, 'Sets the sysctl value net.core.rmem_max. This sets the max OS ' - 'receive buffer size in bytes for all types of connections') + 'receive buffer size in bytes for all types of connections', +) flags.DEFINE_integer( - 'wmem_max', None, + 'wmem_max', + None, 'Sets the sysctl value net.core.wmem_max. This sets the max OS ' - 'send buffer size in bytes for all types of connections') + 'send buffer size in bytes for all types of connections', +) -flags.DEFINE_boolean('gce_hpc_tools', False, - 'Whether to apply the hpc-tools environment script.') +flags.DEFINE_boolean( + 'gce_hpc_tools', False, 'Whether to apply the hpc-tools environment script.' +) -flags.DEFINE_boolean('disable_smt', False, - 'Whether to disable SMT (Simultaneous Multithreading) ' - 'in BIOS.') +flags.DEFINE_boolean( + 'disable_smt', + False, + 'Whether to disable SMT (Simultaneous Multithreading) in BIOS.', +) flags.DEFINE_boolean( 'use_numcpu_multi_files', @@ -206,11 +247,14 @@ ) _DISABLE_YUM_CRON = flags.DEFINE_boolean( - 'disable_yum_cron', True, 'Whether to disable the cron-run yum service.') + 'disable_yum_cron', True, 'Whether to disable the cron-run yum service.' +) _KERNEL_MODULES_TO_ADD = flags.DEFINE_list( - 'kernel_modules_to_add', [], 'Kernel modules to add to Linux VMs') + 'kernel_modules_to_add', [], 'Kernel modules to add to Linux VMs' +) _KERNEL_MODULES_TO_REMOVE = flags.DEFINE_list( - 'kernel_modules_to_remove', [], 'Kernel modules to remove from Linux VMs') + 'kernel_modules_to_remove', [], 'Kernel modules to remove from Linux VMs' +) _DISABLE_CSTATE_BY_NAME_AND_DEEPER = flags.DEFINE_string( 'disable_cstate_by_name_and_deeper', @@ -265,14 +309,18 @@ def AddLine(self, full_line: str) -> None: def _AddMitigation(self, file_name, line): match = re.match('^Mitigation: (.*)', line) or re.match( - '^([^:]+): Mitigation: (.*)$', line) + '^([^:]+): Mitigation: (.*)$', line + ) if match: self.mitigations[file_name] = ':'.join(match.groups()) return True def _AddVulnerability(self, file_name, line): - match = re.match('^Vulnerable: (.*)', line) or re.match( - '^Vulnerable$', line) or re.match('^([^:]+): Vulnerable$', line) + match = ( + re.match('^Vulnerable: (.*)', line) + or re.match('^Vulnerable$', line) + or re.match('^([^:]+): Vulnerable$', line) + ) if match: self.vulnerabilities[file_name] = ':'.join(match.groups()) return True @@ -364,7 +412,8 @@ class BaseLinuxMixin(virtual_machine.BaseOsMixin): # regex to get the network devices from "ip link show" _IP_LINK_RE_DEVICE_MTU = re.compile( - r'^\d+: (?P\S+):.*mtu (?P\d+)') + r'^\d+: (?P\S+):.*mtu (?P\d+)' + ) # device prefixes to ignore from "ip link show" # TODO(spencerkim): Record ib device metadata. _IGNORE_NETWORK_DEVICE_PREFIXES = ('lo', 'docker', 'ib') @@ -417,8 +466,9 @@ def _SetTransparentHugepages(self): return setting = 'always' if FLAGS.enable_transparent_hugepages else 'never' self.RemoteCommand( - 'echo %s | sudo tee /sys/kernel/mm/transparent_hugepage/enabled' % - setting) + 'echo %s | sudo tee /sys/kernel/mm/transparent_hugepage/enabled' + % setting + ) self.os_metadata['transparent_hugepage'] = setting def _DisableCstates(self): @@ -588,8 +638,10 @@ def _WaitForCommand(): # In case the error was with the wrapper script itself, print the log. stdout, _ = self.RemoteCommand('cat %s' % wrapper_log, stack_level=2) if stdout.strip(): - logging.warning('Exception during RobustRemoteCommand. ' - 'Wrapper script log:\n%s', stdout) + logging.warning( + 'Exception during RobustRemoteCommand. Wrapper script log:\n%s', + stdout, + ) raise def SetupRemoteFirewall(self): @@ -603,16 +655,20 @@ def SetupProxy(self): commands = [] if FLAGS.http_proxy: - commands.append("echo 'http_proxy=%s' | sudo tee -a %s" % ( - FLAGS.http_proxy, env_file)) + commands.append( + "echo 'http_proxy=%s' | sudo tee -a %s" % (FLAGS.http_proxy, env_file) + ) if FLAGS.https_proxy: - commands.append("echo 'https_proxy=%s' | sudo tee -a %s" % ( - FLAGS.https_proxy, env_file)) + commands.append( + "echo 'https_proxy=%s' | sudo tee -a %s" + % (FLAGS.https_proxy, env_file) + ) if FLAGS.ftp_proxy: - commands.append("echo 'ftp_proxy=%s' | sudo tee -a %s" % ( - FLAGS.ftp_proxy, env_file)) + commands.append( + "echo 'ftp_proxy=%s' | sudo tee -a %s" % (FLAGS.ftp_proxy, env_file) + ) if commands: self.RemoteCommand(';'.join(commands)) @@ -651,8 +707,10 @@ def PrepareVMEnvironment(self): def _CreateInstallDir(self): self.RemoteCommand( - ('sudo mkdir -p {0}; ' - 'sudo chmod a+rwxt {0}').format(linux_packages.INSTALL_DIR)) + ('sudo mkdir -p {0}; sudo chmod a+rwxt {0}').format( + linux_packages.INSTALL_DIR + ) + ) # LinuxMixins do not implement _Start or _Stop def _Start(self): @@ -668,8 +726,7 @@ def SetFiles(self): for pair in FLAGS.set_files: path, value = pair.split('=') - self.RemoteCommand('echo "%s" | sudo tee %s' % - (value, path)) + self.RemoteCommand('echo "%s" | sudo tee %s' % (value, path)) def _DisableCpus(self): """Apply num_disable_cpus to the VM. @@ -683,12 +740,13 @@ def _DisableCpus(self): self.num_disable_cpus = FLAGS.num_disable_cpus - if (self.num_disable_cpus <= 0 or - self.num_disable_cpus >= self.num_cpus): - raise ValueError('num_disable_cpus must be between 1 and ' - '(num_cpus - 1) inclusive. ' - 'num_disable_cpus: %i, num_cpus: %i' % - (self.num_disable_cpus, self.num_cpus)) + if self.num_disable_cpus <= 0 or self.num_disable_cpus >= self.num_cpus: + raise ValueError( + 'num_disable_cpus must be between 1 and ' + '(num_cpus - 1) inclusive. ' + 'num_disable_cpus: %i, num_cpus: %i' + % (self.num_disable_cpus, self.num_cpus) + ) # We can't disable cpu 0, starting from the last cpu in /proc/cpuinfo. # On multiprocessor systems, we also attempt to disable cpus on each @@ -705,8 +763,8 @@ def _DisableCpus(self): # Sort cpus based on 'core id' on each numa node for numa in cpu_mapping: cpu_mapping[numa] = sorted( - cpu_mapping[numa], - key=lambda cpu_info: (cpu_info[1], cpu_info[0])) + cpu_mapping[numa], key=lambda cpu_info: (cpu_info[1], cpu_info[0]) + ) def _GetNextCPUToDisable(num_disable_cpus): """Get the next CPU id to disable.""" @@ -720,8 +778,9 @@ def _GetNextCPUToDisable(num_disable_cpus): break for cpu_id in _GetNextCPUToDisable(self.num_disable_cpus): - self.RemoteCommand('sudo bash -c "echo 0 > ' - f'/sys/devices/system/cpu/cpu{cpu_id}/online"') + self.RemoteCommand( + f'sudo bash -c "echo 0 > /sys/devices/system/cpu/cpu{cpu_id}/online"' + ) self._proccpu_cache = None self._lscpu_cache = None @@ -735,7 +794,9 @@ def FillDisk(self): out_file = posixpath.join(self.scratch_disks[0].mount_point, 'fill_file') self.RobustRemoteCommand( 'dd if=/dev/zero of={out_file} bs=1G count={fill_size}'.format( - out_file=out_file, fill_size=FLAGS.disk_fill_size)) + out_file=out_file, fill_size=FLAGS.disk_fill_size + ) + ) def _ApplySysctlPersistent(self, sysctl_params): """Apply "key=value" pairs to /etc/sysctl.conf and load via sysctl -p. @@ -749,8 +810,9 @@ def _ApplySysctlPersistent(self, sysctl_params): return for key, value in sysctl_params.items(): - self.RemoteCommand('sudo bash -c \'echo "%s=%s" >> /etc/sysctl.conf\'' - % (key, value)) + self.RemoteCommand( + 'sudo bash -c \'echo "%s=%s" >> /etc/sysctl.conf\'' % (key, value) + ) # See https://www.golinuxcloud.com/sysctl-reload-without-reboot/ self.RemoteCommand('sudo sysctl -p') @@ -771,8 +833,8 @@ def ApplySysctlPersistent(self, sysctl_params, should_reboot=False): def DoSysctls(self): """Apply --sysctl to the VM. - The Sysctl pairs are written persistently so that if a reboot - occurs, the flags are not lost. + The Sysctl pairs are written persistently so that if a reboot + occurs, the flags are not lost. """ sysctl_params = {} for pair in FLAGS.sysctl: @@ -787,7 +849,8 @@ def DoConfigureNetworkForBBR(self): if not self.kernel_release.AtLeast(4, 9): raise flags.ValidationError( - 'BBR requires a linux image with kernel 4.9 or newer') + 'BBR requires a linux image with kernel 4.9 or newer' + ) # if the current congestion control mechanism is already BBR # then nothing needs to be done (avoid unnecessary reboot) @@ -796,7 +859,7 @@ def DoConfigureNetworkForBBR(self): self._ApplySysctlPersistent({ 'net.core.default_qdisc': 'fq', - 'net.ipv4.tcp_congestion_control': 'bbr' + 'net.ipv4.tcp_congestion_control': 'bbr', }) def DoConfigureTCPWindow(self): @@ -829,8 +892,7 @@ def DoConfigureTCPWindow(self): max_receive = rmem_values[2] max_send = wmem_values[2] max_not_sent = notsent_lowat_values[0] - logging.info( - 'notsent[0]: %s', notsent_lowat_values[0]) + logging.info('notsent[0]: %s', notsent_lowat_values[0]) # if flags are set, override current values from vm if FLAGS.tcp_max_receive_buffer: max_receive = FLAGS.tcp_max_receive_buffer @@ -850,12 +912,8 @@ def DoConfigureTCPWindow(self): self.os_metadata['rmem_max'] = rmem_max self.os_metadata['wmem_max'] = wmem_max - rmem_string = '{} {} {}'.format(rmem_values[0], - rmem_values[1], - max_receive) - wmem_string = '{} {} {}'.format(wmem_values[0], - wmem_values[1], - max_send) + rmem_string = '{} {} {}'.format(rmem_values[0], rmem_values[1], max_receive) + wmem_string = '{} {} {}'.format(wmem_values[0], wmem_values[1], max_send) logging.info('rmem_string: ' + rmem_string + ' wmem_string: ' + wmem_string) not_sent_string = '{}'.format(max_not_sent) @@ -864,7 +922,7 @@ def DoConfigureTCPWindow(self): 'net.ipv4.tcp_wmem': wmem_string, 'net.ipv4.tcp_notsent_lowat': not_sent_string, 'net.core.rmem_max': rmem_max, - 'net.core.wmem_max': wmem_max + 'net.core.wmem_max': wmem_max, }) def _RebootIfNecessary(self): @@ -877,7 +935,8 @@ def TcpCongestionControl(self): """Return the congestion control used for tcp.""" try: resp, _ = self.RemoteCommand( - 'cat /proc/sys/net/ipv4/tcp_congestion_control') + 'cat /proc/sys/net/ipv4/tcp_congestion_control' + ) return resp.rstrip('\n') except errors.VirtualMachine.RemoteCommandError: return 'unknown' @@ -913,15 +972,17 @@ def kernel_release(self) -> KernelRelease: """Return kernel release number.""" if not self._kernel_release: self._kernel_release = KernelRelease( - self.RemoteCommand('uname -r')[0].strip()) + self.RemoteCommand('uname -r')[0].strip() + ) return self._kernel_release @property def kernel_command_line(self) -> str: """Return the kernel command line.""" if not self._kernel_command_line: - self._kernel_command_line = self.RemoteCommand( - 'cat /proc/cmdline')[0].strip() + self._kernel_command_line = self.RemoteCommand('cat /proc/cmdline')[ + 0 + ].strip() return self._kernel_command_line @property @@ -939,8 +1000,11 @@ def partition_table(self) -> Dict[str, int]: partition_tables = self.RemoteCommand(cmd)[0] try: self._partition_table = { - dev: int(size) for (dev, size) in regex_util.ExtractAllMatches( - r'Disk\s*(.*):[\s\w\.]*,\s(\d*)\sbytes', partition_tables)} + dev: int(size) + for (dev, size) in regex_util.ExtractAllMatches( + r'Disk\s*(.*):[\s\w\.]*,\s(\d*)\sbytes', partition_tables + ) + } except regex_util.NoMatchError: # TODO(user): Use alternative methods to retrieve partition table. logging.warning('Partition table not found with "%s".', cmd) @@ -952,8 +1016,10 @@ def WaitForBootCompletion(self): """Waits until the VM has booted.""" # Test for listening on the port first, because this will happen strictly # first. - if (FLAGS.cluster_boot_test_port_listening and - self.port_listening_time is None): + if ( + FLAGS.cluster_boot_test_port_listening + and self.port_listening_time is None + ): self.TestConnectRemoteAccessPort() self.port_listening_time = time.time() boot_methods = self.boot_completion_ip_subset @@ -972,11 +1038,13 @@ def WaitForBootCompletion(self): connect_threads = [ (self._WaitForSshExternal, [], {}), (self._WaitForSshInternal, [], {}), - ] + ] background_tasks.RunParallelThreads(connect_threads, len(connect_threads)) else: - raise ValueError('Unknown --boot_completion_ip_subset: ' - + self.boot_completion_ip_subset) + raise ValueError( + 'Unknown --boot_completion_ip_subset: ' + + self.boot_completion_ip_subset + ) if self.bootable_time is None: self.bootable_time = time.time() @@ -1002,8 +1070,9 @@ def _WaitForSSH(self, ip_address: Union[str, None] = None): """Waits until the VM is ready.""" # Always wait for remote host command to succeed, because it is necessary to # run benchmarks - resp, _ = self.RemoteHostCommand('hostname', retries=1, - ip_address=ip_address) + resp, _ = self.RemoteHostCommand( + 'hostname', retries=1, ip_address=ip_address + ) if self.hostname is None: self.hostname = resp[:-1] @@ -1012,8 +1081,10 @@ def RecordAdditionalMetadata(self): super().RecordAdditionalMetadata() if not self.bootable_time: - logging.warning('RecordAdditionalMetadata: skipping additional metadata' - ' capture due to an unreachable VM.') + logging.warning( + 'RecordAdditionalMetadata: skipping additional metadata' + ' capture due to an unreachable VM.' + ) return self.tcp_congestion_control = self.TcpCongestionControl() @@ -1026,23 +1097,28 @@ def RecordAdditionalMetadata(self): self.os_metadata.update(self.partition_table) if FLAGS.append_kernel_command_line: self.os_metadata['kernel_command_line'] = self.kernel_command_line - self.os_metadata[ - 'append_kernel_command_line'] = FLAGS.append_kernel_command_line + self.os_metadata['append_kernel_command_line'] = ( + FLAGS.append_kernel_command_line + ) # TODO(pclay): consider publishing full lsmod as a sample. It's probably too # spammy for metadata if _KERNEL_MODULES_TO_ADD.value: self.os_metadata['added_kernel_modules'] = ','.join( - _KERNEL_MODULES_TO_ADD.value) + _KERNEL_MODULES_TO_ADD.value + ) if _KERNEL_MODULES_TO_REMOVE.value: self.os_metadata['removed_kernel_modules'] = ','.join( - _KERNEL_MODULES_TO_REMOVE.value) + _KERNEL_MODULES_TO_REMOVE.value + ) devices = self._get_network_device_mtus() all_mtus = set(devices.values()) if len(all_mtus) > 1: logging.warning( 'MTU must only have 1 unique MTU value not: %s. MTU now a ' - 'concatenation of values.', all_mtus) + 'concatenation of values.', + all_mtus, + ) self.os_metadata['mtu'] = '-'.join(list(all_mtus)) elif not all_mtus: logging.warning('No unique network devices') @@ -1058,8 +1134,10 @@ def _get_network_device_mtus(self) -> Dict[str, str]: m = self._IP_LINK_RE_DEVICE_MTU.match(line) if m: device_name = m['device_name'] - if not any(device_name.startswith(prefix) - for prefix in self._IGNORE_NETWORK_DEVICE_PREFIXES): + if not any( + device_name.startswith(prefix) + for prefix in self._IGNORE_NETWORK_DEVICE_PREFIXES + ): self._network_device_mtus[device_name] = m['mtu'] return self._network_device_mtus @@ -1070,8 +1148,7 @@ def VMLastBootTime(self): See https://unix.stackexchange.com/questions/165002/how-to-reliably-get-timestamp-at-which-the-system-booted. """ - stdout, _ = self.RemoteHostCommand( - 'stat -c %z /proc/', retries=1) + stdout, _ = self.RemoteHostCommand('stat -c %z /proc/', retries=1) if stdout.startswith('1970-01-01'): # Fix for ARM returning epochtime date_fmt = '+%Y-%m-%d %H:%M:%S.%s %z' @@ -1129,26 +1206,31 @@ def FormatDisk(self, device_path, disk_type=None): # TODO(user): Allow custom disk formatting options. if FLAGS.disk_fs_type == 'xfs': block_size = FLAGS.disk_block_size or 512 - fmt_cmd = ('sudo mkfs.xfs -f -i size={0} {1}'.format( - block_size, device_path)) + fmt_cmd = 'sudo mkfs.xfs -f -i size={0} {1}'.format( + block_size, device_path + ) else: block_size = FLAGS.disk_block_size or 4096 - fmt_cmd = ('sudo mke2fs -F -E lazy_itable_init=0,discard -O ' - '^has_journal -t ext4 -b {0} {1}'.format( - block_size, device_path)) + fmt_cmd = ( + 'sudo mke2fs -F -E lazy_itable_init=0,discard -O ' + '^has_journal -t ext4 -b {0} {1}'.format(block_size, device_path) + ) self.os_metadata['disk_filesystem_type'] = FLAGS.disk_fs_type self.os_metadata['disk_filesystem_blocksize'] = block_size self.RemoteHostCommand(umount_cmd + fmt_cmd) @vm_util.Retry( timeout=vm_util.DEFAULT_TIMEOUT, - retryable_exceptions=(errors.VirtualMachine.RemoteCommandError,)) - def MountDisk(self, - device_path, - mount_path, - disk_type=None, - mount_options=disk.DEFAULT_MOUNT_OPTIONS, - fstab_options=disk.DEFAULT_FSTAB_OPTIONS): + retryable_exceptions=(errors.VirtualMachine.RemoteCommandError,), + ) + def MountDisk( + self, + device_path, + mount_path, + disk_type=None, + mount_options=disk.DEFAULT_MOUNT_OPTIONS, + fstab_options=disk.DEFAULT_FSTAB_OPTIONS, + ): """Mounts a formatted disk in the VM.""" mount_options = '-o %s' % mount_options if mount_options else '' if disk.NFS == disk_type: @@ -1160,20 +1242,26 @@ def MountDisk(self, else: fs_type = FLAGS.disk_fs_type fstab_options = fstab_options or '' - mnt_cmd = ('sudo mkdir -p {mount_path};' - 'sudo mount {mount_options} {device_path} {mount_path} && ' - 'sudo chown $USER:$USER {mount_path};').format( - mount_path=mount_path, - device_path=device_path, - mount_options=mount_options) + mnt_cmd = ( + 'sudo mkdir -p {mount_path};' + 'sudo mount {mount_options} {device_path} {mount_path} && ' + 'sudo chown $USER:$USER {mount_path};' + ).format( + mount_path=mount_path, + device_path=device_path, + mount_options=mount_options, + ) self.RemoteHostCommand(mnt_cmd) # add to /etc/fstab to mount on reboot - mnt_cmd = ('echo "{device_path} {mount_path} {fs_type} {fstab_options}" ' - '| sudo tee -a /etc/fstab').format( - device_path=device_path, - mount_path=mount_path, - fs_type=fs_type, - fstab_options=fstab_options) + mnt_cmd = ( + 'echo "{device_path} {mount_path} {fs_type} {fstab_options}" ' + '| sudo tee -a /etc/fstab' + ).format( + device_path=device_path, + mount_path=mount_path, + fs_type=fs_type, + fstab_options=fstab_options, + ) self.RemoteHostCommand(mnt_cmd) def LogVmDebugInfo(self): @@ -1242,7 +1330,7 @@ def RemoteCommand(self, *args, **kwargs) -> Tuple[str, str]: Args: *args: Arguments passed directly to RemoteCommandWithReturnCode. **kwargs: Keyword arguments passed directly to - RemoteCommandWithReturnCode. + RemoteCommandWithReturnCode. Returns: A tuple of stdout, stderr from running the command. @@ -1291,8 +1379,8 @@ def RemoteHostCommandWithReturnCode( Args: command: A valid bash command. retries: The maximum number of times RemoteCommand should retry SSHing - when it receives a 255 return code. If None, it defaults to the value - of the flag ssh_retries. + when it receives a 255 return code. If None, it defaults to the value of + the flag ssh_retries. ignore_failure: Ignore any failure if set to true. login_shell: Run command in a login shell. timeout: The timeout for IssueCommand. @@ -1320,8 +1408,9 @@ def RemoteHostCommandWithReturnCode( ip_address = self.GetConnectionIp() user_host = '%s@%s' % (self.user_name, ip_address) ssh_cmd = ['ssh', '-A', '-p', str(self.ssh_port), user_host] - ssh_private_key = (self.ssh_private_key if self.is_static else - vm_util.GetPrivateKeyPath()) + ssh_private_key = ( + self.ssh_private_key if self.is_static else vm_util.GetPrivateKeyPath() + ) ssh_cmd.extend(vm_util.GetSshOptions(ssh_private_key)) if should_pre_log: logger.info( @@ -1354,9 +1443,11 @@ def RemoteHostCommandWithReturnCode( if retcode: full_cmd = ' '.join(ssh_cmd) - error_text = ('Got non-zero return code (%s) executing %s\n' - 'Full command: %s\nSTDOUT: %sSTDERR: %s' % - (retcode, command, full_cmd, stdout, stderr)) + error_text = ( + 'Got non-zero return code (%s) executing %s\n' + 'Full command: %s\nSTDOUT: %sSTDERR: %s' + % (retcode, command, full_cmd, stdout, stderr) + ) if not ignore_failure: raise errors.VirtualMachine.RemoteCommandError(error_text) @@ -1371,7 +1462,7 @@ def RemoteHostCommand(self, *args, **kwargs) -> Tuple[str, str]: Args: *args: Arguments passed directly to RemoteHostCommandWithReturnCode. **kwargs: Keyword arguments passed directly to - RemoteHostCommandWithReturnCode. + RemoteHostCommandWithReturnCode. Returns: A tuple of stdout, stderr from running the command. @@ -1385,7 +1476,8 @@ def RemoteHostCommand(self, *args, **kwargs) -> Tuple[str, str]: def _CheckRebootability(self): if not self.IS_REBOOTABLE: raise errors.VirtualMachine.VirtualMachineError( - "Trying to reboot a VM that isn't rebootable.") + "Trying to reboot a VM that isn't rebootable." + ) def _Reboot(self): """OS-specific implementation of reboot command.""" @@ -1420,8 +1512,8 @@ def MoveHostFile(self, target, source_path, remote_path=''): Args: target: The target BaseVirtualMachine object. source_path: The location of the file on the REMOTE machine. - remote_path: The destination of the file on the TARGET machine, default - is the home directory. + remote_path: The destination of the file on the TARGET machine, default is + the home directory. """ self.AuthenticateVm() @@ -1431,10 +1523,14 @@ def MoveHostFile(self, target, source_path, remote_path=''): # ie: the key is added to know known_hosts which allows # OpenMPI to operate correctly. remote_location = '%s@%s:%s' % ( - target.user_name, target.ip_address, remote_path) - self.RemoteHostCommand('scp -P %s -o StrictHostKeyChecking=no -i %s %s %s' % - (target.ssh_port, REMOTE_KEY_PATH, source_path, - remote_location)) + target.user_name, + target.ip_address, + remote_path, + ) + self.RemoteHostCommand( + 'scp -P %s -o StrictHostKeyChecking=no -i %s %s %s' + % (target.ssh_port, REMOTE_KEY_PATH, source_path, remote_location) + ) def AuthenticateVm(self): """Authenticate a remote machine to access all peers.""" @@ -1451,7 +1547,8 @@ def AuthenticateVm(self): ) return self.RemoteCommand( - 'echo "Host *\n StrictHostKeyChecking no\n" > ~/.ssh/config') + 'echo "Host *\n StrictHostKeyChecking no\n" > ~/.ssh/config' + ) self.RemoteCommand('chmod 600 ~/.ssh/config') self.has_private_key = True @@ -1465,7 +1562,8 @@ def TestAuthentication(self, peer): raise errors.VirtualMachine.AuthError( 'Authentication check failed. If you are running with Static VMs, ' 'please make sure that %s can ssh into %s without supplying any ' - 'arguments except the ip address.' % (self, peer)) + 'arguments except the ip address.' % (self, peer) + ) def CheckJavaVersion(self): """Check the version of java on remote machine. @@ -1473,9 +1571,9 @@ def CheckJavaVersion(self): Returns: The version of Java installed on remote machine. """ - version, _ = self.RemoteCommand('java -version 2>&1 >/dev/null | ' - 'grep version | ' - 'awk \'{print $3}\'') + version, _ = self.RemoteCommand( + "java -version 2>&1 >/dev/null | grep version | awk '{print $3}'" + ) return version[:-1] def RemoveFile(self, filename): @@ -1490,8 +1588,8 @@ def GetDeviceSizeFromPath(self, path): """Gets the size of the a drive that contains the path specified. Args: - path: The function will return the amount of space on the file system - that contains this file name. + path: The function will return the amount of space on the file system that + contains this file name. Returns: The size in 1K blocks of the file system containing the file. @@ -1599,8 +1697,7 @@ def _GetNumCpus(self): if FLAGS.use_numcpu_multi_files: return self._GetNumCpusFromMultiFiles() - stdout, _ = self.RemoteCommand( - 'cat /proc/cpuinfo | grep processor | wc -l') + stdout, _ = self.RemoteCommand('cat /proc/cpuinfo | grep processor | wc -l') return int(stdout) def _GetTotalFreeMemoryKb(self): @@ -1681,7 +1778,7 @@ def _GetTotalMemoryKb(self): Otherwise, extracts the memory capacity using /proc/meminfo. This method does not cache results (unlike "total_memory_kb"). """ - meminfo_command = 'cat /proc/meminfo | grep MemTotal | awk \'{print $2}\'' + meminfo_command = "cat /proc/meminfo | grep MemTotal | awk '{print $2}'" stdout, _ = self.RemoteCommand(meminfo_command) meminfo_memory_kb = int(stdout) @@ -1706,15 +1803,16 @@ def StripeDisks(self, devices, striped_device): striped_device: The path to the device that will be created. """ self.Install('mdadm') - stripe_cmd = ('yes | sudo mdadm --create %s --level=stripe --raid-devices=' - '%s %s' % (striped_device, len(devices), ' '.join(devices))) + stripe_cmd = ( + 'yes | sudo mdadm --create %s --level=stripe --raid-devices=%s %s' + % (striped_device, len(devices), ' '.join(devices)) + ) self.RemoteHostCommand(stripe_cmd) # Save the RAID layout on the disk self.RemoteHostCommand('sudo mkdir -p /etc/mdadm') self.RemoteHostCommand('sudo touch /etc/mdadm/mdadm.conf') - cmd = ('sudo mdadm --detail --scan | ' + - 'sudo tee -a /etc/mdadm/mdadm.conf') + cmd = 'sudo mdadm --detail --scan | ' + 'sudo tee -a /etc/mdadm/mdadm.conf' self.RemoteHostCommand(cmd) # Make the disk available during reboot for VMs running Debian based Linux @@ -1723,8 +1821,10 @@ def StripeDisks(self, devices, striped_device): self.RemoteHostCommand(init_ram_fs_cmd) # Automatically mount the disk after reboot - cmd = ('echo \'/dev/md0 /mnt/md0 ext4 defaults,nofail' - ',discard 0 0\' | sudo tee -a /etc/fstab') + cmd = ( + "echo '/dev/md0 /mnt/md0 ext4 defaults,nofail" + ",discard 0 0' | sudo tee -a /etc/fstab" + ) self.RemoteHostCommand(cmd) def BurnCpu(self, burn_cpu_threads=None, burn_cpu_seconds=None): @@ -1741,7 +1841,8 @@ def BurnCpu(self, burn_cpu_threads=None, burn_cpu_seconds=None): end_time = time.time() + burn_cpu_seconds self.RemoteCommand( 'nohup sysbench --num-threads=%s --test=cpu --cpu-max-prime=10000000 ' - 'run 1> /dev/null 2> /dev/null &' % burn_cpu_threads) + 'run 1> /dev/null 2> /dev/null &' % burn_cpu_threads + ) if time.time() < end_time: time.sleep(end_time - time.time()) self.RemoteCommand('pkill -9 sysbench') @@ -1763,7 +1864,9 @@ def SetReadAhead(self, num_sectors, devices): """ self.RemoteCommand( 'sudo blockdev --setra {0} {1}; sudo blockdev --setfra {0} {1};'.format( - num_sectors, ' '.join(devices))) + num_sectors, ' '.join(devices) + ) + ) def GetSha256sum(self, path, filename): """Gets the sha256sum hash for a filename in a path on the VM. @@ -1776,7 +1879,8 @@ def GetSha256sum(self, path, filename): string; The sha256sum hash. """ stdout, _ = self.RemoteCommand( - 'sha256sum %s' % posixpath.join(path, filename)) + 'sha256sum %s' % posixpath.join(path, filename) + ) sha256sum, _ = stdout.split() return sha256sum @@ -1792,7 +1896,8 @@ def AppendKernelCommandLine(self, command_line, reboot=True): reboot: Whether or not to reboot to have the change take effect. """ raise NotImplementedError( - 'Kernel command-line appending for given Linux flavor not implemented.') + 'Kernel command-line appending for given Linux flavor not implemented.' + ) def _DoAppendKernelCommandLine(self): """If the flag is set, attempts to append the provided kernel command line. @@ -1801,12 +1906,15 @@ def _DoAppendKernelCommandLine(self): needs reboot bit instead of immediately rebooting. """ if FLAGS.disable_smt and self.CheckLsCpu().threads_per_core != 1: - FLAGS.append_kernel_command_line = ' '.join( - (FLAGS.append_kernel_command_line, - 'nosmt')) if FLAGS.append_kernel_command_line else 'nosmt' + FLAGS.append_kernel_command_line = ( + ' '.join((FLAGS.append_kernel_command_line, 'nosmt')) + if FLAGS.append_kernel_command_line + else 'nosmt' + ) if FLAGS.append_kernel_command_line: self.AppendKernelCommandLine( - FLAGS.append_kernel_command_line, reboot=False) + FLAGS.append_kernel_command_line, reboot=False + ) self._needs_reboot = True def ModifyKernelModules(self): @@ -1845,7 +1953,8 @@ def cpu_vulnerabilities(self) -> CpuVulnerabilities: """ text, _ = self.RemoteCommand( 'sudo grep . /sys/devices/system/cpu/vulnerabilities/*', - ignore_failure=True) + ignore_failure=True, + ) vuln = CpuVulnerabilities() if not text: logging.warning('No text response when getting CPU vulnerabilities') @@ -1874,7 +1983,8 @@ def GetNVMEDeviceInfo(self): rows = stdout.splitlines() delimiter_row = rows[1] # row 0 is the column headers delimiter_index = [0] + [ - i for i in range(len(delimiter_row)) if delimiter_row[i] == ' '] + i for i in range(len(delimiter_row)) if delimiter_row[i] == ' ' + ] for row in rows[2:]: device = {} device_info = [ @@ -1911,9 +2021,12 @@ def OnStartup(self): super(ClearMixin, self).OnStartup() self.RemoteHostCommand('sudo swupd autoupdate --disable') self.RemoteHostCommand('sudo mkdir -p /etc/sudoers.d') - self.RemoteHostCommand('echo \'Defaults:{0} !requiretty\' | ' - 'sudo tee /etc/sudoers.d/pkb'.format(self.user_name), - login_shell=True) + self.RemoteHostCommand( + "echo 'Defaults:{0} !requiretty' | sudo tee /etc/sudoers.d/pkb".format( + self.user_name + ), + login_shell=True, + ) def PackageCleanup(self): """Cleans up all installed packages. @@ -1926,21 +2039,26 @@ def PackageCleanup(self): def SnapshotPackages(self): """See base class.""" - self.RemoteCommand('sudo swupd bundle-list > {0}/bundle_list'.format( - linux_packages.INSTALL_DIR)) + self.RemoteCommand( + 'sudo swupd bundle-list > {0}/bundle_list'.format( + linux_packages.INSTALL_DIR + ) + ) def RestorePackages(self): """See base class.""" self.RemoteCommand( - 'sudo swupd bundle-list | grep --fixed-strings --line-regexp --invert-match --file ' - '{0}/bundle_list | xargs --no-run-if-empty sudo swupd bundle-remove' - .format(linux_packages.INSTALL_DIR), - ignore_failure=True) + 'sudo swupd bundle-list | grep --fixed-strings --line-regexp' + ' --invert-match --file {0}/bundle_list | xargs --no-run-if-empty sudo' + ' swupd bundle-remove'.format(linux_packages.INSTALL_DIR), + ignore_failure=True, + ) def HasPackage(self, package): """Returns True iff the package is available for installation.""" return self.TryRemoteCommand( - 'sudo swupd bundle-list --all | grep {0}'.format(package)) + 'sudo swupd bundle-list --all | grep {0}'.format(package) + ) def InstallPackages(self, packages: str) -> None: """Installs packages using the swupd bundle manager.""" @@ -1959,7 +2077,9 @@ def Install(self, package_name): else: raise KeyError( 'Package {0} has no install method for Clear Linux.'.format( - package_name)) + package_name + ) + ) self._installed_packages.add(package_name) def Uninstall(self, package_name): @@ -1984,7 +2104,8 @@ def GetOsInfo(self): """See base class.""" stdout, _ = self.RemoteCommand('swupd info | grep Installed') return 'Clear Linux build: {0}'.format( - regex_util.ExtractGroup(CLEAR_BUILD_REGEXP, stdout)) + regex_util.ExtractGroup(CLEAR_BUILD_REGEXP, stdout) + ) def SetupProxy(self): """Sets up proxy configuration variables for the cloud environment.""" @@ -1993,20 +2114,28 @@ def SetupProxy(self): commands = [] if FLAGS.http_proxy: - commands.append("echo 'export http_proxy=%s' | sudo tee -a %s" % ( - FLAGS.http_proxy, profile_file)) + commands.append( + "echo 'export http_proxy=%s' | sudo tee -a %s" + % (FLAGS.http_proxy, profile_file) + ) if FLAGS.https_proxy: - commands.append("echo 'https_proxy=%s' | sudo tee -a %s" % ( - FLAGS.https_proxy, profile_file)) + commands.append( + "echo 'https_proxy=%s' | sudo tee -a %s" + % (FLAGS.https_proxy, profile_file) + ) if FLAGS.ftp_proxy: - commands.append("echo 'ftp_proxy=%s' | sudo tee -a %s" % ( - FLAGS.ftp_proxy, profile_file)) + commands.append( + "echo 'ftp_proxy=%s' | sudo tee -a %s" + % (FLAGS.ftp_proxy, profile_file) + ) if FLAGS.no_proxy: - commands.append("echo 'export no_proxy=%s' | sudo tee -a %s" % ( - FLAGS.no_proxy, profile_file)) + commands.append( + "echo 'export no_proxy=%s' | sudo tee -a %s" + % (FLAGS.no_proxy, profile_file) + ) if commands: self.RemoteCommand(';'.join(commands)) @@ -2016,7 +2145,7 @@ def RemoteCommand(self, command, **kwargs): Args: command: Arguments passed directly to RemoteHostCommandWithReturnCode. **kwargs: Keyword arguments passed directly to - RemoteHostCommandWithReturnCode. + RemoteHostCommandWithReturnCode. Returns: A tuple of stdout and stderr from running the command. @@ -2078,19 +2207,23 @@ class BaseRhelMixin(BaseLinuxMixin): def OnStartup(self): """Eliminates the need to have a tty to run sudo commands.""" super(BaseRhelMixin, self).OnStartup() - self.RemoteHostCommand('echo \'Defaults:%s !requiretty\' | ' - 'sudo tee /etc/sudoers.d/pkb' % self.user_name, - login_shell=True) + self.RemoteHostCommand( + "echo 'Defaults:%s !requiretty' | sudo tee /etc/sudoers.d/pkb" + % self.user_name, + login_shell=True, + ) if FLAGS.gce_hpc_tools: self.InstallGcpHpcTools() # yum cron can stall causing yum commands to hang if _DISABLE_YUM_CRON.value: if self.PACKAGE_MANAGER == YUM: - self.RemoteHostCommand('sudo systemctl disable yum-cron.service', - ignore_failure=True) + self.RemoteHostCommand( + 'sudo systemctl disable yum-cron.service', ignore_failure=True + ) elif self.PACKAGE_MANAGER == DNF: - self.RemoteHostCommand('sudo systemctl disable dnf-automatic.timer', - ignore_failure=True) + self.RemoteHostCommand( + 'sudo systemctl disable dnf-automatic.timer', ignore_failure=True + ) def InstallGcpHpcTools(self): """Installs the GCP HPC tools.""" @@ -2107,16 +2240,18 @@ def PackageCleanup(self): def SnapshotPackages(self): """Grabs a snapshot of the currently installed packages.""" - self.RemoteCommand('rpm -qa > %s/rpm_package_list' - % linux_packages.INSTALL_DIR) + self.RemoteCommand( + 'rpm -qa > %s/rpm_package_list' % linux_packages.INSTALL_DIR + ) def RestorePackages(self): """Restores the currently installed packages to those snapshotted.""" self.RemoteCommand( 'rpm -qa | grep --fixed-strings --line-regexp --invert-match --file ' - '%s/rpm_package_list | xargs --no-run-if-empty sudo rpm -e' % - linux_packages.INSTALL_DIR, - ignore_failure=True) + '%s/rpm_package_list | xargs --no-run-if-empty sudo rpm -e' + % linux_packages.INSTALL_DIR, + ignore_failure=True, + ) def HasPackage(self, package): """Returns True iff the package is available for installation.""" @@ -2151,8 +2286,9 @@ def Install(self, package_name): elif hasattr(package, 'Install'): package.Install(self) else: - raise KeyError('Package %s has no install method for RHEL.' % - package_name) + raise KeyError( + 'Package %s has no install method for RHEL.' % package_name + ) self._installed_packages.add(package_name) def Uninstall(self, package_name): @@ -2192,14 +2328,17 @@ def SetupProxy(self): yum_proxy_file = '/etc/dnf/dnf.conf' if FLAGS.http_proxy: - self.RemoteCommand("echo -e 'proxy= %s' | sudo tee -a %s" % ( - FLAGS.http_proxy, yum_proxy_file)) + self.RemoteCommand( + "echo -e 'proxy= %s' | sudo tee -a %s" + % (FLAGS.http_proxy, yum_proxy_file) + ) def AppendKernelCommandLine(self, command_line, reboot=True): """Appends the provided command-line to the VM and reboots by default.""" self.RemoteCommand( r'echo GRUB_CMDLINE_LINUX_DEFAULT=\"\${GRUB_CMDLINE_LINUX_DEFAULT} %s\"' - ' | sudo tee -a /etc/default/grub' % command_line) + ' | sudo tee -a /etc/default/grub' % command_line + ) self.RemoteCommand('sudo grub2-mkconfig -o /boot/grub2/grub.cfg') self.RemoteCommand('sudo grub2-mkconfig -o /etc/grub2.cfg') if reboot: @@ -2208,6 +2347,7 @@ def AppendKernelCommandLine(self, command_line, reboot=True): class AmazonLinux2Mixin(BaseRhelMixin): """Class holding Amazon Linux 2 VM methods and attributes.""" + OS_TYPE = os_types.AMAZONLINUX2 PACKAGE_MANAGER = YUM @@ -2219,11 +2359,13 @@ def SetupPackageManager(self): class AmazonNeuronMixin(AmazonLinux2Mixin): """Class holding Neuron VM methods and attributes.""" + OS_TYPE = os_types.AMAZON_NEURON class AmazonLinux2023Mixin(BaseRhelMixin): """Class holding Amazon Linux 2023 VM methods and attributes.""" + OS_TYPE = os_types.AMAZONLINUX2023 # Note no EPEL support # https://docs.aws.amazon.com/linux/al2023/ug/compare-with-al2.html#epel @@ -2231,6 +2373,7 @@ class AmazonLinux2023Mixin(BaseRhelMixin): class Rhel7Mixin(BaseRhelMixin): """Class holding RHEL 7 specific VM methods and attributes.""" + OS_TYPE = os_types.RHEL7 PACKAGE_MANAGER = YUM @@ -2239,11 +2382,13 @@ def SetupPackageManager(self): # https://docs.fedoraproject.org/en-US/epel/#_rhel_7 # yum exits 1 if the program is installed so check first self.RemoteCommand( - f'rpm -q epel-release || sudo yum install -y {_EPEL_URL.format(7)}') + f'rpm -q epel-release || sudo yum install -y {_EPEL_URL.format(7)}' + ) class Rhel8Mixin(BaseRhelMixin): """Class holding RHEL 8 specific VM methods and attributes.""" + OS_TYPE = os_types.RHEL8 def SetupPackageManager(self): @@ -2254,6 +2399,7 @@ def SetupPackageManager(self): class Rhel9Mixin(BaseRhelMixin): """Class holding RHEL 9 specific VM methods and attributes.""" + OS_TYPE = os_types.RHEL9 PYTHON_2_PACKAGE = None @@ -2285,6 +2431,7 @@ def SetupPackageManager(self): class CentOs7Mixin(BaseRhelMixin): """Class holding CentOS 7 specific VM methods and attributes.""" + OS_TYPE = os_types.CENTOS7 PACKAGE_MANAGER = YUM @@ -2293,11 +2440,13 @@ def SetupPackageManager(self): # https://docs.fedoraproject.org/en-US/epel/#_centos_7 # yum exits 1 if the program is installed so check first self.RemoteCommand( - 'rpm -q epel-release || sudo yum install -y epel-release') + 'rpm -q epel-release || sudo yum install -y epel-release' + ) class CentOs8Mixin(BaseRhelMixin, virtual_machine.DeprecatedOsMixin): """Class holding CentOS 8 specific VM methods and attributes.""" + OS_TYPE = os_types.CENTOS8 END_OF_LIFE = '2021-12-31' ALTERNATIVE_OS = f'{os_types.ROCKY_LINUX8} or {os_types.CENTOS_STREAM8}' @@ -2307,11 +2456,13 @@ def SetupPackageManager(self): # https://docs.fedoraproject.org/en-US/epel/#almalinux_8_rocky_linux_8 self.RemoteCommand( 'sudo dnf config-manager --set-enabled powertools && ' - 'sudo dnf install -y epel-release') + 'sudo dnf install -y epel-release' + ) class CentOsStream8Mixin(BaseRhelMixin): """Class holding CentOS Stream 8 specific VM methods and attributes.""" + OS_TYPE = os_types.CENTOS_STREAM8 def SetupPackageManager(self): @@ -2319,11 +2470,13 @@ def SetupPackageManager(self): # https://docs.fedoraproject.org/en-US/epel/#_centos_stream_8 self.RemoteCommand( 'sudo dnf config-manager --set-enabled powertools && ' - 'sudo dnf install -y epel-release epel-next-release') + 'sudo dnf install -y epel-release epel-next-release' + ) class CentOsStream9Mixin(BaseRhelMixin): """Class holding CentOS Stream 9 specific VM methods and attributes.""" + OS_TYPE = os_types.CENTOS_STREAM9 PYTHON_2_PACKAGE = None @@ -2332,11 +2485,13 @@ def SetupPackageManager(self): # https://docs.fedoraproject.org/en-US/epel/#_centos_stream_9 self.RemoteCommand( 'sudo dnf config-manager --set-enabled crb && ' - 'sudo dnf install -y epel-release epel-next-release') + 'sudo dnf install -y epel-release epel-next-release' + ) class RockyLinux8Mixin(BaseRhelMixin): """Class holding Rocky Linux 8 specific VM methods and attributes.""" + OS_TYPE = os_types.ROCKY_LINUX8 def SetupPackageManager(self): @@ -2344,11 +2499,13 @@ def SetupPackageManager(self): # https://docs.fedoraproject.org/en-US/epel/#_almalinux_8_rocky_linux_8 self.RemoteCommand( 'sudo dnf config-manager --set-enabled powertools && ' - 'sudo dnf install -y epel-release') + 'sudo dnf install -y epel-release' + ) class RockyLinux9Mixin(BaseRhelMixin): """Class holding Rocky Linux 8 specific VM methods and attributes.""" + OS_TYPE = os_types.ROCKY_LINUX9 def SetupPackageManager(self): @@ -2356,11 +2513,13 @@ def SetupPackageManager(self): # https://docs.fedoraproject.org/en-US/epel/#_almalinux_9_rocky_linux_98 self.RemoteCommand( 'sudo dnf config-manager --set-enabled crb &&' - 'sudo dnf install -y epel-release') + 'sudo dnf install -y epel-release' + ) class CoreOsMixin(BaseContainerLinuxMixin): """Class holding CoreOS Container Linux specific VM methods and attributes.""" + OS_TYPE = os_types.CORE_OS BASE_OS_TYPE = os_types.CORE_OS @@ -2398,16 +2557,20 @@ def SnapshotPackages(self): """Grabs a snapshot of the currently installed packages.""" self.RemoteCommand( 'dpkg --get-selections > %s/dpkg_selections' - % linux_packages.INSTALL_DIR) + % linux_packages.INSTALL_DIR + ) def RestorePackages(self): """Restores the currently installed packages to those snapshotted.""" self.RemoteCommand('sudo dpkg --clear-selections') self.RemoteCommand( 'sudo dpkg --set-selections < %s/dpkg_selections' - % linux_packages.INSTALL_DIR) - self.RemoteCommand('sudo DEBIAN_FRONTEND=\'noninteractive\' ' - 'apt-get --purge -y dselect-upgrade') + % linux_packages.INSTALL_DIR + ) + self.RemoteCommand( + "sudo DEBIAN_FRONTEND='noninteractive' " + 'apt-get --purge -y dselect-upgrade' + ) def HasPackage(self, package): """Returns True iff the package is available for installation.""" @@ -2417,8 +2580,8 @@ def HasPackage(self, package): # apt-cache show will exit 0 for purely virtual packages. # It does always log `N: No packages found` to STDOUT in that case though stdout, stderr, retcode = self.RemoteCommandWithReturnCode( - 'apt-cache --quiet=0 show ' + package, - ignore_failure=True) + 'apt-cache --quiet=0 show ' + package, ignore_failure=True + ) return not retcode and 'No packages found' not in (stdout + stderr) @vm_util.Retry() @@ -2431,17 +2594,24 @@ def InstallPackages(self, packages): self.AptUpdate() self._apt_updated = True try: - install_command = ('sudo DEBIAN_FRONTEND=\'noninteractive\' ' - '/usr/bin/apt-get -y install %s' % (packages)) + install_command = ( + "sudo DEBIAN_FRONTEND='noninteractive' /usr/bin/apt-get -y install %s" + % (packages) + ) self.RemoteCommand(install_command) except errors.VirtualMachine.RemoteCommandError as e: # TODO(user): Remove code below after Azure fix their package repository, # or add code to recover the sources.list self.RemoteCommand( 'sudo sed -i.bk "s/azure.archive.ubuntu.com/archive.ubuntu.com/g" ' - '/etc/apt/sources.list') - logging.info('Installing "%s" failed on %s. This may be transient. ' - 'Updating package list.', packages, self) + '/etc/apt/sources.list' + ) + logging.info( + 'Installing "%s" failed on %s. This may be transient. ' + 'Updating package list.', + packages, + self, + ) self.AptUpdate() raise e @@ -2461,8 +2631,9 @@ def Install(self, package_name): elif hasattr(package, 'Install'): package.Install(self) else: - raise KeyError('Package %s has no install method for Debian.' % - package_name) + raise KeyError( + 'Package %s has no install method for Debian.' % package_name + ) self._installed_packages.add(package_name) def Uninstall(self, package_name): @@ -2507,12 +2678,16 @@ def SetupProxy(self): commands = [] if FLAGS.http_proxy: - commands.append("echo -e 'Acquire::http::proxy \"%s\";' |" - 'sudo tee -a %s' % (FLAGS.http_proxy, apt_proxy_file)) + commands.append( + 'echo -e \'Acquire::http::proxy "%s";\' |sudo tee -a %s' + % (FLAGS.http_proxy, apt_proxy_file) + ) if FLAGS.https_proxy: - commands.append("echo -e 'Acquire::https::proxy \"%s\";' |" - 'sudo tee -a %s' % (FLAGS.https_proxy, apt_proxy_file)) + commands.append( + 'echo -e \'Acquire::https::proxy "%s";\' |sudo tee -a %s' + % (FLAGS.https_proxy, apt_proxy_file) + ) if commands: self.RemoteCommand(';'.join(commands)) @@ -2523,15 +2698,18 @@ def IncreaseSSHConnection(self, target): Args: target: int. The max number of ssh connection. """ - self.RemoteCommand(r'sudo sed -i -e "s/.*MaxStartups.*/MaxStartups {0}/" ' - '/etc/ssh/sshd_config'.format(target)) + self.RemoteCommand( + r'sudo sed -i -e "s/.*MaxStartups.*/MaxStartups {0}/" ' + '/etc/ssh/sshd_config'.format(target) + ) self.RemoteCommand('sudo service ssh restart') def AppendKernelCommandLine(self, command_line, reboot=True): """Appends the provided command-line to the VM and reboots by default.""" self.RemoteCommand( r'echo GRUB_CMDLINE_LINUX_DEFAULT=\"\${GRUB_CMDLINE_LINUX_DEFAULT} %s\"' - r' | sudo tee -a /etc/default/grub' % command_line) + r' | sudo tee -a /etc/default/grub' % command_line + ) self.RemoteCommand('sudo update-grub') if reboot: self.Reboot() @@ -2539,6 +2717,7 @@ def AppendKernelCommandLine(self, command_line, reboot=True): class Debian9Mixin(BaseDebianMixin): """Class holding Debian9 specific VM methods and attributes.""" + OS_TYPE = os_types.DEBIAN9 # https://packages.debian.org/stretch/python PYTHON_2_PACKAGE = 'python' @@ -2546,16 +2725,19 @@ class Debian9Mixin(BaseDebianMixin): class Debian10Mixin(BaseDebianMixin): """Class holding Debian 10 specific VM methods and attributes.""" + OS_TYPE = os_types.DEBIAN10 class Debian10BackportsMixin(Debian10Mixin): """Debian 10 with backported kernel.""" + OS_TYPE = os_types.DEBIAN10_BACKPORTS class Debian11Mixin(BaseDebianMixin): """Class holding Debian 11 specific VM methods and attributes.""" + OS_TYPE = os_types.DEBIAN11 def PrepareVMEnvironment(self): @@ -2567,6 +2749,7 @@ def PrepareVMEnvironment(self): class Debian12Mixin(BaseDebianMixin): """Class holding Debian 12 specific VM methods and attributes.""" + OS_TYPE = os_types.DEBIAN12 def PrepareVMEnvironment(self): @@ -2578,6 +2761,7 @@ def PrepareVMEnvironment(self): class Debian11BackportsMixin(Debian11Mixin): """Debian 11 with backported kernel.""" + OS_TYPE = os_types.DEBIAN11_BACKPORTS @@ -2588,8 +2772,9 @@ def AppendKernelCommandLine(self, command_line, reboot=True): """Appends the provided command-line to the VM and reboots by default.""" self.RemoteCommand( r'echo GRUB_CMDLINE_LINUX_DEFAULT=\"\${GRUB_CMDLINE_LINUX_DEFAULT} %s\"' - r' | sudo tee -a /etc/default/grub.d/50-cloudimg-settings.cfg' % - command_line) + r' | sudo tee -a /etc/default/grub.d/50-cloudimg-settings.cfg' + % command_line + ) self.RemoteCommand('sudo update-grub') if reboot: self.Reboot() @@ -2597,6 +2782,7 @@ def AppendKernelCommandLine(self, command_line, reboot=True): class Ubuntu1604Mixin(BaseUbuntuMixin, virtual_machine.DeprecatedOsMixin): """Class holding Ubuntu1604 specific VM methods and attributes.""" + OS_TYPE = os_types.UBUNTU1604 PYTHON_2_PACKAGE = 'python' END_OF_LIFE = '2021-05-01' @@ -2605,6 +2791,7 @@ class Ubuntu1604Mixin(BaseUbuntuMixin, virtual_machine.DeprecatedOsMixin): class Ubuntu1804Mixin(BaseUbuntuMixin, virtual_machine.DeprecatedOsMixin): """Class holding Ubuntu1804 specific VM methods and attributes.""" + OS_TYPE = os_types.UBUNTU1804 # https://packages.ubuntu.com/bionic/python PYTHON_2_PACKAGE = 'python' @@ -2617,18 +2804,22 @@ def UpdateEnvironmentPath(self): See https://bugs.launchpad.net/snappy/+bug/1659719. """ self.RemoteCommand( - r'sudo sed -i "1 i\export PATH=$PATH:/snap/bin" ~/.bashrc') + r'sudo sed -i "1 i\export PATH=$PATH:/snap/bin" ~/.bashrc' + ) self.RemoteCommand( - r'sudo sed -i "1 i\export PATH=$PATH:/snap/bin" /etc/bash.bashrc') + r'sudo sed -i "1 i\export PATH=$PATH:/snap/bin" /etc/bash.bashrc' + ) class Ubuntu1804EfaMixin(Ubuntu1804Mixin): """Class holding EFA specific VM methods and attributes.""" + OS_TYPE = os_types.UBUNTU1804_EFA class Ubuntu2004Mixin(BaseUbuntuMixin): """Class holding Ubuntu2004 specific VM methods and attributes.""" + OS_TYPE = os_types.UBUNTU2004 # https://packages.ubuntu.com/focal/python2 PYTHON_2_PACKAGE = 'python2' @@ -2639,18 +2830,22 @@ def UpdateEnvironmentPath(self): See https://bugs.launchpad.net/snappy/+bug/1659719. """ self.RemoteCommand( - r'sudo sed -i "1 i\export PATH=$PATH:/snap/bin" ~/.bashrc') + r'sudo sed -i "1 i\export PATH=$PATH:/snap/bin" ~/.bashrc' + ) self.RemoteCommand( - r'sudo sed -i "1 i\export PATH=$PATH:/snap/bin" /etc/bash.bashrc') + r'sudo sed -i "1 i\export PATH=$PATH:/snap/bin" /etc/bash.bashrc' + ) class Ubuntu2004EfaMixin(Ubuntu2004Mixin): """Class holding EFA specific VM methods and attributes.""" + OS_TYPE = os_types.UBUNTU2004_EFA class Ubuntu2204Mixin(BaseUbuntuMixin): """Class holding Ubuntu2204 specific VM methods and attributes.""" + OS_TYPE = os_types.UBUNTU2204 @@ -2662,6 +2857,7 @@ class Ubuntu2304Mixin(BaseUbuntuMixin): class Ubuntu1604Cuda9Mixin(Ubuntu1604Mixin): """Class holding NVIDIA CUDA specific VM methods and attributes.""" + OS_TYPE = os_types.UBUNTU1604_CUDA9 @@ -2706,13 +2902,10 @@ def PrepareVMEnvironment(self): def InitDocker(self): """Initializes the docker container daemon.""" - init_docker_cmd = ['sudo docker run -d ' - '--rm ' - '--net=host ' - '--workdir=%s ' - '-v %s:%s ' % (CONTAINER_WORK_DIR, - vm_util.VM_TMP_DIR, - CONTAINER_MOUNT_DIR)] + init_docker_cmd = [ + 'sudo docker run -d --rm --net=host --workdir=%s -v %s:%s ' + % (CONTAINER_WORK_DIR, vm_util.VM_TMP_DIR, CONTAINER_MOUNT_DIR) + ] for sd in self.scratch_disks: init_docker_cmd.append('-v %s:%s ' % (sd.mount_point, sd.mount_point)) init_docker_cmd.append('%s sleep infinity ' % self.BASE_DOCKER_IMAGE) @@ -2746,6 +2939,7 @@ def ContainerCopy(self, file_name, container_path='', copy_to=True): file_name: Name of the file in the host's vm_util.VM_TMP_DIR. container_path: Optional path of where to copy file on container. copy_to: True to copy to container, False to copy from container. + Raises: RemoteExceptionError: If the source container_path is blank. """ @@ -2760,15 +2954,18 @@ def ContainerCopy(self, file_name, container_path='', copy_to=True): self.RemoteCommand(command) else: if container_path == '': - raise errors.VirtualMachine.RemoteExceptionError('Cannot copy ' - 'from blank target') + raise errors.VirtualMachine.RemoteExceptionError( + 'Cannot copy from blank target' + ) destination_path = posixpath.join(CONTAINER_MOUNT_DIR, file_name) command = 'cp %s %s' % (container_path, destination_path) self.RemoteCommand(command) @vm_util.Retry( - poll_interval=1, max_retries=3, - retryable_exceptions=(errors.VirtualMachine.RemoteCommandError,)) + poll_interval=1, + max_retries=3, + retryable_exceptions=(errors.VirtualMachine.RemoteCommandError,), + ) def RemoteCopy(self, file_path, remote_path='', copy_to=True): """Copies a file to or from the container in the remote VM. @@ -2797,8 +2994,8 @@ def MoveFile(self, target, source_path, remote_path=''): Args: target: The target ContainerizedVirtualMachine object. source_path: The location of the file on the REMOTE machine. - remote_path: The destination of the file on the TARGET machine, default - is the root directory. + remote_path: The destination of the file on the TARGET machine, default is + the root directory. """ file_name = posixpath.basename(source_path) @@ -2910,8 +3107,9 @@ def __init__(self, lscpu): def GetInt(key): if key in self.data and self.data[key].isdigit(): return int(self.data[key]) - raise ValueError('Could not find integer "{}" in {}'.format( - key, sorted(self.data))) + raise ValueError( + 'Could not find integer "{}" in {}'.format(key, sorted(self.data)) + ) self.numa_node_count = GetInt('NUMA node(s)') self.cores_per_socket = GetInt('Core(s) per socket') @@ -2987,8 +3185,7 @@ def GetValues(self): Dict of [cpuinfo key:value string] """ cpuinfo = { - key: ';'.join(sorted(values)) - for key, values in self.attributes.items() + key: ';'.join(sorted(values)) for key, values in self.attributes.items() } cpuinfo['proccpu'] = ','.join(sorted(self.attributes.keys())) return cpuinfo @@ -3060,13 +3257,14 @@ def JujuAddMachine(self, unit): Args: unit: An object representing the unit's BaseVirtualMachine. """ - resp, _ = self.RemoteHostCommand('juju add-machine ssh:%s' % - unit.internal_ip) + resp, _ = self.RemoteHostCommand( + 'juju add-machine ssh:%s' % unit.internal_ip + ) # We don't know what the machine's going to be used for yet, # but track it's placement for easier access later. # We're looking for the output: created machine %d - machine_id = _[_.rindex(' '):].strip() + machine_id = _[_.rindex(' ') :].strip() self.machines[machine_id] = unit def JujuConfigureEnvironment(self): @@ -3099,8 +3297,7 @@ def JujuStatus(self, pattern=''): Args: pattern: Optionally match machines/services with a pattern. """ - output, _ = self.RemoteHostCommand('juju status %s --format=json' % - pattern) + output, _ = self.RemoteHostCommand('juju status %s --format=json' % pattern) return output.strip() def JujuVersion(self): @@ -3116,7 +3313,8 @@ def JujuSet(self, service, params=[]): params: A list of key=values pairs. """ output, _ = self.RemoteHostCommand( - 'juju set %s %s' % (service, ' '.join(params))) + 'juju set %s %s' % (service, ' '.join(params)) + ) return output.strip() @vm_util.Retry(poll_interval=30, timeout=3600) @@ -3129,26 +3327,30 @@ def JujuWait(self): # Accept blocked because the service may be waiting on relation if ss not in ['active', 'unknown']: raise errors.Juju.TimeoutException( - 'Service %s is not ready; status is %s' % (service, ss)) + 'Service %s is not ready; status is %s' % (service, ss) + ) if ss in ['error']: # The service has failed to deploy. debuglog = self.JujuRun('juju debug-log --limit 200') logging.warning(debuglog) raise errors.Juju.UnitErrorException( - 'Service %s is in an error state' % service) + 'Service %s is in an error state' % service + ) for unit in status['services'][service]['units']: unit_data = status['services'][service]['units'][unit] ag = unit_data['agent-state'] if ag != 'started': raise errors.Juju.TimeoutException( - 'Service %s is not ready; agent-state is %s' % (service, ag)) + 'Service %s is not ready; agent-state is %s' % (service, ag) + ) ws = unit_data['workload-status']['current'] if ws not in ['active', 'unknown']: raise errors.Juju.TimeoutException( - 'Service %s is not ready; workload-state is %s' % (service, ws)) + 'Service %s is not ready; workload-state is %s' % (service, ws) + ) def JujuDeploy(self, charm, vm_group): """Deploy (and scale) this service to the machines in its vm group. @@ -3166,15 +3368,17 @@ def JujuDeploy(self, charm, vm_group): # Deploy the first machine resp, _ = self.RemoteHostCommand( - 'juju deploy %s --to %s' % (charm, machines.pop())) + 'juju deploy %s --to %s' % (charm, machines.pop()) + ) # Get the name of the service - service = charm[charm.rindex('/') + 1:] + service = charm[charm.rindex('/') + 1 :] # Deploy to the remaining machine(s) for machine in machines: resp, _ = self.RemoteHostCommand( - 'juju add-unit %s --to %s' % (service, machine)) + 'juju add-unit %s --to %s' % (service, machine) + ) def JujuRelate(self, service1, service2): """Create a relation between two services. @@ -3184,7 +3388,8 @@ def JujuRelate(self, service1, service2): service2: The second service to relate. """ resp, _ = self.RemoteHostCommand( - 'juju add-relation %s %s' % (service1, service2)) + 'juju add-relation %s %s' % (service1, service2) + ) def Install(self, package_name): """Installs a PerfKit package on the VM.""" @@ -3197,16 +3402,21 @@ def Install(self, package_name): package.JujuInstall(self.controller, self.vm_group) self.controller._installed_packages.add(package_name) except AttributeError as e: - logging.warning('Failed to install package %s, falling back to Apt (%s)', - package_name, e) + logging.warning( + 'Failed to install package %s, falling back to Apt (%s)', + package_name, + e, + ) if package_name not in self._installed_packages: if hasattr(package, 'AptInstall'): package.AptInstall(self) elif hasattr(package, 'Install'): package.Install(self) else: - raise KeyError('Package %s has no install method for Juju machines.' % - package_name) + raise KeyError( + 'Package %s has no install method for Juju machines.' + % package_name + ) self._installed_packages.add(package_name) def SetupPackageManager(self): @@ -3234,6 +3444,7 @@ def PrepareVMEnvironment(self): self.JujuAddMachine(unit) -class BaseLinuxVirtualMachine(BaseLinuxMixin, - virtual_machine.BaseVirtualMachine): +class BaseLinuxVirtualMachine( + BaseLinuxMixin, virtual_machine.BaseVirtualMachine +): """Linux VM for use with pytyping.""" diff --git a/perfkitbenchmarker/log_util.py b/perfkitbenchmarker/log_util.py index 79e47b6e61..3f916ca2ea 100644 --- a/perfkitbenchmarker/log_util.py +++ b/perfkitbenchmarker/log_util.py @@ -35,7 +35,7 @@ DEBUG: logging.DEBUG, INFO: logging.INFO, WARNING: logging.WARNING, - ERROR: logging.ERROR + ERROR: logging.ERROR, } # Paths for log writing and exporting. @@ -45,15 +45,18 @@ _PKB_LOG_BUCKET = flags.DEFINE_string( - 'pkb_log_bucket', None, + 'pkb_log_bucket', + None, 'Name of the GCS bucket that PKB logs should route to. If this is not ' 'specified, then PKB logs will remain on the VM. This bucket must exist ' 'and the caller must have write permissions on the bucket for a successful ' - 'export.') + 'export.', +) class ThreadLogContext(object): """Per-thread context for log message prefix labels.""" + def __init__(self, thread_log_context=None): """Constructs a ThreadLogContext by copying a previous ThreadLogContext. @@ -97,9 +100,11 @@ def ExtendLabel(self, label_extension): class _ThreadData(threading.local): + def __init__(self): self.pkb_thread_log_context = ThreadLogContext() + thread_local = _ThreadData() @@ -127,6 +132,7 @@ class PkbLogFilter(logging.Filter): Sets the LogRecord's pkb_label attribute with the ThreadLogContext label. """ + def filter(self, record): record.pkb_label = GetThreadLogContext().label return True @@ -137,8 +143,9 @@ def ConfigureBasicLogging(): logging.basicConfig(format='%(levelname)-8s %(message)s', level=logging.INFO) -def ConfigureLogging(stderr_log_level, log_path, run_uri, - file_log_level=logging.DEBUG): +def ConfigureLogging( + stderr_log_level, log_path, run_uri, file_log_level=logging.DEBUG +): """Configure logging. Note that this will destroy existing logging configuration! @@ -160,19 +167,26 @@ def ConfigureLogging(stderr_log_level, log_path, run_uri, # Set the GCS destination path global variable so it can be used by PKB. global log_cloud_path run_date = datetime.date.today() - log_cloud_path = (f'gs://{_PKB_LOG_BUCKET.value}/' - + f'{run_date.year:04d}/{run_date.month:02d}/' - + f'{run_date.day:02d}/' - + f'{run_uri}-{LOG_FILE_NAME}') + log_cloud_path = ( + f'gs://{_PKB_LOG_BUCKET.value}/' + + f'{run_date.year:04d}/{run_date.month:02d}/' + + f'{run_date.day:02d}/' + + f'{run_uri}-{LOG_FILE_NAME}' + ) # Build the format strings for the stderr and log file message formatters. - stderr_format = ('%(asctime)s {} %(threadName)s %(pkb_label)s' - '%(levelname)-8s %(message)s').format(run_uri) - stderr_color_format = ('%(log_color)s%(asctime)s {} %(threadName)s ' - '%(pkb_label)s%(levelname)-8s%(reset)s ' - '%(message)s').format(run_uri) - file_format = ('%(asctime)s {} %(threadName)s %(pkb_label)s' - '%(filename)s:%(lineno)d %(levelname)-8s %(message)s') + stderr_format = ( + '%(asctime)s {} %(threadName)s %(pkb_label)s%(levelname)-8s %(message)s' + ).format(run_uri) + stderr_color_format = ( + '%(log_color)s%(asctime)s {} %(threadName)s ' + '%(pkb_label)s%(levelname)-8s%(reset)s ' + '%(message)s' + ).format(run_uri) + file_format = ( + '%(asctime)s {} %(threadName)s %(pkb_label)s' + '%(filename)s:%(lineno)d %(levelname)-8s %(message)s' + ) file_format = file_format.format(run_uri) # Reset root logger settings. @@ -210,5 +224,12 @@ def ConfigureLogging(stderr_log_level, log_path, run_uri, def CollectPKBLogs() -> None: """Move PKB log files over to a GCS bucket (`pkb_log_bucket` flag).""" if _PKB_LOG_BUCKET.value: - vm_util.IssueRetryableCommand(['gsutil', '-h', 'Content-Type:text/plain', - 'mv', '-Z', log_local_path, log_cloud_path]) + vm_util.IssueRetryableCommand([ + 'gsutil', + '-h', + 'Content-Type:text/plain', + 'mv', + '-Z', + log_local_path, + log_cloud_path, + ]) diff --git a/perfkitbenchmarker/messaging_service.py b/perfkitbenchmarker/messaging_service.py index 069e231982..dc9a2444ab 100644 --- a/perfkitbenchmarker/messaging_service.py +++ b/perfkitbenchmarker/messaging_service.py @@ -17,9 +17,11 @@ MESSAGING_SERVICE_SCRIPTS_VM_PKB = os.path.join('~', 'perfkitbenchmarker') MESSAGING_SERVICE_SCRIPTS_VM_BIN_DIR = '~' MESSAGING_SERVICE_SCRIPTS_VM_LIB_DIR = os.path.join( - '~', 'perfkitbenchmarker', 'scripts', 'messaging_service_scripts') + '~', 'perfkitbenchmarker', 'scripts', 'messaging_service_scripts' +) MESSAGING_SERVICE_SCRIPTS_VM_COMMON_DIR = os.path.join( - MESSAGING_SERVICE_SCRIPTS_VM_LIB_DIR, 'common') + MESSAGING_SERVICE_SCRIPTS_VM_LIB_DIR, 'common' +) MESSAGING_SERVICE_SCRIPTS_COMMON_PREFIX = 'messaging_service_scripts/common/' MESSAGING_SERVICE_SCRIPTS_COMMON_FILES = [ '__init__.py', @@ -42,7 +44,8 @@ def GetMessagingServiceClass(cloud, delivery): """Gets the underlying Messaging Service class.""" return resource.GetResourceClass( - BaseMessagingService, CLOUD=cloud, DELIVERY=delivery) + BaseMessagingService, CLOUD=cloud, DELIVERY=delivery + ) class BaseMessagingService(resource.BaseResource): @@ -67,8 +70,9 @@ def FromSpec(cls, messaging_service_spec): return cls() def setVms(self, vm_groups): - self.client_vm = vm_groups['clients' if 'clients' in - vm_groups else 'default'][0] + self.client_vm = vm_groups[ + 'clients' if 'clients' in vm_groups else 'default' + ][0] def PrepareClientVm(self): self._InstallCommonClientPackages() @@ -83,23 +87,33 @@ def _InstallCommonClientPackages(self): # Upload common scripts self.client_vm.RemoteCommand( - f'mkdir -p {MESSAGING_SERVICE_SCRIPTS_VM_LIB_DIR}') - self.client_vm.RemoteCommand(' '.join([ - 'find', MESSAGING_SERVICE_SCRIPTS_VM_PKB, '-type', 'd', '-exec', - 'touch', "'{}/__init__.py'", '\\;' - ])) + f'mkdir -p {MESSAGING_SERVICE_SCRIPTS_VM_LIB_DIR}' + ) + self.client_vm.RemoteCommand( + ' '.join([ + 'find', + MESSAGING_SERVICE_SCRIPTS_VM_PKB, + '-type', + 'd', + '-exec', + 'touch', + "'{}/__init__.py'", + '\\;', + ]) + ) self._CopyFiles( MESSAGING_SERVICE_SCRIPTS_COMMON_PREFIX, MESSAGING_SERVICE_SCRIPTS_COMMON_FILES, - MESSAGING_SERVICE_SCRIPTS_VM_COMMON_DIR) + MESSAGING_SERVICE_SCRIPTS_VM_COMMON_DIR, + ) def _CopyFiles(self, prefix, data_srcs, vm_dest_dir): for subpath in data_srcs: dirname = os.path.dirname(os.path.join(vm_dest_dir, subpath)) self.client_vm.RemoteCommand(f'mkdir -p {dirname}') self.client_vm.PushDataFile( - os.path.join(prefix, subpath), - os.path.join(vm_dest_dir, subpath)) + os.path.join(prefix, subpath), os.path.join(vm_dest_dir, subpath) + ) @abc.abstractmethod def _InstallCloudClients(self): @@ -111,9 +125,14 @@ def _InstallCloudClients(self): raise NotImplementedError @abc.abstractmethod - def Run(self, benchmark_scenario: str, number_of_messages: int, - message_size: int, warmup_messages: int, - streaming_pull: bool = False) -> Dict[str, Any]: + def Run( + self, + benchmark_scenario: str, + number_of_messages: int, + message_size: int, + warmup_messages: int, + streaming_pull: bool = False, + ) -> Dict[str, Any]: """Runs remote commands on client VM - benchmark's run phase. Runs a benchmark that consists of first publishing messages and then diff --git a/perfkitbenchmarker/network.py b/perfkitbenchmarker/network.py index 331828b3df..0fa3957136 100644 --- a/perfkitbenchmarker/network.py +++ b/perfkitbenchmarker/network.py @@ -30,8 +30,9 @@ from perfkitbenchmarker import resource -flags.DEFINE_integer('mtu', None, - 'Network MTU to set, if any. Only enabled for GCP.') +flags.DEFINE_integer( + 'mtu', None, 'Network MTU to set, if any. Only enabled for GCP.' +) class NetType(enum.Enum): @@ -59,8 +60,9 @@ def GetFirewall(cls): raise errors.Error('Firewalls should have CLOUD attributes.') benchmark_spec = context.GetThreadBenchmarkSpec() if benchmark_spec is None: - raise errors.Error('GetFirewall called in a thread without a ' - 'BenchmarkSpec.') + raise errors.Error( + 'GetFirewall called in a thread without a BenchmarkSpec.' + ) with benchmark_spec.firewalls_lock: key = cls.CLOUD if key not in benchmark_spec.firewalls: @@ -112,6 +114,7 @@ def __repr__(self): class BaseVpnGateway(object, metaclass=abc.ABCMeta): """An object representing the Base VPN Gateway.""" + CLOUD = None def __init__(self, zone=None, cidr=None): @@ -177,7 +180,8 @@ def __init__(self, spec): def _GetNetworkSpecFromVm(vm): """Returns a BaseNetworkSpec created from VM attributes.""" return BaseNetworkSpec( - zone=vm.zone, cidr=vm.cidr, machine_type=vm.machine_type) + zone=vm.zone, cidr=vm.cidr, machine_type=vm.machine_type + ) @classmethod def _GetKeyFromNetworkSpec(cls, spec): @@ -212,6 +216,7 @@ def FormatCidrString(cidr_raw): Args: cidr_raw: The unformatted CIDR string. + Returns: A CIDR string suitable for use in resource names. Raises: @@ -240,8 +245,9 @@ def GetNetworkFromNetworkSpec(cls, spec): """ benchmark_spec = context.GetThreadBenchmarkSpec() if benchmark_spec is None: - raise errors.Error('GetNetwork called in a thread without a ' - 'BenchmarkSpec.') + raise errors.Error( + 'GetNetwork called in a thread without a BenchmarkSpec.' + ) key = cls._GetKeyFromNetworkSpec(spec) # Grab the list of other networks to setup firewalls, forwarding, etc. @@ -329,9 +335,8 @@ def GetCidrBlock(regional_index=0, subnet_index=0, mask_size=24): Args: regional_index: Int. The IP Address allocation dependent on the region. Default index is 0. - subnet_index: Int. The IP Address section dependent on the subnet. - Default index is 0. - mask_size: Int. Mask size to request from cidr block. - Default index is 24. + subnet_index: Int. The IP Address section dependent on the subnet. Default + index is 0. + mask_size: Int. Mask size to request from cidr block. Default index is 24. """ return '10.{}.{}.0/{}'.format(regional_index, subnet_index, mask_size) diff --git a/perfkitbenchmarker/nfs_service.py b/perfkitbenchmarker/nfs_service.py index d576f0dfc8..2420efefef 100644 --- a/perfkitbenchmarker/nfs_service.py +++ b/perfkitbenchmarker/nfs_service.py @@ -94,16 +94,22 @@ def __init__(self, disk_spec: disk.BaseNFSDiskSpec, zone): # NFS service does not have to have a list of nfs_tiers nor does it have # to be implemented by a provider raise errors.Config.InvalidValue( - ('nfs_tier "%s" not in acceptable list "%s" ' - 'for cloud %s') % (self.nfs_tier, self.NFS_TIERS, self.CLOUD)) - logging.debug('%s NFS service with nfs_tier %s zone %s default version %s', - self.CLOUD, self.nfs_tier, self.zone, - self.DEFAULT_NFS_VERSION) + 'nfs_tier "%s" not in acceptable list "%s" for cloud %s' + % (self.nfs_tier, self.NFS_TIERS, self.CLOUD) + ) + logging.debug( + '%s NFS service with nfs_tier %s zone %s default version %s', + self.CLOUD, + self.nfs_tier, + self.zone, + self.DEFAULT_NFS_VERSION, + ) def CreateNfsDisk(self): mount_point = '%s:%s' % (self.GetRemoteAddress(), self.server_directory) - return disk.NfsDisk(self.disk_spec, mount_point, self.DEFAULT_NFS_VERSION, - self.nfs_tier) + return disk.NfsDisk( + self.disk_spec, mount_point, self.DEFAULT_NFS_VERSION, self.nfs_tier + ) @abc.abstractmethod def _IsReady(self): @@ -118,6 +124,7 @@ def GetRemoteAddress(self): class StaticNfsService(BaseNfsService): """Object allowing VMs to connect to a preprovisioned NFS endpoint.""" + CLOUD = 'Static' def __init__(self, disk_spec): @@ -146,6 +153,7 @@ def GetRemoteAddress(self): class UnmanagedNfsService(BaseNfsService): """Object allowing VMs to connect to a local NFS disk.""" + CLOUD = UNMANAGED # Allows anybody to write to the NFS mount. @@ -153,8 +161,10 @@ class UnmanagedNfsService(BaseNfsService): 'sudo mkdir -p {export_dir}', 'sudo chown $USER:$USER {export_dir}', 'sudo chmod 755 {export_dir}', - ('echo "{export_dir} *(rw,sync,no_subtree_check,no_root_squash)" | ' - 'sudo tee -a /etc/exports'), + ( + 'echo "{export_dir} *(rw,sync,no_subtree_check,no_root_squash)" | ' + 'sudo tee -a /etc/exports' + ), 'sudo exportfs -a', ]) @@ -182,8 +192,9 @@ def __init__( self.server_directory = '/pkb-nfs-server-directory' logging.info('Exporting server directory %s', self.server_directory) if check_export_not_same_mount and disk_spec: - assert self.server_directory != disk_spec.mount_point, ( - 'export server directory must be different from mount point') + assert ( + self.server_directory != disk_spec.mount_point + ), 'export server directory must be different from mount point' def GetRemoteAddress(self): """The NFS server's address.""" @@ -196,14 +207,17 @@ def _ExportNfsDir(self, export_dir_path): export_dir_path: Path to the directory to export. """ if self.server_vm.TryRemoteCommand( - f'grep "^{export_dir_path} " /etc/exports'): + f'grep "^{export_dir_path} " /etc/exports' + ): logging.info('Already NFS exported directory %s', export_dir_path) else: self.server_vm.RemoteCommand( - self._EXPORT_FS_COMMAND.format(export_dir=export_dir_path)) + self._EXPORT_FS_COMMAND.format(export_dir=export_dir_path) + ) nfs_name = self._NFS_NAME[self.server_vm.BASE_OS_TYPE] self.server_vm.RemoteCommand( - self._NFS_RESTART_CMD.format(nfs_name=nfs_name)) + self._NFS_RESTART_CMD.format(nfs_name=nfs_name) + ) def _Create(self): assert self.server_vm, 'NFS server VM not created.' @@ -211,7 +225,8 @@ def _Create(self): self._ExportNfsDir(self.server_directory) # Restart NFS service upon reboot if required (Centos7) self.server_vm.RemoteCommand( - 'sudo systemctl enable nfs', ignore_failure=True) + 'sudo systemctl enable nfs', ignore_failure=True + ) def _Delete(self): pass @@ -238,12 +253,15 @@ def NfsMount(server_ip, client_vm, client_path, server_path=None) -> None: as the client_path. """ client_vm.Install('nfs_utils') - fstab_line = (f'{server_ip}:{server_path or client_path} ' - f'{client_path} nfs defaults 0 0') - client_vm.RemoteCommand(f'sudo mkdir -p {client_path}; ' - f'sudo chown {client_vm.user_name} {client_path}; ' - f'echo "{fstab_line}\n" | sudo tee -a /etc/fstab; ' - 'sudo mount -a') + fstab_line = ( + f'{server_ip}:{server_path or client_path} {client_path} nfs defaults 0 0' + ) + client_vm.RemoteCommand( + f'sudo mkdir -p {client_path}; ' + f'sudo chown {client_vm.user_name} {client_path}; ' + f'echo "{fstab_line}\n" | sudo tee -a /etc/fstab; ' + 'sudo mount -a' + ) def NfsExportAndMount(vms, client_path, server_path=None) -> None: diff --git a/perfkitbenchmarker/num_gpus_map_util.py b/perfkitbenchmarker/num_gpus_map_util.py index 52a058a706..dd27d620f1 100644 --- a/perfkitbenchmarker/num_gpus_map_util.py +++ b/perfkitbenchmarker/num_gpus_map_util.py @@ -27,5 +27,5 @@ 'p2.16xlarge': 16, 'Standard_NC6': 1, 'Standard_NC12': 2, - 'Standard_NC24': 4 + 'Standard_NC24': 4, } diff --git a/perfkitbenchmarker/object_storage_service.py b/perfkitbenchmarker/object_storage_service.py index 883a0b1d1a..5c821fb2d5 100644 --- a/perfkitbenchmarker/object_storage_service.py +++ b/perfkitbenchmarker/object_storage_service.py @@ -25,13 +25,17 @@ from perfkitbenchmarker import errors import six -flags.DEFINE_string('object_storage_credential_file', None, - 'Directory of credential file.') -flags.DEFINE_string('boto_file_location', None, - 'The location of the boto file.') +flags.DEFINE_string( + 'object_storage_credential_file', None, 'Directory of credential file.' +) +flags.DEFINE_string( + 'boto_file_location', None, 'The location of the boto file.' +) OBJECT_TTL_DAYS = flags.DEFINE_integer( - 'object_ttl_days', None, - 'The object TTL in days to set on any bucket created.') + 'object_ttl_days', + None, + 'The object TTL in days to set on any bucket created.', +) FLAGS = flags.FLAGS @@ -44,6 +48,7 @@ class AutoRegisterObjectStorageMeta(abc.ABCMeta): """Metaclass for auto registration.""" + STORAGE_NAME = None def __init__(cls, name, bases, dct): @@ -51,13 +56,17 @@ def __init__(cls, name, bases, dct): if cls.STORAGE_NAME in _OBJECT_STORAGE_REGISTRY: logging.info( "Duplicate storage implementations for name '%s'. " - 'Replacing %s with %s', cls.STORAGE_NAME, - _OBJECT_STORAGE_REGISTRY[cls.STORAGE_NAME].__name__, cls.__name__) + 'Replacing %s with %s', + cls.STORAGE_NAME, + _OBJECT_STORAGE_REGISTRY[cls.STORAGE_NAME].__name__, + cls.__name__, + ) _OBJECT_STORAGE_REGISTRY[cls.STORAGE_NAME] = cls class ObjectStorageService( - six.with_metaclass(AutoRegisterObjectStorageMeta, object)): + six.with_metaclass(AutoRegisterObjectStorageMeta, object) +): """Base class for ObjectStorageServices.""" # Keeping the location in the service object is not very clean, but @@ -97,7 +106,7 @@ def MakeBucket(self, bucket, raise_on_failure=True, tag_bucket=True): Args: bucket: the name of the bucket to create. raise_on_failure: Whether to raise errors.Benchmarks.BucketCreationError - if the bucket fails to be created. + if the bucket fails to be created. tag_bucket: Whether to put default labels like timeout on the bucket. """ pass @@ -285,10 +294,9 @@ def UpdateSampleMetadata(self, samples): """ pass - def GetDownloadUrl(self, - bucket: str, - object_name: str, - use_https=True) -> str: + def GetDownloadUrl( + self, bucket: str, object_name: str, use_https=True + ) -> str: """Get the URL to download objects over HTTP(S). Args: @@ -358,13 +366,12 @@ def GetObjectStorageClass(storage_name) -> type(ObjectStorageService): def FindCredentialFile(default_location): """Return the path to the credential file.""" - credential_file = ( - FLAGS.object_storage_credential_file or default_location) + credential_file = FLAGS.object_storage_credential_file or default_location credential_file = os.path.expanduser(credential_file) - if not (os.path.isfile(credential_file) or - os.path.isdir(credential_file)): + if not (os.path.isfile(credential_file) or os.path.isdir(credential_file)): raise errors.Benchmarks.MissingObjectCredentialException( - 'Credential cannot be found in %s' % credential_file) + 'Credential cannot be found in %s' % credential_file + ) return credential_file @@ -385,4 +392,5 @@ def FindBotoFile() -> str: return str(path) raise errors.Benchmarks.MissingObjectCredentialException( - 'Boto file cannot be found in %s.' % paths_to_check) + 'Boto file cannot be found in %s.' % paths_to_check + ) diff --git a/perfkitbenchmarker/os_types.py b/perfkitbenchmarker/os_types.py index 17c664f072..a0f32125e0 100644 --- a/perfkitbenchmarker/os_types.py +++ b/perfkitbenchmarker/os_types.py @@ -61,14 +61,30 @@ WINDOWS2016_DESKTOP = 'windows2016_desktop' WINDOWS2019_DESKTOP = 'windows2019_desktop' WINDOWS2022_DESKTOP = 'windows2022_desktop' -WINDOWS2019_SQLSERVER_2017_STANDARD = 'windows2019_desktop_sqlserver_2017_standard' -WINDOWS2019_SQLSERVER_2017_ENTERPRISE = 'windows2019_desktop_sqlserver_2017_enterprise' -WINDOWS2019_SQLSERVER_2019_STANDARD = 'windows2019_desktop_sqlserver_2019_standard' -WINDOWS2019_SQLSERVER_2019_ENTERPRISE = 'windows2019_desktop_sqlserver_2019_enterprise' -WINDOWS2022_SQLSERVER_2019_STANDARD = 'windows2022_desktop_sqlserver_2019_standard' -WINDOWS2022_SQLSERVER_2019_ENTERPRISE = 'windows2022_desktop_sqlserver_2019_enterprise' -WINDOWS2022_SQLSERVER_2022_STANDARD = 'windows2022_desktop_sqlserver_2022_standard' -WINDOWS2022_SQLSERVER_2022_ENTERPRISE = 'windows2022_desktop_sqlserver_2022_enterprise' +WINDOWS2019_SQLSERVER_2017_STANDARD = ( + 'windows2019_desktop_sqlserver_2017_standard' +) +WINDOWS2019_SQLSERVER_2017_ENTERPRISE = ( + 'windows2019_desktop_sqlserver_2017_enterprise' +) +WINDOWS2019_SQLSERVER_2019_STANDARD = ( + 'windows2019_desktop_sqlserver_2019_standard' +) +WINDOWS2019_SQLSERVER_2019_ENTERPRISE = ( + 'windows2019_desktop_sqlserver_2019_enterprise' +) +WINDOWS2022_SQLSERVER_2019_STANDARD = ( + 'windows2022_desktop_sqlserver_2019_standard' +) +WINDOWS2022_SQLSERVER_2019_ENTERPRISE = ( + 'windows2022_desktop_sqlserver_2019_enterprise' +) +WINDOWS2022_SQLSERVER_2022_STANDARD = ( + 'windows2022_desktop_sqlserver_2022_standard' +) +WINDOWS2022_SQLSERVER_2022_ENTERPRISE = ( + 'windows2022_desktop_sqlserver_2022_enterprise' +) # Base-only OS types DEBIAN = 'debian' @@ -145,13 +161,13 @@ WINDOWS2022_SQLSERVER_2019_STANDARD, WINDOWS2022_SQLSERVER_2019_ENTERPRISE, WINDOWS2022_SQLSERVER_2022_STANDARD, - WINDOWS2022_SQLSERVER_2022_ENTERPRISE + WINDOWS2022_SQLSERVER_2022_ENTERPRISE, ] CENTOS_TYPES = [CENTOS7, CENTOS8, CENTOS_STREAM8, CENTOS_STREAM9] WINDOWS_OS_TYPES = ( - WINDOWS_CORE_OS_TYPES + WINDOWS_DESKOP_OS_TYPES + - WINDOWS_SQLSERVER_OS_TYPES) + WINDOWS_CORE_OS_TYPES + WINDOWS_DESKOP_OS_TYPES + WINDOWS_SQLSERVER_OS_TYPES +) ALL = LINUX_OS_TYPES + WINDOWS_OS_TYPES BASE_OS_TYPES = [CLEAR, CORE_OS, DEBIAN, RHEL, WINDOWS] @@ -159,4 +175,4 @@ # May change from time to time. DEFAULT = UBUNTU2004 -flags.DEFINE_enum('os_type', DEFAULT, ALL, 'The VM\'s OS type.') +flags.DEFINE_enum('os_type', DEFAULT, ALL, "The VM's OS type.") diff --git a/perfkitbenchmarker/package_lookup.py b/perfkitbenchmarker/package_lookup.py index f204bc3249..005ddb08c8 100644 --- a/perfkitbenchmarker/package_lookup.py +++ b/perfkitbenchmarker/package_lookup.py @@ -43,5 +43,6 @@ def PackageModule(package_name): """ if not _global_package_module_function: raise errors.Setup.InvalidSetupError( - 'Cannot call package_lookup.py; Was SetUpPKB called?') + 'Cannot call package_lookup.py; Was SetUpPKB called?' + ) return _global_package_module_function(package_name) diff --git a/perfkitbenchmarker/pkb.py b/perfkitbenchmarker/pkb.py index 333f025523..effc87cccc 100644 --- a/perfkitbenchmarker/pkb.py +++ b/perfkitbenchmarker/pkb.py @@ -115,10 +115,12 @@ # Keeping this flag here rather than flags.py to avoid a circular dependency # on benchmark_status. _RETRY_SUBSTATUSES = flags.DEFINE_multi_enum( - 'retry_substatuses', benchmark_status.FailedSubstatus.RETRYABLE_SUBSTATUSES, + 'retry_substatuses', + benchmark_status.FailedSubstatus.RETRYABLE_SUBSTATUSES, benchmark_status.FailedSubstatus.RETRYABLE_SUBSTATUSES, 'The failure substatuses to retry on. By default, failed runs are run with ' - 'the same previous config.') + 'the same previous config.', +) COMPLETION_STATUS_FILE_NAME = 'completion_statuses.json' REQUIRED_INFO = ['scratch_disk', 'num_machines'] @@ -127,11 +129,17 @@ FLAGS = flags.FLAGS # Define patterns for help text processing. -BASE_RELATIVE = '../' # Relative path from markdown output to PKB home for link writing. +BASE_RELATIVE = ( # Relative path from markdown output to PKB home for link writing. + '../' +) MODULE_REGEX = r'^\s+?(.*?):.*' # Pattern that matches module names. FLAGS_REGEX = r'(^\s\s--.*?(?=^\s\s--|\Z))+?' # Pattern that matches each flag. -FLAGNAME_REGEX = r'^\s+?(--.*?)(:.*\Z)' # Pattern that matches flag name in each flag. -DOCSTRING_REGEX = r'"""(.*?|$)"""' # Pattern that matches triple quoted comments. +FLAGNAME_REGEX = ( # Pattern that matches flag name in each flag. + r'^\s+?(--.*?)(:.*\Z)' +) +DOCSTRING_REGEX = ( # Pattern that matches triple quoted comments. + r'"""(.*?|$)"""' +) _TEARDOWN_EVENT = multiprocessing.Event() _ANY_ZONE = 'any' @@ -142,22 +150,26 @@ @flags.multi_flags_validator( ['smart_quota_retry', 'smart_capacity_retry', 'retries', 'zones', 'zone'], - message='Smart zone retries requires exactly one single zone from --zones ' - 'or --zone, as well as retry count > 0.') + message=( + 'Smart zone retries requires exactly one single zone from --zones ' + 'or --zone, as well as retry count > 0.' + ), +) def ValidateSmartZoneRetryFlags(flags_dict): """Validates smart zone retry flags.""" if flags_dict['smart_quota_retry'] or flags_dict['smart_capacity_retry']: if flags_dict['retries'] == 0: return False - return (len(flags_dict['zones']) == 1 and - not flags_dict['zone']) or (len(flags_dict['zone']) == 1 and - not flags_dict['zones']) + return (len(flags_dict['zones']) == 1 and not flags_dict['zone']) or ( + len(flags_dict['zone']) == 1 and not flags_dict['zones'] + ) return True @flags.multi_flags_validator( ['retries', 'run_stage'], - message='Retries requires running all stages of the benchmark.') + message='Retries requires running all stages of the benchmark.', +) def ValidateRetriesAndRunStages(flags_dict): if flags_dict['retries'] > 0 and flags_dict['run_stage'] != stages.STAGES: return False @@ -172,18 +184,21 @@ def _InjectBenchmarkInfoIntoDocumentation(): # The following appends descriptions of the benchmarks and descriptions of # the benchmark sets to the help text. benchmark_sets_list = [ - '%s: %s' % - (set_name, benchmark_sets.BENCHMARK_SETS[set_name]['message']) - for set_name in benchmark_sets.BENCHMARK_SETS] + '%s: %s' % (set_name, benchmark_sets.BENCHMARK_SETS[set_name]['message']) + for set_name in benchmark_sets.BENCHMARK_SETS + ] sys.modules['__main__'].__doc__ = ( 'PerfKitBenchmarker version: {version}\n\n{doc}\n' 'Benchmarks (default requirements):\n' - '\t{benchmark_doc}').format( - version=version.VERSION, - doc=__doc__, - benchmark_doc=_GenerateBenchmarkDocumentation()) - sys.modules['__main__'].__doc__ += ('\n\nBenchmark Sets:\n\t%s' - % '\n\t'.join(benchmark_sets_list)) + '\t{benchmark_doc}' + ).format( + version=version.VERSION, + doc=__doc__, + benchmark_doc=_GenerateBenchmarkDocumentation(), + ) + sys.modules['__main__'].__doc__ += '\n\nBenchmark Sets:\n\t%s' % '\n\t'.join( + benchmark_sets_list + ) def _ParseFlags(argv): @@ -201,8 +216,8 @@ def _PrintHelp(matches=None): """Prints help for flags defined in matching modules. Args: - matches: regex string or None. Filters help to only those whose name - matched the regex. If None then all flags are printed. + matches: regex string or None. Filters help to only those whose name matched + the regex. If None then all flags are printed. """ if not matches: print(FLAGS) @@ -223,6 +238,7 @@ def _PrintHelpMD(matches=None): Args: matches: regex string or None. Filters help to only those whose name matched the regex. If None then all flags are printed. + Raises: RuntimeError: If unable to find module help. Eg: @@ -258,7 +274,8 @@ def _PrintHelpMD(matches=None): ) if not match: raise RuntimeError( - f'Unable to find "{module_regex}" in "{helptext_raw}"') + f'Unable to find "{module_regex}" in "{helptext_raw}"' + ) module = match.group(1) module_link = module.replace('.', '/') + '.py' # Put flag name in a markdown code block for visibility. @@ -305,15 +322,21 @@ def _InitializeRunUri(): FLAGS.run_uri = run_uri logging.warning( 'No run_uri specified. Attempting to run the following stages with ' - '--run_uri=%s: %s', FLAGS.run_uri, ', '.join(FLAGS.run_stage)) + '--run_uri=%s: %s', + FLAGS.run_uri, + ', '.join(FLAGS.run_stage), + ) else: raise errors.Setup.NoRunURIError( - 'No run_uri specified. Could not run the following stages: %s' % - ', '.join(FLAGS.run_stage)) + 'No run_uri specified. Could not run the following stages: %s' + % ', '.join(FLAGS.run_stage) + ) elif not FLAGS.run_uri.isalnum() or len(FLAGS.run_uri) > MAX_RUN_URI_LENGTH: - raise errors.Setup.BadRunURIError('run_uri must be alphanumeric and less ' - 'than or equal to %d characters in ' - 'length.' % MAX_RUN_URI_LENGTH) + raise errors.Setup.BadRunURIError( + 'run_uri must be alphanumeric and less ' + 'than or equal to %d characters in ' + 'length.' % MAX_RUN_URI_LENGTH + ) def _CreateBenchmarkSpecs(): @@ -333,10 +356,16 @@ def _CreateBenchmarkSpecs(): with flag_util.OverrideFlags(FLAGS, user_config.get('flags')): config_dict = benchmark_module.GetConfig(user_config) config_spec_class = getattr( - benchmark_module, 'BENCHMARK_CONFIG_SPEC_CLASS', - benchmark_config_spec.BenchmarkConfigSpec) - config = config_spec_class(name, expected_os_types=expected_os_types, - flag_values=FLAGS, **config_dict) + benchmark_module, + 'BENCHMARK_CONFIG_SPEC_CLASS', + benchmark_config_spec.BenchmarkConfigSpec, + ) + config = config_spec_class( + name, + expected_os_types=expected_os_types, + flag_values=FLAGS, + **config_dict, + ) # Assign a unique ID to each benchmark run. This differs even between two # runs of the same benchmark within a single PKB run. @@ -353,7 +382,8 @@ def _CreateBenchmarkSpecs(): raise specs.append( - bm_spec.BenchmarkSpec.GetBenchmarkSpec(benchmark_module, config, uid)) + bm_spec.BenchmarkSpec.GetBenchmarkSpec(benchmark_module, config, uid) + ) return specs @@ -455,7 +485,7 @@ def DoProvisionPhase(spec, timer): events.after_phase.send(stages.PROVISION, benchmark_spec=spec) -class InterruptChecker(): +class InterruptChecker: """An class that check interrupt on VM.""" def __init__(self, vms): @@ -526,8 +556,10 @@ def DoPreparePhase(spec, timer): spec.BenchmarkPrepare(spec) spec.StartBackgroundWorkload() if FLAGS.after_prepare_sleep_time: - logging.info('Sleeping for %s seconds after the prepare phase.', - FLAGS.after_prepare_sleep_time) + logging.info( + 'Sleeping for %s seconds after the prepare phase.', + FLAGS.after_prepare_sleep_time, + ) time.sleep(FLAGS.after_prepare_sleep_time) events.after_phase.send(stages.PREPARE, benchmark_spec=spec) @@ -566,8 +598,10 @@ def _IsRunStageFinished(): consecutive_failures += 1 if consecutive_failures > FLAGS.run_stage_retries: raise - logging.exception('Run failed (consecutive_failures=%s); retrying.', - consecutive_failures) + logging.exception( + 'Run failed (consecutive_failures=%s); retrying.', + consecutive_failures, + ) else: consecutive_failures = 0 finally: @@ -577,8 +611,9 @@ def _IsRunStageFinished(): s.metadata['run_number'] = run_number # Add boot time metrics on the first run iteration. - if run_number == 0 and (FLAGS.boot_samples or - spec.name == cluster_boot_benchmark.BENCHMARK_NAME): + if run_number == 0 and ( + FLAGS.boot_samples or spec.name == cluster_boot_benchmark.BENCHMARK_NAME + ): samples.extend(cluster_boot_benchmark.GetTimeToBoot(spec.vms)) # In order to collect GPU samples one of the VMs must have both an Nvidia @@ -607,15 +642,20 @@ def _IsRunStageFinished(): events.benchmark_samples_created.send(benchmark_spec=spec, samples=samples) events.all_samples_created.send(benchmark_spec=spec, samples=samples) collector.AddSamples(samples, spec.name, spec) - if (FLAGS.publish_after_run and FLAGS.publish_period is not None and - FLAGS.publish_period < (time.time() - last_publish_time)): + if ( + FLAGS.publish_after_run + and FLAGS.publish_period is not None + and FLAGS.publish_period < (time.time() - last_publish_time) + ): collector.PublishSamples() last_publish_time = time.time() run_number += 1 if _IsRunStageFinished(): if FLAGS.after_run_sleep_time: - logging.info('Sleeping for %s seconds after the run phase.', - FLAGS.after_run_sleep_time) + logging.info( + 'Sleeping for %s seconds after the run phase.', + FLAGS.after_run_sleep_time, + ) time.sleep(FLAGS.after_run_sleep_time) break @@ -635,8 +675,11 @@ def DoCleanupPhase(spec, timer): six.moves.input('Hit enter to begin Cleanup.') logging.info('Cleaning up benchmark %s', spec.name) events.before_phase.send(stages.CLEANUP, benchmark_spec=spec) - if (spec.always_call_cleanup or any([vm.is_static for vm in spec.vms]) or - spec.dpb_service is not None): + if ( + spec.always_call_cleanup + or any([vm.is_static for vm in spec.vms]) + or spec.dpb_service is not None + ): spec.StopBackgroundWorkload() with timer.Measure('Benchmark Cleanup'): spec.BenchmarkCleanup(spec) @@ -651,10 +694,10 @@ def DoTeardownPhase(spec, collector, timer): Args: spec: The BenchmarkSpec created for the benchmark. - collector: The SampleCollector object to add samples to - (if collecting delete samples) - timer: An IntervalTimer that measures the start and stop times of - resource teardown. + collector: The SampleCollector object to add samples to (if collecting + delete samples) + timer: An IntervalTimer that measures the start and stop times of resource + teardown. """ logging.info('Tearing down resources for benchmark %s', spec.name) events.before_phase.send(stages.TEARDOWN, benchmark_spec=spec) @@ -672,12 +715,14 @@ def DoTeardownPhase(spec, collector, timer): def _SkipPendingRunsFile(): if FLAGS.skip_pending_runs_file and isfile(FLAGS.skip_pending_runs_file): - logging.warning('%s exists. Skipping benchmark.', - FLAGS.skip_pending_runs_file) + logging.warning( + '%s exists. Skipping benchmark.', FLAGS.skip_pending_runs_file + ) return True else: return False + _SKIP_PENDING_RUNS_CHECKS = [] @@ -692,15 +737,13 @@ def RegisterSkipPendingRunsCheck(func): @events.before_phase.connect def _PublishStageStartedSamples( - sender: str, - benchmark_spec: bm_spec.BenchmarkSpec): + sender: str, benchmark_spec: bm_spec.BenchmarkSpec +): """Publish the start of each stage.""" if sender == stages.PROVISION and pkb_flags.CREATE_STARTED_RUN_SAMPLE.value: _PublishRunStartedSample(benchmark_spec) if pkb_flags.CREATE_STARTED_STAGE_SAMPLES.value: - _PublishEventSample( - benchmark_spec, - f'{sender.capitalize()} Stage Started') + _PublishEventSample(benchmark_spec, f'{sender.capitalize()} Stage Started') def _PublishRunStartedSample(spec): @@ -712,9 +755,7 @@ def _PublishRunStartedSample(spec): Args: spec: The BenchmarkSpec object with run information. """ - metadata = { - 'flags': str(flag_util.GetProvidedCommandLineFlags()) - } + metadata = {'flags': str(flag_util.GetProvidedCommandLineFlags())} # Publish the path to this spec's PKB logs at the start of the runs. if log_util.log_cloud_path: metadata['pkb_log_path'] = log_util.log_cloud_path @@ -722,10 +763,12 @@ def _PublishRunStartedSample(spec): _PublishEventSample(spec, 'Run Started', metadata) -def _PublishEventSample(spec: bm_spec.BenchmarkSpec, - event: str, - metadata: Optional[Dict[str, Any]] = None, - collector: Optional[publisher.SampleCollector] = None): +def _PublishEventSample( + spec: bm_spec.BenchmarkSpec, + event: str, + metadata: Optional[Dict[str, Any]] = None, + collector: Optional[publisher.SampleCollector] = None, +): """Publishes a sample indicating the progress of the benchmark. Value of sample is time of event in unix seconds @@ -742,7 +785,9 @@ def _PublishEventSample(spec: bm_spec.BenchmarkSpec, collector = publisher.SampleCollector() collector.AddSamples( [sample.Sample(event, time.time(), 'seconds', metadata or {})], - spec.name, spec) + spec.name, + spec, + ) collector.PublishSamples() @@ -790,7 +835,8 @@ def RunBenchmark(spec, collector): current_run_stage = stages.PROVISION # Modify the logger prompt for messages logged within this function. label_extension = '{}({}/{})'.format( - spec.name, spec.sequence_number, spec.total_benchmarks) + spec.name, spec.sequence_number, spec.total_benchmarks + ) context.SetThreadBenchmarkSpec(spec) log_context = log_util.GetThreadLogContext() with log_context.ExtendLabel(label_extension): @@ -800,7 +846,6 @@ def RunBenchmark(spec, collector): interrupt_checker = None try: with end_to_end_timer.Measure('End to End'): - _SetRestoreSpec(spec) _SetFreezePath(spec) @@ -833,13 +878,17 @@ def RunBenchmark(spec, collector): DoTeardownPhase(spec, collector, detailed_timer) # Add timing samples. - if (FLAGS.run_stage == stages.STAGES and - timing_util.EndToEndRuntimeMeasurementEnabled()): + if ( + FLAGS.run_stage == stages.STAGES + and timing_util.EndToEndRuntimeMeasurementEnabled() + ): collector.AddSamples( - end_to_end_timer.GenerateSamples(), spec.name, spec) + end_to_end_timer.GenerateSamples(), spec.name, spec + ) if timing_util.RuntimeMeasurementsEnabled(): collector.AddSamples( - detailed_timer.GenerateSamples(), spec.name, spec) + detailed_timer.GenerateSamples(), spec.name, spec + ) # Add resource related samples. collector.AddSamples(spec.GetSamples(), spec.name, spec) @@ -851,28 +900,32 @@ def RunBenchmark(spec, collector): # TODO(dlott) Move to exception chaining with Python3 support if _IsException(e, errors.Benchmarks.InsufficientCapacityCloudFailure): spec.failed_substatus = ( - benchmark_status.FailedSubstatus.INSUFFICIENT_CAPACITY) + benchmark_status.FailedSubstatus.INSUFFICIENT_CAPACITY + ) elif _IsException(e, errors.Benchmarks.QuotaFailure): spec.failed_substatus = benchmark_status.FailedSubstatus.QUOTA - elif (_IsException(e, errors.Benchmarks.KnownIntermittentError) or - _IsException(e, errors.Resource.CreationInternalError) or - _IsException(e, errors.Resource.ProvisionTimeoutError)): + elif ( + _IsException(e, errors.Benchmarks.KnownIntermittentError) + or _IsException(e, errors.Resource.CreationInternalError) + or _IsException(e, errors.Resource.ProvisionTimeoutError) + ): spec.failed_substatus = ( - benchmark_status.FailedSubstatus.KNOWN_INTERMITTENT) + benchmark_status.FailedSubstatus.KNOWN_INTERMITTENT + ) elif _IsException(e, errors.Benchmarks.UnsupportedConfigError): spec.failed_substatus = benchmark_status.FailedSubstatus.UNSUPPORTED elif _IsException(e, errors.Resource.RestoreError): spec.failed_substatus = ( - benchmark_status.FailedSubstatus.RESTORE_FAILED) + benchmark_status.FailedSubstatus.RESTORE_FAILED + ) elif _IsException(e, errors.Resource.FreezeError): - spec.failed_substatus = ( - benchmark_status.FailedSubstatus.FREEZE_FAILED) + spec.failed_substatus = benchmark_status.FailedSubstatus.FREEZE_FAILED elif isinstance(e, KeyboardInterrupt): spec.failed_substatus = ( - benchmark_status.FailedSubstatus.PROCESS_KILLED) + benchmark_status.FailedSubstatus.PROCESS_KILLED + ) else: - spec.failed_substatus = ( - benchmark_status.FailedSubstatus.UNCATEGORIZED) + spec.failed_substatus = benchmark_status.FailedSubstatus.UNCATEGORIZED spec.status_detail = str(e) # Resource cleanup (below) can take a long time. Log the error to give @@ -886,8 +939,10 @@ def RunBenchmark(spec, collector): if stages.CLEANUP in FLAGS.run_stage and spec.always_call_cleanup: DoCleanupPhase(spec, detailed_timer) - if (FLAGS.always_teardown_on_exception and - stages.TEARDOWN not in FLAGS.run_stage): + if ( + FLAGS.always_teardown_on_exception + and stages.TEARDOWN not in FLAGS.run_stage + ): # Note that if TEARDOWN is specified, it will happen below. DoTeardownPhase(spec, collector, detailed_timer) raise @@ -912,7 +967,8 @@ def PublishFailedRunSample( spec: bm_spec.BenchmarkSpec, error_message: str, run_stage_that_failed: str, - collector: publisher.SampleCollector): + collector: publisher.SampleCollector, +): """Publish a sample.Sample representing a failed run stage. The sample metric will have the name 'Run Failed'; @@ -925,8 +981,8 @@ def PublishFailedRunSample( Args: spec: benchmark_spec - error_message: error message that was caught, resulting in the - run stage failure. + error_message: error message that was caught, resulting in the run stage + failure. run_stage_that_failed: run stage that failed by raising an Exception collector: the collector to publish to. """ @@ -934,12 +990,13 @@ def PublishFailedRunSample( # metadata. We may want to only include flags specific to the benchmark that # failed. This can be acomplished using gflag's FlagsByModuleDict(). metadata = { - 'error_message': error_message[0:FLAGS.failed_run_samples_error_length], + 'error_message': error_message[0 : FLAGS.failed_run_samples_error_length], 'run_stage': run_stage_that_failed, - 'flags': str(flag_util.GetProvidedCommandLineFlags()) + 'flags': str(flag_util.GetProvidedCommandLineFlags()), } background_tasks.RunThreaded( - lambda vm: vm.UpdateInterruptibleVmStatus(use_api=True), spec.vms) + lambda vm: vm.UpdateInterruptibleVmStatus(use_api=True), spec.vms + ) interruptible_vm_count = 0 interrupted_vm_count = 0 @@ -949,8 +1006,7 @@ def PublishFailedRunSample( interruptible_vm_count += 1 if vm.WasInterrupted(): interrupted_vm_count += 1 - spec.failed_substatus = ( - benchmark_status.FailedSubstatus.INTERRUPTED) + spec.failed_substatus = benchmark_status.FailedSubstatus.INTERRUPTED status_code = vm.GetVmStatusCode() if status_code: vm_status_codes.append(status_code) @@ -959,24 +1015,29 @@ def PublishFailedRunSample( metadata['failed_substatus'] = spec.failed_substatus if interruptible_vm_count: - metadata.update({'interruptible_vms': interruptible_vm_count, - 'interrupted_vms': interrupted_vm_count, - 'vm_status_codes': vm_status_codes}) + metadata.update({ + 'interruptible_vms': interruptible_vm_count, + 'interrupted_vms': interrupted_vm_count, + 'vm_status_codes': vm_status_codes, + }) if interrupted_vm_count: logging.error( '%d interruptible VMs were interrupted in this failed PKB run.', - interrupted_vm_count) + interrupted_vm_count, + ) _PublishEventSample(spec, 'Run Failed', metadata, collector) def _ShouldRetry(spec: bm_spec.BenchmarkSpec) -> bool: """Returns whether the benchmark run should be retried.""" - return (spec.status == benchmark_status.FAILED and - spec.failed_substatus in _RETRY_SUBSTATUSES.value) + return ( + spec.status == benchmark_status.FAILED + and spec.failed_substatus in _RETRY_SUBSTATUSES.value + ) def RunBenchmarkTask( - spec: bm_spec.BenchmarkSpec + spec: bm_spec.BenchmarkSpec, ) -> Tuple[Sequence[bm_spec.BenchmarkSpec], List[sample.SampleDict]]: """Task that executes RunBenchmark. @@ -1016,10 +1077,17 @@ def RunBenchmarkTask( return result_specs, collector.samples return [spec], [] - run_start_msg = ('\n' + '-' * 85 + '\n' + - 'Starting benchmark %s attempt %s of %s' + '\n' + '-' * 85) - logging.info(run_start_msg, benchmark_info, current_run_count + 1, - max_run_count) + run_start_msg = ( + '\n' + + '-' * 85 + + '\n' + + 'Starting benchmark %s attempt %s of %s' + + '\n' + + '-' * 85 + ) + logging.info( + run_start_msg, benchmark_info, current_run_count + 1, max_run_count + ) collector = publisher.SampleCollector() # Make a new copy of the benchmark_spec for each run since currently a # benchmark spec isn't compatible with multiple runs. In particular, the @@ -1042,7 +1110,8 @@ def RunBenchmarkTask( if _ShouldRetry(spec_for_run) and current_run_count != max_run_count - 1: logging.info( 'Benchmark should be retried. Waiting %s seconds before running.', - pkb_flags.RETRY_DELAY_SECONDS.value) + pkb_flags.RETRY_DELAY_SECONDS.value, + ) time.sleep(pkb_flags.RETRY_DELAY_SECONDS.value) # Handle smart retries if specified. @@ -1050,8 +1119,10 @@ def RunBenchmarkTask( else: logging.info( - 'Benchmark should not be retried. ' - 'Finished %s runs of %s', current_run_count + 1, max_run_count) + 'Benchmark should not be retried. Finished %s runs of %s', + current_run_count + 1, + max_run_count, + ) break # We need to return both the spec and samples so that we know @@ -1060,7 +1131,7 @@ def RunBenchmarkTask( return result_specs, collector.samples -class ZoneRetryManager(): +class ZoneRetryManager: """Encapsulates state and functions for zone retries. Attributes: @@ -1070,8 +1141,10 @@ class ZoneRetryManager(): def __init__(self, machine_type: str): self._CheckFlag(machine_type) - if (not pkb_flags.SMART_CAPACITY_RETRY.value - and not pkb_flags.SMART_QUOTA_RETRY.value): + if ( + not pkb_flags.SMART_CAPACITY_RETRY.value + and not pkb_flags.SMART_QUOTA_RETRY.value + ): return self._machine_type = machine_type self._zones_tried: Set[str] = set() @@ -1083,7 +1156,8 @@ def _CheckMachineTypeIsSupported(self, machine_type: str) -> None: if not machine_type: raise errors.Config.MissingOption( 'machine_type flag must be specified on the command line ' - 'if zone=any feature is used.') + 'if zone=any feature is used.' + ) def _GetCurrentZoneFlag(self): return FLAGS[self._zone_flag].value[0] @@ -1101,7 +1175,8 @@ def _SetOriginalZoneAndFlag(self) -> None: """Records the flag name and zone value that the benchmark started with.""" # This is guaranteed to set values due to flag validator. self._supported_zones = self._utils.GetZonesFromMachineType( - self._machine_type) + self._machine_type + ) if self._GetCurrentZoneFlag() == _ANY_ZONE: if pkb_flags.MAX_RETRIES.value < 1: FLAGS['retries'].parse(len(self._supported_zones)) @@ -1118,22 +1193,28 @@ def HandleSmartRetries(self, spec: bm_spec.BenchmarkSpec) -> None: Args: spec: benchmark spec. """ - if (pkb_flags.SMART_QUOTA_RETRY.value and spec.failed_substatus - == benchmark_status.FailedSubstatus.QUOTA): + if ( + pkb_flags.SMART_QUOTA_RETRY.value + and spec.failed_substatus == benchmark_status.FailedSubstatus.QUOTA + ): self._AssignZoneToNewRegion() - elif (pkb_flags.SMART_CAPACITY_RETRY.value and spec.failed_substatus in { + elif pkb_flags.SMART_CAPACITY_RETRY.value and spec.failed_substatus in { benchmark_status.FailedSubstatus.UNSUPPORTED, - benchmark_status.FailedSubstatus.INSUFFICIENT_CAPACITY - }): + benchmark_status.FailedSubstatus.INSUFFICIENT_CAPACITY, + }: self._AssignZoneToSameRegion() def _AssignZoneToNewRegion(self) -> None: """Changes zone to be a new zone in the different region.""" region = self._utils.GetRegionFromZone(self._GetCurrentZoneFlag()) self._regions_tried.add(region) - regions_to_try = set( - self._utils.GetRegionFromZone(zone) - for zone in self._supported_zones) - self._regions_tried + regions_to_try = ( + set( + self._utils.GetRegionFromZone(zone) + for zone in self._supported_zones + ) + - self._regions_tried + ) # Restart from empty if we've exhausted all alternatives. if not regions_to_try: self._regions_tried.clear() @@ -1146,7 +1227,8 @@ def _AssignZoneToNewRegion(self) -> None: def _AssignZoneToSameRegion(self) -> None: """Changes zone to be a new zone in the same region.""" supported_zones_in_region = self._utils.GetZonesInRegion( - self._original_region).intersection(self._supported_zones) + self._original_region + ).intersection(self._supported_zones) self._ChooseAndSetNewZone(supported_zones_in_region) def _ChooseAndSetNewZone(self, possible_zones: Set[str]) -> None: @@ -1201,7 +1283,8 @@ def SetUpPKB(): stderr_log_level=log_util.LOG_LEVELS[FLAGS.log_level], log_path=vm_util.PrependTempDir(log_util.LOG_FILE_NAME), run_uri=FLAGS.run_uri, - file_log_level=log_util.LOG_LEVELS[FLAGS.file_log_level]) + file_log_level=log_util.LOG_LEVELS[FLAGS.file_log_level], + ) logging.info('PerfKitBenchmarker version: %s', version.VERSION) # Log all provided flag values. @@ -1217,19 +1300,22 @@ def SetUpPKB(): for executable in REQUIRED_EXECUTABLES: if not vm_util.ExecutableOnPath(executable): raise errors.Setup.MissingExecutableError( - 'Could not find required executable "%s"' % executable) + 'Could not find required executable "%s"' % executable + ) # Check mutually exclusive flags if FLAGS.run_stage_iterations > 1 and FLAGS.run_stage_time > 0: raise errors.Setup.InvalidFlagConfigurationError( - 'Flags run_stage_iterations and run_stage_time are mutually exclusive') + 'Flags run_stage_iterations and run_stage_time are mutually exclusive' + ) vm_util.SSHKeyGen() if FLAGS.static_vm_file: with open(FLAGS.static_vm_file) as fp: static_virtual_machine.StaticVirtualMachine.ReadStaticVirtualMachineFile( - fp) + fp + ) events.initialization_complete.send(parsed_flags=FLAGS) @@ -1241,10 +1327,11 @@ def SetUpPKB(): # in parallel. if not FLAGS.max_concurrent_threads: FLAGS.max_concurrent_threads = max( - background_tasks.MAX_CONCURRENT_THREADS, - FLAGS.num_vms) - logging.info('Setting --max_concurrent_threads=%d.', - FLAGS.max_concurrent_threads) + background_tasks.MAX_CONCURRENT_THREADS, FLAGS.num_vms + ) + logging.info( + 'Setting --max_concurrent_threads=%d.', FLAGS.max_concurrent_threads + ) def RunBenchmarkTasksInSeries(tasks): @@ -1278,13 +1365,13 @@ def RunBenchmarks(): benchmark_spec_lists = None collector = publisher.SampleCollector() try: - tasks = [(RunBenchmarkTask, (spec,), {}) - for spec in benchmark_specs] + tasks = [(RunBenchmarkTask, (spec,), {}) for spec in benchmark_specs] if FLAGS.run_processes is None: spec_sample_tuples = RunBenchmarkTasksInSeries(tasks) else: spec_sample_tuples = background_tasks.RunParallelProcesses( - tasks, FLAGS.run_processes, FLAGS.run_processes_delay) + tasks, FLAGS.run_processes, FLAGS.run_processes_delay + ) benchmark_spec_lists, sample_lists = list(zip(*spec_sample_tuples)) for sample_list in sample_lists: collector.samples.extend(sample_list) @@ -1298,31 +1385,38 @@ def RunBenchmarks(): if benchmark_specs: logging.info(benchmark_status.CreateSummary(benchmark_specs)) - logging.info('Complete logs can be found at: %s', - log_util.log_local_path) - logging.info('Completion statuses can be found at: %s', - vm_util.PrependTempDir(COMPLETION_STATUS_FILE_NAME)) + logging.info('Complete logs can be found at: %s', log_util.log_local_path) + logging.info( + 'Completion statuses can be found at: %s', + vm_util.PrependTempDir(COMPLETION_STATUS_FILE_NAME), + ) if stages.TEARDOWN not in FLAGS.run_stage: logging.info( - 'To run again with this setup, please use --run_uri=%s', FLAGS.run_uri) + 'To run again with this setup, please use --run_uri=%s', FLAGS.run_uri + ) if FLAGS.archive_bucket: - archive.ArchiveRun(vm_util.GetTempDir(), FLAGS.archive_bucket, - gsutil_path=FLAGS.gsutil_path, - prefix=FLAGS.run_uri + '_') + archive.ArchiveRun( + vm_util.GetTempDir(), + FLAGS.archive_bucket, + gsutil_path=FLAGS.gsutil_path, + prefix=FLAGS.run_uri + '_', + ) # Write completion status file(s) - completion_status_file_name = ( - vm_util.PrependTempDir(COMPLETION_STATUS_FILE_NAME)) + completion_status_file_name = vm_util.PrependTempDir( + COMPLETION_STATUS_FILE_NAME + ) with open(completion_status_file_name, 'w') as status_file: _WriteCompletionStatusFile(benchmark_specs, status_file) if FLAGS.completion_status_file: with open(FLAGS.completion_status_file, 'w') as status_file: _WriteCompletionStatusFile(benchmark_specs, status_file) - all_benchmarks_succeeded = all(spec.status == benchmark_status.SUCCEEDED - for spec in benchmark_specs) + all_benchmarks_succeeded = all( + spec.status == benchmark_status.SUCCEEDED for spec in benchmark_specs + ) # Upload PKB logs to GCS after all benchmark runs are complete. log_util.CollectPKBLogs() return 0 if all_benchmarks_succeeded else 1 @@ -1331,10 +1425,12 @@ def RunBenchmarks(): def _GenerateBenchmarkDocumentation(): """Generates benchmark documentation to show in --help.""" benchmark_docs = [] - for benchmark_module in (linux_benchmarks.BENCHMARKS + - windows_benchmarks.BENCHMARKS): + for benchmark_module in ( + linux_benchmarks.BENCHMARKS + windows_benchmarks.BENCHMARKS + ): benchmark_config = configs.LoadMinimalConfig( - benchmark_module.BENCHMARK_CONFIG, benchmark_module.BENCHMARK_NAME) + benchmark_module.BENCHMARK_CONFIG, benchmark_module.BENCHMARK_NAME + ) vm_groups = benchmark_config.get('vm_groups', {}) total_vm_count = 0 vm_str = '' @@ -1351,11 +1447,15 @@ def _GenerateBenchmarkDocumentation(): name = benchmark_module.BENCHMARK_NAME if benchmark_module in windows_benchmarks.BENCHMARKS: name += ' (Windows)' - benchmark_docs.append('%s: %s (%s VMs%s)' % - (name, - benchmark_config['description'], - vm_str or total_vm_count, - scratch_disk_str)) + benchmark_docs.append( + '%s: %s (%s VMs%s)' + % ( + name, + benchmark_config['description'], + vm_str or total_vm_count, + scratch_disk_str, + ) + ) return '\n\t'.join(benchmark_docs) @@ -1378,7 +1478,7 @@ def _GetGccMetadata(vm): return { 'name': vm.name, 'versiondump': build_tools.GetVersion(vm, 'gcc'), - 'versioninfo': build_tools.GetVersionInfo(vm, 'gcc') + 'versioninfo': build_tools.GetVersionInfo(vm, 'gcc'), } linux_vms = [vm for vm in vms if vm.OS_TYPE in os_types.LINUX_OS_TYPES] @@ -1400,7 +1500,7 @@ def _GetGlibcMetadata(vm): return { 'name': vm.name, # TODO(user): Add glibc versiondump. - 'versioninfo': _GetGlibcVersionInfo(vm) + 'versioninfo': _GetGlibcVersionInfo(vm), } linux_vms = [vm for vm in vms if vm.OS_TYPE in os_types.LINUX_OS_TYPES] @@ -1443,8 +1543,11 @@ def _ParseMeminfo(meminfo_txt: str) -> Tuple[Dict[str, int], List[str]]: @events.benchmark_samples_created.connect -def _CollectMeminfoHandler(unused_sender, benchmark_spec: bm_spec.BenchmarkSpec, - samples: List[sample.Sample]) -> None: +def _CollectMeminfoHandler( + unused_sender, + benchmark_spec: bm_spec.BenchmarkSpec, + samples: List[sample.Sample], +) -> None: """Optionally creates /proc/meminfo samples. If the flag --collect_meminfo is set appends a sample.Sample of /proc/meminfo diff --git a/perfkitbenchmarker/placement_group.py b/perfkitbenchmarker/placement_group.py index fda37fdd04..bd9a61b08c 100644 --- a/perfkitbenchmarker/placement_group.py +++ b/perfkitbenchmarker/placement_group.py @@ -28,15 +28,16 @@ PLACEMENT_GROUP_CLOSEST_SUPPORTED = 'closest_supported' PLACEMENT_GROUP_NONE = 'none' -PLACEMENT_GROUP_OPTIONS = frozenset([ - PLACEMENT_GROUP_CLOSEST_SUPPORTED, - PLACEMENT_GROUP_NONE -]) +PLACEMENT_GROUP_OPTIONS = frozenset( + [PLACEMENT_GROUP_CLOSEST_SUPPORTED, PLACEMENT_GROUP_NONE] +) # Default placement group style is specified by Cloud Specific Placement Group. flags.DEFINE_string( - 'placement_group_style', None, - 'The vm placement group option to use. Default set by cloud.') + 'placement_group_style', + None, + 'The vm placement group option to use. Default set by cloud.', +) def GetPlacementGroupSpecClass(cloud): @@ -46,8 +47,7 @@ def GetPlacementGroupSpecClass(cloud): def GetPlacementGroupClass(cloud): """Returns the PlacementGroup class corresponding to 'cloud'.""" - return resource.GetResourceClass(BasePlacementGroup, - CLOUD=cloud) + return resource.GetResourceClass(BasePlacementGroup, CLOUD=cloud) class BasePlacementGroupSpec(spec.BaseSpec): @@ -67,10 +67,10 @@ def _ApplyFlags(cls, config_values, flag_values): Can be overridden by derived classes to add support for specific flags. Args: - config_values: dict mapping config option names to provided values. May - be modified by this function. + config_values: dict mapping config option names to provided values. May be + modified by this function. flag_values: flags.FlagValues. Runtime flags that may override the - provided config values. + provided config values. """ super(BasePlacementGroupSpec, cls)._ApplyFlags(config_values, flag_values) if FLAGS.placement_group_style: diff --git a/perfkitbenchmarker/provider_info.py b/perfkitbenchmarker/provider_info.py index ab727874d4..813e9b69fe 100644 --- a/perfkitbenchmarker/provider_info.py +++ b/perfkitbenchmarker/provider_info.py @@ -16,7 +16,6 @@ This contains the BaseProviderInfo class which is used for IsBenchmarkSupported - """ import six @@ -37,8 +36,21 @@ # for performing on premise to cloud benchmarks DOCKER = 'Docker' -VALID_CLOUDS = (GCP, AZURE, AWS, IBMCLOUD, DIGITALOCEAN, KUBERNETES, OPENSTACK, - RACKSPACE, CLOUDSTACK, ALICLOUD, MESOS, PROFITBRICKS, DOCKER) +VALID_CLOUDS = ( + GCP, + AZURE, + AWS, + IBMCLOUD, + DIGITALOCEAN, + KUBERNETES, + OPENSTACK, + RACKSPACE, + CLOUDSTACK, + ALICLOUD, + MESOS, + PROFITBRICKS, + DOCKER, +) _PROVIDER_INFO_REGISTRY = {} diff --git a/perfkitbenchmarker/providers/__init__.py b/perfkitbenchmarker/providers/__init__.py index 1a2eec0dc2..5fcf944764 100644 --- a/perfkitbenchmarker/providers/__init__.py +++ b/perfkitbenchmarker/providers/__init__.py @@ -41,7 +41,7 @@ def LoadProviderFlags(providers): Args: providers: series of strings. Each element is a value from VALID_CLOUDS - indicating a cloud provider for which to import the flags module. + indicating a cloud provider for which to import the flags module. """ for provider_name in providers: normalized_name = _GetProviderPackageName(provider_name) @@ -55,7 +55,8 @@ def LoadProviderFlags(providers): def LoadProviderUtils(cloud: str) -> types.ModuleType: util_module_name = '.'.join( - (__name__, _GetProviderPackageName(cloud), 'util')) + (__name__, _GetProviderPackageName(cloud), 'util') + ) return importlib.import_module(util_module_name) @@ -69,9 +70,9 @@ def LoadProvider(provider_name, ignore_package_requirements=True): Args: provider_name: string chosen from VALID_CLOUDS. The name of the provider - whose modules should be loaded. + whose modules should be loaded. ignore_package_requirements: boolean. If True, the provider's Python package - requirements file is ignored. + requirements file is ignored. """ if provider_name in _imported_providers: return @@ -86,8 +87,11 @@ def LoadProvider(provider_name, ignore_package_requirements=True): # themselves so that they can be instantiated during resource provisioning. provider_package_path = os.path.join(__path__[0], normalized_name) try: - modules = tuple(import_util.LoadModulesForPath( - [provider_package_path], __name__ + '.' + normalized_name)) + modules = tuple( + import_util.LoadModulesForPath( + [provider_package_path], __name__ + '.' + normalized_name + ) + ) if not modules: raise ImportError('No modules found for provider %s.' % provider_name) except Exception: diff --git a/perfkitbenchmarker/providers/alicloud/ali_disk.py b/perfkitbenchmarker/providers/alicloud/ali_disk.py index c52c1675e6..80ee7ede46 100644 --- a/perfkitbenchmarker/providers/alicloud/ali_disk.py +++ b/perfkitbenchmarker/providers/alicloud/ali_disk.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Module containing classes related to AliCloud disks. -""" +"""Module containing classes related to AliCloud disks.""" import json import logging @@ -56,7 +55,8 @@ def _Create(self): '--RegionId %s' % self.region, '--ZoneId %s' % self.zone, '--Size %s' % self.disk_size, - '--DiskCategory %s' % DISK_TYPE[self.disk_type]] + '--DiskCategory %s' % DISK_TYPE[self.disk_type], + ] create_cmd = util.GetEncodedCmd(create_cmd) stdout, _, _ = vm_util.IssueCommand(create_cmd, raise_on_failure=False) response = json.loads(stdout) @@ -71,9 +71,13 @@ def _Delete(self): delete_cmd = util.ALI_PREFIX + [ 'ecs', 'DeleteDisk', - '--DiskId %s' % self.id] - logging.info('Deleting AliCloud disk %s. This may fail if the disk is not ' - 'yet detached, but will be retried.', self.id) + '--DiskId %s' % self.id, + ] + logging.info( + 'Deleting AliCloud disk %s. This may fail if the disk is not ' + 'yet detached, but will be retried.', + self.id, + ) delete_cmd = util.GetEncodedCmd(delete_cmd) vm_util.IssueRetryableCommand(delete_cmd) @@ -87,7 +91,8 @@ def Attach(self, vm): self.attached_vm_id = vm.id if self.attached_vm_id not in AliDisk.vm_devices: AliDisk.vm_devices[self.attached_vm_id] = set( - string.ascii_lowercase[1:]) + string.ascii_lowercase[1:] + ) self.device_letter = min(AliDisk.vm_devices[self.attached_vm_id]) AliDisk.vm_devices[self.attached_vm_id].remove(self.device_letter) @@ -96,7 +101,8 @@ def Attach(self, vm): 'AttachDisk', '--InstanceId %s' % self.attached_vm_id, '--DiskId %s' % self.id, - '--Device %s' % self.GetVirtualDevicePath()] + '--Device %s' % self.GetVirtualDevicePath(), + ] attach_cmd = util.GetEncodedCmd(attach_cmd) vm_util.IssueRetryableCommand(attach_cmd) @@ -106,7 +112,8 @@ def Detach(self): 'ecs', 'DetachDisk', '--InstanceId %s' % self.attached_vm_id, - '--DiskId %s' % self.id] + '--DiskId %s' % self.id, + ] detach_cmd = util.GetEncodedCmd(detach_cmd) vm_util.IssueRetryableCommand(detach_cmd) @@ -127,14 +134,16 @@ def GetVirtualDevicePath(self): @vm_util.Retry(poll_interval=5, max_retries=30, log_errors=False) def WaitForDiskStatus(self, status_list): """Waits until disk is attach to the instance.""" - logging.info('Waits until the disk\'s status is one of statuses: %s', - status_list) + logging.info( + "Waits until the disk's status is one of statuses: %s", status_list + ) describe_cmd = util.ALI_PREFIX + [ 'ecs', 'DescribeDisks', '--RegionId %s' % self.region, '--ZoneId %s' % self.zone, - '--DiskIds \'["%s"]\'' % self.id] + '--DiskIds \'["%s"]\'' % self.id, + ] attach_cmd = util.GetEncodedCmd(describe_cmd) stdout, _ = vm_util.IssueRetryableCommand(attach_cmd) response = json.loads(stdout) diff --git a/perfkitbenchmarker/providers/alicloud/ali_network.py b/perfkitbenchmarker/providers/alicloud/ali_network.py index 62b3f00cfc..1068e94fd8 100644 --- a/perfkitbenchmarker/providers/alicloud/ali_network.py +++ b/perfkitbenchmarker/providers/alicloud/ali_network.py @@ -53,7 +53,8 @@ def _Create(self): 'CreateVpc', '--VpcName %s' % self.name, '--RegionId %s' % self.region, - '--CidrBlock 10.0.0.0/16'] + '--CidrBlock 10.0.0.0/16', + ] create_cmd = util.GetEncodedCmd(create_cmd) stdout, _, _ = vm_util.IssueCommand(create_cmd, raise_on_failure=False) response = json.loads(stdout) @@ -69,7 +70,8 @@ def _Exists(self): 'ecs', 'DescribeVpcs', '--RegionId %s' % self.region, - '--VpcId %s' % self.id] + '--VpcId %s' % self.id, + ] describe_cmd = util.GetEncodedCmd(describe_cmd) stdout, _ = vm_util.IssueRetryableCommand(describe_cmd) response = json.loads(stdout) @@ -80,13 +82,15 @@ def _Exists(self): @vm_util.Retry(poll_interval=5, max_retries=30, log_errors=False) def _WaitForVpcStatus(self, status_list): """Waits until VPC's status is in status_list""" - logging.info('Waits until the status of VPC is in status_list: %s', - status_list) + logging.info( + 'Waits until the status of VPC is in status_list: %s', status_list + ) describe_cmd = util.ALI_PREFIX + [ 'ecs', 'DescribeVpcs', '--RegionId %s' % self.region, - '--VpcId %s' % self.id] + '--VpcId %s' % self.id, + ] describe_cmd = util.GetEncodedCmd(describe_cmd) stdout, _ = vm_util.IssueRetryableCommand(describe_cmd) response = json.loads(stdout) @@ -101,7 +105,8 @@ def _Delete(self): 'ecs', 'DeleteVpc', '--RegionId %s' % self.region, - '--VpcId %s' % self.id] + '--VpcId %s' % self.id, + ] delete_cmd = util.GetEncodedCmd(delete_cmd) vm_util.IssueCommand(delete_cmd, raise_on_failure=False) @@ -143,7 +148,8 @@ def _Delete(self): 'ecs', 'DeleteVSwitch', '--RegionId %s' % self.region, - '--VSwitchId %s' % self.id] + '--VSwitchId %s' % self.id, + ] delete_cmd = util.GetEncodedCmd(delete_cmd) vm_util.IssueCommand(delete_cmd, raise_on_failure=False) @@ -154,7 +160,8 @@ def _Exists(self): 'DescribeVSwitches', '--RegionId %s' % self.region, '--VpcId %s' % self.vpc_id, - '--ZoneId %s' % self.zone] + '--ZoneId %s' % self.zone, + ] describe_cmd = util.GetEncodedCmd(describe_cmd) stdout, _ = vm_util.IssueRetryableCommand(describe_cmd) response = json.loads(stdout) @@ -179,7 +186,8 @@ def _Create(self): 'ecs', 'CreateSecurityGroup', '--SecurityGroupName %s' % self.name, - '--RegionId %s' % self.region] + '--RegionId %s' % self.region, + ] if self.use_vpc: create_cmd.append('--VpcId %s' % self.vpc_id) create_cmd = util.GetEncodedCmd(create_cmd) @@ -188,8 +196,9 @@ def _Create(self): def _PostCreate(self): """Tag security group with default tags.""" - util.AddDefaultTags(self.group_id, util.ResourceTypes.SECURITYGROUP, - self.region) + util.AddDefaultTags( + self.group_id, util.ResourceTypes.SECURITYGROUP, self.region + ) def _Delete(self): """Deletes the security group.""" @@ -197,7 +206,8 @@ def _Delete(self): 'ecs', 'DeleteSecurityGroup', '--RegionId %s' % self.region, - '--SecurityGroupId %s' % self.group_id] + '--SecurityGroupId %s' % self.group_id, + ] delete_cmd = util.GetEncodedCmd(delete_cmd) vm_util.IssueRetryableCommand(delete_cmd) @@ -207,7 +217,8 @@ def _Exists(self): 'ecs', 'DescribeSecurityGroups', '--RegionId %s' % self.region, - '--SecurityGroupId %s' % self.group_id] + '--SecurityGroupId %s' % self.group_id, + ] show_cmd = util.GetEncodedCmd(show_cmd) stdout, _ = vm_util.IssueRetryableCommand(show_cmd) response = json.loads(stdout) @@ -243,7 +254,8 @@ def AllowIcmp(self, vm): '--PortRange -1/-1', '--SourceCidrIp 0.0.0.0/0', '--RegionId %s' % vm.region, - '--SecurityGroupId %s' % vm.group_id] + '--SecurityGroupId %s' % vm.group_id, + ] if FLAGS.ali_use_vpc: authorize_cmd.append('--NicType intranet') authorize_cmd = util.GetEncodedCmd(authorize_cmd) @@ -289,7 +301,8 @@ def _AllowPort(self, vm, port): '--PortRange %s/%s' % (port, port), '--SourceCidrIp 0.0.0.0/0', '--RegionId %s' % vm.region, - '--SecurityGroupId %s' % vm.group_id] + '--SecurityGroupId %s' % vm.group_id, + ] if FLAGS.ali_use_vpc: authorize_cmd.append('--NicType intranet') authorize_cmd = util.GetEncodedCmd(authorize_cmd) @@ -308,8 +321,7 @@ class AliNetwork(network.BaseNetwork): def __init__(self, spec): super(AliNetwork, self).__init__(spec) - self.name = ( - 'perfkit-%s-%s' % (FLAGS.run_uri, str(uuid.uuid4())[-12:])) + self.name = 'perfkit-%s-%s' % (FLAGS.run_uri, str(uuid.uuid4())[-12:]) self.region = util.GetRegionByZone(spec.zone) self.use_vpc = FLAGS.ali_use_vpc if self.use_vpc: @@ -317,8 +329,9 @@ def __init__(self, spec): self.vswitch = None self.security_group = None else: - self.security_group = \ - AliSecurityGroup(self.name, self.region, use_vpc=False) + self.security_group = AliSecurityGroup( + self.name, self.region, use_vpc=False + ) @vm_util.Retry() def Create(self): @@ -331,10 +344,9 @@ def Create(self): self.vswitch.Create() if self.security_group is None: - self.security_group = AliSecurityGroup(self.name, - self.region, - use_vpc=True, - vpc_id=self.vpc.id) + self.security_group = AliSecurityGroup( + self.name, self.region, use_vpc=True, vpc_id=self.vpc.id + ) self.security_group.Create() else: self.security_group.Create() diff --git a/perfkitbenchmarker/providers/alicloud/ali_virtual_machine.py b/perfkitbenchmarker/providers/alicloud/ali_virtual_machine.py index 74eadd08f2..9a4f48a27e 100644 --- a/perfkitbenchmarker/providers/alicloud/ali_virtual_machine.py +++ b/perfkitbenchmarker/providers/alicloud/ali_virtual_machine.py @@ -53,7 +53,8 @@ 'ecs.m1.medium': 4, } INSTANCE_EXISTS_STATUSES = frozenset( - ['Starting', 'Running', 'Stopping', 'Stopped']) + ['Starting', 'Running', 'Stopping', 'Stopped'] +) INSTANCE_DELETED_STATUSES = frozenset([]) INSTANCE_KNOWN_STATUSES = INSTANCE_EXISTS_STATUSES | INSTANCE_DELETED_STATUSES @@ -92,13 +93,15 @@ def __init__(self, vm_spec): @vm_util.Retry(poll_interval=1, log_errors=False) def _WaitForInstanceStatus(self, status_list): """Waits until the instance's status is in status_list.""" - logging.info('Waits until the instance\'s status is one of statuses: %s', - status_list) + logging.info( + "Waits until the instance's status is one of statuses: %s", status_list + ) describe_cmd = util.ALI_PREFIX + [ 'ecs', 'DescribeInstances', '--RegionId %s' % self.region, - '--InstanceIds \'["%s"]\'' % self.id] + '--InstanceIds \'["%s"]\'' % self.id, + ] describe_cmd = util.GetEncodedCmd(describe_cmd) stdout, _ = vm_util.IssueRetryableCommand(describe_cmd) response = json.loads(stdout) @@ -110,13 +113,15 @@ def _WaitForInstanceStatus(self, status_list): @vm_util.Retry(poll_interval=5, max_retries=30, log_errors=False) def _WaitForEipStatus(self, status_list): """Waits until the instance's status is in status_list.""" - logging.info('Waits until the eip\'s status is one of statuses: %s', - status_list) + logging.info( + "Waits until the eip's status is one of statuses: %s", status_list + ) describe_cmd = util.ALI_PREFIX + [ 'ecs', 'DescribeEipAddresses', '--RegionId %s' % self.region, - '--AllocationId %s' % self.eip_id] + '--AllocationId %s' % self.eip_id, + ] describe_cmd = util.GetEncodedCmd(describe_cmd) stdout, _ = vm_util.IssueRetryableCommand(describe_cmd) response = json.loads(stdout) @@ -133,7 +138,8 @@ def _AllocatePubIp(self, region, instance_id): 'AllocateEipAddress', '--RegionId %s' % region, '--InternetChargeType PayByTraffic', - '--Bandwidth %s' % self.eip_address_bandwidth] + '--Bandwidth %s' % self.eip_address_bandwidth, + ] allocatip_cmd = util.GetEncodedCmd(allocatip_cmd) stdout, _ = vm_util.IssueRetryableCommand(allocatip_cmd) response = json.loads(stdout) @@ -148,7 +154,8 @@ def _AllocatePubIp(self, region, instance_id): '--RegionId %s' % region, '--AllocationId %s' % self.eip_id, '--InstanceId %s' % instance_id, - '--InstanceType EcsInstance'] + '--InstanceType EcsInstance', + ] associate_cmd = util.GetEncodedCmd(associate_cmd) vm_util.IssueRetryableCommand(associate_cmd) @@ -159,7 +166,8 @@ def _AllocatePubIp(self, region, instance_id): 'ecs', 'AllocatePublicIpAddress', '--RegionId %s' % region, - '--InstanceId %s' % instance_id] + '--InstanceId %s' % instance_id, + ] allocatip_cmd = util.GetEncodedCmd(allocatip_cmd) stdout, _ = vm_util.IssueRetryableCommand(allocatip_cmd) response = json.loads(stdout) @@ -178,7 +186,8 @@ def _GetDefaultImage(cls, region): 'ecs', 'DescribeImages', '--RegionId %s' % region, - '--ImageName \'%s\'' % cls.IMAGE_NAME_FILTER] + "--ImageName '%s'" % cls.IMAGE_NAME_FILTER, + ] describe_cmd = util.GetEncodedCmd(describe_cmd) stdout, _ = vm_util.IssueRetryableCommand(describe_cmd) @@ -198,17 +207,22 @@ def _PostCreate(self): 'ecs', 'DescribeInstances', '--RegionId %s' % self.region, - '--InstanceIds \'["%s"]\'' % self.id] - logging.info('Getting instance %s public IP. This will fail until ' - 'a public IP is available, but will be retried.', self.id) + '--InstanceIds \'["%s"]\'' % self.id, + ] + logging.info( + 'Getting instance %s public IP. This will fail until ' + 'a public IP is available, but will be retried.', + self.id, + ) describe_cmd = util.GetEncodedCmd(describe_cmd) stdout, _ = vm_util.IssueRetryableCommand(describe_cmd) response = json.loads(stdout) instance = response['Instances']['Instance'][0] if self.network.use_vpc: pub_ip_address = instance['EipAddress']['IpAddress'] - self.internal_ip = \ - instance['VpcAttributes']['PrivateIpAddress']['IpAddress'][0] + self.internal_ip = instance['VpcAttributes']['PrivateIpAddress'][ + 'IpAddress' + ][0] else: pub_ip_address = instance['PublicIpAddress']['IpAddress'][0] self.internal_ip = instance['InnerIpAddress']['IpAddress'][0] @@ -227,7 +241,8 @@ def _PostCreate(self): describe_cmd = util.ALI_PREFIX + [ 'ecs', 'DescribeDisks', - '--InstanceId %s' % self.id] + '--InstanceId %s' % self.id, + ] describe_cmd_encoded = util.GetEncodedCmd(describe_cmd) stdout, _ = vm_util.IssueRetryableCommand(describe_cmd_encoded) @@ -236,7 +251,8 @@ def _PostCreate(self): while response['NextToken']: describe_cmd_encoded = util.GetEncodedCmd( - describe_cmd + ['--NextToken', response['NextToken']]) + describe_cmd + ['--NextToken', response['NextToken']] + ) stdout, _ = vm_util.IssueRetryableCommand(describe_cmd_encoded) response = json.loads(stdout) @@ -248,10 +264,12 @@ def _PostCreate(self): util.AddDefaultTags(disk_id, util.ResourceTypes.DISK, self.region) # Tag KeyPair - util.AddTags(self.key_pair_name, util.ResourceTypes.KEYPAIR, self.region, - **tags) - util.AddDefaultTags(self.key_pair_name, util.ResourceTypes.KEYPAIR, - self.region) + util.AddTags( + self.key_pair_name, util.ResourceTypes.KEYPAIR, self.region, **tags + ) + util.AddDefaultTags( + self.key_pair_name, util.ResourceTypes.KEYPAIR, self.region + ) def _CreateDependencies(self): """Create VM dependencies.""" @@ -281,14 +299,16 @@ def _Create(self): '--SecurityGroupId %s' % self.network.security_group.group_id, '--KeyPairName %s' % self.key_pair_name, '--SystemDisk.Category %s' % self.system_disk_type, - '--SystemDisk.Size %s' % self.system_disk_size] + '--SystemDisk.Size %s' % self.system_disk_size, + ] if FLAGS.data_disk_type == disk.LOCAL: disk_cmd = [ '--DataDisk1Category ephemeral_ssd', '--DataDisk1Size %s' % self.scratch_disk_size, - '--DataDisk1Device %s%s' % (util.GetDrivePathPrefix(), - DRIVE_START_LETTER)] + '--DataDisk1Device %s%s' + % (util.GetDrivePathPrefix(), DRIVE_START_LETTER), + ] create_cmd.extend(disk_cmd) if FLAGS.ali_io_optimized is not None: @@ -300,15 +320,19 @@ def _Create(self): create_cmd.extend([ '--InternetChargeType PayByTraffic', '--InternetMaxBandwidthIn %s' % self.bandwidth_in, - '--InternetMaxBandwidthOut %s' % self.bandwidth_out]) + '--InternetMaxBandwidthOut %s' % self.bandwidth_out, + ]) # Create user and add SSH key public_key = AliCloudKeyFileManager.GetPublicKey() - user_data = util.ADD_USER_TEMPLATE.format(user_name=self.user_name, - public_key=public_key) + user_data = util.ADD_USER_TEMPLATE.format( + user_name=self.user_name, public_key=public_key + ) logging.debug('encoding startup script: %s', user_data) - create_cmd.extend(['--UserData', six.ensure_str( - base64.b64encode(user_data.encode('utf-8')))]) + create_cmd.extend([ + '--UserData', + six.ensure_str(base64.b64encode(user_data.encode('utf-8'))), + ]) create_cmd = util.GetEncodedCmd(create_cmd) stdout, _ = vm_util.IssueRetryableCommand(create_cmd) @@ -320,7 +344,8 @@ def _Create(self): start_cmd = util.ALI_PREFIX + [ 'ecs', 'StartInstance', - '--InstanceId %s' % self.id] + '--InstanceId %s' % self.id, + ] start_cmd = util.GetEncodedCmd(start_cmd) vm_util.IssueRetryableCommand(start_cmd) @@ -329,7 +354,8 @@ def _Delete(self): stop_cmd = util.ALI_PREFIX + [ 'ecs', 'StopInstance', - '--InstanceId %s' % self.id] + '--InstanceId %s' % self.id, + ] stop_cmd = util.GetEncodedCmd(stop_cmd) vm_util.IssueRetryableCommand(stop_cmd) @@ -338,7 +364,8 @@ def _Delete(self): delete_cmd = util.ALI_PREFIX + [ 'ecs', 'DeleteInstance', - '--InstanceId %s' % self.id] + '--InstanceId %s' % self.id, + ] delete_cmd = util.GetEncodedCmd(delete_cmd) vm_util.IssueRetryableCommand(delete_cmd) @@ -348,7 +375,8 @@ def _Delete(self): 'ecs', 'ReleaseEipAddress', '--RegionId %s' % self.region, - '--AllocationId %s' % self.eip_id] + '--AllocationId %s' % self.eip_id, + ] release_eip_cmd = util.GetEncodedCmd(release_eip_cmd) vm_util.IssueRetryableCommand(release_eip_cmd) @@ -358,7 +386,8 @@ def _Exists(self): 'ecs', 'DescribeInstances', '--RegionId %s' % self.region, - '--InstanceIds \'["%s"]\'' % str(self.id)] + '--InstanceIds \'["%s"]\'' % str(self.id), + ] describe_cmd = util.GetEncodedCmd(describe_cmd) stdout, _ = vm_util.IssueRetryableCommand(describe_cmd) response = json.loads(stdout) @@ -388,9 +417,13 @@ def CreateScratchDisk(self, disk_spec): data_disk.device_letter = DRIVE_START_LETTER self.FormatDisk(data_disk.GetDevicePath(), disk_spec.disk_type) - self.MountDisk(data_disk.GetDevicePath(), disk_spec.mount_point, - disk_spec.disk_type, data_disk.mount_options, - data_disk.fstab_options) + self.MountDisk( + data_disk.GetDevicePath(), + disk_spec.mount_point, + disk_spec.disk_type, + data_disk.mount_options, + data_disk.fstab_options, + ) def AddMetadata(self, **kwargs): """Adds metadata to the VM.""" @@ -399,6 +432,7 @@ def AddMetadata(self, **kwargs): class AliCloudKeyFileManager(object): """Object for managing AliCloud Keyfiles.""" + _lock = threading.Lock() imported_keyfile_set = set() deleted_keyfile_set = set() @@ -415,9 +449,13 @@ def ImportKeyfile(cls, region): import_cmd = util.ALI_PREFIX + [ 'ecs', 'ImportKeyPair', - '--RegionId', region, - '--KeyPairName', key_name, - '--PublicKeyBody', json.dumps(public_key)] + '--RegionId', + region, + '--KeyPairName', + key_name, + '--PublicKeyBody', + json.dumps(public_key), + ] vm_util.IssueRetryableCommand(import_cmd) cls.run_uri_key_names[FLAGS.run_uri] = key_name return key_name @@ -431,8 +469,11 @@ def DeleteKeyfile(cls, region, key_name): delete_cmd = util.ALI_PREFIX + [ 'ecs', 'DeleteKeyPairs', - '--RegionId', region, - '--KeyPairNames', json.dumps([key_name])] + '--RegionId', + region, + '--KeyPairNames', + json.dumps([key_name]), + ] vm_util.IssueRetryableCommand(delete_cmd) del cls.run_uri_key_names[FLAGS.run_uri] @@ -442,27 +483,30 @@ def GetKeyNameForRun(cls): @classmethod def GetPublicKey(cls): - cat_cmd = ['cat', - vm_util.GetPublicKeyPath()] + cat_cmd = ['cat', vm_util.GetPublicKeyPath()] keyfile, _ = vm_util.IssueRetryableCommand(cat_cmd) return keyfile.strip() -class Debian113BasedAliVirtualMachine(AliVirtualMachine, - linux_virtual_machine.Debian11Mixin): +class Debian113BasedAliVirtualMachine( + AliVirtualMachine, linux_virtual_machine.Debian11Mixin +): IMAGE_NAME_FILTER = 'debian_11_3_x64_20G*alibase*.vhd' -class Ubuntu1604BasedAliVirtualMachine(AliVirtualMachine, - linux_virtual_machine.Ubuntu1604Mixin): +class Ubuntu1604BasedAliVirtualMachine( + AliVirtualMachine, linux_virtual_machine.Ubuntu1604Mixin +): IMAGE_NAME_FILTER = 'ubuntu_16_04_x64*alibase*.vhd' -class Ubuntu1804BasedAliVirtualMachine(AliVirtualMachine, - linux_virtual_machine.Ubuntu1804Mixin): +class Ubuntu1804BasedAliVirtualMachine( + AliVirtualMachine, linux_virtual_machine.Ubuntu1804Mixin +): IMAGE_NAME_FILTER = 'ubuntu_18_04_x64*alibase*.vhd' -class CentOs7BasedAliVirtualMachine(AliVirtualMachine, - linux_virtual_machine.CentOs7Mixin): +class CentOs7BasedAliVirtualMachine( + AliVirtualMachine, linux_virtual_machine.CentOs7Mixin +): IMAGE_NAME_FILTER = 'centos_7_05_x64*alibase*.vhd' diff --git a/perfkitbenchmarker/providers/alicloud/flags.py b/perfkitbenchmarker/providers/alicloud/flags.py index 74e1587a44..6fc7104063 100644 --- a/perfkitbenchmarker/providers/alicloud/flags.py +++ b/perfkitbenchmarker/providers/alicloud/flags.py @@ -14,27 +14,37 @@ from absl import flags -flags.DEFINE_string('ali_user_name', 'ubuntu', - 'This determines the user name that Perfkit will ' - 'attempt to use. This must be changed in order to ' - 'use any image other than ubuntu.') +flags.DEFINE_string( + 'ali_user_name', + 'ubuntu', + 'This determines the user name that Perfkit will ' + 'attempt to use. This must be changed in order to ' + 'use any image other than ubuntu.', +) flags.DEFINE_integer('ali_bandwidth_in', 100, 'Inbound Bandwidth') flags.DEFINE_integer('ali_bandwidth_out', 100, 'Outbound Bandwidth') -flags.DEFINE_string('ali_io_optimized', None, - 'IO optimized for disk in AliCloud. The default is ' - 'None which means no IO optimized ' - '"optimized" means use IO optimized. If you ' - 'choose optimized, you must specify the system disk type') -flags.DEFINE_string('ali_system_disk_type', 'cloud_ssd', - 'System disk category for AliCloud. The default is ' - '"cloud" for General cloud disk, ' - '"cloud_ssd" for cloud ssd disk, ' - '"cloud_essd" for enhanced cloud ssd disk, ' - '"cloud_efficiency" for efficiency cloud disk, ' - '"ephemeral_ssd" for local ssd disk') -flags.DEFINE_integer('ali_system_disk_size', 50, - 'System disk size in GB. Default is 50 GB.') -flags.DEFINE_boolean('ali_use_vpc', True, - 'Use VPC to create networks') -flags.DEFINE_integer('ali_eip_address_bandwidth', 100, - 'The rate limit of the EIP in Mbps.') +flags.DEFINE_string( + 'ali_io_optimized', + None, + 'IO optimized for disk in AliCloud. The default is ' + 'None which means no IO optimized ' + '"optimized" means use IO optimized. If you ' + 'choose optimized, you must specify the system disk type', +) +flags.DEFINE_string( + 'ali_system_disk_type', + 'cloud_ssd', + 'System disk category for AliCloud. The default is ' + '"cloud" for General cloud disk, ' + '"cloud_ssd" for cloud ssd disk, ' + '"cloud_essd" for enhanced cloud ssd disk, ' + '"cloud_efficiency" for efficiency cloud disk, ' + '"ephemeral_ssd" for local ssd disk', +) +flags.DEFINE_integer( + 'ali_system_disk_size', 50, 'System disk size in GB. Default is 50 GB.' +) +flags.DEFINE_boolean('ali_use_vpc', True, 'Use VPC to create networks') +flags.DEFINE_integer( + 'ali_eip_address_bandwidth', 100, 'The rate limit of the EIP in Mbps.' +) diff --git a/perfkitbenchmarker/providers/alicloud/provider_info.py b/perfkitbenchmarker/providers/alicloud/provider_info.py index 0718529fc9..21f76a2975 100644 --- a/perfkitbenchmarker/providers/alicloud/provider_info.py +++ b/perfkitbenchmarker/providers/alicloud/provider_info.py @@ -18,6 +18,5 @@ class AliCloudProviderInfo(provider_info.BaseProviderInfo): - UNSUPPORTED_BENCHMARKS = ['mysql_service'] CLOUD = provider_info.ALICLOUD diff --git a/perfkitbenchmarker/providers/alicloud/util.py b/perfkitbenchmarker/providers/alicloud/util.py index a3146c69b9..348b51049c 100644 --- a/perfkitbenchmarker/providers/alicloud/util.py +++ b/perfkitbenchmarker/providers/alicloud/util.py @@ -44,6 +44,7 @@ class ResourceTypes: """AliCloud resource types.""" + INSTANCE = 'instance' IMAGE = 'image' SNAPSHOT = 'snapshot' @@ -79,8 +80,10 @@ def _BuildTagsList(**kwargs): tags_list = [] for index, (key, value) in enumerate(six.iteritems(kwargs)): tags_list.extend([ - '--Tag.{0}.Key'.format(index + 1), str(key), - '--Tag.{0}.Value'.format(index + 1), str(value) + '--Tag.{0}.Key'.format(index + 1), + str(key), + '--Tag.{0}.Value'.format(index + 1), + str(value), ]) return tags_list @@ -99,10 +102,14 @@ def AddTags(resource_id, resource_type, region, **kwargs): return tag_cmd = ALI_PREFIX + [ - 'ecs', 'AddTags', - '--RegionId', region, - '--ResourceId', resource_id, - '--ResourceType', resource_type + 'ecs', + 'AddTags', + '--RegionId', + region, + '--ResourceId', + resource_id, + '--ResourceType', + resource_type, ] tag_cmd.extend(_BuildTagsList(**kwargs)) vm_util.IssueRetryableCommand(tag_cmd) @@ -153,10 +160,14 @@ def VPCAddDefaultTags(resource_id, resource_type, region, timeout_minutes=None): return tag_cmd = ALI_PREFIX + [ - 'vpc', 'TagResources', - '--RegionId', region, - '--ResourceId.1', resource_id, - '--ResourceType', resource_type + 'vpc', + 'TagResources', + '--RegionId', + region, + '--ResourceId.1', + resource_id, + '--ResourceType', + resource_type, ] tag_cmd.extend(_BuildTagsList(**tags)) vm_util.IssueRetryableCommand(tag_cmd) diff --git a/perfkitbenchmarker/providers/aws/athena.py b/perfkitbenchmarker/providers/aws/athena.py index 535e9cee8b..b8546dda44 100644 --- a/perfkitbenchmarker/providers/aws/athena.py +++ b/perfkitbenchmarker/providers/aws/athena.py @@ -34,16 +34,41 @@ AWS_ATHENA_CMD_POSTFIX = ['--output', 'json'] # TODO(user): Derive the full table set from the TPC suite. TPC_H_TABLES = [ - 'customer', 'lineitem', 'nation', 'orders', 'part', 'partsupp', 'region', - 'supplier' + 'customer', + 'lineitem', + 'nation', + 'orders', + 'part', + 'partsupp', + 'region', + 'supplier', ] TPC_DS_TABLES = [ - 'call_center', 'catalog_page', 'catalog_returns', 'catalog_sales', - 'customer', 'customer_address', 'customer_demographics', 'date_dim', - 'dbgen_version', 'household_demographics', 'income_band', 'inventory', - 'item', 'promotion', 'reason', 'ship_mode', 'store', 'store_returns', - 'store_sales', 'time_dim', 'warehouse', 'web_page', 'web_returns', - 'web_sales', 'web_site' + 'call_center', + 'catalog_page', + 'catalog_returns', + 'catalog_sales', + 'customer', + 'customer_address', + 'customer_demographics', + 'date_dim', + 'dbgen_version', + 'household_demographics', + 'income_band', + 'inventory', + 'item', + 'promotion', + 'reason', + 'ship_mode', + 'store', + 'store_returns', + 'store_sales', + 'time_dim', + 'warehouse', + 'web_page', + 'web_returns', + 'web_sales', + 'web_site', ] FLAGS = flags.FLAGS @@ -53,8 +78,9 @@ class AthenaQueryError(RuntimeError): pass -def GetAthenaClientInterface(database: str, output_bucket: str, - region: str) -> edw_service.EdwClientInterface: +def GetAthenaClientInterface( + database: str, output_bucket: str, region: str +) -> edw_service.EdwClientInterface: """Builds and Returns the requested Athena client Interface. Args: @@ -71,8 +97,10 @@ def GetAthenaClientInterface(database: str, output_bucket: str, """ if FLAGS.athena_client_interface == 'JAVA': return JavaClientInterface(database, output_bucket, region) - raise RuntimeError('Unknown Athena Client Interface requested.' + - FLAGS.athena_client_interface) + raise RuntimeError( + 'Unknown Athena Client Interface requested.' + + FLAGS.athena_client_interface + ) class GenericClientInterface(edw_service.EdwClientInterface): @@ -96,13 +124,12 @@ def GetMetadata(self) -> Dict[str, str]: client_workgroup = FLAGS.athena_workgroup or 'dynamic' return { 'client': f'{FLAGS.athena_client_interface}_{client_workgroup}', - 'client_region': self.region + 'client_region': self.region, } class JavaClientInterface(GenericClientInterface): - """Java Client Interface class for Athena. - """ + """Java Client Interface class for Athena.""" def Prepare(self, package_name: str) -> None: """Prepares the client vm to execute query. @@ -120,7 +147,8 @@ def Prepare(self, package_name: str) -> None: self.client_vm.Install('openjdk') # Push the executable jar to the working directory on client vm self.client_vm.InstallPreprovisionedPackageData( - package_name, [LATEST_CLIENT_JAR], '') + package_name, [LATEST_CLIENT_JAR], '' + ) def ExecuteQuery(self, query_name: Text) -> Tuple[float, Dict[str, str]]: """Executes a query and returns performance details. @@ -135,14 +163,16 @@ def ExecuteQuery(self, query_name: Text) -> Tuple[float, Dict[str, str]]: successful query the value is expected to be positive. run_metadata: A dictionary of query execution attributes eg. script name """ - query_command = (f'java -cp {LATEST_CLIENT_JAR} ' - 'com.google.cloud.performance.edw.Single ' - f'--region {self.region} ' - f'--database {self.database} ' - f'--output_location {self.output_bucket} ' - f'--query_file {query_name} ' - f'--query_timeout_secs {FLAGS.athena_query_timeout} ' - f'--collect_metrics {FLAGS.athena_metrics_collection}') + query_command = ( + f'java -cp {LATEST_CLIENT_JAR} ' + 'com.google.cloud.performance.edw.Single ' + f'--region {self.region} ' + f'--database {self.database} ' + f'--output_location {self.output_bucket} ' + f'--query_file {query_name} ' + f'--query_timeout_secs {FLAGS.athena_query_timeout} ' + f'--collect_metrics {FLAGS.athena_metrics_collection}' + ) if not FLAGS.athena_metrics_collection: # execute the query in requested persistent workgroup @@ -150,8 +180,9 @@ def ExecuteQuery(self, query_name: Text) -> Tuple[float, Dict[str, str]]: query_command = f'{query_command} --delete_workgroup False' else: # the dynamic workgroup may have to live beyond the benchmark - query_command = (f'{query_command} ' - f'--delete_workgroup {FLAGS.athena_workgroup_delete}') + query_command = ( + f'{query_command} --delete_workgroup {FLAGS.athena_workgroup_delete}' + ) stdout, _ = self.client_vm.RemoteCommand(query_command) details = copy.copy(self.GetMetadata()) # Copy the base metadata @@ -209,7 +240,8 @@ def RunScriptCommand(script_command): """ start_time = datetime.datetime.now() stdout, _, retcode = vm_util.IssueCommand( - script_command, raise_on_failure=False) + script_command, raise_on_failure=False + ) if retcode: raise AthenaQueryError end_time = datetime.datetime.now() @@ -226,17 +258,19 @@ def __init__(self, edw_service_spec): super(Athena, self).__init__(edw_service_spec) self.region = util.GetRegionFromZone(FLAGS.zone[0]) self.output_bucket = '-'.join( - [FLAGS.athena_output_location_prefix, self.region, FLAGS.run_uri]) - self.client_interface = GetAthenaClientInterface(self.cluster_identifier, - self.output_bucket, - self.region) + [FLAGS.athena_output_location_prefix, self.region, FLAGS.run_uri] + ) + self.client_interface = GetAthenaClientInterface( + self.cluster_identifier, self.output_bucket, self.region + ) self.s3_service = s3.S3Service() self.s3_service.PrepareService(self.region) self.s3_service.MakeBucket(self.output_bucket) if FLAGS.provision_athena: self.data_bucket = 'pkb' + self.cluster_identifier.replace('_', '') self.tables = ( - TPC_H_TABLES if FLAGS.edw_tpc_dsb_type == 'tpc_h' else TPC_DS_TABLES) + TPC_H_TABLES if FLAGS.edw_tpc_dsb_type == 'tpc_h' else TPC_DS_TABLES + ) self.athena_db_create_time = 0 self.athena_table_create_time = 0 @@ -253,15 +287,17 @@ def BuildAthenaCommand(self, query_string, database=None): cmd = [] cmd.extend(AWS_ATHENA_CMD_PREFIX) cmd.extend([ - '--region', self.region, + '--region', + self.region, 'start-query-execution', - '--query-string', query_string + '--query-string', + query_string, ]) if database: - cmd.extend(['--query-execution-context', ('Database=%s' % database)]) + cmd.extend(['--query-execution-context', 'Database=%s' % database]) cmd.extend([ '--result-configuration', - ('OutputLocation=s3://%s' % self.output_bucket) + ('OutputLocation=s3://%s' % self.output_bucket), ]) cmd.extend(AWS_ATHENA_CMD_POSTFIX) return cmd @@ -275,45 +311,58 @@ def _EmptyDatabase(): If the database and/or tables don't already exist, the drop commands will simply fail, which won't raise errors. """ - drop_script_path = data.ResourcePath('edw/athena/%s/ddl/drop.sql' % - FLAGS.edw_tpc_dsb_type) + drop_script_path = data.ResourcePath( + 'edw/athena/%s/ddl/drop.sql' % FLAGS.edw_tpc_dsb_type + ) drop_script_contents = ReadScript(drop_script_path) # Drop all tables so the database can be dropped. for table in self.tables: # Remove the folder backing each parquet table so they can be refreshed. - vm_util.IssueCommand([ - 'aws', 's3', 'rm', - 's3://%s/%s_parquet' % (self.data_bucket, table), '--recursive' - ], raise_on_failure=False) + vm_util.IssueCommand( + [ + 'aws', + 's3', + 'rm', + 's3://%s/%s_parquet' % (self.data_bucket, table), + '--recursive', + ], + raise_on_failure=False, + ) # The parquet tables don't have the type suffix so that the queries can # run as written without having to change the table names. for suffix in ['_csv', '']: - script_contents = PrepareQueryString(drop_script_contents, - {'{table}': table + suffix}) + script_contents = PrepareQueryString( + drop_script_contents, {'{table}': table + suffix} + ) script_command = self.BuildAthenaCommand( - script_contents, database=self.cluster_identifier) + script_contents, database=self.cluster_identifier + ) RunScriptCommand(script_command) drop_database_query_string = PrepareQueryString( 'drop database database_name', - {'database_name': self.cluster_identifier}) + {'database_name': self.cluster_identifier}, + ) script_command = self.BuildAthenaCommand(drop_database_query_string) RunScriptCommand(script_command) def _CreateDatabase(): create_database_query_string = PrepareQueryString( 'create database database_name', - {'database_name': self.cluster_identifier}) + {'database_name': self.cluster_identifier}, + ) script_command = self.BuildAthenaCommand(create_database_query_string) return RunScriptCommand(script_command) def _CreateTable(table_create_sql_template): template_script_path = data.ResourcePath(table_create_sql_template) template_script_contents = ReadScript(template_script_path) - script_contents = PrepareQueryString(template_script_contents, - {'{bucket}': self.data_bucket}) + script_contents = PrepareQueryString( + template_script_contents, {'{bucket}': self.data_bucket} + ) script_command = self.BuildAthenaCommand( - script_contents, database=self.cluster_identifier) + script_contents, database=self.cluster_identifier + ) return RunScriptCommand(script_command) def _CreateAllTables(): @@ -321,8 +370,10 @@ def _CreateAllTables(): cumulative_table_create_time = 0 for table in self.tables: for suffix in ['_csv', '_parquet']: - script = 'edw/athena/%s/ddl/%s.sql' % (FLAGS.edw_tpc_dsb_type, - table + suffix) + script = 'edw/athena/%s/ddl/%s.sql' % ( + FLAGS.edw_tpc_dsb_type, + table + suffix, + ) _, table_create_time = _CreateTable(script) cumulative_table_create_time += table_create_time return cumulative_table_create_time diff --git a/perfkitbenchmarker/providers/aws/aws_aurora_db.py b/perfkitbenchmarker/providers/aws/aws_aurora_db.py index 0bbb4ac2a7..ac6c8ea8e3 100644 --- a/perfkitbenchmarker/providers/aws/aws_aurora_db.py +++ b/perfkitbenchmarker/providers/aws/aws_aurora_db.py @@ -38,11 +38,13 @@ _AURORA_ENGINES = [ sql_engine_utils.AURORA_MYSQL, - sql_engine_utils.AURORA_POSTGRES] + sql_engine_utils.AURORA_POSTGRES, +] class AwsAuroraRelationalDb(aws_relational_db.BaseAwsRelationalDb): """Implements the aurora database for AWS.""" + CLOUD = 'AWS' IS_MANAGED = True ENGINE = _AURORA_ENGINES @@ -64,13 +66,16 @@ def _Create(self): """ zones_needed_for_high_availability = len(self.zones) > 1 if zones_needed_for_high_availability != self.spec.high_availability: - raise Exception('When db_high_availability is true, multiple ' - 'zones must be specified. When ' - 'db_high_availability is false, one zone ' - 'should be specified. ' - 'db_high_availability: {0} ' - 'zone count: {1} '.format( - zones_needed_for_high_availability, len(self.zones))) + raise Exception( + 'When db_high_availability is true, multiple ' + 'zones must be specified. When ' + 'db_high_availability is false, one zone ' + 'should be specified. ' + 'db_high_availability: {0} ' + 'zone count: {1} '.format( + zones_needed_for_high_availability, len(self.zones) + ) + ) # Create the cluster. cmd = ( @@ -96,7 +101,6 @@ def _Create(self): vm_util.IssueCommand(cmd) for zone in self.zones: - # The first instance is assumed to be writer - # and so use the instance_id for that id. if zone == self.zones[0]: @@ -106,17 +110,23 @@ def _Create(self): self.all_instance_ids.append(instance_identifier) - cmd = util.AWS_PREFIX + [ - 'rds', 'create-db-instance', - '--db-instance-identifier=%s' % instance_identifier, - '--db-cluster-identifier=%s' % self.cluster_id, - '--engine=%s' % self.spec.engine, - '--engine-version=%s' % self.spec.engine_version, - '--no-auto-minor-version-upgrade', - '--db-instance-class=%s' % self.spec.db_spec.machine_type, - '--region=%s' % self.region, - '--availability-zone=%s' % zone, '--tags' - ] + util.MakeFormattedDefaultTags() + cmd = ( + util.AWS_PREFIX + + [ + 'rds', + 'create-db-instance', + '--db-instance-identifier=%s' % instance_identifier, + '--db-cluster-identifier=%s' % self.cluster_id, + '--engine=%s' % self.spec.engine, + '--engine-version=%s' % self.spec.engine_version, + '--no-auto-minor-version-upgrade', + '--db-instance-class=%s' % self.spec.db_spec.machine_type, + '--region=%s' % self.region, + '--availability-zone=%s' % zone, + '--tags', + ] + + util.MakeFormattedDefaultTags() + ) vm_util.IssueCommand(cmd) def _PostCreate(self): @@ -136,9 +146,9 @@ def _UpdateClusterClass(self, instance_class: str) -> None: def _UpdateInstanceClass(self, instance_id: str, instance_class: str) -> None: """Updates DBInstanceClass for the writer instance.""" - current_instance_class = self._DescribeInstance(instance_id)[ - 'DBInstances' - ][0]['DBInstanceClass'] + current_instance_class = self._DescribeInstance(instance_id)['DBInstances'][ + 0 + ]['DBInstanceClass'] if current_instance_class != instance_class: logging.info( 'Updating capacity from %s to %s', @@ -167,9 +177,10 @@ def UpdateCapacityForRun(self) -> None: def _DescribeCluster(self): cmd = util.AWS_PREFIX + [ - 'rds', 'describe-db-clusters', + 'rds', + 'describe-db-clusters', '--db-cluster-identifier=%s' % self.cluster_id, - '--region=%s' % self.region + '--region=%s' % self.region, ] stdout, _, retcode = vm_util.IssueCommand(cmd, raise_on_failure=False) if retcode != 0: @@ -198,10 +209,11 @@ def _FailoverHA(self): """Fail over from master to replica.""" new_primary_id = self.all_instance_ids[1] cmd = util.AWS_PREFIX + [ - 'rds', 'failover-db-cluster', + 'rds', + 'failover-db-cluster', '--db-cluster-identifier=%s' % self.cluster_id, '--target-db-instance-identifier=%s' % new_primary_id, - '--region=%s' % self.region + '--region=%s' % self.region, ] vm_util.IssueCommand(cmd) @@ -223,10 +235,12 @@ def _Delete(self): poll_interval=60, fuzz=0, timeout=3600, - retryable_exceptions=(errors.Resource.RetryableDeletionError,)) + retryable_exceptions=(errors.Resource.RetryableDeletionError,), + ) def WaitUntilClusterDeleted(): if self._ClusterExists(): raise errors.Resource.RetryableDeletionError('Not yet deleted') + WaitUntilClusterDeleted() @staticmethod @@ -235,6 +249,7 @@ def GetDefaultEngineVersion(engine): Args: engine (string): type of database (my_sql or postgres). + Returns: (string): Default engine version. Raises: diff --git a/perfkitbenchmarker/providers/aws/aws_capacity_reservation.py b/perfkitbenchmarker/providers/aws/aws_capacity_reservation.py index 16366faaa5..f8ae8b7a17 100644 --- a/perfkitbenchmarker/providers/aws/aws_capacity_reservation.py +++ b/perfkitbenchmarker/providers/aws/aws_capacity_reservation.py @@ -67,13 +67,15 @@ class CreationError(Exception): class AwsCapacityReservation(capacity_reservation.BaseCapacityReservation): """An object representing an AWS EC2 CapacityReservation.""" + CLOUD = provider_info.AWS def __init__(self, vm_group): if not vm_group: raise InvalidVmGroupSizeError( 'AwsCapacityReservation must be initialized with at least one ' - 'VM in the vm_group.') + 'VM in the vm_group.' + ) super(AwsCapacityReservation, self).__init__(vm_group) self.zone_or_region = vm_group[0].zone @@ -109,8 +111,8 @@ def _Create(self): instance_platform = 'Windows' else: raise UnsupportedOsTypeError( - 'Unsupported os_type for AWS CapacityReservation: %s.' - % self.os_type) + 'Unsupported os_type for AWS CapacityReservation: %s.' % self.os_type + ) # If the user did not specify an AZ, we need to try to create the # CapacityReservation in a specifc AZ until it succeeds. @@ -121,9 +123,9 @@ def _Create(self): else: zones_to_try = [self.zone_or_region] - end_date = ( - datetime.datetime.utcnow() + - datetime.timedelta(minutes=FLAGS.timeout_minutes)) + end_date = datetime.datetime.utcnow() + datetime.timedelta( + minutes=FLAGS.timeout_minutes + ) for zone in zones_to_try: cmd = util.AWS_PREFIX + [ 'ec2', @@ -137,23 +139,32 @@ def _Create(self): '--end-date-type=limited', '--end-date=%s' % end_date.isoformat(), ] - stdout, stderr, retcode = vm_util.IssueCommand(cmd, - raise_on_failure=False) + stdout, stderr, retcode = vm_util.IssueCommand( + cmd, raise_on_failure=False + ) if retcode: - logging.info('Unable to create CapacityReservation in %s. ' - 'This may be retried. Details: %s', zone, stderr) + logging.info( + 'Unable to create CapacityReservation in %s. ' + 'This may be retried. Details: %s', + zone, + stderr, + ) if _INSUFFICIENT_CAPACITY in stderr: logging.error(util.STOCKOUT_MESSAGE) raise errors.Benchmarks.InsufficientCapacityCloudFailure( - util.STOCKOUT_MESSAGE + ' CapacityReservation in ' + zone) + util.STOCKOUT_MESSAGE + ' CapacityReservation in ' + zone + ) continue json_output = json.loads(stdout) - self.capacity_reservation_id = ( - json_output['CapacityReservation']['CapacityReservationId']) + self.capacity_reservation_id = json_output['CapacityReservation'][ + 'CapacityReservationId' + ] self._UpdateVmsInGroup(self.capacity_reservation_id, zone) return - raise CreationError('Unable to create CapacityReservation in any of the ' - 'following zones: %s.' % zones_to_try) + raise CreationError( + 'Unable to create CapacityReservation in any of the ' + 'following zones: %s.' % zones_to_try + ) def _Delete(self): """Deletes the capacity reservation.""" @@ -190,9 +201,9 @@ def _UpdateVmsInGroup(self, capacity_reservation_id, zone): Args: capacity_reservation_id: ID of the reservation created by this instance. - zone: Zone chosen by this class, or if it was supplied, the zone - provided by the user. In the latter case, setting the zone is equivalent - to a no-op. + zone: Zone chosen by this class, or if it was supplied, the zone provided + by the user. In the latter case, setting the zone is equivalent to a + no-op. """ for vm in self.vm_group: vm.capacity_reservation_id = capacity_reservation_id diff --git a/perfkitbenchmarker/providers/aws/aws_cluster_parameter_group.py b/perfkitbenchmarker/providers/aws/aws_cluster_parameter_group.py index d02290bfb2..b570cf5f37 100644 --- a/perfkitbenchmarker/providers/aws/aws_cluster_parameter_group.py +++ b/perfkitbenchmarker/providers/aws/aws_cluster_parameter_group.py @@ -34,16 +34,23 @@ def __init__(self, cmd_prefix): def _Create(self): cmd = self.cmd_prefix + [ - 'redshift', 'create-cluster-parameter-group', '--parameter-group-name', - self.name, '--parameter-group-family', 'redshift-1.0', '--description', - 'Cluster Parameter group for run uri {}'.format(FLAGS.run_uri) + 'redshift', + 'create-cluster-parameter-group', + '--parameter-group-name', + self.name, + '--parameter-group-family', + 'redshift-1.0', + '--description', + 'Cluster Parameter group for run uri {}'.format(FLAGS.run_uri), ] vm_util.IssueCommand(cmd) def _Delete(self): """Delete a redshift cluster parameter group.""" cmd = self.cmd_prefix + [ - 'redshift', 'delete-cluster-parameter-group', '--parameter-group-name', - self.name + 'redshift', + 'delete-cluster-parameter-group', + '--parameter-group-name', + self.name, ] vm_util.IssueCommand(cmd, raise_on_failure=False) diff --git a/perfkitbenchmarker/providers/aws/aws_cluster_subnet_group.py b/perfkitbenchmarker/providers/aws/aws_cluster_subnet_group.py index abdaed9bef..6cf265f24f 100644 --- a/perfkitbenchmarker/providers/aws/aws_cluster_subnet_group.py +++ b/perfkitbenchmarker/providers/aws/aws_cluster_subnet_group.py @@ -39,17 +39,23 @@ def __init__(self, cmd_prefix): def _Create(self): cmd = self.cmd_prefix + [ - 'redshift', 'create-cluster-subnet-group', - '--cluster-subnet-group-name', self.name, '--description', - 'Cluster Subnet Group for run uri {}'.format( - FLAGS.run_uri), '--subnet-ids', self.subnet_id + 'redshift', + 'create-cluster-subnet-group', + '--cluster-subnet-group-name', + self.name, + '--description', + 'Cluster Subnet Group for run uri {}'.format(FLAGS.run_uri), + '--subnet-ids', + self.subnet_id, ] vm_util.IssueCommand(cmd) def _Delete(self): """Delete a redshift cluster subnet group.""" cmd = self.cmd_prefix + [ - 'redshift', 'delete-cluster-subnet-group', - '--cluster-subnet-group-name', self.name + 'redshift', + 'delete-cluster-subnet-group', + '--cluster-subnet-group-name', + self.name, ] vm_util.IssueCommand(cmd, raise_on_failure=False) diff --git a/perfkitbenchmarker/providers/aws/aws_container_service.py b/perfkitbenchmarker/providers/aws/aws_container_service.py index d9dea943be..edac340270 100644 --- a/perfkitbenchmarker/providers/aws/aws_container_service.py +++ b/perfkitbenchmarker/providers/aws/aws_container_service.py @@ -51,11 +51,16 @@ def _Create(self): self.user_managed = True return create_cmd = util.AWS_PREFIX + [ - 'ecr', 'create-repository', '--region', self.region, - '--repository-name', self.name + 'ecr', + 'create-repository', + '--region', + self.region, + '--repository-name', + self.name, ] _, stderr, retcode = vm_util.IssueCommand( - create_cmd, raise_on_failure=False) + create_cmd, raise_on_failure=False + ) if retcode: if 'InsufficientInstanceCapacity' in stderr: raise errors.Benchmarks.InsufficientCapacityCloudFailure(stderr) @@ -63,13 +68,19 @@ def _Create(self): raise errors.Benchmarks.QuotaFailure(stderr) raise errors.Resource.CreationError( 'Failed to create EKS Cluster: {} return code: {}'.format( - retcode, stderr)) + retcode, stderr + ) + ) def _Exists(self): """Returns True if the repository exists.""" describe_cmd = util.AWS_PREFIX + [ - 'ecr', 'describe-repositories', '--region', self.region, - '--repository-names', self.name + 'ecr', + 'describe-repositories', + '--region', + self.region, + '--repository-names', + self.name, ] stdout, _, _ = vm_util.IssueCommand(describe_cmd, raise_on_failure=False) if not stdout or not json.loads(stdout)['repositories']: @@ -79,8 +90,13 @@ def _Exists(self): def _Delete(self): """Deletes the repository.""" delete_cmd = util.AWS_PREFIX + [ - 'ecr', 'delete-repository', '--region', self.region, - '--repository-name', self.name, '--force' + 'ecr', + 'delete-repository', + '--region', + self.region, + '--repository-name', + self.name, + '--force', ] vm_util.IssueCommand(delete_cmd, raise_on_failure=False) @@ -104,7 +120,8 @@ def _Delete(self): def PrePush(self, image): """Prepares registry to push a given image.""" repository_name = '{namespace}/{name}'.format( - namespace=self.name, name=image.name) + namespace=self.name, name=image.name + ) repository = EcrRepository(repository_name, self.region) self.repositories.append(repository) repository.Create() @@ -165,13 +182,23 @@ def _CreateDependencies(self): def _Create(self): """Create the task definition.""" register_cmd = util.AWS_PREFIX + [ - '--region', self.region, 'ecs', 'register-task-definition', '--family', - self.name, '--execution-role-arn', 'ecsTaskExecutionRole', - '--network-mode', 'awsvpc', '--requires-compatibilities=FARGATE', + '--region', + self.region, + 'ecs', + 'register-task-definition', + '--family', + self.name, + '--execution-role-arn', + 'ecsTaskExecutionRole', + '--network-mode', + 'awsvpc', + '--requires-compatibilities=FARGATE', '--cpu', - str(int(1024 * self.cpus)), '--memory', - str(self.memory), '--container-definitions', - self._GetContainerDefinitions() + str(int(1024 * self.cpus)), + '--memory', + str(self.memory), + '--container-definitions', + self._GetContainerDefinitions(), ] stdout, _, _ = vm_util.IssueCommand(register_cmd) response = json.loads(stdout) @@ -182,8 +209,12 @@ def _Delete(self): if self.arn is None: return deregister_cmd = util.AWS_PREFIX + [ - '--region', self.region, 'ecs', 'deregister-task-definition', - '--task-definition', self.arn + '--region', + self.region, + 'ecs', + 'deregister-task-definition', + '--task-definition', + self.arn, ] vm_util.IssueCommand(deregister_cmd) @@ -195,16 +226,16 @@ def _GetContainerDefinitions(self): 'essential': True, 'portMappings': [{ 'containerPort': self.container_port, - 'protocol': 'TCP' + 'protocol': 'TCP', }], 'logConfiguration': { 'logDriver': 'awslogs', 'options': { 'awslogs-group': 'pkb', 'awslogs-region': self.region, - 'awslogs-stream-prefix': 'pkb' - } - } + 'awslogs-stream-prefix': 'pkb', + }, + }, }] return json.dumps(definitions) @@ -222,7 +253,8 @@ def __init__(self, name, container_spec, cluster): self.subnet_id = cluster.network.subnet.id self.ip_address = None self.security_group_id = ( - cluster.network.regional_network.vpc.default_security_group_id) + cluster.network.regional_network.vpc.default_security_group_id + ) def _GetNetworkConfig(self): network_config = { @@ -254,11 +286,20 @@ def _GetOverrides(self): def _Create(self): """Creates the task.""" run_cmd = util.AWS_PREFIX + [ - '--region', self.region, 'ecs', 'run-task', '--cluster', - self.cluster_name, '--task-definition', self.task_def.arn, - '--launch-type', 'FARGATE', '--network-configuration', - self._GetNetworkConfig(), '--overrides', - self._GetOverrides() + '--region', + self.region, + 'ecs', + 'run-task', + '--cluster', + self.cluster_name, + '--task-definition', + self.task_def.arn, + '--launch-type', + 'FARGATE', + '--network-configuration', + self._GetNetworkConfig(), + '--overrides', + self._GetOverrides(), ] stdout, _, _ = vm_util.IssueCommand(run_cmd) response = json.loads(stdout) @@ -278,16 +319,28 @@ def _Delete(self): if self.arn is None: return stop_cmd = util.AWS_PREFIX + [ - '--region', self.region, 'ecs', 'stop-task', '--cluster', - self.cluster_name, '--task', self.arn + '--region', + self.region, + 'ecs', + 'stop-task', + '--cluster', + self.cluster_name, + '--task', + self.arn, ] vm_util.IssueCommand(stop_cmd) def _GetTask(self): """Returns a dictionary representation of the task.""" describe_cmd = util.AWS_PREFIX + [ - '--region', self.region, 'ecs', 'describe-tasks', '--cluster', - self.cluster_name, '--tasks', self.arn + '--region', + self.region, + 'ecs', + 'describe-tasks', + '--cluster', + self.cluster_name, + '--tasks', + self.arn, ] stdout, _, _ = vm_util.IssueCommand(describe_cmd) response = json.loads(stdout) @@ -302,12 +355,14 @@ def WaitForExit(self, timeout=None): @vm_util.Retry( timeout=timeout, - retryable_exceptions=(container_service.RetriableContainerException,)) + retryable_exceptions=(container_service.RetriableContainerException,), + ) def _WaitForExit(): task = self._GetTask() if task['lastStatus'] != 'STOPPED': raise container_service.RetriableContainerException( - 'Task is not STOPPED.') + 'Task is not STOPPED.' + ) return task return _WaitForExit() @@ -318,7 +373,8 @@ def GetLogs(self): task_id = self.arn.split('/')[-1] log_stream = 'pkb/{name}/{task_id}'.format(name=self.name, task_id=task_id) return six.text_type( - aws_logs.GetLogStreamAsString(self.region, log_stream, 'pkb')) + aws_logs.GetLogStreamAsString(self.region, log_stream, 'pkb') + ) class EcsService(container_service.BaseContainerService): @@ -334,19 +390,23 @@ def __init__(self, name, container_spec, cluster): self.cluster_name = cluster.name self.subnet_id = cluster.network.subnet.id self.security_group_id = ( - cluster.network.regional_network.vpc.default_security_group_id) + cluster.network.regional_network.vpc.default_security_group_id + ) self.load_balancer = aws_load_balancer.LoadBalancer( - [cluster.network.subnet]) + [cluster.network.subnet] + ) self.target_group = aws_load_balancer.TargetGroup( - cluster.network.regional_network.vpc, self.container_port) + cluster.network.regional_network.vpc, self.container_port + ) self.port = 80 def _CreateDependencies(self): """Creates the load balancer for the service.""" self.load_balancer.Create() self.target_group.Create() - listener = aws_load_balancer.Listener(self.load_balancer, self.target_group, - self.port) + listener = aws_load_balancer.Listener( + self.load_balancer, self.target_group, self.port + ) listener.Create() self.ip_address = self.load_balancer.dns_name @@ -386,13 +446,27 @@ def _Create(self): def _Delete(self): """Deletes the service.""" update_cmd = util.AWS_PREFIX + [ - '--region', self.region, 'ecs', 'update-service', '--cluster', - self.cluster_name, '--service', self.name, '--desired-count', '0' + '--region', + self.region, + 'ecs', + 'update-service', + '--cluster', + self.cluster_name, + '--service', + self.name, + '--desired-count', + '0', ] vm_util.IssueCommand(update_cmd) delete_cmd = util.AWS_PREFIX + [ - '--region', self.region, 'ecs', 'delete-service', '--cluster', - self.cluster_name, '--service', self.name + '--region', + self.region, + 'ecs', + 'delete-service', + '--cluster', + self.cluster_name, + '--service', + self.name, ] vm_util.IssueCommand(delete_cmd, raise_on_failure=False) @@ -445,8 +519,12 @@ def __init__(self, cluster_spec): def _Create(self): """Creates the cluster.""" create_cmd = util.AWS_PREFIX + [ - '--region', self.region, 'ecs', 'create-cluster', '--cluster-name', - self.name + '--region', + self.region, + 'ecs', + 'create-cluster', + '--cluster-name', + self.name, ] stdout, _, _ = vm_util.IssueCommand(create_cmd) response = json.loads(stdout) @@ -457,8 +535,12 @@ def _Exists(self): if not self.arn: return False describe_cmd = util.AWS_PREFIX + [ - '--region', self.region, 'ecs', 'describe-clusters', '--clusters', - self.arn + '--region', + self.region, + 'ecs', + 'describe-clusters', + '--clusters', + self.arn, ] stdout, _, _ = vm_util.IssueCommand(describe_cmd) response = json.loads(stdout) @@ -470,7 +552,12 @@ def _Exists(self): def _Delete(self): """Deletes the cluster.""" delete_cmd = util.AWS_PREFIX + [ - '--region', self.region, 'ecs', 'delete-cluster', '--cluster', self.name + '--region', + self.region, + 'ecs', + 'delete-cluster', + '--cluster', + self.name, ] vm_util.IssueCommand(delete_cmd, raise_on_failure=False) @@ -492,9 +579,9 @@ def DeployContainerService(self, name, container_spec): task_def.Create() service = EcsService(name, container_spec, self) self.services[name] = service - self.firewall.AllowPortInSecurityGroup(service.region, - service.security_group_id, - service.container_port) + self.firewall.AllowPortInSecurityGroup( + service.region, service.security_group_id, service.container_port + ) service.Create() @@ -524,11 +611,13 @@ def _Create(self): """Creates the cluster.""" # Create the cluster spec but don't provision any resources. create_cmd = [ - FLAGS.kops, 'create', 'cluster', + FLAGS.kops, + 'create', + 'cluster', '--name=%s' % self.name, '--zones=%s' % self.default_node_pool.zone, '--node-count=%s' % self.default_node_pool.num_nodes, - '--node-size=%s' % self.default_node_pool.machine_type + '--node-size=%s' % self.default_node_pool.machine_type, ] env = os.environ.copy() env['KUBECONFIG'] = FLAGS.kubeconfig @@ -547,7 +636,7 @@ def _Create(self): 'perfkitbenchmarker-run': FLAGS.run_uri, 'benchmark': benchmark_spec.name, 'perfkit_uuid': benchmark_spec.uuid, - 'benchmark_uid': benchmark_spec.uid + 'benchmark_uid': benchmark_spec.uid, } # Replace the cluster spec. @@ -565,21 +654,27 @@ def _Delete(self): """Deletes the cluster.""" super()._Delete() delete_cmd = [ - FLAGS.kops, 'delete', 'cluster', + FLAGS.kops, + 'delete', + 'cluster', '--name=%s' % self.name, - '--state=s3://%s' % self.config_bucket, '--yes' + '--state=s3://%s' % self.config_bucket, + '--yes', ] vm_util.IssueCommand(delete_cmd, raise_on_failure=False) def _IsReady(self): """Returns True if the cluster is ready, else False.""" validate_cmd = [ - FLAGS.kops, 'validate', 'cluster', + FLAGS.kops, + 'validate', + 'cluster', '--name=%s' % self.name, - '--state=s3://%s' % self.config_bucket + '--state=s3://%s' % self.config_bucket, ] env = os.environ.copy() env['KUBECONFIG'] = FLAGS.kubeconfig _, _, retcode = vm_util.IssueCommand( - validate_cmd, env=env, raise_on_failure=False) + validate_cmd, env=env, raise_on_failure=False + ) return not retcode diff --git a/perfkitbenchmarker/providers/aws/aws_dax.py b/perfkitbenchmarker/providers/aws/aws_dax.py index 41c6a0c67c..7b03769878 100644 --- a/perfkitbenchmarker/providers/aws/aws_dax.py +++ b/perfkitbenchmarker/providers/aws/aws_dax.py @@ -51,16 +51,22 @@ def __init__(self, benchmark_uid, zone, network): self.subnet_id = network.subnet.id self.account = util.GetAccount() self.iam_role = aws_iam_role.AwsIamRole( - self.account, _DAX_ROLE_NAME_TEMPLATE.format(uid=self.benchmark_uid), - _DAX_POLICY_NAME_TEMPLATE.format(uid=self.benchmark_uid), _DAX_SERVICE, + self.account, + _DAX_ROLE_NAME_TEMPLATE.format(uid=self.benchmark_uid), + _DAX_POLICY_NAME_TEMPLATE.format(uid=self.benchmark_uid), + _DAX_SERVICE, _DAX_ACTION, _DYNAMODB_RESOURCE_TEMPLATE.format( - region=self.region, account=self.account)) + region=self.region, account=self.account + ), + ) self.cluster_endpoint = None self.subnet_group_name = _DAX_SUBNET_GROUP_TEMPLATE.format( - uid=self.benchmark_uid) + uid=self.benchmark_uid + ) self.cluster_name = _DAX_CLUSTER_NAME_TEMPLATE.format( - uid=self.benchmark_uid) + uid=self.benchmark_uid + ) def _CreateDependencies(self): """See base class. @@ -70,8 +76,12 @@ def _CreateDependencies(self): self.iam_role.Create() cmd = util.AWS_PREFIX + [ - 'dax', 'create-subnet-group', '--subnet-group-name', - self.subnet_group_name, '--subnet-ids', self.subnet_id + 'dax', + 'create-subnet-group', + '--subnet-group-name', + self.subnet_group_name, + '--subnet-ids', + self.subnet_id, ] _, stderror, retcode = vm_util.IssueCommand(cmd, raise_on_failure=True) @@ -81,11 +91,22 @@ def _CreateDependencies(self): def _Create(self): """See base class.""" cmd = util.AWS_PREFIX + [ - 'dax', 'create-cluster', '--cluster-name', self.cluster_name, - '--node-type', FLAGS.aws_dax_node_type, '--replication-factor', - str(FLAGS.aws_dax_replication_factor), '--iam-role-arn', - self.iam_role.GetRoleArn(), '--subnet-group', self.subnet_group_name, - '--sse-specification', 'Enabled=true', '--region', self.region + 'dax', + 'create-cluster', + '--cluster-name', + self.cluster_name, + '--node-type', + FLAGS.aws_dax_node_type, + '--replication-factor', + str(FLAGS.aws_dax_replication_factor), + '--iam-role-arn', + self.iam_role.GetRoleArn(), + '--subnet-group', + self.subnet_group_name, + '--sse-specification', + 'Enabled=true', + '--region', + self.region, ] _, stderror, retcode = vm_util.IssueCommand(cmd, raise_on_failure=True) @@ -95,8 +116,10 @@ def _Create(self): def _DeleteDependencies(self): """See base class.""" cmd = util.AWS_PREFIX + [ - 'dax', 'delete-subnet-group', '--subnet-group-name', - self.subnet_group_name + 'dax', + 'delete-subnet-group', + '--subnet-group-name', + self.subnet_group_name, ] _, stderror, retcode = vm_util.IssueCommand(cmd, raise_on_failure=False) @@ -108,7 +131,10 @@ def _DeleteDependencies(self): def _Delete(self): """See base class.""" cmd = util.AWS_PREFIX + [ - 'dax', 'delete-cluster', '--cluster-name', self.cluster_name + 'dax', + 'delete-cluster', + '--cluster-name', + self.cluster_name, ] _, stderror, retcode = vm_util.IssueCommand(cmd, raise_on_failure=False) if retcode != 0: @@ -117,7 +143,10 @@ def _Delete(self): def _Exists(self): """See base class.""" cmd = util.AWS_PREFIX + [ - 'dax', 'describe-clusters', '--cluster-names', self.cluster_name + 'dax', + 'describe-clusters', + '--cluster-names', + self.cluster_name, ] _, _, retcode = vm_util.IssueCommand(cmd, raise_on_failure=False) return not retcode @@ -129,7 +158,10 @@ def _IsReady(self): True if the DAX cluster is ready. """ cmd = util.AWS_PREFIX + [ - 'dax', 'describe-clusters', '--cluster-names', self.cluster_name + 'dax', + 'describe-clusters', + '--cluster-names', + self.cluster_name, ] stdout, _, retcode = vm_util.IssueCommand(cmd, raise_on_failure=False) @@ -142,8 +174,9 @@ def _IsReady(self): if status == _DAX_STATUS_AVAILABLE and not self.cluster_endpoint: endpoint = result['Clusters'][0]['ClusterDiscoveryEndpoint'] - self.cluster_endpoint = '{}:{}'.format(endpoint['Address'], - endpoint['Port']) + self.cluster_endpoint = '{}:{}'.format( + endpoint['Address'], endpoint['Port'] + ) return status == _DAX_STATUS_AVAILABLE def _PostCreate(self): @@ -154,9 +187,14 @@ def _PostCreate(self): for security_group in self.vpc.GetSecurityGroups(): if security_group['GroupName'] == 'default': cmd = util.AWS_PREFIX + [ - 'ec2', 'authorize-security-group-ingress', '--group-id', - security_group['GroupId'], '--protocol', 'tcp', '--port', - str(_DAX_TCP_PORT) + 'ec2', + 'authorize-security-group-ingress', + '--group-id', + security_group['GroupId'], + '--protocol', + 'tcp', + '--port', + str(_DAX_TCP_PORT), ] _, stderror, retcode = vm_util.IssueCommand(cmd, raise_on_failure=True) @@ -167,5 +205,6 @@ def GetClusterEndpoint(self): """Returns the DAX cluster's endpoint.""" if not self._IsReady(): raise errors.Benchmarks.PrepareException( - 'GetEndpoint when preparing dax cluster: cluster not ready yet.') + 'GetEndpoint when preparing dax cluster: cluster not ready yet.' + ) return self.cluster_endpoint diff --git a/perfkitbenchmarker/providers/aws/aws_disk.py b/perfkitbenchmarker/providers/aws/aws_disk.py index e36154e2a5..5b800260d0 100644 --- a/perfkitbenchmarker/providers/aws/aws_disk.py +++ b/perfkitbenchmarker/providers/aws/aws_disk.py @@ -39,6 +39,7 @@ class AwsStateRetryableError(Exception): """Error for retrying when an AWS disk is in a transitional state.""" + VOLUME_EXISTS_STATUSES = frozenset(['creating', 'available', 'in-use', 'error']) VOLUME_DELETED_STATUSES = frozenset(['deleting', 'deleted']) VOLUME_KNOWN_STATUSES = VOLUME_EXISTS_STATUSES | VOLUME_DELETED_STATUSES @@ -76,14 +77,8 @@ class AwsStateRetryableError(Exception): disk.MEDIA: disk.SSD, disk.REPLICATION: disk.ZONE, }, - ST1: { - disk.MEDIA: disk.HDD, - disk.REPLICATION: disk.ZONE - }, - SC1: { - disk.MEDIA: disk.HDD, - disk.REPLICATION: disk.ZONE - } + ST1: {disk.MEDIA: disk.HDD, disk.REPLICATION: disk.ZONE}, + SC1: {disk.MEDIA: disk.HDD, disk.REPLICATION: disk.ZONE}, } LOCAL_SSD_METADATA = { @@ -127,7 +122,15 @@ class AwsStateRetryableError(Exception): 't1', ] NON_LOCAL_NVME_TYPES = LOCAL_HDD_PREFIXES + [ - 'c3', 'cr1', 'g2', 'i2', 'm3', 'r3', 'x1', 'x1e'] + 'c3', + 'cr1', + 'g2', + 'i2', + 'm3', + 'r3', + 'x1', + 'x1e', +] # Following dictionary based on # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/InstanceStorage.html @@ -335,8 +338,7 @@ def LocalDriveIsNvme(machine_type): def EbsDriveIsNvme(machine_type): """Check if the machine type uses NVMe driver.""" instance_family = machine_type.split('.')[0].lower() - return (instance_family not in NON_EBS_NVME_TYPES or - 'metal' in machine_type) + return instance_family not in NON_EBS_NVME_TYPES or 'metal' in machine_type AWS = 'AWS' @@ -359,10 +361,10 @@ def _ApplyFlags(cls, config_values, flag_values): Can be overridden by derived classes to add support for specific flags. Args: - config_values: dict mapping config option names to provided values. May - be modified by this function. + config_values: dict mapping config option names to provided values. May be + modified by this function. flag_values: flags.FlagValues. Runtime flags that may override the - provided config values. + provided config values. """ super(AwsDiskSpec, cls)._ApplyFlags(config_values, flag_values) if flag_values['aws_provisioned_iops'].present: @@ -381,16 +383,16 @@ def _GetOptionDecoderConstructions(cls): """ result = super(AwsDiskSpec, cls)._GetOptionDecoderConstructions() result.update({ - 'iops': (option_decoders.IntDecoder, { - 'default': None, - 'none_ok': True - }) + 'iops': ( + option_decoders.IntDecoder, + {'default': None, 'none_ok': True}, + ) }) result.update({ - 'throughput': (option_decoders.IntDecoder, { - 'default': None, - 'none_ok': True - }) + 'throughput': ( + option_decoders.IntDecoder, + {'default': None, 'none_ok': True}, + ) }) return result @@ -398,6 +400,7 @@ def _GetOptionDecoderConstructions(cls): @dataclasses.dataclass class AWSDiskIdentifiers: """Identifiers of an AWS disk assigned by AWS at creation time.""" + volume_id: Optional[str] path: Optional[str] @@ -422,9 +425,11 @@ def __init__(self, disk_spec, zone, machine_type, disk_spec_id=None): if self.disk_type != disk.LOCAL: self.metadata.update(DISK_METADATA.get(self.disk_type, {})) else: - self.metadata.update((LOCAL_HDD_METADATA - if LocalDiskIsHDD(machine_type) - else LOCAL_SSD_METADATA)) + self.metadata.update(( + LOCAL_HDD_METADATA + if LocalDiskIsHDD(machine_type) + else LOCAL_SSD_METADATA + )) if self.iops: self.metadata['iops'] = self.iops if self.throughput: @@ -432,8 +437,9 @@ def __init__(self, disk_spec, zone, machine_type, disk_spec_id=None): self.disk_spec_id = disk_spec_id def AssignDeviceLetter(self, letter_suggestion, nvme_boot_drive_index): - if (LocalDriveIsNvme(self.machine_type) and - EbsDriveIsNvme(self.machine_type)): + if LocalDriveIsNvme(self.machine_type) and EbsDriveIsNvme( + self.machine_type + ): first_device_letter = 'b' local_drive_number = ord(letter_suggestion) - ord(first_device_letter) logging.info('local drive number is: %d', local_drive_number) @@ -452,7 +458,8 @@ def _Create(self): 'create-volume', '--region=%s' % self.region, '--size=%s' % self.disk_size, - '--volume-type=%s' % self.disk_type] + '--volume-type=%s' % self.disk_type, + ] if not util.IsRegion(self.zone): create_cmd.append('--availability-zone=%s' % self.zone) if self.disk_type in [IO1, IO2]: @@ -481,9 +488,13 @@ def _Delete(self): 'ec2', 'delete-volume', '--region=%s' % self.region, - '--volume-id=%s' % self.id] - logging.info('Deleting AWS volume %s. This may fail if the disk is not ' - 'yet detached, but will be retried.', self.id) + '--volume-id=%s' % self.id, + ] + logging.info( + 'Deleting AWS volume %s. This may fail if the disk is not ' + 'yet detached, but will be retried.', + self.id, + ) vm_util.IssueCommand(delete_cmd, raise_on_failure=False) def _Exists(self): @@ -492,7 +503,8 @@ def _Exists(self): 'ec2', 'describe-volumes', '--region=%s' % self.region, - '--filter=Name=volume-id,Values=%s' % self.id] + '--filter=Name=volume-id,Values=%s' % self.id, + ] stdout, _ = util.IssueRetryableCommand(describe_cmd) response = json.loads(stdout) volumes = response['Volumes'] @@ -506,7 +518,8 @@ def _Exists(self): @vm_util.Retry( poll_interval=0.5, log_errors=True, - retryable_exceptions=(AwsStateRetryableError,)) + retryable_exceptions=(AwsStateRetryableError,), + ) def _WaitForAttachedState(self): """Returns if the state of the disk is attached. @@ -529,9 +542,12 @@ def _WaitForAttachedState(self): response = json.loads(stdout) status = response['Volumes'][0]['Attachments'][0]['State'] if status.lower() != 'attached': - logging.info('Disk (id:%s) attaching to ' - 'VM (id:%s) has status %s.', - self.id, self.attached_vm_id, status) + logging.info( + 'Disk (id:%s) attaching to VM (id:%s) has status %s.', + self.id, + self.attached_vm_id, + status, + ) raise AwsStateRetryableError() volume_id = response['Volumes'][0]['Attachments'][0]['VolumeId'] @@ -571,16 +587,21 @@ def Attach(self, vm): self.device_letter = AwsDisk.GenerateDeviceLetter(vm.name) device_name = ( - self.GenerateDeviceNamePrefix(self.disk_type) + self.device_letter) + self.GenerateDeviceNamePrefix(self.disk_type) + self.device_letter + ) attach_cmd = util.AWS_PREFIX + [ 'ec2', 'attach-volume', '--region=%s' % self.region, '--instance-id=%s' % vm.id, '--volume-id=%s' % self.id, - '--device=%s' % device_name] - logging.info('Attaching AWS volume %s. This may fail if the disk is not ' - 'ready, but will be retried.', self.id) + '--device=%s' % device_name, + ] + logging.info( + 'Attaching AWS volume %s. This may fail if the disk is not ' + 'ready, but will be retried.', + self.id, + ) util.IssueRetryableCommand(attach_cmd) volume_id, device_name = self._WaitForAttachedState() vm.LogDeviceByName(device_name, volume_id) @@ -594,12 +615,14 @@ def Detach(self): 'detach-volume', '--region=%s' % self.region, '--instance-id=%s' % self.attached_vm_id, - '--volume-id=%s' % self.id] + '--volume-id=%s' % self.id, + ] util.IssueRetryableCommand(detach_cmd) with self._lock: assert self.attached_vm_id in AwsDisk.available_device_letters_by_vm AwsDisk.available_device_letters_by_vm[self.attached_vm_id].add( - self.device_letter) + self.device_letter + ) self.attached_vm_id = None self.device_letter = None diff --git a/perfkitbenchmarker/providers/aws/aws_dynamodb.py b/perfkitbenchmarker/providers/aws/aws_dynamodb.py index 2731ee9bb7..947481f311 100644 --- a/perfkitbenchmarker/providers/aws/aws_dynamodb.py +++ b/perfkitbenchmarker/providers/aws/aws_dynamodb.py @@ -34,23 +34,32 @@ FLAGS = flags.FLAGS flags.DEFINE_string( - 'aws_dynamodb_primarykey', None, + 'aws_dynamodb_primarykey', + None, 'The primaryKey of dynamodb table. This switches to sortkey if using sort.' 'If testing GSI/LSI, use the range keyname of the index you want to test.' - 'Defaults to primary_key') + 'Defaults to primary_key', +) flags.DEFINE_boolean( - 'aws_dynamodb_use_sort', None, - 'Determine whether to use sort key or not. Defaults to False.') + 'aws_dynamodb_use_sort', + None, + 'Determine whether to use sort key or not. Defaults to False.', +) flags.DEFINE_string( - 'aws_dynamodb_sortkey', None, + 'aws_dynamodb_sortkey', + None, 'The sortkey of dynamodb table. This switches to primarykey if using sort.' 'If testing GSI/LSI, use the primary keyname of the index you want to test.' - 'Defaults to sort_key.') + 'Defaults to sort_key.', +) flags.DEFINE_enum( - 'aws_dynamodb_attributetype', None, ['S', 'N', 'B'], + 'aws_dynamodb_attributetype', + None, + ['S', 'N', 'B'], 'The type of attribute, default to S (String).' 'Alternates are N (Number) and B (Binary).' - 'Defaults to S.') + 'Defaults to S.', +) _BILLING_MODE = flags.DEFINE_enum( 'aws_dynamodb_billing_mode', None, @@ -64,25 +73,39 @@ ' autoscaling policy.' ), ) -flags.DEFINE_integer('aws_dynamodb_read_capacity', None, - 'Set RCU for dynamodb table. Defaults to 25.') -flags.DEFINE_integer('aws_dynamodb_write_capacity', None, - 'Set WCU for dynamodb table. Defaults to 25.') -flags.DEFINE_integer('aws_dynamodb_lsi_count', None, - 'Set amount of Local Secondary Indexes. Only set 0-5.' - 'Defaults to 0.') -flags.DEFINE_integer('aws_dynamodb_gsi_count', None, - 'Set amount of Global Secondary Indexes. Only set 0-5.' - 'Defaults to 0.') +flags.DEFINE_integer( + 'aws_dynamodb_read_capacity', + None, + 'Set RCU for dynamodb table. Defaults to 25.', +) +flags.DEFINE_integer( + 'aws_dynamodb_write_capacity', + None, + 'Set WCU for dynamodb table. Defaults to 25.', +) +flags.DEFINE_integer( + 'aws_dynamodb_lsi_count', + None, + 'Set amount of Local Secondary Indexes. Only set 0-5.Defaults to 0.', +) +flags.DEFINE_integer( + 'aws_dynamodb_gsi_count', + None, + 'Set amount of Global Secondary Indexes. Only set 0-5.Defaults to 0.', +) # For info on autoscaling parameters, see # https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/AutoScaling.html _AUTOSCALING_MAX_WCU = flags.DEFINE_integer( - 'aws_dynamodb_autoscaling_wcu_max', None, 'Maximum WCU for autoscaling.') + 'aws_dynamodb_autoscaling_wcu_max', None, 'Maximum WCU for autoscaling.' +) _AUTOSCALING_MAX_RCU = flags.DEFINE_integer( - 'aws_dynamodb_autoscaling_rcu_max', None, 'Maximum RCU for autoscaling.') + 'aws_dynamodb_autoscaling_rcu_max', None, 'Maximum RCU for autoscaling.' +) _AUTOSCALING_CPU_TARGET = flags.DEFINE_integer( - 'aws_dynamodb_autoscaling_target', None, - 'The target utilization percent for autoscaling.') + 'aws_dynamodb_autoscaling_target', + None, + 'The target utilization percent for autoscaling.', +) # Throughput constants _FREE_TIER_RCU = 25 @@ -148,8 +171,10 @@ def _ValidateConfig(cls, config_values) -> None: if 'lsi_count' in config_values: if not -1 < config_values['lsi_count'] < 6: raise errors.Config.InvalidValue('lsi_count must be from 0-5') - if (not config_values.get('use_sort', False) and - config_values['lsi_count'] != 0): + if ( + not config_values.get('use_sort', False) + and config_values['lsi_count'] != 0 + ): raise errors.Config.InvalidValue('lsi_count requires use_sort=True') if not -1 < config_values.get('gsi_count', 0) < 6: raise errors.Config.InvalidValue('gsi_count must be from 0-5') @@ -204,6 +229,7 @@ def __repr__(self) -> str: class AwsDynamoDBInstance(non_relational_db.BaseNonRelationalDb): """Class for working with DynamoDB.""" + SERVICE_TYPE = non_relational_db.DYNAMODB def __init__( @@ -232,7 +258,8 @@ def __init__( self.rcu = rcu or _FREE_TIER_RCU self.wcu = wcu or _FREE_TIER_WCU self.throughput = ( - f'ReadCapacityUnits={self.rcu},WriteCapacityUnits={self.wcu}') + f'ReadCapacityUnits={self.rcu},WriteCapacityUnits={self.wcu}' + ) self.primary_key = primary_key or 'primary_key' self.sort_key = sort_key or 'sort_key' @@ -271,23 +298,19 @@ def _CreateLocalSecondaryIndex(self) -> List[str]: for lsi in range(0, self.lsi_count): lsi_item = json.dumps({ 'IndexName': f'lsiidx{str(lsi)}', - 'KeySchema': [{ - 'AttributeName': self.primary_key, - 'KeyType': 'HASH' - }, { - 'AttributeName': f'lattr{str(lsi)}', - 'KeyType': 'RANGE' - }], - 'Projection': { - 'ProjectionType': 'KEYS_ONLY' - } + 'KeySchema': [ + {'AttributeName': self.primary_key, 'KeyType': 'HASH'}, + {'AttributeName': f'lattr{str(lsi)}', 'KeyType': 'RANGE'}, + ], + 'Projection': {'ProjectionType': 'KEYS_ONLY'}, }) lsi_entry.append(lsi_item) attr_list.append( json.dumps({ 'AttributeName': f'lattr{str(lsi)}', - 'AttributeType': self.attribute_type - })) + 'AttributeType': self.attribute_type, + }) + ) lsi_items.append('[' + ','.join(lsi_entry) + ']') lsi_items.append(','.join(attr_list)) return lsi_items @@ -300,32 +323,29 @@ def _CreateGlobalSecondaryIndex(self) -> List[str]: for gsi in range(0, self.gsi_count): gsi_item = json.dumps({ 'IndexName': f'gsiidx{str(gsi)}', - 'KeySchema': [{ - 'AttributeName': f'gsikey{str(gsi)}', - 'KeyType': 'HASH' - }, { - 'AttributeName': f'gattr{str(gsi)}', - 'KeyType': 'RANGE' - }], - 'Projection': { - 'ProjectionType': 'KEYS_ONLY' - }, + 'KeySchema': [ + {'AttributeName': f'gsikey{str(gsi)}', 'KeyType': 'HASH'}, + {'AttributeName': f'gattr{str(gsi)}', 'KeyType': 'RANGE'}, + ], + 'Projection': {'ProjectionType': 'KEYS_ONLY'}, 'ProvisionedThroughput': { 'ReadCapacityUnits': 5, - 'WriteCapacityUnits': 5 - } + 'WriteCapacityUnits': 5, + }, }) gsi_entry.append(gsi_item) attr_list.append( json.dumps({ 'AttributeName': f'gattr{str(gsi)}', - 'AttributeType': self.attribute_type - })) + 'AttributeType': self.attribute_type, + }) + ) attr_list.append( json.dumps({ 'AttributeName': f'gsikey{str(gsi)}', - 'AttributeType': self.attribute_type - })) + 'AttributeType': self.attribute_type, + }) + ) gsi_items.append('[' + ','.join(gsi_entry) + ']') gsi_items.append(','.join(attr_list)) return gsi_items @@ -346,14 +366,13 @@ def _PrimaryKeyJson(self) -> str: def _PrimaryAttrsJson(self) -> str: return json.dumps({ 'AttributeName': self.primary_key, - 'AttributeType': self.attribute_type + 'AttributeType': self.attribute_type, }) def _SortAttrsJson(self) -> str: - return json.dumps({ - 'AttributeName': self.sort_key, - 'AttributeType': self.attribute_type - }) + return json.dumps( + {'AttributeName': self.sort_key, 'AttributeType': self.attribute_type} + ) def _SortKeyJson(self) -> str: return json.dumps({'AttributeName': self.sort_key, 'KeyType': 'RANGE'}) @@ -373,25 +392,26 @@ def _Create(self) -> None: if self.billing_mode == _PROVISIONED: cmd.extend(['--provisioned-throughput', self.throughput]) if self.lsi_count > 0 and self.use_sort: - self._SetAttrDefnArgs(cmd, [ - self._PrimaryAttrsJson(), - self._SortAttrsJson(), self.lsi_indexes[1] - ]) + self._SetAttrDefnArgs( + cmd, + [ + self._PrimaryAttrsJson(), + self._SortAttrsJson(), + self.lsi_indexes[1], + ], + ) cmd.append('--local-secondary-indexes') cmd.append(self.lsi_indexes[0]) - self._SetKeySchemaArgs( - cmd, [self._PrimaryKeyJson(), - self._SortKeyJson()]) + self._SetKeySchemaArgs(cmd, [self._PrimaryKeyJson(), self._SortKeyJson()]) elif self.use_sort: self._SetAttrDefnArgs( - cmd, [self._PrimaryAttrsJson(), - self._SortAttrsJson()]) - self._SetKeySchemaArgs( - cmd, [self._PrimaryKeyJson(), - self._SortKeyJson()]) + cmd, [self._PrimaryAttrsJson(), self._SortAttrsJson()] + ) + self._SetKeySchemaArgs(cmd, [self._PrimaryKeyJson(), self._SortKeyJson()]) if self.gsi_count > 0: self._SetAttrDefnArgs( - cmd, cmd[10].strip('[]').split(',') + [self.gsi_indexes[1]]) + cmd, cmd[10].strip('[]').split(',') + [self.gsi_indexes[1]] + ) cmd.append('--global-secondary-indexes') cmd.append(self.gsi_indexes[0]) _, stderror, retcode = vm_util.IssueCommand(cmd, raise_on_failure=False) @@ -400,9 +420,11 @@ def _Create(self) -> None: def IsAutoscaling(self) -> bool: """Returns whether this instance uses (or should use) an autoscaling policy.""" - return all([_AUTOSCALING_CPU_TARGET.value, - _AUTOSCALING_MAX_RCU.value, - _AUTOSCALING_MAX_WCU.value]) + return all([ + _AUTOSCALING_CPU_TARGET.value, + _AUTOSCALING_MAX_RCU.value, + _AUTOSCALING_MAX_WCU.value, + ]) def IsServerless(self) -> bool: """Returns whether this instance uses autoscaling or on-demand.""" @@ -411,35 +433,49 @@ def IsServerless(self) -> bool: def _PostCreate(self): if not self.IsAutoscaling(): return - self._CreateAutoscalingPolicy(_RCU_SCALABLE_DIMENSION, - _RCU_SCALING_METRIC, - self.rcu, _AUTOSCALING_MAX_RCU.value) - self._CreateAutoscalingPolicy(_WCU_SCALABLE_DIMENSION, - _WCU_SCALING_METRIC, - self.wcu, _AUTOSCALING_MAX_WCU.value) - - def _CreateScalableTarget(self, - scalable_dimension: str, - min_capacity: int, - max_capacity: int) -> None: + self._CreateAutoscalingPolicy( + _RCU_SCALABLE_DIMENSION, + _RCU_SCALING_METRIC, + self.rcu, + _AUTOSCALING_MAX_RCU.value, + ) + self._CreateAutoscalingPolicy( + _WCU_SCALABLE_DIMENSION, + _WCU_SCALING_METRIC, + self.wcu, + _AUTOSCALING_MAX_WCU.value, + ) + + def _CreateScalableTarget( + self, scalable_dimension: str, min_capacity: int, max_capacity: int + ) -> None: """Creates a scalable target which controls QPS limits for the table.""" logging.info('Registering scalable target.') cmd = util.AWS_PREFIX + [ 'application-autoscaling', 'register-scalable-target', - '--service-namespace', 'dynamodb', - '--resource-id', self._resource_id, - '--scalable-dimension', scalable_dimension, - '--min-capacity', str(min_capacity), - '--max-capacity', str(max_capacity), - '--region', self.region] + '--service-namespace', + 'dynamodb', + '--resource-id', + self._resource_id, + '--scalable-dimension', + scalable_dimension, + '--min-capacity', + str(min_capacity), + '--max-capacity', + str(max_capacity), + '--region', + self.region, + ] vm_util.IssueCommand(cmd, raise_on_failure=True) - def _CreateAutoscalingPolicy(self, - scalable_dimension: str, - scaling_metric: str, - min_capacity: int, - max_capacity: int) -> None: + def _CreateAutoscalingPolicy( + self, + scalable_dimension: str, + scaling_metric: str, + min_capacity: int, + max_capacity: int, + ) -> None: """Creates an autoscaling policy for the table.""" self._CreateScalableTarget(scalable_dimension, min_capacity, max_capacity) logging.info('Creating autoscaling policy.') @@ -449,19 +485,25 @@ def _CreateAutoscalingPolicy(self, }, 'ScaleOutCooldown': _DEFAULT_SCALE_OUT_COOLDOWN, 'ScaleInCooldown': _DEFAULT_SCALE_IN_COOLDOWN, - 'TargetValue': _AUTOSCALING_CPU_TARGET.value + 'TargetValue': _AUTOSCALING_CPU_TARGET.value, } cmd = util.AWS_PREFIX + [ 'application-autoscaling', 'put-scaling-policy', - '--service-namespace', 'dynamodb', - '--resource-id', self._resource_id, - '--scalable-dimension', scalable_dimension, - '--policy-name', f'{FLAGS.run_uri}-policy', - '--policy-type', 'TargetTrackingScaling', + '--service-namespace', + 'dynamodb', + '--resource-id', + self._resource_id, + '--scalable-dimension', + scalable_dimension, + '--policy-name', + f'{FLAGS.run_uri}-policy', + '--policy-type', + 'TargetTrackingScaling', '--target-tracking-scaling-policy-configuration', json.dumps(scaling_policy), - '--region', self.region + '--region', + self.region, ] vm_util.IssueCommand(cmd, raise_on_failure=True) @@ -470,9 +512,12 @@ def _HasAutoscalingPolicies(self) -> bool: cmd = util.AWS_PREFIX + [ 'application-autoscaling', 'describe-scaling-policies', - '--service-namespace', 'dynamodb', - '--resource-id', self._resource_id, - '--region', self.region + '--service-namespace', + 'dynamodb', + '--resource-id', + self._resource_id, + '--region', + self.region, ] stdout, _, _ = vm_util.IssueCommand(cmd, raise_on_failure=True) result = json.loads(stdout).get('ScalingPolicies', []) @@ -483,8 +528,11 @@ def _Delete(self) -> None: cmd = util.AWS_PREFIX + [ 'dynamodb', 'delete-table', - '--region', self.region, - '--table-name', self.table_name] + '--region', + self.region, + '--table-name', + self.table_name, + ] logging.info('Attempting deletion: ') vm_util.IssueCommand(cmd, raise_on_failure=False) @@ -494,8 +542,11 @@ def _IsReady(self) -> bool: cmd = util.AWS_PREFIX + [ 'dynamodb', 'describe-table', - '--region', self.region, - '--table-name', self.table_name] + '--region', + self.region, + '--table-name', + self.table_name, + ] stdout, _, _ = vm_util.IssueCommand(cmd) result = json.loads(stdout) return result['Table']['TableStatus'] == 'ACTIVE' @@ -515,8 +566,11 @@ def _DescribeTable(self) -> Dict[Any, Any]: cmd = util.AWS_PREFIX + [ 'dynamodb', 'describe-table', - '--region', self.region, - '--table-name', self.table_name] + '--region', + self.region, + '--table-name', + self.table_name, + ] stdout, stderr, retcode = vm_util.IssueCommand(cmd, raise_on_failure=False) if retcode != 0: logging.info('Could not find table %s, %s', self.table_name, stderr) @@ -545,28 +599,28 @@ def GetResourceMetadata(self) -> Dict[str, Any]: } if self.IsAutoscaling(): metadata.update({ - 'aws_dynamodb_autoscaling': - True, - 'aws_dynamodb_autoscaling_rcu_min_capacity': - self.rcu, - 'aws_dynamodb_autoscaling_rcu_max_capacity': - _AUTOSCALING_MAX_RCU.value or self.rcu, - 'aws_dynamodb_autoscaling_wcu_min_capacity': - self.wcu, - 'aws_dynamodb_autoscaling_wcu_max_capacity': - _AUTOSCALING_MAX_WCU.value or self.wcu, - 'aws_dynamodb_autoscaling_scale_in_cooldown': - _DEFAULT_SCALE_IN_COOLDOWN, - 'aws_dynamodb_autoscaling_scale_out_cooldown': - _DEFAULT_SCALE_OUT_COOLDOWN, - 'aws_dynamodb_autoscaling_target': - _DEFAULT_AUTOSCALING_TARGET + 'aws_dynamodb_autoscaling': True, + 'aws_dynamodb_autoscaling_rcu_min_capacity': self.rcu, + 'aws_dynamodb_autoscaling_rcu_max_capacity': ( + _AUTOSCALING_MAX_RCU.value or self.rcu + ), + 'aws_dynamodb_autoscaling_wcu_min_capacity': self.wcu, + 'aws_dynamodb_autoscaling_wcu_max_capacity': ( + _AUTOSCALING_MAX_WCU.value or self.wcu + ), + 'aws_dynamodb_autoscaling_scale_in_cooldown': ( + _DEFAULT_SCALE_IN_COOLDOWN + ), + 'aws_dynamodb_autoscaling_scale_out_cooldown': ( + _DEFAULT_SCALE_OUT_COOLDOWN + ), + 'aws_dynamodb_autoscaling_target': _DEFAULT_AUTOSCALING_TARGET, }) return metadata - def SetThroughput(self, - rcu: Optional[int] = None, - wcu: Optional[int] = None) -> None: + def SetThroughput( + self, rcu: Optional[int] = None, wcu: Optional[int] = None + ) -> None: """Updates the table's rcu and wcu.""" if not rcu: rcu = self.rcu @@ -577,14 +631,21 @@ def SetThroughput(self, logging.info('Table already has requested rcu %s and wcu %s', rcu, wcu) return cmd = util.AWS_PREFIX + [ - 'dynamodb', 'update-table', - '--table-name', self.table_name, - '--region', self.region, + 'dynamodb', + 'update-table', + '--table-name', + self.table_name, + '--region', + self.region, '--provisioned-throughput', f'ReadCapacityUnits={rcu},WriteCapacityUnits={wcu}', ] - logging.info('Setting %s table provisioned throughput to %s rcu and %s wcu', - self.table_name, rcu, wcu) + logging.info( + 'Setting %s table provisioned throughput to %s rcu and %s wcu', + self.table_name, + rcu, + wcu, + ) util.IssueRetryableCommand(cmd) while not self._IsReady(): continue @@ -594,8 +655,11 @@ def _GetThroughput(self) -> Tuple[int, int]: output = self._DescribeTable()['ProvisionedThroughput'] return output['ReadCapacityUnits'], output['WriteCapacityUnits'] - @vm_util.Retry(poll_interval=1, max_retries=3, - retryable_exceptions=(errors.Resource.CreationError)) + @vm_util.Retry( + poll_interval=1, + max_retries=3, + retryable_exceptions=(errors.Resource.CreationError), + ) def _GetTagResourceCommand(self, tags: Sequence[str]) -> Sequence[str]: """Returns the tag-resource command with the provided tags. @@ -613,11 +677,21 @@ def _GetTagResourceCommand(self, tags: Sequence[str]) -> Sequence[str]: """ if not self._Exists(): raise errors.Resource.CreationError( - f'Cannot get resource arn of non-existent instance {self.table_name}') - return util.AWS_PREFIX + [ - 'dynamodb', 'tag-resource', '--resource-arn', self.resource_arn, - '--region', self.region, '--tags' - ] + list(tags) + f'Cannot get resource arn of non-existent instance {self.table_name}' + ) + return ( + util.AWS_PREFIX + + [ + 'dynamodb', + 'tag-resource', + '--resource-arn', + self.resource_arn, + '--region', + self.region, + '--tags', + ] + + list(tags) + ) def UpdateWithDefaultTags(self) -> None: """Adds default tags to the table.""" @@ -630,8 +704,11 @@ def UpdateTimeout(self, timeout_minutes: int) -> None: """Updates the timeout associated with the table.""" tags = util.MakeFormattedDefaultTags(timeout_minutes) cmd = self._GetTagResourceCommand(tags) - logging.info('Updating timeout tags on table %s with timeout minutes %s', - self.table_name, timeout_minutes) + logging.info( + 'Updating timeout tags on table %s with timeout minutes %s', + self.table_name, + timeout_minutes, + ) util.IssueRetryableCommand(cmd) def _Freeze(self) -> None: @@ -642,12 +719,12 @@ def _Freeze(self) -> None: https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ProvisionedThroughput.html. """ if self._HasAutoscalingPolicies(): - self._CreateScalableTarget(_RCU_SCALABLE_DIMENSION, - _FREE_TIER_RCU, - _AUTOSCALING_MAX_RCU.value) - self._CreateScalableTarget(_WCU_SCALABLE_DIMENSION, - _FREE_TIER_WCU, - _AUTOSCALING_MAX_WCU.value) + self._CreateScalableTarget( + _RCU_SCALABLE_DIMENSION, _FREE_TIER_RCU, _AUTOSCALING_MAX_RCU.value + ) + self._CreateScalableTarget( + _WCU_SCALABLE_DIMENSION, _FREE_TIER_WCU, _AUTOSCALING_MAX_WCU.value + ) return if self.billing_mode == _ON_DEMAND: return @@ -669,12 +746,12 @@ def _Restore(self) -> None: self.wcu = FLAGS.aws_dynamodb_write_capacity if self._HasAutoscalingPolicies(): # If the flags are not provided, this should default to previous levels. - self._CreateScalableTarget(_RCU_SCALABLE_DIMENSION, - self.rcu, - _AUTOSCALING_MAX_RCU.value) - self._CreateScalableTarget(_WCU_SCALABLE_DIMENSION, - self.wcu, - _AUTOSCALING_MAX_WCU.value) + self._CreateScalableTarget( + _RCU_SCALABLE_DIMENSION, self.rcu, _AUTOSCALING_MAX_RCU.value + ) + self._CreateScalableTarget( + _WCU_SCALABLE_DIMENSION, self.wcu, _AUTOSCALING_MAX_WCU.value + ) return if self.billing_mode == _ON_DEMAND: return diff --git a/perfkitbenchmarker/providers/aws/aws_elasticache_redis.py b/perfkitbenchmarker/providers/aws/aws_elasticache_redis.py index bee2ec9e7f..5f2625a0da 100644 --- a/perfkitbenchmarker/providers/aws/aws_elasticache_redis.py +++ b/perfkitbenchmarker/providers/aws/aws_elasticache_redis.py @@ -79,8 +79,7 @@ def CheckPrerequisites(benchmark_config): else: if ( not FLAGS.elasticache_failover_zone - or FLAGS.elasticache_failover_zone[:-1] - != FLAGS.cloud_redis_region + or FLAGS.elasticache_failover_zone[:-1] != FLAGS.cloud_redis_region ): raise errors.Config.InvalidValue( 'Invalid failover zone. A failover zone in %s must be specified. ' diff --git a/perfkitbenchmarker/providers/aws/aws_elasticached_memcached.py b/perfkitbenchmarker/providers/aws/aws_elasticached_memcached.py index ac54c2e6e0..04eb6c5d14 100644 --- a/perfkitbenchmarker/providers/aws/aws_elasticached_memcached.py +++ b/perfkitbenchmarker/providers/aws/aws_elasticached_memcached.py @@ -46,8 +46,10 @@ def __init__(self, spec): @staticmethod def CheckPrerequisites(benchmark_config): - if (FLAGS.managed_memory_store_version and - FLAGS.managed_memory_store_version not in MEMCACHED_VERSIONS): + if ( + FLAGS.managed_memory_store_version + and FLAGS.managed_memory_store_version not in MEMCACHED_VERSIONS + ): raise errors.Config.InvalidValue('Invalid Memcached version.') def GetResourceMetadata(self): @@ -65,31 +67,56 @@ def GetResourceMetadata(self): def _CreateDependencies(self): """Create the subnet dependencies.""" subnet_id = self.spec.vms[0].network.subnet.id - cmd = ['aws', 'elasticache', 'create-cache-subnet-group', - '--region', self.region, - '--cache-subnet-group-name', self.subnet_group_name, - '--cache-subnet-group-description', '"memcached benchmark subnet"', - '--subnet-ids', subnet_id] + cmd = [ + 'aws', + 'elasticache', + 'create-cache-subnet-group', + '--region', + self.region, + '--cache-subnet-group-name', + self.subnet_group_name, + '--cache-subnet-group-description', + '"memcached benchmark subnet"', + '--subnet-ids', + subnet_id, + ] vm_util.IssueCommand(cmd) def _DeleteDependencies(self): """Delete the subnet dependencies.""" - cmd = ['aws', 'elasticache', 'delete-cache-subnet-group', - '--region', self.region, - '--cache-subnet-group-name', self.subnet_group_name] + cmd = [ + 'aws', + 'elasticache', + 'delete-cache-subnet-group', + '--region', + self.region, + '--cache-subnet-group-name', + self.subnet_group_name, + ] vm_util.IssueCommand(cmd, raise_on_failure=False) def _Create(self): """Creates the cache cluster.""" - cmd = ['aws', 'elasticache', 'create-cache-cluster', - '--engine', 'memcached', - '--region', self.region, - '--cache-cluster-id', self.name, - '--preferred-availability-zone', self.zone, - '--num-cache-nodes', str(managed_memory_store.MEMCACHED_NODE_COUNT), - '--cache-node-type', self.node_type, - '--cache-subnet-group-name', self.subnet_group_name] + cmd = [ + 'aws', + 'elasticache', + 'create-cache-cluster', + '--engine', + 'memcached', + '--region', + self.region, + '--cache-cluster-id', + self.name, + '--preferred-availability-zone', + self.zone, + '--num-cache-nodes', + str(managed_memory_store.MEMCACHED_NODE_COUNT), + '--cache-node-type', + self.node_type, + '--cache-subnet-group-name', + self.subnet_group_name, + ] if self.version: cmd += ['--engine-version', self.version] @@ -100,9 +127,15 @@ def _Create(self): def _Delete(self): """Deletes the cache cluster.""" - cmd = ['aws', 'elasticache', 'delete-cache-cluster', - '--region', self.region, - '--cache-cluster-id', self.name] + cmd = [ + 'aws', + 'elasticache', + 'delete-cache-cluster', + '--region', + self.region, + '--cache-cluster-id', + self.name, + ] vm_util.IssueCommand(cmd, raise_on_failure=False) def _IsDeleting(self): @@ -122,7 +155,10 @@ def _Exists(self): """Returns true if the cluster exists and is not being deleted.""" cluster_info = self._DescribeInstance() return cluster_info.get('CacheClusterStatus', '') not in [ - '', 'deleting', 'create-failed'] + '', + 'deleting', + 'create-failed', + ] def _DescribeInstance(self): """Calls describe on cluster. @@ -130,9 +166,15 @@ def _DescribeInstance(self): Returns: dict mapping string cluster_info property key to value. """ - cmd = ['aws', 'elasticache', 'describe-cache-clusters', - '--region', self.region, - '--cache-cluster-id', self.name] + cmd = [ + 'aws', + 'elasticache', + 'describe-cache-clusters', + '--region', + self.region, + '--cache-cluster-id', + self.name, + ] stdout, stderr, retcode = vm_util.IssueCommand(cmd, raise_on_failure=False) if retcode != 0: logging.info('Could not find cluster %s, %s', self.name, stderr) @@ -153,7 +195,8 @@ def _PopulateEndpoint(self): cluster_info = self._DescribeInstance() if not cluster_info: raise errors.Resource.RetryableGetError( - 'Failed to retrieve information on {0}.'.format(self.name)) + 'Failed to retrieve information on {0}.'.format(self.name) + ) endpoint = cluster_info['ConfigurationEndpoint'] self._ip = endpoint['Address'] diff --git a/perfkitbenchmarker/providers/aws/aws_emr.py b/perfkitbenchmarker/providers/aws/aws_emr.py index 25695c0559..37dea67571 100644 --- a/perfkitbenchmarker/providers/aws/aws_emr.py +++ b/perfkitbenchmarker/providers/aws/aws_emr.py @@ -67,13 +67,19 @@ def __init__(self, cmd_prefix, group_id): self.cmd_prefix = cmd_prefix def _Delete(self): - cmd = self.cmd_prefix + ['ec2', 'delete-security-group', - '--group-id=' + self.group_id] + cmd = self.cmd_prefix + [ + 'ec2', + 'delete-security-group', + '--group-id=' + self.group_id, + ] vm_util.IssueCommand(cmd, raise_on_failure=False) def _Exists(self): - cmd = self.cmd_prefix + ['ec2', 'describe-security-groups', - '--group-id=' + self.group_id] + cmd = self.cmd_prefix + [ + 'ec2', + 'describe-security-groups', + '--group-id=' + self.group_id, + ] _, _, retcode = vm_util.IssueCommand(cmd, raise_on_failure=False) # if the security group doesn't exist, the describe command gives an error. return retcode == 0 @@ -91,8 +97,7 @@ class AwsEMR(spark_service.BaseSparkService): project: Enclosing project for the cluster. cmd_prefix: emr prefix, including region network: network to use; set if needed by machine type - bucket_to_delete: bucket name to delete when cluster is - terminated. + bucket_to_delete: bucket name to delete when cluster is terminated. """ CLOUD = provider_info.AWS @@ -111,11 +116,13 @@ def __init__(self, spark_service_spec): self.cmd_prefix += ['--region', region] # Certain machine types require subnets. - if (self.spec.static_cluster_id is None and - (worker_machine_type[0:2] in NEEDS_SUBNET or - leader_machine_type[0:2] in NEEDS_SUBNET)): + if self.spec.static_cluster_id is None and ( + worker_machine_type[0:2] in NEEDS_SUBNET + or leader_machine_type[0:2] in NEEDS_SUBNET + ): vm_config = virtual_machine.GetVmClass(self.CLOUD, os_types.DEFAULT)( - self.spec.master_group.vm_spec) + self.spec.master_group.vm_spec + ) self.network = aws_network.AwsNetwork.GetNetwork(vm_config) else: self.network = None @@ -138,35 +145,49 @@ def _Create(self): instance_groups = [] for group_type, group_spec in [ ('CORE', self.spec.worker_group), - ('MASTER', self.spec.master_group)]: - instance_properties = {'InstanceCount': group_spec.vm_count, - 'InstanceGroupType': group_type, - 'InstanceType': group_spec.vm_spec.machine_type, - 'Name': group_type + ' group'} + ('MASTER', self.spec.master_group), + ]: + instance_properties = { + 'InstanceCount': group_spec.vm_count, + 'InstanceGroupType': group_type, + 'InstanceType': group_spec.vm_spec.machine_type, + 'Name': group_type + ' group', + } if group_spec.disk_spec: # Make sure nothing we are ignoring is included in the disk spec assert group_spec.disk_spec.device_path is None assert group_spec.disk_spec.disk_number is None assert group_spec.disk_spec.mount_point is None assert group_spec.disk_spec.iops is None - ebs_configuration = {'EbsBlockDeviceConfigs': [ - {'VolumeSpecification': - {'SizeInGB': group_spec.disk_spec.disk_size, - 'VolumeType': group_spec.disk_spec.disk_type}, - 'VolumesPerInstance': - group_spec.disk_spec.num_striped_disks}]} + ebs_configuration = { + 'EbsBlockDeviceConfigs': [{ + 'VolumeSpecification': { + 'SizeInGB': group_spec.disk_spec.disk_size, + 'VolumeType': group_spec.disk_spec.disk_type, + }, + 'VolumesPerInstance': group_spec.disk_spec.num_striped_disks, + }] + } instance_properties.update({'EbsConfiguration': ebs_configuration}) instance_groups.append(instance_properties) # we need to store the cluster id. - cmd = self.cmd_prefix + ['emr', 'create-cluster', '--name', name, - '--release-label', RELEASE_LABEL, - '--use-default-roles', - '--instance-groups', - json.dumps(instance_groups), - '--application', 'Name=Spark', - 'Name=Hadoop', - '--log-uri', logs_bucket] + cmd = self.cmd_prefix + [ + 'emr', + 'create-cluster', + '--name', + name, + '--release-label', + RELEASE_LABEL, + '--use-default-roles', + '--instance-groups', + json.dumps(instance_groups), + '--application', + 'Name=Spark', + 'Name=Hadoop', + '--log-uri', + logs_bucket, + ] if self.network: cmd += ['--ec2-attributes', 'SubnetId=' + self.network.subnet.id] stdout, _, _ = vm_util.IssueCommand(cmd) @@ -178,16 +199,24 @@ def _Create(self): def _AddTag(self, key, value): """Add the key value pair as a tag to the emr cluster.""" - cmd = self.cmd_prefix + ['emr', 'add-tags', - '--resource-id', self.cluster_id, - '--tag', - '{}={}'.format(key, value)] + cmd = self.cmd_prefix + [ + 'emr', + 'add-tags', + '--resource-id', + self.cluster_id, + '--tag', + '{}={}'.format(key, value), + ] vm_util.IssueCommand(cmd) def _DeleteSecurityGroups(self): """Delete the security groups associated with this cluster.""" - cmd = self.cmd_prefix + ['emr', 'describe-cluster', - '--cluster-id', self.cluster_id] + cmd = self.cmd_prefix + [ + 'emr', + 'describe-cluster', + '--cluster-id', + self.cluster_id, + ] stdout, _, _ = vm_util.IssueCommand(cmd) cluster_desc = json.loads(stdout) sec_object = cluster_desc['Cluster']['Ec2InstanceAttributes'] @@ -202,11 +231,14 @@ def _DeleteSecurityGroups(self): # remove all references to the manager group from the worker group. for proto, port in [('tcp', '0-65535'), ('udp', '0-65535'), ('icmp', '-1')]: for group1, group2 in [(worker_sg, manager_sg), (manager_sg, worker_sg)]: - cmd = self.cmd_prefix + ['ec2', 'revoke-security-group-ingress', - '--group-id=' + group1, - '--source-group=' + group2, - '--protocol=' + proto, - '--port=' + port] + cmd = self.cmd_prefix + [ + 'ec2', + 'revoke-security-group-ingress', + '--group-id=' + group1, + '--source-group=' + group2, + '--protocol=' + proto, + '--port=' + port, + ] vm_util.IssueCommand(cmd) # Now we need to delete the manager, then the worker. @@ -217,22 +249,34 @@ def _DeleteSecurityGroups(self): def _Delete(self): """Deletes the cluster.""" - cmd = self.cmd_prefix + ['emr', 'terminate-clusters', '--cluster-ids', - self.cluster_id] + cmd = self.cmd_prefix + [ + 'emr', + 'terminate-clusters', + '--cluster-ids', + self.cluster_id, + ] vm_util.IssueCommand(cmd, raise_on_failure=False) def _DeleteDependencies(self): if self.network: self._DeleteSecurityGroups() if self.bucket_to_delete: - bucket_del_cmd = self.cmd_prefix + ['s3', 'rb', '--force', - self.bucket_to_delete] + bucket_del_cmd = self.cmd_prefix + [ + 's3', + 'rb', + '--force', + self.bucket_to_delete, + ] vm_util.IssueCommand(bucket_del_cmd) def _Exists(self): """Check to see whether the cluster exists.""" - cmd = self.cmd_prefix + ['emr', 'describe-cluster', - '--cluster-id', self.cluster_id] + cmd = self.cmd_prefix + [ + 'emr', + 'describe-cluster', + '--cluster-id', + self.cluster_id, + ] stdout, _, retcode = vm_util.IssueCommand(cmd, raise_on_failure=False) if retcode != 0: return False @@ -244,26 +288,36 @@ def _Exists(self): def _IsReady(self): """Check to see if the cluster is ready.""" - cmd = self.cmd_prefix + ['emr', 'describe-cluster', '--cluster-id', - self.cluster_id] + cmd = self.cmd_prefix + [ + 'emr', + 'describe-cluster', + '--cluster-id', + self.cluster_id, + ] stdout, _, _ = vm_util.IssueCommand(cmd) result = json.loads(stdout) if result['Cluster']['Status']['State'] == 'TERMINATED_WITH_ERRORS': reason = result['Cluster']['Status']['StateChangeReason']['Message'] message = reason if reason.startswith('Subnet is required'): - message = ('Cluster creation failed because this machine type requires ' - 'a subnet. To ensure PKB creates a subnet for this machine ' - 'type, update the NEEDS_SUBNET variable of ' - 'providers/aws/aws_emr.py to contain prefix of this machine ' - 'type. Raw AWS message={0}'.format(reason)) + message = ( + 'Cluster creation failed because this machine type requires ' + 'a subnet. To ensure PKB creates a subnet for this machine ' + 'type, update the NEEDS_SUBNET variable of ' + 'providers/aws/aws_emr.py to contain prefix of this machine ' + 'type. Raw AWS message={0}'.format(reason) + ) raise Exception(message) return result['Cluster']['Status']['State'] == READY_STATE def _GetLogBase(self): """Gets the base uri for the logs.""" - cmd = self.cmd_prefix + ['emr', 'describe-cluster', '--cluster-id', - self.cluster_id] + cmd = self.cmd_prefix + [ + 'emr', + 'describe-cluster', + '--cluster-id', + self.cluster_id, + ] stdout, _, _ = vm_util.IssueCommand(cmd) result = json.loads(stdout) if 'LogUri' in result['Cluster']: @@ -286,13 +340,20 @@ def _IsStepDone(self, step_id): Args: step_id: The step id to query. + Returns: A dictionary describing the step if the step the step is complete, None otherwise. """ - cmd = self.cmd_prefix + ['emr', 'describe-step', '--cluster-id', - self.cluster_id, '--step-id', step_id] + cmd = self.cmd_prefix + [ + 'emr', + 'describe-step', + '--cluster-id', + self.cluster_id, + '--step-id', + step_id, + ] stdout, _, _ = vm_util.IssueCommand(cmd) result = json.loads(stdout) state = result['Step']['Status']['State'] @@ -319,9 +380,15 @@ def _MakeSparkStep(self, jarfile, classname, job_arguments): step_list = ['Type=Spark', 'Args=' + arg_string] return step_list - def SubmitJob(self, jarfile, classname, job_poll_interval=JOB_WAIT_SLEEP, - job_arguments=None, job_stdout_file=None, - job_type=spark_service.SPARK_JOB_TYPE): + def SubmitJob( + self, + jarfile, + classname, + job_poll_interval=JOB_WAIT_SLEEP, + job_arguments=None, + job_stdout_file=None, + job_type=spark_service.SPARK_JOB_TYPE, + ): """Submit the job. Submit the job and wait for it to complete. If job_stdout_file is not @@ -331,12 +398,11 @@ def SubmitJob(self, jarfile, classname, job_poll_interval=JOB_WAIT_SLEEP, Args: jarfile: Jar file containing the class to submit. classname: Name of the class. - job_poll_interval: Submit job will poll until the job is done; this is - the time between checks. + job_poll_interval: Submit job will poll until the job is done; this is the + time between checks. job_arguments: Arguments to pass to the job. - job_stdout_file: Name of a file in which to put the job's standard out. - If there is data here already, it will be overwritten. - + job_stdout_file: Name of a file in which to put the job's standard out. If + there is data here already, it will be overwritten. """ @vm_util.Retry(poll_interval=job_poll_interval, fuzz=0) @@ -344,8 +410,11 @@ def WaitForFile(filename): if not self._CheckForFile(filename): raise Exception('File not found yet') - @vm_util.Retry(timeout=FLAGS.aws_emr_job_wait_time, - poll_interval=job_poll_interval, fuzz=0) + @vm_util.Retry( + timeout=FLAGS.aws_emr_job_wait_time, + poll_interval=job_poll_interval, + fuzz=0, + ) def WaitForStep(step_id): result = self._IsStepDone(step_id) if result is None: @@ -359,8 +428,14 @@ def WaitForStep(step_id): else: raise Exception('Job type %s unsupported for EMR' % job_type) step_string = ','.join(step_list) - cmd = self.cmd_prefix + ['emr', 'add-steps', '--cluster-id', - self.cluster_id, '--steps', step_string] + cmd = self.cmd_prefix + [ + 'emr', + 'add-steps', + '--cluster-id', + self.cluster_id, + '--steps', + step_string, + ] stdout, _, _ = vm_util.IssueCommand(cmd) result = json.loads(stdout) step_id = result['StepIds'][0] @@ -380,14 +455,16 @@ def WaitForStep(step_id): if job_stdout_file: log_base = self._GetLogBase() if log_base is None: - logging.warning('SubmitJob requested output, but EMR cluster was not ' - 'created with logging') + logging.warning( + 'SubmitJob requested output, but EMR cluster was not ' + 'created with logging' + ) return metrics # log_base ends in a slash. - s3_stdout = '{0}{1}/steps/{2}/stdout.gz'.format(log_base, - self.cluster_id, - step_id) + s3_stdout = '{0}{1}/steps/{2}/stdout.gz'.format( + log_base, self.cluster_id, step_id + ) WaitForFile(s3_stdout) dest_file = '{0}.gz'.format(job_stdout_file) cp_cmd = ['aws', 's3', 'cp', s3_stdout, dest_file] diff --git a/perfkitbenchmarker/providers/aws/aws_glue_crawler.py b/perfkitbenchmarker/providers/aws/aws_glue_crawler.py index 2fb2e4d47b..9ccb3ec0b4 100644 --- a/perfkitbenchmarker/providers/aws/aws_glue_crawler.py +++ b/perfkitbenchmarker/providers/aws/aws_glue_crawler.py @@ -51,12 +51,14 @@ def _Create(self) -> None: database_input = { 'Name': self.db_name, 'Description': '\n'.join( - f'{k}={v}' for k, v in util.MakeDefaultTags().items()), + f'{k}={v}' for k, v in util.MakeDefaultTags().items() + ), } cmd = util.AWS_PREFIX + [ 'glue', 'create-database', - '--database-input', json.dumps(database_input), + '--database-input', + json.dumps(database_input), f'--region={self.region}', ] vm_util.IssueCommand(cmd) @@ -69,13 +71,18 @@ def _Create(self) -> None: cmd = util.AWS_PREFIX + [ 'glue', 'create-crawler', - '--name', self.crawler_name, - '--role', self.role, - '--database-name', self.db_name, - '--targets', json.dumps(targets), - '--region', self.region, - '--tags', ','.join( - f'{k}={v}' for k, v in util.MakeDefaultTags().items()), + '--name', + self.crawler_name, + '--role', + self.role, + '--database-name', + self.db_name, + '--targets', + json.dumps(targets), + '--region', + self.region, + '--tags', + ','.join(f'{k}={v}' for k, v in util.MakeDefaultTags().items()), ] vm_util.IssueCommand(cmd) @@ -85,10 +92,13 @@ def _Exists(self) -> bool: def _IsReady(self, raise_on_crawl_failure=False) -> bool: stdout, _, _ = self._GetCrawler() data = json.loads(stdout) - if (data['Crawler'].get('LastCrawl', {}).get('Status') == self.FAILED and - raise_on_crawl_failure): + if ( + data['Crawler'].get('LastCrawl', {}).get('Status') == self.FAILED + and raise_on_crawl_failure + ): raise CrawlFailedError( - data['Crawler'].get('LastCrawl', {}).get('ErrorMessage', '')) + data['Crawler'].get('LastCrawl', {}).get('ErrorMessage', '') + ) return data['Crawler']['State'] == self.READY def _Delete(self) -> None: @@ -96,8 +106,10 @@ def _Delete(self) -> None: cmd = util.AWS_PREFIX + [ 'glue', 'delete-database', - '--name', self.db_name, - '--region', self.region, + '--name', + self.db_name, + '--region', + self.region, ] vm_util.IssueCommand(cmd, raise_on_failure=False) @@ -105,8 +117,10 @@ def _Delete(self) -> None: cmd = util.AWS_PREFIX + [ 'glue', 'delete-crawler', - '--name', self.crawler_name, - '--region', self.region, + '--name', + self.crawler_name, + '--region', + self.region, ] vm_util.IssueCommand(cmd, raise_on_failure=False) @@ -123,21 +137,27 @@ def DiscoverData(self) -> float: cmd = util.AWS_PREFIX + [ 'glue', 'start-crawler', - '--name', self.crawler_name, - '--region', self.region, + '--name', + self.crawler_name, + '--region', + self.region, ] vm_util.IssueCommand(cmd) self._WaitUntilCrawlerReady() cmd = util.AWS_PREFIX + [ 'glue', 'get-crawler-metrics', - '--crawler-name-list', self.crawler_name, - '--region', self.region, + '--crawler-name-list', + self.crawler_name, + '--region', + self.region, ] output, _, _ = vm_util.IssueCommand(cmd) data = json.loads(output) - assert (isinstance(data['CrawlerMetricsList'], list) and - len(data['CrawlerMetricsList']) == 1) + assert ( + isinstance(data['CrawlerMetricsList'], list) + and len(data['CrawlerMetricsList']) == 1 + ) return data['CrawlerMetricsList'][0]['LastRuntimeSeconds'] def GetMetadata(self) -> Dict[str, Any]: @@ -154,11 +174,13 @@ def GetMetadata(self) -> Dict[str, Any]: timeout=CRAWL_TIMEOUT, poll_interval=CRAWL_POLL_INTERVAL, fuzz=0, - retryable_exceptions=CrawlNotCompletedError,) + retryable_exceptions=CrawlNotCompletedError, + ) def _WaitUntilCrawlerReady(self): if not self._IsReady(raise_on_crawl_failure=True): raise CrawlNotCompletedError( - f'Crawler {self.crawler_name} still running.') + f'Crawler {self.crawler_name} still running.' + ) def _DbExists(self) -> Optional[bool]: """Whether the database exists or not. @@ -171,8 +193,10 @@ def _DbExists(self) -> Optional[bool]: cmd = util.AWS_PREFIX + [ 'glue', 'get-database', - '--name', self.db_name, - '--region', self.region, + '--name', + self.db_name, + '--region', + self.region, ] _, stderr, retcode = vm_util.IssueCommand(cmd, raise_on_failure=False) if not retcode: @@ -197,7 +221,9 @@ def _GetCrawler(self, raise_on_failure=True) -> Tuple[str, str, int]: cmd = util.AWS_PREFIX + [ 'glue', 'get-crawler', - '--name', self.crawler_name, - '--region', self.region, + '--name', + self.crawler_name, + '--region', + self.region, ] return vm_util.IssueCommand(cmd, raise_on_failure=raise_on_failure) diff --git a/perfkitbenchmarker/providers/aws/aws_iam_role.py b/perfkitbenchmarker/providers/aws/aws_iam_role.py index c0b2150aef..fceb11321a 100644 --- a/perfkitbenchmarker/providers/aws/aws_iam_role.py +++ b/perfkitbenchmarker/providers/aws/aws_iam_role.py @@ -67,14 +67,16 @@ class AwsIamRole(resource.BaseResource): """Class representing an AWS IAM role.""" - def __init__(self, - account, - role_name, - policy_name, - service, - action, - resource_arn, - policy_version=None): + def __init__( + self, + account, + role_name, + policy_name, + service, + action, + resource_arn, + policy_version=None, + ): super(AwsIamRole, self).__init__() self.account = account self.role_name = role_name @@ -84,9 +86,11 @@ def __init__(self, self.resource_arn = resource_arn self.policy_version = policy_version or _POLICY_VERSION self.role_arn = _ROLE_ARN_TEMPLATE.format( - account=self.account, role_name=self.role_name) + account=self.account, role_name=self.role_name + ) self.policy_arn = _POLICY_ARN_TEMPLATE.format( - account=self.account, policy_name=self.policy_name) + account=self.account, policy_name=self.policy_name + ) def _Create(self): """See base class.""" @@ -94,12 +98,17 @@ def _Create(self): with open(_TRUST_RELATIONSHIP_FILE, 'w+') as relationship_file: relationship_file.write( _TRUST_RELATIONSHIP_TEMPLATE.format( - version=self.policy_version, service=self.service)) + version=self.policy_version, service=self.service + ) + ) cmd = util.AWS_PREFIX + [ - 'iam', 'create-role', '--role-name', self.role_name, + 'iam', + 'create-role', + '--role-name', + self.role_name, '--assume-role-policy-document', - 'file://{}'.format(_TRUST_RELATIONSHIP_FILE) + 'file://{}'.format(_TRUST_RELATIONSHIP_FILE), ] _, stderror, retcode = vm_util.IssueCommand(cmd, raise_on_failure=True) @@ -112,10 +121,16 @@ def _Create(self): _ROLE_POLICY_TEMPLATE.format( version=self.policy_version, action=self.action, - resource_arn=self.resource_arn)) + resource_arn=self.resource_arn, + ) + ) cmd = util.AWS_PREFIX + [ - 'iam', 'create-policy', '--policy-name', 'PolicyFor' + self.role_name, - '--policy-document', 'file://{}'.format(_ROLE_POLICY_FILE) + 'iam', + 'create-policy', + '--policy-name', + 'PolicyFor' + self.role_name, + '--policy-document', + 'file://{}'.format(_ROLE_POLICY_FILE), ] _, stderror, retcode = vm_util.IssueCommand(cmd, raise_on_failure=True) @@ -123,8 +138,12 @@ def _Create(self): logging.warning('Failed to create policy! %s', stderror) cmd = util.AWS_PREFIX + [ - 'iam', 'attach-role-policy', '--role-name', self.role_name, - '--policy-arn', self.policy_arn + 'iam', + 'attach-role-policy', + '--role-name', + self.role_name, + '--policy-arn', + self.policy_arn, ] _, stderror, retcode = vm_util.IssueCommand(cmd, raise_on_failure=True) @@ -139,8 +158,12 @@ def _Create(self): def _Delete(self): """See base class.""" cmd = util.AWS_PREFIX + [ - 'iam', 'detach-role-policy', '--role-name', self.role_name, - '--policy-arn', self.policy_arn + 'iam', + 'detach-role-policy', + '--role-name', + self.role_name, + '--policy-arn', + self.policy_arn, ] _, stderror, retcode = vm_util.IssueCommand(cmd, raise_on_failure=False) @@ -148,7 +171,10 @@ def _Delete(self): logging.warning('Failed to delete role policy! %s', stderror) cmd = util.AWS_PREFIX + [ - 'iam', 'delete-policy', '--policy-arn', self.policy_arn + 'iam', + 'delete-policy', + '--policy-arn', + self.policy_arn, ] _, stderror, retcode = vm_util.IssueCommand(cmd, raise_on_failure=False) @@ -156,7 +182,10 @@ def _Delete(self): logging.warning('Failed to delete policy! %s', stderror) cmd = util.AWS_PREFIX + [ - 'iam', 'delete-role', '--role-name', self.role_name + 'iam', + 'delete-role', + '--role-name', + self.role_name, ] _, stderror, retcode = vm_util.IssueCommand(cmd, raise_on_failure=False) @@ -176,7 +205,10 @@ def _RoleExists(self): def _PolicyExists(self): """Returns true if the IAM policy used by the role exists.""" cmd = util.AWS_PREFIX + [ - 'iam', 'get-policy', '--policy-arn', self.policy_arn + 'iam', + 'get-policy', + '--policy-arn', + self.policy_arn, ] stdout, _, retcode = vm_util.IssueCommand(cmd, raise_on_failure=False) return retcode == 0 and stdout and json.loads(stdout)['Policy'] diff --git a/perfkitbenchmarker/providers/aws/aws_load_balancer.py b/perfkitbenchmarker/providers/aws/aws_load_balancer.py index cdf33a2e83..ddcf8442c2 100644 --- a/perfkitbenchmarker/providers/aws/aws_load_balancer.py +++ b/perfkitbenchmarker/providers/aws/aws_load_balancer.py @@ -44,13 +44,20 @@ def __init__(self, vpc, port): def _Create(self): """Create the target group.""" create_cmd = util.AWS_PREFIX + [ - '--region', self.region, - 'elbv2', 'create-target-group', - '--target-type', 'ip', - '--name', self.name, - '--protocol', self.protocol, - '--port', str(self.port), - '--vpc-id', self.vpc_id + '--region', + self.region, + 'elbv2', + 'create-target-group', + '--target-type', + 'ip', + '--name', + self.name, + '--protocol', + self.protocol, + '--port', + str(self.port), + '--vpc-id', + self.vpc_id, ] stdout, _, _ = vm_util.IssueCommand(create_cmd) response = json.loads(stdout) @@ -61,9 +68,12 @@ def _Delete(self): if self.arn is None: return delete_cmd = util.AWS_PREFIX + [ - '--region', self.region, - 'elbv2', 'delete-target-group', - '--target-group-arn', self.arn + '--region', + self.region, + 'elbv2', + 'delete-target-group', + '--target-group-arn', + self.arn, ] vm_util.IssueCommand(delete_cmd, raise_on_failure=False) @@ -87,12 +97,21 @@ def __init__(self, subnets): def _Create(self): """Create the load balancer.""" - create_cmd = util.AWS_PREFIX + [ - '--region', self.region, - 'elbv2', 'create-load-balancer', - '--name', self.name, - '--type', self.type, - '--tags'] + util.MakeFormattedDefaultTags() + create_cmd = ( + util.AWS_PREFIX + + [ + '--region', + self.region, + 'elbv2', + 'create-load-balancer', + '--name', + self.name, + '--type', + self.type, + '--tags', + ] + + util.MakeFormattedDefaultTags() + ) # Add --subnets argument to the command. create_cmd.append('--subnets') create_cmd.extend(self.subnet_ids) @@ -107,9 +126,12 @@ def _Delete(self): if self.arn is None: return delete_cmd = util.AWS_PREFIX + [ - '--region', self.region, - 'elbv2', 'delete-load-balancer', - '--load-balancer-arn', self.arn + '--region', + self.region, + 'elbv2', + 'delete-load-balancer', + '--load-balancer-arn', + self.arn, ] vm_util.IssueCommand(delete_cmd, raise_on_failure=False) @@ -127,21 +149,24 @@ def __init__(self, load_balancer, target_group, port): def _GetDefaultActions(self): """Returns a JSON representation of the default actions for the listener.""" - actions = [{ - 'Type': 'forward', - 'TargetGroupArn': self.target_group_arn - }] + actions = [{'Type': 'forward', 'TargetGroupArn': self.target_group_arn}] return json.dumps(actions) def _Create(self): """Create the listener.""" create_cmd = util.AWS_PREFIX + [ - '--region', self.region, - 'elbv2', 'create-listener', - '--load-balancer-arn', self.load_balancer_arn, - '--protocol', self.protocol, - '--port', str(self.port), - '--default-actions', self._GetDefaultActions() + '--region', + self.region, + 'elbv2', + 'create-listener', + '--load-balancer-arn', + self.load_balancer_arn, + '--protocol', + self.protocol, + '--port', + str(self.port), + '--default-actions', + self._GetDefaultActions(), ] vm_util.IssueCommand(create_cmd) diff --git a/perfkitbenchmarker/providers/aws/aws_logs.py b/perfkitbenchmarker/providers/aws/aws_logs.py index cf9c8cdd4c..1f2171c419 100644 --- a/perfkitbenchmarker/providers/aws/aws_logs.py +++ b/perfkitbenchmarker/providers/aws/aws_logs.py @@ -32,42 +32,57 @@ def __init__(self, region, name, retention_in_days=7): def _Create(self): """Create the log group.""" create_cmd = util.AWS_PREFIX + [ - '--region', self.region, - 'logs', 'create-log-group', - '--log-group-name', self.name + '--region', + self.region, + 'logs', + 'create-log-group', + '--log-group-name', + self.name, ] vm_util.IssueCommand(create_cmd) def _Delete(self): """Delete the log group.""" delete_cmd = util.AWS_PREFIX + [ - '--region', self.region, - 'logs', 'delete-log-group', - '--log-group-name', self.name + '--region', + self.region, + 'logs', + 'delete-log-group', + '--log-group-name', + self.name, ] vm_util.IssueCommand(delete_cmd, raise_on_failure=False) def Exists(self): """Returns True if the log group exists.""" describe_cmd = util.AWS_PREFIX + [ - '--region', self.region, - 'logs', 'describe-log-groups', - '--log-group-name-prefix', self.name, - '--no-paginate' + '--region', + self.region, + 'logs', + 'describe-log-groups', + '--log-group-name-prefix', + self.name, + '--no-paginate', ] stdout, _, _ = vm_util.IssueCommand(describe_cmd) log_groups = json.loads(stdout)['logGroups'] - group = next((group for group in log_groups - if group['logGroupName'] == self.name), None) + group = next( + (group for group in log_groups if group['logGroupName'] == self.name), + None, + ) return bool(group) def _PostCreate(self): """Set the retention policy.""" put_cmd = util.AWS_PREFIX + [ - '--region', self.region, - 'logs', 'put-retention-policy', - '--log-group-name', self.name, - '--retention-in-days', str(self.retention_in_days) + '--region', + self.region, + 'logs', + 'put-retention-policy', + '--log-group-name', + self.name, + '--retention-in-days', + str(self.retention_in_days), ] vm_util.IssueCommand(put_cmd) @@ -75,11 +90,15 @@ def _PostCreate(self): def GetLogs(region, stream_name, group_name, token=None): """Fetches the JSON formatted log stream starting at the token.""" get_cmd = util.AWS_PREFIX + [ - '--region', region, - 'logs', 'get-log-events', + '--region', + region, + 'logs', + 'get-log-events', '--start-from-head', - '--log-group-name', group_name, - '--log-stream-name', stream_name, + '--log-group-name', + group_name, + '--log-stream-name', + stream_name, ] if token: get_cmd.extend(['--next-token', token]) diff --git a/perfkitbenchmarker/providers/aws/aws_network.py b/perfkitbenchmarker/providers/aws/aws_network.py index 287fca031e..0b0e4bb54e 100644 --- a/perfkitbenchmarker/providers/aws/aws_network.py +++ b/perfkitbenchmarker/providers/aws/aws_network.py @@ -38,24 +38,32 @@ from perfkitbenchmarker.providers.aws import util _AWS_VPC = flags.DEFINE_string( - 'aws_vpc', None, - 'The static AWS VPC id to use. If unset, creates a new VPC.') + 'aws_vpc', + None, + 'The static AWS VPC id to use. If unset, creates a new VPC.', +) _AWS_SUBNET = flags.DEFINE_string( - 'aws_subnet', None, + 'aws_subnet', + None, 'The static AWS subnet id to use. Set value to "default" to use ' - 'default subnet. If unset, creates a new subnet.') + 'default subnet. If unset, creates a new subnet.', +) AWS_ENI_COUNT = flags.DEFINE_integer( - 'aws_eni_count', 1, - 'The number of ENIs per instance.') + 'aws_eni_count', 1, 'The number of ENIs per instance.' +) AWS_NETWORK_CARD_COUNT = flags.DEFINE_integer( - 'aws_network_card_count', 1, - 'The number of network cards per instance.') + 'aws_network_card_count', 1, 'The number of network cards per instance.' +) flags.DEFINE_bool('aws_efa', False, 'Whether to use an Elastic Fiber Adapter.') -flags.DEFINE_string('aws_efa_version', '1.12.1', - 'Version of AWS EFA to use (must also pass in --aws_efa).') +flags.DEFINE_string( + 'aws_efa_version', + '1.12.1', + 'Version of AWS EFA to use (must also pass in --aws_efa).', +) flags.DEFINE_integer('aws_efa_count', 1, 'The number of EFAs per instance.') -flags.DEFINE_multi_enum('aws_endpoint', [], ['s3'], - 'List of AWS endpoints to create') +flags.DEFINE_multi_enum( + 'aws_endpoint', [], ['s3'], 'List of AWS endpoints to create' +) FLAGS = flags.FLAGS @@ -127,9 +135,9 @@ def AllowIcmp(self, vm): '--group-id=%s' % vm.group_id, '--protocol=icmp', '--port=-1', - '--cidr=%s' % source] - util.IssueRetryableCommand( - authorize_cmd) + '--cidr=%s' % source, + ] + util.IssueRetryableCommand(authorize_cmd) self.firewall_icmp_set.add(entry) def AllowPort(self, vm, start_port, end_port=None, source_range=None): @@ -145,15 +153,13 @@ def AllowPort(self, vm, start_port, end_port=None, source_range=None): """ if vm.is_static or vm.network.is_static: return - self.AllowPortInSecurityGroup(vm.region, vm.group_id, start_port, end_port, - source_range) - - def AllowPortInSecurityGroup(self, - region, - security_group, - start_port, - end_port=None, - source_range=None): + self.AllowPortInSecurityGroup( + vm.region, vm.group_id, start_port, end_port, source_range + ) + + def AllowPortInSecurityGroup( + self, region, security_group, start_port, end_port=None, source_range=None + ): """Opens a port on the firewall for a security group. Args: @@ -238,14 +244,17 @@ def _Create(self): 'ec2', 'create-vpc', '--region=%s' % self.region, - '--cidr-block=%s' % self.cidr] + '--cidr-block=%s' % self.cidr, + ] stdout, stderr, retcode = vm_util.IssueCommand( - create_cmd, raise_on_failure=False) + create_cmd, raise_on_failure=False + ) if 'VpcLimitExceeded' in stderr: raise errors.Benchmarks.QuotaFailure(stderr) if retcode: raise errors.Resource.CreationError( - 'Failed to create Vpc: %s return code: %s' % (retcode, stderr)) + 'Failed to create Vpc: %s return code: %s' % (retcode, stderr) + ) response = json.loads(stdout) self.id = response['Vpc']['VpcId'] @@ -261,11 +270,15 @@ def _SetSecurityGroupId(self): """Looks up the VPC default security group.""" groups = self.GetSecurityGroups('default') if len(groups) != 1: - raise ValueError('Expected one security group, got {} in {}'.format( - len(groups), groups)) + raise ValueError( + 'Expected one security group, got {} in {}'.format( + len(groups), groups + ) + ) self.default_security_group_id = groups[0]['GroupId'] - logging.info('Default security group ID: %s', - self.default_security_group_id) + logging.info( + 'Default security group ID: %s', self.default_security_group_id + ) if FLAGS.aws_efa: self._AllowSelfOutBound() @@ -273,9 +286,11 @@ def GetSecurityGroups(self, group_name=None): cmd = util.AWS_PREFIX + [ 'ec2', 'describe-security-groups', - '--region', self.region, + '--region', + self.region, '--filters', - 'Name=vpc-id,Values=' + self.id] + 'Name=vpc-id,Values=' + self.id, + ] if group_name: cmd.append('Name=group-name,Values={}'.format(group_name)) stdout, _, _ = vm_util.IssueCommand(cmd) @@ -287,7 +302,8 @@ def _Exists(self): 'ec2', 'describe-vpcs', '--region=%s' % self.region, - '--filter=Name=vpc-id,Values=%s' % self.id] + '--filter=Name=vpc-id,Values=%s' % self.id, + ] stdout, _ = util.IssueRetryableCommand(describe_cmd) response = json.loads(stdout) vpcs = response['Vpcs'] @@ -306,9 +322,11 @@ def _EnableDnsHostnames(self): 'ec2', 'modify-vpc-attribute', '--region=%s' % self.region, - '--vpc-id', self.id, + '--vpc-id', + self.id, '--enable-dns-hostnames', - '{ "Value": true }'] + '{ "Value": true }', + ] util.IssueRetryableCommand(enable_hostnames_command) @@ -326,7 +344,8 @@ def _Delete(self): 'ec2', 'delete-vpc', '--region=%s' % self.region, - '--vpc-id=%s' % self.id] + '--vpc-id=%s' % self.id, + ] vm_util.IssueCommand(delete_cmd, raise_on_failure=False) def NextSubnetCidrBlock(self): @@ -343,10 +362,12 @@ def NextSubnetCidrBlock(self): """ with self._subnet_index_lock: if self._subnet_index >= (1 << 8) - 1: - raise ValueError('Exceeded subnet limit ({0}).'.format( - self._subnet_index)) - cidr = network.GetCidrBlock(self.regional_network_index, - self._subnet_index) + raise ValueError( + 'Exceeded subnet limit ({0}).'.format(self._subnet_index) + ) + cidr = network.GetCidrBlock( + self.regional_network_index, self._subnet_index + ) self._subnet_index += 1 return cidr @@ -357,9 +378,16 @@ def _AllowSelfOutBound(self): Details: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa-start.html """ cmd = util.AWS_PREFIX + [ - 'ec2', 'authorize-security-group-egress', - '--region', self.region, '--group-id', self.default_security_group_id, - '--protocol', 'all', '--source-group', self.default_security_group_id + 'ec2', + 'authorize-security-group-egress', + '--region', + self.region, + '--group-id', + self.default_security_group_id, + '--protocol', + 'all', + '--source-group', + self.default_security_group_id, ] try: vm_util.IssueCommand(cmd) @@ -375,11 +403,12 @@ def AllowVpcPeerInBound(self, peer_vpc): peer_vpc: AwsVpc. Peer vpc to allow inbound traffic from. """ cmd = util.AWS_PREFIX + [ - 'ec2', 'authorize-security-group-ingress', + 'ec2', + 'authorize-security-group-ingress', '--region=%s' % self.region, '--group-id=%s' % self.default_security_group_id, '--protocol=%s' % 'all', - '--cidr=%s' % peer_vpc.cidr + '--cidr=%s' % peer_vpc.cidr, ] vm_util.IssueRetryableCommand(cmd) @@ -403,7 +432,8 @@ def _Create(self): 'create-subnet', '--region=%s' % self.region, '--vpc-id=%s' % self.vpc_id, - '--cidr-block=%s' % self.cidr_block] + '--cidr-block=%s' % self.cidr_block, + ] if not util.IsRegion(self.zone): create_cmd.append('--availability-zone=%s' % self.zone) @@ -414,14 +444,17 @@ def _Create(self): def _Delete(self): """Deletes the subnet.""" - logging.info('Deleting subnet %s. This may fail if all instances in the ' - 'subnet have not completed termination, but will be retried.', - self.id) + logging.info( + 'Deleting subnet %s. This may fail if all instances in the ' + 'subnet have not completed termination, but will be retried.', + self.id, + ) delete_cmd = util.AWS_PREFIX + [ 'ec2', 'delete-subnet', '--region=%s' % self.region, - '--subnet-id=%s' % self.id] + '--subnet-id=%s' % self.id, + ] vm_util.IssueCommand(delete_cmd, raise_on_failure=False) def _Exists(self): @@ -438,9 +471,10 @@ def GetDict(self): AssertionError: If there is more than one subnet. """ describe_cmd = util.AWS_PREFIX + [ - 'ec2', 'describe-subnets', + 'ec2', + 'describe-subnets', '--region=%s' % self.region, - '--filter=Name=vpc-id,Values=%s' % self.vpc_id + '--filter=Name=vpc-id,Values=%s' % self.vpc_id, ] if self.id: describe_cmd.append('--filter=Name=subnet-id,Values=%s' % self.id) @@ -471,7 +505,8 @@ def _Create(self): create_cmd = util.AWS_PREFIX + [ 'ec2', 'create-internet-gateway', - '--region=%s' % self.region] + '--region=%s' % self.region, + ] stdout, _, _ = vm_util.IssueCommand(create_cmd) response = json.loads(stdout) self.id = response['InternetGateway']['InternetGatewayId'] @@ -483,7 +518,8 @@ def _Delete(self): 'ec2', 'delete-internet-gateway', '--region=%s' % self.region, - '--internet-gateway-id=%s' % self.id] + '--internet-gateway-id=%s' % self.id, + ] vm_util.IssueCommand(delete_cmd, raise_on_failure=False) def _Exists(self): @@ -505,16 +541,18 @@ def GetDict(self): '--region=%s' % self.region, ] if self.id: - describe_cmd.append('--filter=Name=internet-gateway-id,Values=%s' % - self.id) + describe_cmd.append( + '--filter=Name=internet-gateway-id,Values=%s' % self.id + ) elif self.vpc_id: # Only query with self.vpc_id if the self.id is NOT set -- after calling # Detach() this object will set still have a vpc_id but will be filtered # out in a query if using attachment.vpc-id. # Using self.vpc_id instead of self.attached as the init phase always # sets it to False. - describe_cmd.append('--filter=Name=attachment.vpc-id,Values=%s' % - self.vpc_id) + describe_cmd.append( + '--filter=Name=attachment.vpc-id,Values=%s' % self.vpc_id + ) else: raise errors.Error('Must have a VPC id or a gateway id') stdout, _ = util.IssueRetryableCommand(describe_cmd) @@ -532,7 +570,8 @@ def Attach(self, vpc_id): 'attach-internet-gateway', '--region=%s' % self.region, '--internet-gateway-id=%s' % self.id, - '--vpc-id=%s' % self.vpc_id] + '--vpc-id=%s' % self.vpc_id, + ] util.IssueRetryableCommand(attach_cmd) self.attached = True @@ -542,8 +581,10 @@ def Detach(self): def _suppress_failure(stdout, stderr, retcode): """Suppresses Detach failure when internet gateway is in a bad state.""" del stdout # unused - if retcode and ('InvalidInternetGatewayID.NotFound' in stderr or - 'Gateway.NotAttached' in stderr): + if retcode and ( + 'InvalidInternetGatewayID.NotFound' in stderr + or 'Gateway.NotAttached' in stderr + ): return True return False @@ -553,7 +594,8 @@ def _suppress_failure(stdout, stderr, retcode): 'detach-internet-gateway', '--region=%s' % self.region, '--internet-gateway-id=%s' % self.id, - '--vpc-id=%s' % self.vpc_id] + '--vpc-id=%s' % self.vpc_id, + ] util.IssueRetryableCommand(detach_cmd, suppress_failure=_suppress_failure) self.attached = False @@ -592,7 +634,8 @@ def GetDict(self): 'ec2', 'describe-route-tables', '--region=%s' % self.region, - '--filters=Name=vpc-id,Values=%s' % self.vpc_id] + '--filters=Name=vpc-id,Values=%s' % self.vpc_id, + ] stdout, _ = util.IssueRetryableCommand(describe_cmd) return json.loads(stdout)['RouteTables'] @@ -617,7 +660,8 @@ def CreateRoute(self, internet_gateway_id): '--region=%s' % self.region, '--route-table-id=%s' % self.id, '--gateway-id=%s' % internet_gateway_id, - '--destination-cidr-block=0.0.0.0/0'] + '--destination-cidr-block=0.0.0.0/0', + ] util.IssueRetryableCommand(create_cmd) def CreateVpcPeeringRoute(self, vpc_peering_id, destination_cidr): @@ -628,7 +672,8 @@ def CreateVpcPeeringRoute(self, vpc_peering_id, destination_cidr): '--region=%s' % self.region, '--route-table-id=%s' % self.id, '--vpc-peering-connection-id=%s' % vpc_peering_id, - '--destination-cidr-block=%s' % destination_cidr] + '--destination-cidr-block=%s' % destination_cidr, + ] util.IssueRetryableCommand(create_cmd) @@ -671,10 +716,12 @@ def __init__(self, region, vpc_id=None): # Each regional network needs unique cidr_block for VPC peering. with _AwsRegionalNetwork._regional_network_lock: - self.vpc = AwsVpc(self.region, vpc_id, - _AwsRegionalNetwork._regional_network_count) + self.vpc = AwsVpc( + self.region, vpc_id, _AwsRegionalNetwork._regional_network_count + ) self.cidr_block = network.GetCidrBlock( - _AwsRegionalNetwork._regional_network_count) + _AwsRegionalNetwork._regional_network_count + ) _AwsRegionalNetwork._regional_network_count += 1 @classmethod @@ -692,8 +739,9 @@ def GetForRegion(cls, region, vpc_id=None): """ benchmark_spec = context.GetThreadBenchmarkSpec() if benchmark_spec is None: - raise errors.Error('GetNetwork called in a thread without a ' - 'BenchmarkSpec.') + raise errors.Error( + 'GetNetwork called in a thread without a BenchmarkSpec.' + ) key = cls.CLOUD, REGION, region # Because this method is only called from the AwsNetwork constructor, which # is only called from AwsNetwork.GetNetwork, we already hold the @@ -749,14 +797,19 @@ def __init__(self, zone, vpc_id=None, subnet_id=None, machine_type=None): super(AwsNetworkSpec, self).__init__(zone) self.machine_type = machine_type if vpc_id or subnet_id: - logging.info('Confirming vpc (%s) and subnet (%s) selections', vpc_id, - subnet_id) + logging.info( + 'Confirming vpc (%s) and subnet (%s) selections', vpc_id, subnet_id + ) my_subnet = AwsSubnet(self.zone, vpc_id, subnet_id=subnet_id).GetDict() self.vpc_id = my_subnet['VpcId'] self.subnet_id = my_subnet['SubnetId'] self.cidr_block = my_subnet['CidrBlock'] - logging.info('Using vpc %s subnet %s cidr %s', self.vpc_id, - self.subnet_id, self.cidr_block) + logging.info( + 'Using vpc %s subnet %s cidr %s', + self.vpc_id, + self.subnet_id, + self.cidr_block, + ) else: self.vpc_id = None self.subnet_id = None @@ -779,21 +832,28 @@ def _get_default_vpc_id(region: str) -> str: created. """ vpc_cmd = util.AWS_PREFIX + [ - 'ec2', 'describe-vpcs', - '--region', region, - '--filters', 'Name=isDefault,Values=true' + 'ec2', + 'describe-vpcs', + '--region', + region, + '--filters', + 'Name=isDefault,Values=true', ] stdout, _ = vm_util.IssueRetryableCommand(vpc_cmd) vpcs = json.loads(stdout)['Vpcs'] if vpcs: return vpcs[0]['VpcId'] create_cmd = util.AWS_PREFIX + [ - 'ec2', 'create-default-vpc', '--region', region, + 'ec2', + 'create-default-vpc', + '--region', + region, ] stdout, _, ret = vm_util.IssueCommand(create_cmd, raise_on_failure=False) if ret: raise errors.Benchmarks.UnsupportedConfigError( - f'AWS default VPC does not exist for region {region}.') + f'AWS default VPC does not exist for region {region}.' + ) return json.loads(stdout)['Vpc']['VpcId'] @@ -814,24 +874,31 @@ def _get_default_subnet_id(zone: str) -> str: """ region = util.GetRegionFromZone(zone) subnet_cmd = util.AWS_PREFIX + [ - 'ec2', 'describe-subnets', - '--region', region, '--filter', + 'ec2', + 'describe-subnets', + '--region', + region, + '--filter', f'Name=availabilityZone,Values={zone}', - 'Name=defaultForAz,Values=true' + 'Name=defaultForAz,Values=true', ] stdout, _ = vm_util.IssueRetryableCommand(subnet_cmd) subnets = json.loads(stdout)['Subnets'] if subnets: return subnets[0]['SubnetId'] create_cmd = util.AWS_PREFIX + [ - 'ec2', 'create-default-subnet', - '--region', region, - '--availability-zone', zone + 'ec2', + 'create-default-subnet', + '--region', + region, + '--availability-zone', + zone, ] stdout, _, ret = vm_util.IssueCommand(create_cmd, raise_on_failure=False) if ret: raise errors.Benchmarks.UnsupportedConfigError( - f'AWS default subnet does not exist for zone {zone}.') + f'AWS default subnet does not exist for zone {zone}.' + ) return json.loads(stdout)['Subnet']['SubnetId'] @@ -865,23 +932,30 @@ def __init__(self, spec): super(AwsNetwork, self).__init__(spec) self.region = util.GetRegionFromZone(spec.zone) self.regional_network = _AwsRegionalNetwork.GetForRegion( - self.region, spec.vpc_id) + self.region, spec.vpc_id + ) self.subnet = None self.vpc_peering = None # Placement Group no_placement_group = ( - not FLAGS.placement_group_style or - FLAGS.placement_group_style == placement_group.PLACEMENT_GROUP_NONE) - has_optional_pg = (FLAGS.placement_group_style == - placement_group.PLACEMENT_GROUP_CLOSEST_SUPPORTED) + not FLAGS.placement_group_style + or FLAGS.placement_group_style == placement_group.PLACEMENT_GROUP_NONE + ) + has_optional_pg = ( + FLAGS.placement_group_style + == placement_group.PLACEMENT_GROUP_CLOSEST_SUPPORTED + ) if no_placement_group: self.placement_group = None elif has_optional_pg and not _is_placement_group_compatible( - spec.machine_type): + spec.machine_type + ): logging.warning( 'machine type %s does not support placement groups. ' - 'Placement group style set to none.', spec.machine_type) + 'Placement group style set to none.', + spec.machine_type, + ) self.placement_group = None elif has_optional_pg and len(set(FLAGS.zone)) > 1: logging.warning( @@ -892,16 +966,20 @@ def __init__(self, spec): elif not _is_placement_group_compatible(spec.machine_type): raise errors.Benchmarks.UnsupportedConfigError( f'machine type {spec.machine_type} does not support ' - f'placement groups. Use placement group style none.') + 'placement groups. Use placement group style none.' + ) elif len(set(FLAGS.zone)) > 1: raise errors.Benchmarks.UnsupportedConfigError( 'inter-zone/inter-region tests do not support placement groups. ' - 'Use placement group style closest_supported.') + 'Use placement group style closest_supported.' + ) else: placement_group_spec = aws_placement_group.AwsPlacementGroupSpec( - 'AwsPlacementGroupSpec', flag_values=FLAGS, zone=spec.zone) + 'AwsPlacementGroupSpec', flag_values=FLAGS, zone=spec.zone + ) self.placement_group = aws_placement_group.AwsPlacementGroup( - placement_group_spec) + placement_group_spec + ) self.is_static = False if spec.vpc_id: self.is_static = True @@ -909,7 +987,8 @@ def __init__(self, spec): self.zone, spec.vpc_id, cidr_block=self.regional_network.cidr_block, - subnet_id=spec.subnet_id) + subnet_id=spec.subnet_id, + ) @staticmethod def _GetNetworkSpecFromVm(vm): @@ -928,8 +1007,9 @@ def Create(self): if self.subnet is None: cidr = self.regional_network.vpc.NextSubnetCidrBlock() - self.subnet = AwsSubnet(self.zone, self.regional_network.vpc.id, - cidr_block=cidr) + self.subnet = AwsSubnet( + self.zone, self.regional_network.vpc.id, cidr_block=cidr + ) self.subnet.Create() if self.placement_group: self.placement_group.Create() @@ -957,8 +1037,9 @@ def Peer(self, peering_network): if self.regional_network is peering_network.regional_network: return - spec = network.BaseVPCPeeringSpec(self.regional_network, - peering_network.regional_network) + spec = network.BaseVPCPeeringSpec( + self.regional_network, peering_network.regional_network + ) self.vpc_peering = AwsVpcPeering(spec) peering_network.vpc_peering = self.vpc_peering self.vpc_peering.Create() @@ -985,31 +1066,37 @@ def _Create(self): '--region=%s' % self.network_a.region, '--peer-region=%s' % self.network_b.region, '--vpc-id=%s' % self.network_a.vpc.id, - '--peer-vpc-id=%s' % self.network_b.vpc.id] + '--peer-vpc-id=%s' % self.network_b.vpc.id, + ] stdout, _ = vm_util.IssueRetryableCommand(create_cmd) response = json.loads(stdout) - self.id = response['VpcPeeringConnection'][ - 'VpcPeeringConnectionId'] + self.id = response['VpcPeeringConnection']['VpcPeeringConnectionId'] # Accepts Peering Connection accept_cmd = util.AWS_PREFIX + [ 'ec2', 'accept-vpc-peering-connection', '--region=%s' % self.network_b.region, - '--vpc-peering-connection-id=%s' % self.id] + '--vpc-peering-connection-id=%s' % self.id, + ] vm_util.IssueRetryableCommand(accept_cmd) util.AddDefaultTags(self.id, self.network_a.region) - logging.info('Creating VPC peering between %s and %s', - self.network_a.vpc.cidr, self.network_b.vpc.cidr) + logging.info( + 'Creating VPC peering between %s and %s', + self.network_a.vpc.cidr, + self.network_b.vpc.cidr, + ) # Adds VPC peering to both networks' route tables - self.network_a.route_table.CreateVpcPeeringRoute(self.id, - self.network_b.vpc.cidr) - self.network_b.route_table.CreateVpcPeeringRoute(self.id, - self.network_a.vpc.cidr) + self.network_a.route_table.CreateVpcPeeringRoute( + self.id, self.network_b.vpc.cidr + ) + self.network_b.route_table.CreateVpcPeeringRoute( + self.id, self.network_a.vpc.cidr + ) # Updates security group to allow inbound traffic from peering networks self.network_a.vpc.AllowVpcPeerInBound(self.network_b.vpc) @@ -1021,5 +1108,6 @@ def _Delete(self): 'ec2', 'delete-vpc-peering-connection', '--region=%s' % self.network_a.region, - '--vpc-peering-connection-id=%s' % self.id] + '--vpc-peering-connection-id=%s' % self.id, + ] vm_util.IssueCommand(delete_cmd) diff --git a/perfkitbenchmarker/providers/aws/aws_nfs_service.py b/perfkitbenchmarker/providers/aws/aws_nfs_service.py index 25d45d669e..0fcc2a2a49 100644 --- a/perfkitbenchmarker/providers/aws/aws_nfs_service.py +++ b/perfkitbenchmarker/providers/aws/aws_nfs_service.py @@ -95,8 +95,11 @@ def security_group(self): return None def _Create(self): - logging.info('Creating NFS resource, subnet: %s, security group: %s', - self.subnet_id, self.security_group) + logging.info( + 'Creating NFS resource, subnet: %s, security group: %s', + self.subnet_id, + self.security_group, + ) self._CreateFiler() logging.info('Waiting for filer to start up') self.aws_commands.WaitUntilFilerAvailable(self.filer_id) @@ -115,7 +118,8 @@ def GetRemoteAddress(self): if self.filer_id is None: raise errors.Resource.RetryableGetError('Filer not created') return '{name}.efs.{region}.amazonaws.com'.format( - name=self.filer_id, region=self.region) + name=self.filer_id, region=self.region + ) def _IsReady(self): return self.aws_commands.IsMountAvailable(self.mount_id) @@ -131,14 +135,19 @@ def _CreateFiler(self): self.nfs_tier = filer['PerformanceMode'] self.filer_id = filer['FileSystemId'] self.disk_spec.disk_size = int( - round(filer['SizeInBytes']['Value'] / 10.0 ** 9)) + round(filer['SizeInBytes']['Value'] / 10.0**9) + ) return token = FLAGS.aws_efs_token or 'nfs-token-%s' % FLAGS.run_uri self.filer_id = self.aws_commands.CreateFiler( - token, self.nfs_tier, self.throughput_mode, self.provisioned_throughput) + token, self.nfs_tier, self.throughput_mode, self.provisioned_throughput + ) self.aws_commands.AddTagsToFiler(self.filer_id) - logging.info('Created filer %s with address %s', self.filer_id, - self.GetRemoteAddress()) + logging.info( + 'Created filer %s with address %s', + self.filer_id, + self.GetRemoteAddress(), + ) def _CreateMount(self): """Creates an NFS mount point on an EFS service.""" @@ -149,12 +158,12 @@ def _CreateMount(self): raise errors.Resource.CreationError('Did not create a filer first') logging.info('Creating NFS mount point') self.mount_id = self.aws_commands.CreateMount( - self.filer_id, self.subnet_id, self.security_group) + self.filer_id, self.subnet_id, self.security_group + ) logging.info('Mount target %s starting up', self.mount_id) def _DeleteMount(self): - """Deletes the EFS mount point. - """ + """Deletes the EFS mount point.""" if not self.mount_id: return logging.info('Deleting NFS mount mount %s', self.mount_id) @@ -172,7 +181,8 @@ def _DeleteFiler(self): if self.mount_id: # this isn't retryable as the mount point wasn't deleted raise errors.Resource.RetryableDeletionError( - 'Did not delete mount point first') + 'Did not delete mount point first' + ) logging.info('Deleting NFS filer %s', self.filer_id) self.aws_commands.DeleteFiler(self.filer_id) self.filer_id = None @@ -198,8 +208,9 @@ def GetFiler(self, token): assert len(file_systems) < 2, 'Too many file systems.' return file_systems[0] - def CreateFiler(self, token, nfs_tier, throughput_mode, - provisioned_throughput): + def CreateFiler( + self, token, nfs_tier, throughput_mode, provisioned_throughput + ): args = ['create-file-system', '--creation-token', token] if nfs_tier is not None: args += ['--performance-mode', nfs_tier] @@ -215,23 +226,31 @@ def AddTagsToFiler(self, filer_id): @vm_util.Retry() def WaitUntilFilerAvailable(self, filer_id): - if not self._IsAvailable('describe-file-systems', '--file-system-id', - 'FileSystems', filer_id): + if not self._IsAvailable( + 'describe-file-systems', '--file-system-id', 'FileSystems', filer_id + ): raise errors.Resource.RetryableCreationError( - '{} not ready'.format(filer_id)) + '{} not ready'.format(filer_id) + ) @vm_util.Retry() def DeleteFiler(self, file_system_id): args = self.efs_prefix + [ - 'delete-file-system', '--file-system-id', file_system_id] + 'delete-file-system', + '--file-system-id', + file_system_id, + ] _, stderr, retcode = vm_util.IssueCommand(args, raise_on_failure=False) if retcode and 'FileSystemInUse' in stderr: - raise Exception('Mount Point hasn\'t finished deleting.') + raise Exception("Mount Point hasn't finished deleting.") def CreateMount(self, file_system_id, subnet_id, security_group=None): args = [ - 'create-mount-target', '--file-system-id', file_system_id, - '--subnet-id', subnet_id + 'create-mount-target', + '--file-system-id', + file_system_id, + '--subnet-id', + subnet_id, ] if security_group: args += ['--security-groups', security_group] @@ -241,12 +260,17 @@ def IsMountAvailable(self, mount_target_id): if mount_target_id is None: # caller called _IsReady() before the mount point was created return False - return self._IsAvailable('describe-mount-targets', '--mount-target-id', - 'MountTargets', mount_target_id) + return self._IsAvailable( + 'describe-mount-targets', + '--mount-target-id', + 'MountTargets', + mount_target_id, + ) def DeleteMount(self, mount_target_id): self._IssueAwsCommand( - ['delete-mount-target', '--mount-target-id', mount_target_id], False) + ['delete-mount-target', '--mount-target-id', mount_target_id], False + ) def _IsAvailable(self, describe_cmd, id_attr, response_attribute, id_value): describe = self._IssueAwsCommand([describe_cmd, id_attr, id_value]) diff --git a/perfkitbenchmarker/providers/aws/aws_placement_group.py b/perfkitbenchmarker/providers/aws/aws_placement_group.py index 250ce934ea..1cd39541f1 100644 --- a/perfkitbenchmarker/providers/aws/aws_placement_group.py +++ b/perfkitbenchmarker/providers/aws/aws_placement_group.py @@ -40,7 +40,8 @@ 'Number of partition placement groups. Only used if' '--placement_group_style=partition.', lower_bound=2, - upper_bound=7) + upper_bound=7, +) class AwsPlacementGroupSpec(placement_group.BasePlacementGroupSpec): @@ -63,13 +64,16 @@ def _GetOptionDecoderConstructions(cls): """ result = super(AwsPlacementGroupSpec, cls)._GetOptionDecoderConstructions() result.update({ - 'placement_group_style': (option_decoders.EnumDecoder, { - 'valid_values': - set([CLUSTER, SPREAD, PARTITION] + - list(placement_group.PLACEMENT_GROUP_OPTIONS)), - 'default': - CLUSTER, - }) + 'placement_group_style': ( + option_decoders.EnumDecoder, + { + 'valid_values': set( + [CLUSTER, SPREAD, PARTITION] + + list(placement_group.PLACEMENT_GROUP_OPTIONS) + ), + 'default': CLUSTER, + }, + ) }) return result @@ -83,12 +87,11 @@ def __init__(self, aws_placement_group_spec): """Init method for AwsPlacementGroup. Args: - aws_placement_group_spec: Object containing the - information needed to create an AwsPlacementGroup. + aws_placement_group_spec: Object containing the information needed to + create an AwsPlacementGroup. """ super(AwsPlacementGroup, self).__init__(aws_placement_group_spec) - self.name = ( - 'perfkit-%s-%s' % (FLAGS.run_uri, str(uuid.uuid4())[-12:])) + self.name = 'perfkit-%s-%s' % (FLAGS.run_uri, str(uuid.uuid4())[-12:]) self.region = util.GetRegionFromZone(self.zone) self.strategy = aws_placement_group_spec.placement_group_style # Already checked for compatibility in aws_network.py @@ -100,8 +103,9 @@ def __init__(self, aws_placement_group_spec): def _Create(self): """Creates the Placement Group.""" - formatted_tags = util.FormatTagSpecifications('placement-group', - util.MakeDefaultTags()) + formatted_tags = util.FormatTagSpecifications( + 'placement-group', util.MakeDefaultTags() + ) create_cmd = util.AWS_PREFIX + [ 'ec2', @@ -109,7 +113,7 @@ def _Create(self): '--region=%s' % self.region, '--group-name=%s' % self.name, '--strategy=%s' % self.strategy, - '--tag-specifications=%s' % formatted_tags + '--tag-specifications=%s' % formatted_tags, ] if self.partition_count: @@ -123,7 +127,8 @@ def _Delete(self): 'ec2', 'delete-placement-group', '--region=%s' % self.region, - '--group-name=%s' % self.name] + '--group-name=%s' % self.name, + ] # Failed deletes are ignorable (probably already deleted). vm_util.IssueCommand(delete_cmd, raise_on_failure=False) @@ -133,7 +138,8 @@ def _Exists(self): 'ec2', 'describe-placement-groups', '--region=%s' % self.region, - '--filter=Name=group-name,Values=%s' % self.name] + '--filter=Name=group-name,Values=%s' % self.name, + ] stdout, _ = util.IssueRetryableCommand(describe_cmd) response = json.loads(stdout) placement_groups = response['PlacementGroups'] diff --git a/perfkitbenchmarker/providers/aws/aws_rds_db.py b/perfkitbenchmarker/providers/aws/aws_rds_db.py index 9b0cb92954..0a1dca9efe 100644 --- a/perfkitbenchmarker/providers/aws/aws_rds_db.py +++ b/perfkitbenchmarker/providers/aws/aws_rds_db.py @@ -38,18 +38,21 @@ _SQL_SERVER_ENGINES = ( sql_engine_utils.SQLSERVER_EXPRESS, sql_engine_utils.SQLSERVER_STANDARD, - sql_engine_utils.SQLSERVER_ENTERPRISE) + sql_engine_utils.SQLSERVER_ENTERPRISE, +) _RDS_ENGINES = [ sql_engine_utils.MYSQL, sql_engine_utils.POSTGRES, sql_engine_utils.SQLSERVER_EXPRESS, sql_engine_utils.SQLSERVER_STANDARD, - sql_engine_utils.SQLSERVER_ENTERPRISE] + sql_engine_utils.SQLSERVER_ENTERPRISE, +] class AwsRDSRelationalDb(aws_relational_db.BaseAwsRelationalDb): """Implements the RDS database for AWS.""" + CLOUD = 'AWS' IS_MANAGED = True ENGINE = _RDS_ENGINES @@ -146,9 +149,11 @@ def _SetEndpoint(self): def _FailoverHA(self): """Fail over from master to replica.""" cmd = util.AWS_PREFIX + [ - 'rds', 'reboot-db-instance', - '--db-instance-identifier=%s' % self.instance_id, '--force-failover', - '--region=%s' % self.region + 'rds', + 'reboot-db-instance', + '--db-instance-identifier=%s' % self.instance_id, + '--force-failover', + '--region=%s' % self.region, ] vm_util.IssueCommand(cmd) @@ -158,6 +163,7 @@ def GetDefaultEngineVersion(engine): Args: engine (string): type of database (my_sql or postgres). + Returns: (string): Default engine version. Raises: diff --git a/perfkitbenchmarker/providers/aws/aws_relational_db.py b/perfkitbenchmarker/providers/aws/aws_relational_db.py index ffee14043c..a22fc6749d 100644 --- a/perfkitbenchmarker/providers/aws/aws_relational_db.py +++ b/perfkitbenchmarker/providers/aws/aws_relational_db.py @@ -67,6 +67,7 @@ class AWSMysqlIAASRelationalDb(mysql_iaas_relational_db.MysqlIAASRelationalDb): class AwsRelationalDbParameterError(Exception): """Exceptions for invalid Db parameters.""" + pass @@ -152,14 +153,24 @@ def _CreateDependencies(self): self._CreateDbSubnetGroup(self.subnets_used_by_db) open_port_cmd = util.AWS_PREFIX + [ - 'ec2', 'authorize-security-group-ingress', '--group-id', - self.security_group_id, '--source-group', self.security_group_id, - '--protocol', 'tcp', '--port={0}'.format( - self.port), '--region', self.region + 'ec2', + 'authorize-security-group-ingress', + '--group-id', + self.security_group_id, + '--source-group', + self.security_group_id, + '--protocol', + 'tcp', + '--port={0}'.format(self.port), + '--region', + self.region, ] stdout, stderr, _ = vm_util.IssueCommand(open_port_cmd) - logging.info('Granted DB port ingress, stdout is:\n%s\nstderr is:\n%s', - stdout, stderr) + logging.info( + 'Granted DB port ingress, stdout is:\n%s\nstderr is:\n%s', + stdout, + stderr, + ) def _DeleteDependencies(self): """Method that will be called once after _DeleteResource() is called. @@ -182,13 +193,13 @@ def _CreateSubnetInZone(self, new_subnet_zone): """ cidr = self.client_vm.network.regional_network.vpc.NextSubnetCidrBlock() logging.info('Attempting to create a subnet in zone %s', new_subnet_zone) - new_subnet = ( - aws_network.AwsSubnet(new_subnet_zone, - self.client_vm.network.regional_network.vpc.id, - cidr)) + new_subnet = aws_network.AwsSubnet( + new_subnet_zone, self.client_vm.network.regional_network.vpc.id, cidr + ) new_subnet.Create() - logging.info('Successfully created a new subnet, subnet id is: %s', - new_subnet.id) + logging.info( + 'Successfully created a new subnet, subnet id is: %s', new_subnet.id + ) return new_subnet def _CreateDbSubnetGroup(self, subnets): @@ -200,38 +211,53 @@ def _CreateDbSubnetGroup(self, subnets): """ db_subnet_group_name = 'pkb-db-subnet-group-{0}'.format(FLAGS.run_uri) - create_db_subnet_group_cmd = util.AWS_PREFIX + ([ - 'rds', 'create-db-subnet-group', '--db-subnet-group-name', - db_subnet_group_name, '--db-subnet-group-description', - 'pkb_subnet_group_for_db', '--region', self.region, '--subnet-ids' - ] + [subnet.id - for subnet in subnets] + ['--tags'] + util.MakeFormattedDefaultTags()) + create_db_subnet_group_cmd = util.AWS_PREFIX + ( + [ + 'rds', + 'create-db-subnet-group', + '--db-subnet-group-name', + db_subnet_group_name, + '--db-subnet-group-description', + 'pkb_subnet_group_for_db', + '--region', + self.region, + '--subnet-ids', + ] + + [subnet.id for subnet in subnets] + + ['--tags'] + + util.MakeFormattedDefaultTags() + ) vm_util.IssueCommand(create_db_subnet_group_cmd) # save for cleanup self.db_subnet_group_name = db_subnet_group_name self.security_group_id = ( - self.client_vm.network.regional_network.vpc.default_security_group_id) + self.client_vm.network.regional_network.vpc.default_security_group_id + ) def _ApplyDbFlags(self): """Apply managed flags on RDS.""" if self.spec.db_flags: self.parameter_group = 'pkb-parameter-group-' + FLAGS.run_uri cmd = util.AWS_PREFIX + [ - 'rds', 'create-db-parameter-group', + 'rds', + 'create-db-parameter-group', '--db-parameter-group-name=%s' % self.parameter_group, '--db-parameter-group-family=%s' % self._GetParameterGroupFamily(), - '--region=%s' % self.region, '--description="AWS pkb option group"' + '--region=%s' % self.region, + '--description="AWS pkb option group"', ] vm_util.IssueCommand(cmd) cmd = util.AWS_PREFIX + [ - 'rds', 'modify-db-instance', + 'rds', + 'modify-db-instance', '--db-instance-identifier=%s' % self.instance_id, '--db-parameter-group-name=%s' % self.parameter_group, - '--region=%s' % self.region, '--apply-immediately' + '--region=%s' % self.region, + '--apply-immediately', ] vm_util.IssueCommand(cmd) @@ -241,11 +267,12 @@ def _ApplyDbFlags(self): if len(key_value_pair) != 2: raise AwsRelationalDbParameterError('Malformed parameter %s' % flag) cmd = util.AWS_PREFIX + [ - 'rds', 'modify-db-parameter-group', + 'rds', + 'modify-db-parameter-group', '--db-parameter-group-name=%s' % self.parameter_group, '--parameters=ParameterName=%s,ParameterValue=%s,ApplyMethod=pending-reboot' % (key_value_pair[0], key_value_pair[1]), - '--region=%s' % self.region + '--region=%s' % self.region, ] vm_util.IssueCommand(cmd) @@ -264,14 +291,16 @@ def _GetParameterGroupFamily(self): NotImplementedError: If there is no supported ParameterGroupFamiliy. """ all_supported_versions = ( - MYSQL_SUPPORTED_MAJOR_VERSIONS + POSTGRES_SUPPORTED_MAJOR_VERSIONS) + MYSQL_SUPPORTED_MAJOR_VERSIONS + POSTGRES_SUPPORTED_MAJOR_VERSIONS + ) for version in all_supported_versions: if self.spec.engine_version.startswith(version): return self.spec.engine + version - raise NotImplementedError('The parameter group of engine %s,' - ' version %s is not supported' % - (self.spec.engine, self.spec.engine_version)) + raise NotImplementedError( + 'The parameter group of engine %s, version %s is not supported' + % (self.spec.engine, self.spec.engine_version) + ) def _TeardownNetworking(self): """Tears down all network resources that were created for the database.""" @@ -279,8 +308,12 @@ def _TeardownNetworking(self): subnet_for_db.Delete() if hasattr(self, 'db_subnet_group_name'): delete_db_subnet_group_cmd = util.AWS_PREFIX + [ - 'rds', 'delete-db-subnet-group', '--db-subnet-group-name', - self.db_subnet_group_name, '--region', self.region + 'rds', + 'delete-db-subnet-group', + '--db-subnet-group-name', + self.db_subnet_group_name, + '--region', + self.region, ] vm_util.IssueCommand(delete_db_subnet_group_cmd, raise_on_failure=False) @@ -288,17 +321,23 @@ def _TeardownParameterGroup(self): """Tears down all parameter group that were created for the database.""" if self.parameter_group: delete_db_parameter_group_cmd = util.AWS_PREFIX + [ - 'rds', 'delete-db-parameter-group', '--db-parameter-group-name', - self.parameter_group, '--region', self.region + 'rds', + 'delete-db-parameter-group', + '--db-parameter-group-name', + self.parameter_group, + '--region', + self.region, ] vm_util.IssueCommand( - delete_db_parameter_group_cmd, raise_on_failure=False) + delete_db_parameter_group_cmd, raise_on_failure=False + ) def _DescribeInstance(self, instance_id): cmd = util.AWS_PREFIX + [ - 'rds', 'describe-db-instances', + 'rds', + 'describe-db-instances', '--db-instance-identifier=%s' % instance_id, - '--region=%s' % self.region + '--region=%s' % self.region, ] stdout, _, retcode = vm_util.IssueCommand(cmd, raise_on_failure=False) if retcode != 0: @@ -330,10 +369,15 @@ def _IsInstanceReady(self, instance_id, timeout=IS_READY_TIMEOUT): if json_output: try: state = json_output['DBInstances'][0]['DBInstanceStatus'] - pending_values = ( - json_output['DBInstances'][0]['PendingModifiedValues']) - waiting_param = json_output['DBInstances'][0]['DBParameterGroups'][0][ - 'ParameterApplyStatus'] == 'applying' + pending_values = json_output['DBInstances'][0][ + 'PendingModifiedValues' + ] + waiting_param = ( + json_output['DBInstances'][0]['DBParameterGroups'][0][ + 'ParameterApplyStatus' + ] + == 'applying' + ) logging.info('Instance state: %s', state) if pending_values: logging.info('Pending values: %s', (str(pending_values))) @@ -350,7 +394,8 @@ def _IsInstanceReady(self, instance_id, timeout=IS_READY_TIMEOUT): except: # pylint: disable=bare-except logging.exception( - 'Error attempting to read stdout. Creation failure.') + 'Error attempting to read stdout. Creation failure.' + ) return False time.sleep(5) @@ -363,9 +408,10 @@ def _Reboot(self): raise RuntimeError('Instance is not in a state that can reboot') cmd = util.AWS_PREFIX + [ - 'rds', 'reboot-db-instance', + 'rds', + 'reboot-db-instance', '--db-instance-identifier=%s' % self.instance_id, - '--region=%s' % self.region + '--region=%s' % self.region, ] vm_util.IssueCommand(cmd) @@ -377,7 +423,8 @@ def _SetPrimaryAndSecondaryZones(self): self.primary_zone = self.subnets_used_by_db[0].zone if self.spec.high_availability: self.secondary_zone = ','.join( - [x.zone for x in self.subnets_used_by_db[1:]]) + [x.zone for x in self.subnets_used_by_db[1:]] + ) def _DeleteInstance(self, instance_id): cmd = util.AWS_PREFIX + [ @@ -438,9 +485,13 @@ def _Delete(self): be called multiple times, even if the resource has already been deleted. """ - @vm_util.Retry(poll_interval=60, fuzz=0, timeout=3600, - retryable_exceptions=( - errors.Resource.RetryableDeletionError,)) + + @vm_util.Retry( + poll_interval=60, + fuzz=0, + timeout=3600, + retryable_exceptions=(errors.Resource.RetryableDeletionError,), + ) def WaitUntilInstanceDeleted(instance_id): if self._InstanceExists(instance_id): raise errors.Resource.RetryableDeletionError('Not yet deleted') diff --git a/perfkitbenchmarker/providers/aws/aws_sqs.py b/perfkitbenchmarker/providers/aws/aws_sqs.py index f3a279b5e1..5139aa6b20 100644 --- a/perfkitbenchmarker/providers/aws/aws_sqs.py +++ b/perfkitbenchmarker/providers/aws/aws_sqs.py @@ -15,7 +15,8 @@ FLAGS = flags.FLAGS MESSAGING_SERVICE_SCRIPTS_VM_AWS_DIR = os.path.join( - msgsvc.MESSAGING_SERVICE_SCRIPTS_VM_LIB_DIR, 'aws') + msgsvc.MESSAGING_SERVICE_SCRIPTS_VM_LIB_DIR, 'aws' +) MESSAGING_SERVICE_SCRIPTS_AWS_PREFIX = 'messaging_service_scripts/aws' MESSAGING_SERVICE_SCRIPTS_AWS_FILES = ['__init__.py', 'aws_sqs_client.py'] MESSAGING_SERVICE_SCRIPTS_AWS_BIN = 'messaging_service_scripts/aws_benchmark.py' @@ -38,8 +39,10 @@ def _Create(self): cmd = util.AWS_PREFIX + [ 'sqs', 'create-queue', - '--queue-name', self.queue_name, - '--region', self.region + '--queue-name', + self.queue_name, + '--region', + self.region, ] vm_util.IssueCommand(cmd) @@ -48,8 +51,10 @@ def _Exists(self) -> bool: cmd = util.AWS_PREFIX + [ 'sqs', 'get-queue-url', - '--queue-name', self.queue_name, - '--region', self.region + '--queue-name', + self.queue_name, + '--region', + self.region, ] _, _, retcode = vm_util.IssueCommand(cmd, raise_on_failure=False) return retcode == 0 @@ -59,8 +64,10 @@ def _Delete(self): cmd = util.AWS_PREFIX + [ 'sqs', 'delete-queue', - '--queue-url', self._GetQueue(), - '--region', self.region + '--queue-url', + self._GetQueue(), + '--region', + self.region, ] vm_util.IssueCommand(cmd, raise_on_failure=False) @@ -77,30 +84,39 @@ def _IsDeleting(self): def _InstallCloudClients(self): self.client_vm.RemoteCommand( - 'sudo pip3 install boto3', ignore_failure=False) + 'sudo pip3 install boto3', ignore_failure=False + ) self._CopyFiles( MESSAGING_SERVICE_SCRIPTS_AWS_PREFIX, MESSAGING_SERVICE_SCRIPTS_AWS_FILES, - MESSAGING_SERVICE_SCRIPTS_VM_AWS_DIR) + MESSAGING_SERVICE_SCRIPTS_VM_AWS_DIR, + ) self.client_vm.PushDataFile(MESSAGING_SERVICE_SCRIPTS_AWS_BIN) # copy AWS creds self.client_vm.Install('aws_credentials') - def Run(self, benchmark_scenario: str, number_of_messages: int, - message_size: int, warmup_messages: int, - streaming_pull: bool = False) -> Dict[str, Any]: + def Run( + self, + benchmark_scenario: str, + number_of_messages: int, + message_size: int, + warmup_messages: int, + streaming_pull: bool = False, + ) -> Dict[str, Any]: """Runs remote commands on client VM - benchmark's run phase.""" if streaming_pull: raise ValueError('Unsupported StreamingPull in AWS SQS.') - command = (f'python3 -m aws_benchmark ' - f'--queue_name={self.queue_name} ' - f'--region={self.region} ' - f'--benchmark_scenario={benchmark_scenario} ' - f'--number_of_messages={number_of_messages} ' - f'--message_size={message_size} ' - f'--warmup_messages={warmup_messages}') + command = ( + 'python3 -m aws_benchmark ' + f'--queue_name={self.queue_name} ' + f'--region={self.region} ' + f'--benchmark_scenario={benchmark_scenario} ' + f'--number_of_messages={number_of_messages} ' + f'--message_size={message_size} ' + f'--warmup_messages={warmup_messages}' + ) stdout, _ = self.client_vm.RemoteCommand(command) results = json.loads(stdout) return results @@ -114,8 +130,10 @@ def _GetQueue(self) -> str: cmd = util.AWS_PREFIX + [ 'sqs', 'get-queue-url', - '--queue-name', self.queue_name, - '--region', self.region + '--queue-name', + self.queue_name, + '--region', + self.region, ] stdout, _, _ = vm_util.IssueCommand(cmd) return json.loads(stdout)['QueueUrl'] diff --git a/perfkitbenchmarker/providers/aws/aws_virtual_machine.py b/perfkitbenchmarker/providers/aws/aws_virtual_machine.py index 299d3fc72e..11dc8c880a 100644 --- a/perfkitbenchmarker/providers/aws/aws_virtual_machine.py +++ b/perfkitbenchmarker/providers/aws/aws_virtual_machine.py @@ -33,7 +33,6 @@ from absl import flags from perfkitbenchmarker import disk from perfkitbenchmarker import disk_strategies - from perfkitbenchmarker import errors from perfkitbenchmarker import flags as pkb_flags from perfkitbenchmarker import linux_virtual_machine @@ -64,27 +63,33 @@ TERMINATED = 'terminated' SHUTTING_DOWN = 'shutting-down' INSTANCE_TRANSITIONAL_STATUSES = frozenset(['pending']) -INSTANCE_EXISTS_STATUSES = (INSTANCE_TRANSITIONAL_STATUSES | - frozenset([RUNNING, 'stopping', 'stopped'])) +INSTANCE_EXISTS_STATUSES = INSTANCE_TRANSITIONAL_STATUSES | frozenset( + [RUNNING, 'stopping', 'stopped'] +) INSTANCE_DELETED_STATUSES = frozenset([SHUTTING_DOWN, TERMINATED]) -INSTANCE_KNOWN_STATUSES = (INSTANCE_EXISTS_STATUSES | INSTANCE_DELETED_STATUSES) +INSTANCE_KNOWN_STATUSES = INSTANCE_EXISTS_STATUSES | INSTANCE_DELETED_STATUSES HOST_EXISTS_STATES = frozenset( - ['available', 'under-assessment', 'permanent-failure']) + ['available', 'under-assessment', 'permanent-failure'] +) HOST_RELEASED_STATES = frozenset(['released', 'released-permanent-failure']) KNOWN_HOST_STATES = HOST_EXISTS_STATES | HOST_RELEASED_STATES AWS_INITIATED_SPOT_TERMINATING_TRANSITION_STATUSES = frozenset( - ['marked-for-termination', 'marked-for-stop']) + ['marked-for-termination', 'marked-for-stop'] +) -AWS_INITIATED_SPOT_TERMINAL_STATUSES = frozenset( - ['instance-terminated-by-price', 'instance-terminated-by-service', - 'instance-terminated-no-capacity', - 'instance-terminated-capacity-oversubscribed', - 'instance-terminated-launch-group-constraint']) +AWS_INITIATED_SPOT_TERMINAL_STATUSES = frozenset([ + 'instance-terminated-by-price', + 'instance-terminated-by-service', + 'instance-terminated-no-capacity', + 'instance-terminated-capacity-oversubscribed', + 'instance-terminated-launch-group-constraint', +]) USER_INITIATED_SPOT_TERMINAL_STATUSES = frozenset( - ['request-canceled-and-instance-running', 'instance-terminated-by-user']) + ['request-canceled-and-instance-running', 'instance-terminated-by-user'] +) # From https://wiki.debian.org/Cloud/AmazonEC2Image/Stretch # Marketplace AMI exists, but not in all regions @@ -135,7 +140,7 @@ 'DeviceIndex': 0, 'NetworkCardIndex': 0, 'Groups': '', - 'SubnetId': '' + 'SubnetId': '', } # Parameters for use with Elastic Fiber Adapter @@ -143,13 +148,19 @@ _EFA_PARAMS['InterfaceType'] = 'efa' # Location of EFA installer -_EFA_URL = ('https://s3-us-west-2.amazonaws.com/aws-efa-installer/' - 'aws-efa-installer-{version}.tar.gz') +_EFA_URL = ( + 'https://s3-us-west-2.amazonaws.com/aws-efa-installer/' + 'aws-efa-installer-{version}.tar.gz' +) # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/p5-efa.html # Based on testing, dl1 and trn1 require similar commandline arguments. -_EFA_V2_MACHINE_TYPES = ('p5.48xlarge', 'dl1.24xlarge', - 'trn1.32xlarge', 'trn1n.32xlarge') +_EFA_V2_MACHINE_TYPES = ( + 'p5.48xlarge', + 'dl1.24xlarge', + 'trn1.32xlarge', + 'trn1n.32xlarge', +) class AwsTransitionalVmRetryableError(Exception): @@ -193,17 +204,21 @@ def GetRootBlockDeviceSpecForImage(image_id, region): 'describe-images', '--region=%s' % region, '--image-ids=%s' % image_id, - '--query', 'Images[]'] + '--query', + 'Images[]', + ] stdout, _ = util.IssueRetryableCommand(command) images = json.loads(stdout) assert images assert len(images) == 1, ( - 'Expected to receive only one image description for %s' % image_id) + 'Expected to receive only one image description for %s' % image_id + ) image_spec = images[0] root_device_name = image_spec['RootDeviceName'] block_device_mappings = image_spec['BlockDeviceMappings'] - root_block_device_dict = next((x for x in block_device_mappings if - x['DeviceName'] == root_device_name)) + root_block_device_dict = next( + (x for x in block_device_mappings if x['DeviceName'] == root_device_name) + ) return root_block_device_dict @@ -212,6 +227,7 @@ def GetBlockDeviceMap(vm): Args: vm: vm to build the block device map for. + Returns: The json representation of the block device map. If self.boot_disk_size and self.image are provided, the block device map @@ -228,11 +244,9 @@ def GetBlockDeviceMap(vm): mappings = [] if vm.boot_disk_size is not None: if vm.image is None: - raise ValueError( - 'Image must be provided if boot_disk_size is specified') + raise ValueError('Image must be provided if boot_disk_size is specified') if vm.region is None: - raise ValueError( - 'Region must be provided if image is specified') + raise ValueError('Region must be provided if image is specified') root_block_device = GetRootBlockDeviceSpecForImage(vm.image, vm.region) root_block_device['Ebs']['VolumeSize'] = vm.boot_disk_size # The 'Encrypted' key must be removed or the CLI will complain @@ -250,7 +264,8 @@ def GetBlockDeviceMap(vm): mapping = collections.OrderedDict() device_letter = aws_disk.AwsDisk.GenerateDeviceLetter(vm.name) device_name_prefix = aws_disk.AwsDisk.GenerateDeviceNamePrefix( - disk_spec.disk_type) + disk_spec.disk_type + ) device_name = device_name_prefix + device_letter mapping['DeviceName'] = device_name if disk_spec.disk_type == disk.LOCAL: @@ -266,7 +281,9 @@ def GetBlockDeviceMap(vm): ebs_block['VolumeSize'] = disk_spec.disk_size ebs_block['DeleteOnTermination'] = True if disk_spec.iops and disk_spec.disk_type in [ - aws_disk.IO1, aws_disk.IO2, aws_disk.GP3 + aws_disk.IO1, + aws_disk.IO2, + aws_disk.GP3, ]: ebs_block['Iops'] = disk_spec.iops if disk_spec.throughput and disk_spec.disk_type in [aws_disk.GP3]: @@ -338,7 +355,8 @@ def _Create(self): '--instance-type=%s' % self.machine_type, '--availability-zone=%s' % self.zone, '--auto-placement=off', - '--quantity=1'] + '--quantity=1', + ] vm_util.IssueCommand(create_cmd) def _Delete(self): @@ -347,7 +365,8 @@ def _Delete(self): 'ec2', 'release-hosts', '--region=%s' % self.region, - '--host-ids=%s' % self.id] + '--host-ids=%s' % self.id, + ] vm_util.IssueCommand(delete_cmd, raise_on_failure=False) @vm_util.Retry() @@ -356,7 +375,8 @@ def _Exists(self): 'ec2', 'describe-hosts', '--region=%s' % self.region, - '--filter=Name=client-token,Values=%s' % self.client_token] + '--filter=Name=client-token,Values=%s' % self.client_token, + ] stdout, _, _ = vm_util.IssueCommand(describe_cmd) response = json.loads(stdout) hosts = response['Hosts'] @@ -386,10 +406,10 @@ def _ApplyFlags(cls, config_values, flag_values): Can be overridden by derived classes to add support for specific flags. Args: - config_values: dict mapping config option names to provided values. May - be modified by this function. + config_values: dict mapping config option names to provided values. May be + modified by this function. flag_values: flags.FlagValues. Runtime flags that may override the - provided config values. + provided config values. """ super(AwsVmSpec, cls)._ApplyFlags(config_values, flag_values) if flag_values['aws_boot_disk_size'].present: @@ -400,7 +420,8 @@ def _ApplyFlags(cls, config_values, flag_values): config_values['spot_price'] = flag_values.aws_spot_price if flag_values['aws_spot_block_duration_minutes'].present: config_values['spot_block_duration_minutes'] = int( - flag_values.aws_spot_block_duration_minutes) + flag_values.aws_spot_block_duration_minutes + ) @classmethod def _GetOptionDecoderConstructions(cls): @@ -413,18 +434,16 @@ def _GetOptionDecoderConstructions(cls): """ result = super(AwsVmSpec, cls)._GetOptionDecoderConstructions() result.update({ - 'use_spot_instance': (option_decoders.BooleanDecoder, { - 'default': False - }), - 'spot_price': (option_decoders.FloatDecoder, { - 'default': None - }), - 'spot_block_duration_minutes': (option_decoders.IntDecoder, { - 'default': None - }), - 'boot_disk_size': (option_decoders.IntDecoder, { - 'default': None - }) + 'use_spot_instance': ( + option_decoders.BooleanDecoder, + {'default': False}, + ), + 'spot_price': (option_decoders.FloatDecoder, {'default': None}), + 'spot_block_duration_minutes': ( + option_decoders.IntDecoder, + {'default': None}, + ), + 'boot_disk_size': (option_decoders.IntDecoder, {'default': None}), }) return result @@ -443,6 +462,7 @@ def _GetKeyfileSetKey(region): class AwsKeyFileManager(object): """Object for managing AWS Keyfiles.""" + _lock = threading.Lock() imported_keyfile_set = set() deleted_keyfile_set = set() @@ -453,24 +473,27 @@ def ImportKeyfile(cls, region): with cls._lock: if _GetKeyfileSetKey(region) in cls.imported_keyfile_set: return - cat_cmd = ['cat', - vm_util.GetPublicKeyPath()] + cat_cmd = ['cat', vm_util.GetPublicKeyPath()] keyfile, _ = vm_util.IssueRetryableCommand(cat_cmd) - formatted_tags = util.FormatTagSpecifications('key-pair', - util.MakeDefaultTags()) + formatted_tags = util.FormatTagSpecifications( + 'key-pair', util.MakeDefaultTags() + ) import_cmd = util.AWS_PREFIX + [ - 'ec2', '--region=%s' % region, + 'ec2', + '--region=%s' % region, 'import-key-pair', '--key-name=%s' % cls.GetKeyNameForRun(), '--public-key-material=%s' % keyfile, '--tag-specifications=%s' % formatted_tags, ] _, stderr, retcode = vm_util.IssueCommand( - import_cmd, raise_on_failure=False) + import_cmd, raise_on_failure=False + ) if retcode: if 'KeyPairLimitExceeded' in stderr: raise errors.Benchmarks.QuotaFailure( - 'KeyPairLimitExceeded in %s: %s' % (region, stderr)) + 'KeyPairLimitExceeded in %s: %s' % (region, stderr) + ) else: raise errors.Benchmarks.PrepareException(stderr) @@ -485,9 +508,11 @@ def DeleteKeyfile(cls, region): if _GetKeyfileSetKey(region) in cls.deleted_keyfile_set: return delete_cmd = util.AWS_PREFIX + [ - 'ec2', '--region=%s' % region, + 'ec2', + '--region=%s' % region, 'delete-key-pair', - '--key-name=%s' % cls.GetKeyNameForRun()] + '--key-name=%s' % cls.GetKeyNameForRun(), + ] util.IssueRetryableCommand(delete_cmd) cls.deleted_keyfile_set.add(_GetKeyfileSetKey(region)) if _GetKeyfileSetKey(region) in cls.imported_keyfile_set: @@ -597,21 +622,22 @@ def __init__(self, vm_spec): self.host = None self.id = None self.metadata.update({ - 'spot_instance': - self.use_spot_instance, - 'spot_price': - self.spot_price, - 'spot_block_duration_minutes': - self.spot_block_duration_minutes, - 'placement_group_strategy': + 'spot_instance': self.use_spot_instance, + 'spot_price': self.spot_price, + 'spot_block_duration_minutes': self.spot_block_duration_minutes, + 'placement_group_strategy': ( self.placement_group.strategy - if self.placement_group else placement_group.PLACEMENT_GROUP_NONE, - 'partition_count': - self.placement_group.partition_count - if self.placement_group else 0, - 'aws_credit_specification': + if self.placement_group + else placement_group.PLACEMENT_GROUP_NONE + ), + 'partition_count': ( + self.placement_group.partition_count if self.placement_group else 0 + ), + 'aws_credit_specification': ( FLAGS.aws_credit_specification - if FLAGS.aws_credit_specification else 'none' + if FLAGS.aws_credit_specification + else 'none' + ), }) self.spot_early_termination = False self.spot_status_code = None @@ -628,11 +654,12 @@ def __init__(self, vm_spec): if self.use_dedicated_host and util.IsRegion(self.zone): raise ValueError( 'In order to use dedicated hosts, you must specify an availability ' - 'zone, not a region ("zone" was %s).' % self.zone) + 'zone, not a region ("zone" was %s).' + % self.zone + ) if self.use_dedicated_host and self.use_spot_instance: - raise ValueError( - 'Tenancy=host is not supported for Spot Instances') + raise ValueError('Tenancy=host is not supported for Spot Instances') self.allocation_id = None self.association_id = None self.aws_tags = {} @@ -688,11 +715,19 @@ def GetDefaultImage(cls, machine_type, region): if FLAGS.aws_image_name_filter: raise ValueError( '--aws_image_name_filter is not supported for AWS OS Mixins that ' - 'use SSM to select AMIs. You can still pass --image.') - stdout, _, _ = vm_util.IssueCommand(util.AWS_PREFIX + [ - 'ssm', 'get-parameters', - '--region', region, - '--names', cls.IMAGE_SSM_PATTERN.format(**format_dict)]) + 'use SSM to select AMIs. You can still pass --image.' + ) + stdout, _, _ = vm_util.IssueCommand( + util.AWS_PREFIX + + [ + 'ssm', + 'get-parameters', + '--region', + region, + '--names', + cls.IMAGE_SSM_PATTERN.format(**format_dict), + ] + ) response = json.loads(stdout) if response['InvalidParameters']: raise AwsImageNotFoundError('Invalid SSM parameters:\n' + stdout) @@ -705,7 +740,8 @@ def GetDefaultImage(cls, machine_type, region): raise NotImplementedError('AWS OSMixins require IMAGE_OWNER') if not cls.IMAGE_NAME_FILTER_PATTERN: raise NotImplementedError( - 'AWS OSMixins require IMAGE_NAME_FILTER_PATTERN') + 'AWS OSMixins require IMAGE_NAME_FILTER_PATTERN' + ) if FLAGS.aws_image_name_filter: cls.IMAGE_NAME_FILTER_PATTERN = FLAGS.aws_image_name_filter @@ -719,20 +755,23 @@ def GetDefaultImage(cls, machine_type, region): '--region=%s' % region, 'ec2', 'describe-images', - '--query', ('Images[*].{Name:Name,ImageId:ImageId,' - 'CreationDate:CreationDate}'), + '--query', + 'Images[*].{Name:Name,ImageId:ImageId,CreationDate:CreationDate}', '--filters', 'Name=name,Values=' + image_name_filter, - 'Name=block-device-mapping.volume-type,Values=%s' % - cls.DEFAULT_ROOT_DISK_TYPE, + 'Name=block-device-mapping.volume-type,Values=%s' + % cls.DEFAULT_ROOT_DISK_TYPE, 'Name=virtualization-type,Values=%s' % virt_type, - 'Name=architecture,Values=%s' % processor_architecture] + 'Name=architecture,Values=%s' % processor_architecture, + ] if cls.IMAGE_PRODUCT_CODE_FILTER: - describe_cmd.extend(['Name=product-code,Values=%s' % - cls.IMAGE_PRODUCT_CODE_FILTER]) + describe_cmd.extend( + ['Name=product-code,Values=%s' % cls.IMAGE_PRODUCT_CODE_FILTER] + ) if cls.IMAGE_DESCRIPTION_FILTER: - describe_cmd.extend(['Name=description,Values=%s' % - cls.IMAGE_DESCRIPTION_FILTER]) + describe_cmd.extend( + ['Name=description,Values=%s' % cls.IMAGE_DESCRIPTION_FILTER] + ) if cls.ALLOW_DEPRECATED_IMAGE: describe_cmd.append('--include-deprecated') else: @@ -742,8 +781,9 @@ def GetDefaultImage(cls, machine_type, region): stdout, _ = util.IssueRetryableCommand(describe_cmd) if not stdout: - raise AwsImageNotFoundError('aws describe-images did not produce valid ' - 'output.') + raise AwsImageNotFoundError( + 'aws describe-images did not produce valid output.' + ) if cls.IMAGE_NAME_REGEX: # Further filter images by the IMAGE_NAME_REGEX filter. @@ -757,9 +797,11 @@ def GetDefaultImage(cls, machine_type, region): excluded_images.append(image) if excluded_images: - logging.debug('Excluded the following images with regex "%s": %s', - image_name_regex, - sorted(image['Name'] for image in excluded_images)) + logging.debug( + 'Excluded the following images with regex "%s": %s', + image_name_regex, + sorted(image['Name'] for image in excluded_images), + ) else: images = json.loads(stdout) @@ -774,30 +816,37 @@ def _RunDescribeInstancesCommand(self): 'ec2', 'describe-instances', '--region=%s' % self.region, - '--filter=Name=client-token,Values=%s' % self.client_token] + '--filter=Name=client-token,Values=%s' % self.client_token, + ] stdout, _ = util.IssueRetryableCommand(describe_cmd) return json.loads(stdout) @vm_util.Retry(max_retries=2) def _PostCreate(self): """Get the instance's data and tag it.""" - logging.info('Getting instance %s public IP. This will fail until ' - 'a public IP is available, but will be retried.', self.id) + logging.info( + 'Getting instance %s public IP. This will fail until ' + 'a public IP is available, but will be retried.', + self.id, + ) response = self._RunDescribeInstancesCommand() instance = response['Reservations'][0]['Instances'][0] self.internal_ip = instance['PrivateIpAddress'] for network_interface in instance.get('NetworkInterfaces', []): # Ensures primary NIC is first if network_interface['Attachment']['DeviceIndex'] == 0: - self.internal_ips = ([network_interface['PrivateIpAddress']] + - self.internal_ips) + self.internal_ips = [ + network_interface['PrivateIpAddress'] + ] + self.internal_ips else: self.internal_ips.append(network_interface['PrivateIpAddress']) if util.IsRegion(self.zone): self.zone = str(instance['Placement']['AvailabilityZone']) assert self.group_id == instance['SecurityGroups'][0]['GroupId'], ( - self.group_id, instance['SecurityGroups'][0]['GroupId']) + self.group_id, + instance['SecurityGroups'][0]['GroupId'], + ) if FLAGS.aws_efa: self._ConfigureEfa(instance) elif self.network_eni_count > 1: @@ -834,8 +883,9 @@ def _ConfigureEfa(self, instance): self.RemoteCommand(f'curl -O {url}; tar -xzf {tarfile}') self._InstallEfa() # Run test program to confirm EFA working - self.RemoteCommand('cd aws-efa-installer; ' - 'PATH=${PATH}:/opt/amazon/efa/bin ./efa_test.sh') + self.RemoteCommand( + 'cd aws-efa-installer; PATH=${PATH}:/opt/amazon/efa/bin ./efa_test.sh' + ) def _ConfigureElasticIp(self, instance): """Create and associate Elastic IP. @@ -851,10 +901,10 @@ def _ConfigureElasticIp(self, instance): break assert network_interface_id is not None - stdout, _, _ = vm_util.IssueCommand(util.AWS_PREFIX + - ['ec2', 'allocate-address', - f'--region={self.region}', - '--domain=vpc']) + stdout, _, _ = vm_util.IssueCommand( + util.AWS_PREFIX + + ['ec2', 'allocate-address', f'--region={self.region}', '--domain=vpc'] + ) response = json.loads(stdout) self.ip_address = response['PublicIp'] self.allocation_id = response['AllocationId'] @@ -862,10 +912,15 @@ def _ConfigureElasticIp(self, instance): util.AddDefaultTags(self.allocation_id, self.region) stdout, _, _ = vm_util.IssueCommand( - util.AWS_PREFIX + ['ec2', 'associate-address', - f'--region={self.region}', - f'--allocation-id={self.allocation_id}', - f'--network-interface-id={network_interface_id}']) + util.AWS_PREFIX + + [ + 'ec2', + 'associate-address', + f'--region={self.region}', + f'--allocation-id={self.allocation_id}', + f'--network-interface-id={network_interface_id}', + ] + ) response = json.loads(stdout) self.association_id = response['AssociationId'] @@ -875,8 +930,10 @@ def _InstallEfa(self): See https://aws.amazon.com/hpc/efa/ """ if not self.TryRemoteCommand('ulimit -l | grep unlimited'): - self.RemoteCommand(f'echo "{self.user_name} - memlock unlimited" | ' - 'sudo tee -a /etc/security/limits.conf') + self.RemoteCommand( + f'echo "{self.user_name} - memlock unlimited" | ' + 'sudo tee -a /etc/security/limits.conf' + ) self.RemoteCommand('cd aws-efa-installer; sudo ./efa_installer.sh -y') if not self.TryRemoteCommand('ulimit -l | grep unlimited'): # efa_installer.sh should reboot enabling this change, reboot if necessary @@ -911,8 +968,9 @@ def _GenerateCreateCommand(self): '--image-id=%s' % self.image, '--instance-type=%s' % self.machine_type, '--key-name=%s' % AwsKeyFileManager.GetKeyNameForRun(), - '--tag-specifications=%s' % - util.FormatTagSpecifications('instance', self.aws_tags)] + '--tag-specifications=%s' + % util.FormatTagSpecifications('instance', self.aws_tags), + ] if FLAGS.aws_vm_hibernate: create_cmd.extend([ @@ -927,7 +985,7 @@ def _GenerateCreateCommand(self): '--instance-types', self.machine_type, '--query', - 'InstanceTypes[0].VCpuInfo.DefaultCores' + 'InstanceTypes[0].VCpuInfo.DefaultCores', ] stdout, _, _ = vm_util.IssueCommand(query_cmd) cores = int(json.loads(stdout)) @@ -941,22 +999,24 @@ def _GenerateCreateCommand(self): 'NetworkCardIndex': device_index, 'DeviceIndex': device_index, 'Groups': self.group_id, - 'SubnetId': self.network.subnet.id + 'SubnetId': self.network.subnet.id, }) - if self.machine_type in _EFA_V2_MACHINE_TYPES and efa_params[ - 'DeviceIndex']: + if ( + self.machine_type in _EFA_V2_MACHINE_TYPES + and efa_params['DeviceIndex'] + ): efa_params['DeviceIndex'] = 1 if FLAGS.aws_efa_count == 1 and self.assign_external_ip: efa_params['AssociatePublicIpAddress'] = True - efas.append(','.join(f'{key}={value}' for key, value in - sorted(efa_params.items()))) + efas.append( + ','.join( + f'{key}={value}' for key, value in sorted(efa_params.items()) + ) + ) create_cmd.extend(efas) elif self.network_eni_count > 1: enis = ['--network-interfaces'] - enis_per_network_card = ( - self.network_eni_count // - self.network_card_count - ) + enis_per_network_card = self.network_eni_count // self.network_card_count for device_index in range(self.network_eni_count): eni_params = _ENI_PARAMS.copy() eni_params.update({ @@ -965,8 +1025,11 @@ def _GenerateCreateCommand(self): 'Groups': self.group_id, 'SubnetId': self.network.subnet.id, }) - enis.append(','.join(f'{key}={value}' for key, value in - sorted(eni_params.items()))) + enis.append( + ','.join( + f'{key}={value}' for key, value in sorted(eni_params.items()) + ) + ) create_cmd.extend(enis) else: if self.assign_external_ip: @@ -977,14 +1040,17 @@ def _GenerateCreateCommand(self): if placement: create_cmd.append('--placement=%s' % placement) if FLAGS.aws_credit_specification: - create_cmd.append('--credit-specification=%s' % - FLAGS.aws_credit_specification) + create_cmd.append( + '--credit-specification=%s' % FLAGS.aws_credit_specification + ) if self.user_data: create_cmd.append('--user-data=%s' % self.user_data) if self.capacity_reservation_id: create_cmd.append( '--capacity-reservation-specification=CapacityReservationTarget=' - '{CapacityReservationId=%s}' % self.capacity_reservation_id) + '{CapacityReservationId=%s}' + % self.capacity_reservation_id + ) if self.use_spot_instance: instance_market_options = collections.OrderedDict() spot_options = collections.OrderedDict() @@ -997,22 +1063,26 @@ def _GenerateCreateCommand(self): instance_market_options['MarketType'] = 'spot' instance_market_options['SpotOptions'] = spot_options create_cmd.append( - '--instance-market-options=%s' % json.dumps(instance_market_options)) + '--instance-market-options=%s' % json.dumps(instance_market_options) + ) return create_cmd def _CreateDependencies(self): """Create VM dependencies.""" AwsKeyFileManager.ImportKeyfile(self.region) # GetDefaultImage calls the AWS CLI. - self.image = self.image or self.GetDefaultImage(self.machine_type, - self.region) + self.image = self.image or self.GetDefaultImage( + self.machine_type, self.region + ) self.AllowRemoteAccessPorts() if self.use_dedicated_host: with self._lock: - if (not self.host_list or (self.num_vms_per_host and - self.host_list[-1].fill_fraction + - 1.0 / self.num_vms_per_host > 1.0)): + if not self.host_list or ( + self.num_vms_per_host + and self.host_list[-1].fill_fraction + 1.0 / self.num_vms_per_host + > 1.0 + ): host = AwsDedicatedHost(self.machine_type, self.zone) self.host_list.append(host) host.Create() @@ -1035,15 +1105,17 @@ def _DeleteDependencies(self): def _Create(self): """Create a VM instance.""" - _, stderr, retcode = vm_util.IssueCommand(self.create_cmd, - raise_on_failure=False) + _, stderr, retcode = vm_util.IssueCommand( + self.create_cmd, raise_on_failure=False + ) if self.use_dedicated_host and 'InsufficientCapacityOnHost' in stderr: if self.num_vms_per_host: raise errors.Resource.CreationError( 'Failed to create host: %d vms of type %s per host exceeds ' - 'memory capacity limits of the host' % - (self.num_vms_per_host, self.machine_type)) + 'memory capacity limits of the host' + % (self.num_vms_per_host, self.machine_type) + ) else: logging.warning( 'Creation failed due to insufficient host capacity. A new host ' @@ -1087,14 +1159,16 @@ def _Create(self): raise errors.Benchmarks.UnsupportedConfigError(stderr) if retcode: raise errors.Resource.CreationError( - 'Failed to create VM: %s return code: %s' % (stderr, retcode)) + 'Failed to create VM: %s return code: %s' % (stderr, retcode) + ) if not self.create_return_time: self.create_return_time = time.time() @vm_util.Retry( poll_interval=0.5, log_errors=True, - retryable_exceptions=(AwsTransitionalVmRetryableError,)) + retryable_exceptions=(AwsTransitionalVmRetryableError,), + ) def _WaitForStoppedStatus(self): """Returns the status of the VM. @@ -1148,13 +1222,14 @@ def _Suspend(self): vm_util.IssueCommand(suspend_cmd) except: raise errors.Benchmarks.KnownIntermittentError( - 'Instance is still not ready to hibernate') + 'Instance is still not ready to hibernate' + ) self._PostSuspend() @vm_util.Retry( - poll_interval=0.5, - retryable_exceptions=(AwsTransitionalVmRetryableError,)) + poll_interval=0.5, retryable_exceptions=(AwsTransitionalVmRetryableError,) + ) def _WaitForNewIP(self): """Checks for a new IP address, waiting if the VM is still pending. @@ -1190,39 +1265,53 @@ def _Delete(self): 'ec2', 'terminate-instances', '--region=%s' % self.region, - '--instance-ids=%s' % self.id] + '--instance-ids=%s' % self.id, + ] vm_util.IssueCommand(delete_cmd, raise_on_failure=False) if hasattr(self, 'spot_instance_request_id'): cancel_cmd = util.AWS_PREFIX + [ '--region=%s' % self.region, 'ec2', 'cancel-spot-instance-requests', - '--spot-instance-request-ids=%s' % self.spot_instance_request_id] + '--spot-instance-request-ids=%s' % self.spot_instance_request_id, + ] vm_util.IssueCommand(cancel_cmd, raise_on_failure=False) if FLAGS.aws_efa or self.network_eni_count > 1: if self.association_id: - vm_util.IssueCommand(util.AWS_PREFIX + - ['ec2', 'disassociate-address', - f'--region={self.region}', - f'--association-id={self.association_id}']) + vm_util.IssueCommand( + util.AWS_PREFIX + + [ + 'ec2', + 'disassociate-address', + f'--region={self.region}', + f'--association-id={self.association_id}', + ] + ) if self.allocation_id: - vm_util.IssueCommand(util.AWS_PREFIX + - ['ec2', 'release-address', - f'--region={self.region}', - f'--allocation-id={self.allocation_id}']) + vm_util.IssueCommand( + util.AWS_PREFIX + + [ + 'ec2', + 'release-address', + f'--region={self.region}', + f'--allocation-id={self.allocation_id}', + ] + ) # _Start or _Stop not yet implemented for AWS def _Start(self): """Starts the VM.""" if not self.id: raise errors.Benchmarks.RunError( - 'Expected VM id to be non-null. Please make sure the VM exists.') + 'Expected VM id to be non-null. Please make sure the VM exists.' + ) start_cmd = util.AWS_PREFIX + [ - 'ec2', 'start-instances', + 'ec2', + 'start-instances', f'--region={self.region}', - f'--instance-ids={self.id}' + f'--instance-ids={self.id}', ] vm_util.IssueCommand(start_cmd) @@ -1233,11 +1322,13 @@ def _Stop(self): """Stops the VM.""" if not self.id: raise errors.Benchmarks.RunError( - 'Expected VM id to be non-null. Please make sure the VM exists.') + 'Expected VM id to be non-null. Please make sure the VM exists.' + ) stop_cmd = util.AWS_PREFIX + [ - 'ec2', 'stop-instances', + 'ec2', + 'stop-instances', f'--region={self.region}', - f'--instance-ids={self.id}' + f'--instance-ids={self.id}', ] vm_util.IssueCommand(stop_cmd) @@ -1250,50 +1341,61 @@ def _UpdateInterruptibleVmStatusThroughApi(self): '--region=%s' % self.region, 'ec2', 'describe-spot-instance-requests', - '--spot-instance-request-ids=%s' % self.spot_instance_request_id] + '--spot-instance-request-ids=%s' % self.spot_instance_request_id, + ] stdout, _, _ = vm_util.IssueCommand(describe_cmd) sir_response = json.loads(stdout)['SpotInstanceRequests'] self.spot_status_code = sir_response[0]['Status']['Code'] self.spot_early_termination = ( - self.spot_status_code in AWS_INITIATED_SPOT_TERMINAL_STATUSES) + self.spot_status_code in AWS_INITIATED_SPOT_TERMINAL_STATUSES + ) - @vm_util.Retry(poll_interval=1, - log_errors=False, - retryable_exceptions=(AwsTransitionalVmRetryableError, - LookupError)) + @vm_util.Retry( + poll_interval=1, + log_errors=False, + retryable_exceptions=(AwsTransitionalVmRetryableError, LookupError), + ) def _Exists(self): """Returns whether the VM exists.""" response = self._RunDescribeInstancesCommand() reservations = response['Reservations'] if not reservations or len(reservations) != 1: - logging.info('describe-instances did not return exactly one reservation. ' - 'This sometimes shows up immediately after a successful ' - 'run-instances command. Retrying describe-instances ' - 'command.') + logging.info( + 'describe-instances did not return exactly one reservation. ' + 'This sometimes shows up immediately after a successful ' + 'run-instances command. Retrying describe-instances ' + 'command.' + ) raise AwsTransitionalVmRetryableError() instances = reservations[0]['Instances'] if instances: status = instances[0]['State']['Name'] # In this path run-instances succeeded, a pending instance was created, # but not fulfilled so it moved to terminated. - if (status == TERMINATED and - instances[0]['StateReason']['Code'] == - 'Server.InsufficientInstanceCapacity'): + if ( + status == TERMINATED + and instances[0]['StateReason']['Code'] + == 'Server.InsufficientInstanceCapacity' + ): raise errors.Benchmarks.InsufficientCapacityCloudFailure( - instances[0]['StateReason']['Message']) + instances[0]['StateReason']['Message'] + ) # In this path run-instances succeeded, a pending instance was created, # but instance is shutting down due to internal server error. This is a # retryable command for run-instance. # Client token needs to be refreshed for idempotency. - elif (status == SHUTTING_DOWN and - instances[0]['StateReason']['Code'] == 'Server.InternalError'): + elif ( + status == SHUTTING_DOWN + and instances[0]['StateReason']['Code'] == 'Server.InternalError' + ): self.client_token = str(uuid.uuid4()) # When measuring time-to-delete, we want Exists() to retry until the # VM is in the TERMINATED state. Otherwise, it is fine to say that # the VM no longer exists when it is SHUTTING_DOWN. elif pkb_flags.MEASURE_DELETE.value and status == SHUTTING_DOWN: - logging.info('VM has status %s, retrying describe-instances command', - status) + logging.info( + 'VM has status %s, retrying describe-instances command', status + ) raise AwsTransitionalVmRetryableError() return status in INSTANCE_EXISTS_STATUSES else: @@ -1302,8 +1404,11 @@ def _Exists(self): @vm_util.Retry( poll_interval=1, log_errors=False, - retryable_exceptions=(AwsUnknownStatusError, - AwsTransitionalVmRetryableError,)) + retryable_exceptions=( + AwsUnknownStatusError, + AwsTransitionalVmRetryableError, + ), + ) def _WaitUntilRunning(self): """Waits until the VM is running. @@ -1319,18 +1424,23 @@ def _WaitUntilRunning(self): reservations = response['Reservations'] if not reservations or len(reservations) != 1: if not self.create_start_time: - logging.info('VM does not have a create_start_time; ' - 'provisioning failed.') + logging.info( + 'VM does not have a create_start_time; provisioning failed.' + ) raise AwsVmNotCreatedError() - logging.info('describe-instances did not return exactly one reservation. ' - 'This sometimes shows up immediately after a successful ' - 'run-instances command. Retrying describe-instances ' - 'command.') + logging.info( + 'describe-instances did not return exactly one reservation. ' + 'This sometimes shows up immediately after a successful ' + 'run-instances command. Retrying describe-instances ' + 'command.' + ) raise AwsTransitionalVmRetryableError() instances = reservations[0]['Instances'] if not instances or len(instances) != 1: - logging.info('describe-instances did not return exactly one instance. ' - 'Retrying describe-instances command.') + logging.info( + 'describe-instances did not return exactly one instance. ' + 'Retrying describe-instances command.' + ) raise AwsTransitionalVmRetryableError() status = instances[0]['State']['Name'] self.id = instances[0]['InstanceId'] @@ -1338,22 +1448,28 @@ def _WaitUntilRunning(self): self.spot_instance_request_id = instances[0]['SpotInstanceRequestId'] if status in INSTANCE_TRANSITIONAL_STATUSES: - logging.info('VM has status %s; retrying describe-instances command.', - status) + logging.info( + 'VM has status %s; retrying describe-instances command.', status + ) raise AwsTransitionalVmRetryableError() # In this path run-instances succeeded, a pending instance was created, but # not fulfilled so it moved to terminated. - elif (status == TERMINATED and - instances[0]['StateReason']['Code'] == - 'Server.InsufficientInstanceCapacity'): + elif ( + status == TERMINATED + and instances[0]['StateReason']['Code'] + == 'Server.InsufficientInstanceCapacity' + ): raise errors.Benchmarks.InsufficientCapacityCloudFailure( - instances[0]['StateReason']['Message']) + instances[0]['StateReason']['Message'] + ) # In this path run-instances succeeded, a pending instance was created, but # instance is shutting down due to internal server error. This is a # retryable command for run-instance. # Client token needs to be refreshed for idempotency. - elif (status == SHUTTING_DOWN and - instances[0]['StateReason']['Code'] == 'Server.InternalError'): + elif ( + status == SHUTTING_DOWN + and instances[0]['StateReason']['Code'] == 'Server.InternalError' + ): self.client_token = str(uuid.uuid4()) # Set the running time elif status == RUNNING: @@ -1362,16 +1478,20 @@ def _WaitUntilRunning(self): # TODO(user): Adjust this function to properly handle ALL potential # VM statuses. In the meantime, just retry if the VM is not yet running. else: - raise AwsUnknownStatusError(f'Unknown status: {status}; ' - 'retrying describe-instances command') + raise AwsUnknownStatusError( + f'Unknown status: {status}; retrying describe-instances command' + ) def _GetNvmeBootIndex(self): - if (aws_disk.LocalDriveIsNvme(self.machine_type) and - aws_disk.EbsDriveIsNvme(self.machine_type)): + if aws_disk.LocalDriveIsNvme(self.machine_type) and aws_disk.EbsDriveIsNvme( + self.machine_type + ): # identify boot drive # If this command ever fails consider 'findmnt -nM / -o source' - cmd = ('realpath /dev/disk/by-label/cloudimg-rootfs ' - '| grep --only-matching "nvme[0-9]*"') + cmd = ( + 'realpath /dev/disk/by-label/cloudimg-rootfs ' + '| grep --only-matching "nvme[0-9]*"' + ) boot_drive = self.RemoteCommand(cmd, ignore_failure=True)[0].strip() if boot_drive: # get the boot drive index by dropping the nvme prefix @@ -1412,8 +1532,9 @@ def CreateScratchDisk(self, spec_index, disk_spec): else: # Remote disk numbers start at 1 + max_local disks (0 is the system disk # and local disks occupy [1, max_local_disks]). - data_disk.disk_number = (self.remote_disk_counter + - 1 + self.max_local_disks) + data_disk.disk_number = ( + self.remote_disk_counter + 1 + self.max_local_disks + ) self.remote_disk_counter += 1 disks.append(data_disk) @@ -1444,7 +1565,8 @@ def PopulateNVMEDevicePath(self, scratch_disk, nvme_devices): local_devices.append(device_path) else: raise errors.Benchmarks.UnsupportedConfigError( - f'{model_number} NVME devices is not supported.') + f'{model_number} NVME devices is not supported.' + ) # because local devices are assigned in a round robin manner, # some local devices might already have been assigned (previous spec_index). @@ -1550,20 +1672,25 @@ def DownloadPreprovisionedData( install_path: The install path on this VM. module_name: Name of the module associated with this data file. filename: The name of the file that was downloaded. - timeout: Timeout value for downloading preprovisionedData, Five minutes - by default. + timeout: Timeout value for downloading preprovisionedData, Five minutes by + default. """ self.InstallCli() # TODO(deitz): Add retry logic. - self.RobustRemoteCommand(GenerateDownloadPreprovisionedDataCommand( - install_path, module_name, filename), timeout=timeout) + self.RobustRemoteCommand( + GenerateDownloadPreprovisionedDataCommand( + install_path, module_name, filename + ), + timeout=timeout, + ) def ShouldDownloadPreprovisionedData(self, module_name, filename): """Returns whether or not preprovisioned data is available.""" self.Install('aws_credentials') self.Install('awscli') return FLAGS.aws_preprovisioned_data_bucket and self.TryRemoteCommand( - GenerateStatPreprovisionedDataCommand(module_name, filename)) + GenerateStatPreprovisionedDataCommand(module_name, filename) + ) def IsInterruptible(self): """Returns whether this vm is an interruptible vm (spot vm). @@ -1615,14 +1742,16 @@ def DiskCreatedOnVMCreation(self, data_disk): return self.DiskTypeCreatedOnVMCreation(data_disk.disk_type) -class ClearBasedAwsVirtualMachine(AwsVirtualMachine, - linux_virtual_machine.ClearMixin): +class ClearBasedAwsVirtualMachine( + AwsVirtualMachine, linux_virtual_machine.ClearMixin +): IMAGE_NAME_FILTER_PATTERN = 'clear/images/*/clear-*' DEFAULT_USER_NAME = 'clear' -class CoreOsBasedAwsVirtualMachine(AwsVirtualMachine, - linux_virtual_machine.CoreOsMixin): +class CoreOsBasedAwsVirtualMachine( + AwsVirtualMachine, linux_virtual_machine.CoreOsMixin +): IMAGE_NAME_FILTER_PATTERN = 'fedora-coreos-*' # CoreOS only distinguishes between stable and testing in the description IMAGE_DESCRIPTION_FILTER = 'Fedora CoreOS stable *' @@ -1630,8 +1759,9 @@ class CoreOsBasedAwsVirtualMachine(AwsVirtualMachine, DEFAULT_USER_NAME = 'core' -class Debian9BasedAwsVirtualMachine(AwsVirtualMachine, - linux_virtual_machine.Debian9Mixin): +class Debian9BasedAwsVirtualMachine( + AwsVirtualMachine, linux_virtual_machine.Debian9Mixin +): # From https://wiki.debian.org/Cloud/AmazonEC2Image/Stretch IMAGE_NAME_FILTER_PATTERN = 'debian-stretch-{alternate_architecture}-*' IMAGE_OWNER = DEBIAN_9_IMAGE_PROJECT @@ -1642,8 +1772,9 @@ def _BeforeSuspend(self): raise NotImplementedError() -class Debian10BasedAwsVirtualMachine(AwsVirtualMachine, - linux_virtual_machine.Debian10Mixin): +class Debian10BasedAwsVirtualMachine( + AwsVirtualMachine, linux_virtual_machine.Debian10Mixin +): # From https://wiki.debian.org/Cloud/AmazonEC2Image/Buster IMAGE_NAME_FILTER_PATTERN = 'debian-10-{alternate_architecture}-*' IMAGE_OWNER = DEBIAN_IMAGE_PROJECT @@ -1651,13 +1782,14 @@ class Debian10BasedAwsVirtualMachine(AwsVirtualMachine, class Debian10BackportsBasedAwsVirtualMachine( - Debian10BasedAwsVirtualMachine, - linux_virtual_machine.Debian10BackportsMixin): + Debian10BasedAwsVirtualMachine, linux_virtual_machine.Debian10BackportsMixin +): IMAGE_NAME_FILTER_PATTERN = 'debian-10-backports-{alternate_architecture}-*' -class Debian11BasedAwsVirtualMachine(AwsVirtualMachine, - linux_virtual_machine.Debian11Mixin): +class Debian11BasedAwsVirtualMachine( + AwsVirtualMachine, linux_virtual_machine.Debian11Mixin +): # From https://wiki.debian.org/Cloud/AmazonEC2Image/Bullseye IMAGE_NAME_FILTER_PATTERN = 'debian-11-{alternate_architecture}-*' IMAGE_OWNER = DEBIAN_IMAGE_PROJECT @@ -1665,13 +1797,14 @@ class Debian11BasedAwsVirtualMachine(AwsVirtualMachine, class Debian11BackportsBasedAwsVirtualMachine( - Debian11BasedAwsVirtualMachine, - linux_virtual_machine.Debian11BackportsMixin): + Debian11BasedAwsVirtualMachine, linux_virtual_machine.Debian11BackportsMixin +): IMAGE_NAME_FILTER_PATTERN = 'debian-11-backports-{alternate_architecture}-*' -class Debian12BasedAwsVirtualMachine(AwsVirtualMachine, - linux_virtual_machine.Debian12Mixin): +class Debian12BasedAwsVirtualMachine( + AwsVirtualMachine, linux_virtual_machine.Debian12Mixin +): # From https://wiki.debian.org/Cloud/AmazonEC2Image/Bookworm IMAGE_NAME_FILTER_PATTERN = 'debian-12-{alternate_architecture}-*' IMAGE_OWNER = DEBIAN_IMAGE_PROJECT @@ -1683,41 +1816,55 @@ class UbuntuBasedAwsVirtualMachine(AwsVirtualMachine): DEFAULT_USER_NAME = 'ubuntu' -class Ubuntu1604BasedAwsVirtualMachine(UbuntuBasedAwsVirtualMachine, - linux_virtual_machine.Ubuntu1604Mixin): - IMAGE_NAME_FILTER_PATTERN = 'ubuntu/images/*/ubuntu-xenial-16.04-{alternate_architecture}-server-20*' +class Ubuntu1604BasedAwsVirtualMachine( + UbuntuBasedAwsVirtualMachine, linux_virtual_machine.Ubuntu1604Mixin +): + IMAGE_NAME_FILTER_PATTERN = ( + 'ubuntu/images/*/ubuntu-xenial-16.04-{alternate_architecture}-server-20*' + ) def _InstallEfa(self): super(Ubuntu1604BasedAwsVirtualMachine, self)._InstallEfa() self.Reboot() -class Ubuntu1804BasedAwsVirtualMachine(UbuntuBasedAwsVirtualMachine, - linux_virtual_machine.Ubuntu1804Mixin): - IMAGE_NAME_FILTER_PATTERN = 'ubuntu/images/*/ubuntu-bionic-18.04-{alternate_architecture}-server-20*' +class Ubuntu1804BasedAwsVirtualMachine( + UbuntuBasedAwsVirtualMachine, linux_virtual_machine.Ubuntu1804Mixin +): + IMAGE_NAME_FILTER_PATTERN = ( + 'ubuntu/images/*/ubuntu-bionic-18.04-{alternate_architecture}-server-20*' + ) class Ubuntu1804EfaBasedAwsVirtualMachine( - UbuntuBasedAwsVirtualMachine, linux_virtual_machine.Ubuntu1804EfaMixin): + UbuntuBasedAwsVirtualMachine, linux_virtual_machine.Ubuntu1804EfaMixin +): IMAGE_OWNER = UBUNTU_EFA_IMAGE_PROJECT IMAGE_NAME_FILTER_PATTERN = 'Deep Learning AMI GPU CUDA * (Ubuntu 18.04) *' -class Ubuntu2004BasedAwsVirtualMachine(UbuntuBasedAwsVirtualMachine, - linux_virtual_machine.Ubuntu2004Mixin): - IMAGE_NAME_FILTER_PATTERN = 'ubuntu/images/*/ubuntu-focal-20.04-{alternate_architecture}-server-20*' +class Ubuntu2004BasedAwsVirtualMachine( + UbuntuBasedAwsVirtualMachine, linux_virtual_machine.Ubuntu2004Mixin +): + IMAGE_NAME_FILTER_PATTERN = ( + 'ubuntu/images/*/ubuntu-focal-20.04-{alternate_architecture}-server-20*' + ) class Ubuntu2004EfaBasedAwsVirtualMachine( - UbuntuBasedAwsVirtualMachine, linux_virtual_machine.Ubuntu2004EfaMixin): + UbuntuBasedAwsVirtualMachine, linux_virtual_machine.Ubuntu2004EfaMixin +): IMAGE_OWNER = UBUNTU_EFA_IMAGE_PROJECT DEFAULT_ROOT_DISK_TYPE = 'gp3' IMAGE_NAME_FILTER_PATTERN = 'Deep Learning Base GPU AMI (Ubuntu 20.04) *' -class Ubuntu2204BasedAwsVirtualMachine(UbuntuBasedAwsVirtualMachine, - linux_virtual_machine.Ubuntu2204Mixin): - IMAGE_NAME_FILTER_PATTERN = 'ubuntu/images/*/ubuntu-jammy-22.04-{alternate_architecture}-server-20*' +class Ubuntu2204BasedAwsVirtualMachine( + UbuntuBasedAwsVirtualMachine, linux_virtual_machine.Ubuntu2204Mixin +): + IMAGE_NAME_FILTER_PATTERN = ( + 'ubuntu/images/*/ubuntu-jammy-22.04-{alternate_architecture}-server-20*' + ) class Ubuntu2304BasedAwsVirtualMachine( @@ -1728,43 +1875,56 @@ class Ubuntu2304BasedAwsVirtualMachine( ) -class JujuBasedAwsVirtualMachine(UbuntuBasedAwsVirtualMachine, - linux_virtual_machine.JujuMixin): +class JujuBasedAwsVirtualMachine( + UbuntuBasedAwsVirtualMachine, linux_virtual_machine.JujuMixin +): """Class with configuration for AWS Juju virtual machines.""" - IMAGE_NAME_FILTER_PATTERN = 'ubuntu/images/*/ubuntu-trusty-14.04-{alternate_architecture}-server-20*' + + IMAGE_NAME_FILTER_PATTERN = ( + 'ubuntu/images/*/ubuntu-trusty-14.04-{alternate_architecture}-server-20*' + ) class AmazonLinux2BasedAwsVirtualMachine( - AwsVirtualMachine, linux_virtual_machine.AmazonLinux2Mixin): + AwsVirtualMachine, linux_virtual_machine.AmazonLinux2Mixin +): """Class with configuration for AWS Amazon Linux 2 virtual machines.""" + IMAGE_SSM_PATTERN = '/aws/service/ami-amazon-linux-latest/amzn2-ami-{virt_type}-{architecture}-{disk_type}' -class AmazonNeuronBasedAwsVirtualMachine(AmazonLinux2BasedAwsVirtualMachine, - linux_virtual_machine.AmazonNeuronMixin - ): +class AmazonNeuronBasedAwsVirtualMachine( + AmazonLinux2BasedAwsVirtualMachine, linux_virtual_machine.AmazonNeuronMixin +): """Class with configuration for AWS Amazon Neuron virtual machines.""" + pass class AmazonLinux2023BasedAwsVirtualMachine( - AwsVirtualMachine, linux_virtual_machine.AmazonLinux2023Mixin): + AwsVirtualMachine, linux_virtual_machine.AmazonLinux2023Mixin +): """Class with configuration for AWS Amazon Linux 2023 virtual machines.""" + IMAGE_SSM_PATTERN = '/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-{architecture}' -class Rhel7BasedAwsVirtualMachine(AwsVirtualMachine, - linux_virtual_machine.Rhel7Mixin): +class Rhel7BasedAwsVirtualMachine( + AwsVirtualMachine, linux_virtual_machine.Rhel7Mixin +): """Class with configuration for AWS RHEL 7 virtual machines.""" + # Documentation on finding RHEL images: # https://access.redhat.com/articles/3692431 IMAGE_NAME_FILTER_PATTERN = 'RHEL-7*' IMAGE_OWNER = RHEL_IMAGE_PROJECT -class Rhel8BasedAwsVirtualMachine(AwsVirtualMachine, - linux_virtual_machine.Rhel8Mixin): +class Rhel8BasedAwsVirtualMachine( + AwsVirtualMachine, linux_virtual_machine.Rhel8Mixin +): """Class with configuration for AWS RHEL 8 virtual machines.""" + # Documentation on finding RHEL images: # https://access.redhat.com/articles/3692431 # All RHEL AMIs are HVM. HVM- blocks HVM_BETA. @@ -1772,9 +1932,11 @@ class Rhel8BasedAwsVirtualMachine(AwsVirtualMachine, IMAGE_OWNER = RHEL_IMAGE_PROJECT -class Rhel9BasedAwsVirtualMachine(AwsVirtualMachine, - linux_virtual_machine.Rhel9Mixin): +class Rhel9BasedAwsVirtualMachine( + AwsVirtualMachine, linux_virtual_machine.Rhel9Mixin +): """Class with configuration for AWS RHEL 9 virtual machines.""" + # Documentation on finding RHEL images: # https://access.redhat.com/articles/3692431 # All RHEL AMIs are HVM. HVM- blocks HVM_BETA. @@ -1782,9 +1944,11 @@ class Rhel9BasedAwsVirtualMachine(AwsVirtualMachine, IMAGE_OWNER = RHEL_IMAGE_PROJECT -class CentOs7BasedAwsVirtualMachine(AwsVirtualMachine, - linux_virtual_machine.CentOs7Mixin): +class CentOs7BasedAwsVirtualMachine( + AwsVirtualMachine, linux_virtual_machine.CentOs7Mixin +): """Class with configuration for AWS CentOS 7 virtual machines.""" + # Documentation on finding the CentOS 7 image: # https://wiki.centos.org/Cloud/AWS#x86_64 IMAGE_NAME_FILTER_PATTERN = 'CentOS Linux 7*' @@ -1792,17 +1956,21 @@ class CentOs7BasedAwsVirtualMachine(AwsVirtualMachine, DEFAULT_USER_NAME = 'centos' def _InstallEfa(self): - logging.info('Upgrading Centos7 kernel, installing kernel headers and ' - 'rebooting before installing EFA.') + logging.info( + 'Upgrading Centos7 kernel, installing kernel headers and ' + 'rebooting before installing EFA.' + ) self.RemoteCommand('sudo yum upgrade -y kernel') self.InstallPackages('kernel-devel') self.Reboot() super()._InstallEfa() -class CentOs8BasedAwsVirtualMachine(AwsVirtualMachine, - linux_virtual_machine.CentOs8Mixin): +class CentOs8BasedAwsVirtualMachine( + AwsVirtualMachine, linux_virtual_machine.CentOs8Mixin +): """Class with configuration for AWS CentOS 8 virtual machines.""" + # This describes the official AMIs listed here: # https://wiki.centos.org/Cloud/AWS#Official_CentOS_Linux_:_Public_Images IMAGE_OWNER = CENTOS_IMAGE_PROJECT @@ -1811,8 +1979,10 @@ class CentOs8BasedAwsVirtualMachine(AwsVirtualMachine, class CentOsStream8BasedAwsVirtualMachine( - AwsVirtualMachine, linux_virtual_machine.CentOsStream8Mixin): + AwsVirtualMachine, linux_virtual_machine.CentOsStream8Mixin +): """Class with configuration for AWS CentOS Stream 8 virtual machines.""" + # This describes the official AMIs listed here: # https://wiki.centos.org/Cloud/AWS#Official_CentOS_Linux_:_Public_Images IMAGE_OWNER = CENTOS_IMAGE_PROJECT @@ -1820,44 +1990,56 @@ class CentOsStream8BasedAwsVirtualMachine( DEFAULT_USER_NAME = 'centos' -class RockyLinux8BasedAwsVirtualMachine(AwsVirtualMachine, - linux_virtual_machine.RockyLinux8Mixin): +class RockyLinux8BasedAwsVirtualMachine( + AwsVirtualMachine, linux_virtual_machine.RockyLinux8Mixin +): """Class with configuration for AWS Rocky Linux 8 virtual machines.""" + IMAGE_OWNER = ROCKY_LINUX_IMAGE_PROJECT IMAGE_NAME_FILTER_PATTERN = 'Rocky-8-*' DEFAULT_USER_NAME = 'rocky' -class RockyLinux9BasedAwsVirtualMachine(AwsVirtualMachine, - linux_virtual_machine.RockyLinux9Mixin): +class RockyLinux9BasedAwsVirtualMachine( + AwsVirtualMachine, linux_virtual_machine.RockyLinux9Mixin +): """Class with configuration for AWS Rocky Linux 9 virtual machines.""" + IMAGE_OWNER = ROCKY_LINUX_IMAGE_PROJECT IMAGE_NAME_FILTER_PATTERN = 'Rocky-9-*' DEFAULT_USER_NAME = 'rocky' class CentOsStream9BasedAwsVirtualMachine( - AwsVirtualMachine, linux_virtual_machine.CentOsStream9Mixin): + AwsVirtualMachine, linux_virtual_machine.CentOsStream9Mixin +): """Class with configuration for AWS CentOS Stream 9 virtual machines.""" + # This describes the official AMIs listed here: # https://wiki.centos.org/Cloud/AWS#Official_CentOS_Linux_:_Public_Images IMAGE_OWNER = CENTOS_IMAGE_PROJECT IMAGE_NAME_FILTER_PATTERN = 'CentOS Stream 9*' -class BaseWindowsAwsVirtualMachine(AwsVirtualMachine, - windows_virtual_machine.BaseWindowsMixin): +class BaseWindowsAwsVirtualMachine( + AwsVirtualMachine, windows_virtual_machine.BaseWindowsMixin +): """Support for Windows machines on AWS.""" + DEFAULT_USER_NAME = 'Administrator' def __init__(self, vm_spec): super(BaseWindowsAwsVirtualMachine, self).__init__(vm_spec) - self.user_data = ('%s' % - windows_virtual_machine.STARTUP_SCRIPT) + self.user_data = ( + '%s' % windows_virtual_machine.STARTUP_SCRIPT + ) def DownloadPreprovisionedData( - self, install_path, module_name, filename, - timeout=virtual_machine.PREPROVISIONED_DATA_TIMEOUT + self, + install_path, + module_name, + filename, + timeout=virtual_machine.PREPROVISIONED_DATA_TIMEOUT, ): """Downloads a data file from an AWS S3 bucket with pre-provisioned data. @@ -1867,8 +2049,8 @@ def DownloadPreprovisionedData( install_path: The install path on this VM. module_name: Name of the module associated with this data file. filename: The name of the file that was downloaded. - timeout: Timeout value for downloading preprovisionedData, Five minutes - by default. + timeout: Timeout value for downloading preprovisionedData, Five minutes by + default. """ # TODO(deitz): Add retry logic. temp_local_path = vm_util.GetTempDir() @@ -1879,16 +2061,22 @@ def DownloadPreprovisionedData( timeout=timeout, ) self.PushFile( - vm_util.PrependTempDir(filename), - ntpath.join(install_path, filename) + vm_util.PrependTempDir(filename), ntpath.join(install_path, filename) ) vm_util.IssueCommand(['rm', vm_util.PrependTempDir(filename)]) def ShouldDownloadPreprovisionedData(self, module_name, filename): """Returns whether or not preprovisioned data is available.""" - return FLAGS.aws_preprovisioned_data_bucket and vm_util.IssueCommand( - GenerateStatPreprovisionedDataCommand( - module_name, filename).split(' '), raise_on_failure=False)[-1] == 0 + return ( + FLAGS.aws_preprovisioned_data_bucket + and vm_util.IssueCommand( + GenerateStatPreprovisionedDataCommand(module_name, filename).split( + ' ' + ), + raise_on_failure=False, + )[-1] + == 0 + ) @vm_util.Retry() def _GetDecodedPasswordData(self): @@ -1897,7 +2085,8 @@ def _GetDecodedPasswordData(self): 'ec2', 'get-password-data', '--region=%s' % self.region, - '--instance-id=%s' % self.id] + '--instance-id=%s' % self.id, + ] stdout, _ = util.IssueRetryableCommand(get_password_cmd) response = json.loads(stdout) password_data = response['PasswordData'] @@ -1922,13 +2111,15 @@ def _PostCreate(self): with vm_util.NamedTemporaryFile() as tf: tf.write(decoded_password_data) tf.close() - decrypt_cmd = ['openssl', - 'rsautl', - '-decrypt', - '-in', - tf.name, - '-inkey', - vm_util.GetPrivateKeyPath()] + decrypt_cmd = [ + 'openssl', + 'rsautl', + '-decrypt', + '-in', + tf.name, + '-inkey', + vm_util.GetPrivateKeyPath(), + ] password, _ = vm_util.IssueRetryableCommand(decrypt_cmd) self.password = password @@ -1944,8 +2135,11 @@ def GetResourceMetadata(self): @vm_util.Retry( max_retries=10, - retryable_exceptions=(AwsUnexpectedWindowsAdapterOutputError, - errors.VirtualMachine.RemoteCommandError)) + retryable_exceptions=( + AwsUnexpectedWindowsAdapterOutputError, + errors.VirtualMachine.RemoteCommandError, + ), + ) def DisableInterruptModeration(self): """Disable the networking feature 'Interrupt Moderation'.""" @@ -1953,9 +2147,12 @@ def DisableInterruptModeration(self): net_adapters, _ = self.RemoteCommand('Get-NetAdapter') if 'Intel(R) 82599 Virtual Function' not in net_adapters: raise AwsDriverDoesntSupportFeatureError( - 'Driver not tested with Interrupt Moderation in PKB.') - aws_int_dis_path = ('HKLM\\SYSTEM\\ControlSet001\\Control\\Class\\' - '{4d36e972-e325-11ce-bfc1-08002be10318}\\0011') + 'Driver not tested with Interrupt Moderation in PKB.' + ) + aws_int_dis_path = ( + 'HKLM\\SYSTEM\\ControlSet001\\Control\\Class\\' + '{4d36e972-e325-11ce-bfc1-08002be10318}\\0011' + ) command = 'reg add "%s" /v *InterruptModeration /d 0 /f' % aws_int_dis_path self.RemoteCommand(command) try: @@ -1966,20 +2163,24 @@ def DisableInterruptModeration(self): # broken. pass int_dis_value, _ = self.RemoteCommand( - 'reg query "%s" /v *InterruptModeration' % aws_int_dis_path) + 'reg query "%s" /v *InterruptModeration' % aws_int_dis_path + ) # The second line should look like: # *InterruptModeration REG_SZ 0 registry_query_lines = int_dis_value.splitlines() if len(registry_query_lines) < 3: raise AwsUnexpectedWindowsAdapterOutputError( - 'registry query failed: %s ' % int_dis_value) + 'registry query failed: %s ' % int_dis_value + ) registry_query_result = registry_query_lines[2].split() if len(registry_query_result) < 3: raise AwsUnexpectedWindowsAdapterOutputError( - 'unexpected registry query response: %s' % int_dis_value) + 'unexpected registry query response: %s' % int_dis_value + ) if registry_query_result[2] != '0': raise AwsUnexpectedWindowsAdapterOutputError( - 'InterruptModeration failed to disable') + 'InterruptModeration failed to disable' + ) def DiskDriveIsLocal(self, device, model): """Helper method to determine if a disk drive is a local ssd to stripe.""" @@ -1989,71 +2190,100 @@ def DiskDriveIsLocal(self, device, model): class Windows2016CoreAwsVirtualMachine( - BaseWindowsAwsVirtualMachine, windows_virtual_machine.Windows2016CoreMixin): - IMAGE_SSM_PATTERN = '/aws/service/ami-windows-latest/Windows_Server-2016-English-Core-Base' + BaseWindowsAwsVirtualMachine, windows_virtual_machine.Windows2016CoreMixin +): + IMAGE_SSM_PATTERN = ( + '/aws/service/ami-windows-latest/Windows_Server-2016-English-Core-Base' + ) class Windows2019CoreAwsVirtualMachine( - BaseWindowsAwsVirtualMachine, windows_virtual_machine.Windows2019CoreMixin): - IMAGE_SSM_PATTERN = '/aws/service/ami-windows-latest/Windows_Server-2019-English-Core-Base' + BaseWindowsAwsVirtualMachine, windows_virtual_machine.Windows2019CoreMixin +): + IMAGE_SSM_PATTERN = ( + '/aws/service/ami-windows-latest/Windows_Server-2019-English-Core-Base' + ) class Windows2022CoreAwsVirtualMachine( - BaseWindowsAwsVirtualMachine, windows_virtual_machine.Windows2022CoreMixin): - IMAGE_SSM_PATTERN = '/aws/service/ami-windows-latest/Windows_Server-2022-English-Core-Base' + BaseWindowsAwsVirtualMachine, windows_virtual_machine.Windows2022CoreMixin +): + IMAGE_SSM_PATTERN = ( + '/aws/service/ami-windows-latest/Windows_Server-2022-English-Core-Base' + ) class Windows2016DesktopAwsVirtualMachine( BaseWindowsAwsVirtualMachine, - windows_virtual_machine.Windows2016DesktopMixin): - IMAGE_SSM_PATTERN = '/aws/service/ami-windows-latest/Windows_Server-2016-English-Full-Base' + windows_virtual_machine.Windows2016DesktopMixin, +): + IMAGE_SSM_PATTERN = ( + '/aws/service/ami-windows-latest/Windows_Server-2016-English-Full-Base' + ) class Windows2019DesktopAwsVirtualMachine( BaseWindowsAwsVirtualMachine, - windows_virtual_machine.Windows2019DesktopMixin): - IMAGE_SSM_PATTERN = '/aws/service/ami-windows-latest/Windows_Server-2019-English-Full-Base' + windows_virtual_machine.Windows2019DesktopMixin, +): + IMAGE_SSM_PATTERN = ( + '/aws/service/ami-windows-latest/Windows_Server-2019-English-Full-Base' + ) class Windows2022DesktopAwsVirtualMachine( BaseWindowsAwsVirtualMachine, - windows_virtual_machine.Windows2022DesktopMixin): - IMAGE_SSM_PATTERN = '/aws/service/ami-windows-latest/Windows_Server-2022-English-Full-Base' + windows_virtual_machine.Windows2022DesktopMixin, +): + IMAGE_SSM_PATTERN = ( + '/aws/service/ami-windows-latest/Windows_Server-2022-English-Full-Base' + ) class Windows2019DesktopSQLServer2019StandardAwsVirtualMachine( BaseWindowsAwsVirtualMachine, - windows_virtual_machine.Windows2019SQLServer2019Standard): + windows_virtual_machine.Windows2019SQLServer2019Standard, +): IMAGE_SSM_PATTERN = '/aws/service/ami-windows-latest/Windows_Server-2019-English-Full-SQL_2019_Standard' class Windows2019DesktopSQLServer2019EnterpriseAwsVirtualMachine( BaseWindowsAwsVirtualMachine, - windows_virtual_machine.Windows2019SQLServer2019Enterprise): + windows_virtual_machine.Windows2019SQLServer2019Enterprise, +): IMAGE_SSM_PATTERN = '/aws/service/ami-windows-latest/Windows_Server-2019-English-Full-SQL_2019_Enterprise' class Windows2022DesktopSQLServer2019StandardAwsVirtualMachine( BaseWindowsAwsVirtualMachine, - windows_virtual_machine.Windows2022SQLServer2019Standard): + windows_virtual_machine.Windows2022SQLServer2019Standard, +): IMAGE_SSM_PATTERN = '/aws/service/ami-windows-latest/Windows_Server-2022-English-Full-SQL_2019_Standard' class Windows2022DesktopSQLServer2019EnterpriseAwsVirtualMachine( BaseWindowsAwsVirtualMachine, - windows_virtual_machine.Windows2022SQLServer2019Enterprise): + windows_virtual_machine.Windows2022SQLServer2019Enterprise, +): IMAGE_SSM_PATTERN = '/aws/service/ami-windows-latest/Windows_Server-2022-English-Full-SQL_2019_Enterprise' -def GenerateDownloadPreprovisionedDataCommand(install_path, module_name, - filename): +def GenerateDownloadPreprovisionedDataCommand( + install_path, module_name, filename +): """Returns a string used to download preprovisioned data.""" return 'aws s3 cp --only-show-errors s3://%s/%s/%s %s' % ( - FLAGS.aws_preprovisioned_data_bucket, module_name, filename, - posixpath.join(install_path, filename)) + FLAGS.aws_preprovisioned_data_bucket, + module_name, + filename, + posixpath.join(install_path, filename), + ) def GenerateStatPreprovisionedDataCommand(module_name, filename): """Returns a string used to download preprovisioned data.""" return 'aws s3api head-object --bucket %s --key %s/%s' % ( - FLAGS.aws_preprovisioned_data_bucket, module_name, filename) + FLAGS.aws_preprovisioned_data_bucket, + module_name, + filename, + ) diff --git a/perfkitbenchmarker/providers/aws/aws_vpc_endpoint.py b/perfkitbenchmarker/providers/aws/aws_vpc_endpoint.py index 3be8cc0145..e8cedd87ef 100644 --- a/perfkitbenchmarker/providers/aws/aws_vpc_endpoint.py +++ b/perfkitbenchmarker/providers/aws/aws_vpc_endpoint.py @@ -34,7 +34,8 @@ def GetAwsVpcEndpointClass(aws_service): """Returns the AwsVpcEndpoint class for the given service.""" return resource.GetResourceClass( - AwsVpcEndpoint, CLOUD=provider_info.AWS, AWS_SERVICE=aws_service) + AwsVpcEndpoint, CLOUD=provider_info.AWS, AWS_SERVICE=aws_service + ) def CreateEndpointService(aws_service, vpc): @@ -59,6 +60,7 @@ class AwsVpcEndpoint(resource.BaseResource): vpc: The aws_network.AwsVpc object to make the connection in. The VPC does not initially need an ID but does when Create() is called. """ + REQUIRED_ATTRS = ['CLOUD', 'AWS_SERVICE'] RESOURCE_TYPE = 'AwsVpcEndpoint' CLOUD = provider_info.AWS @@ -70,8 +72,9 @@ def __init__(self, vpc): self._vpc = vpc self.region = self._vpc.region assert self.region, 'VPC region must be set' - self._service_name = 'com.amazonaws.{}.{}'.format(self.region, - self.AWS_SERVICE) + self._service_name = 'com.amazonaws.{}.{}'.format( + self.region, self.AWS_SERVICE + ) # in the Create() method query to see if an endpoint already defined self.id = None @@ -86,10 +89,13 @@ def endpoint_id(self): if not self.vpc_id: # When creating an SDDC there will not be a VPC to have an endpoint return None - ids = self._RunCommand(['describe-vpc-endpoints'] + util.AwsFilter({ - 'vpc-id': self.vpc_id, - 'service-name': self._service_name - }) + ['--query', 'VpcEndpoints[].VpcEndpointId']) + ids = self._RunCommand( + ['describe-vpc-endpoints'] + + util.AwsFilter( + {'vpc-id': self.vpc_id, 'service-name': self._service_name} + ) + + ['--query', 'VpcEndpoints[].VpcEndpointId'] + ) if not ids: # There is a VPC but no endpoint return None @@ -105,9 +111,11 @@ def route_table_id(self): routing tables found. """ assert self.vpc_id, 'No defined VPC id.' - table_ids = self._RunCommand(['describe-route-tables'] + - util.AwsFilter({'vpc-id': self.vpc_id}) + - ['--query', 'RouteTables[].RouteTableId']) + table_ids = self._RunCommand( + ['describe-route-tables'] + + util.AwsFilter({'vpc-id': self.vpc_id}) + + ['--query', 'RouteTables[].RouteTableId'] + ) assert len(table_ids) == 1, 'Only want 1 route table: {}'.format(table_ids) return table_ids[0] @@ -123,9 +131,15 @@ def _Create(self): # Endpoint already created return create_response = self._RunCommand([ - 'create-vpc-endpoint', '--vpc-endpoint-type', 'Gateway', '--vpc-id', - self.vpc_id, '--service-name', self._service_name, '--route-table-ids', - self.route_table_id + 'create-vpc-endpoint', + '--vpc-endpoint-type', + 'Gateway', + '--vpc-id', + self.vpc_id, + '--service-name', + self._service_name, + '--route-table-ids', + self.route_table_id, ]) self.id = create_response['VpcEndpoint']['VpcEndpointId'] @@ -142,7 +156,8 @@ def _Delete(self): endpoint_id = self.id or self.endpoint_id if endpoint_id: self._RunCommand( - ['delete-vpc-endpoints', '--vpc-endpoint-ids', endpoint_id]) + ['delete-vpc-endpoints', '--vpc-endpoint-ids', endpoint_id] + ) def _RunCommand(self, cmds): """Runs the AWS ec2 command in the defined region. @@ -166,4 +181,5 @@ class AwsVpcS3Endpoint(AwsVpcEndpoint): vpc: The aws_network.AwsVpc object to make the connection in. The VPC does not initially need an ID but does when Create() is called. """ + AWS_SERVICE = 's3' diff --git a/perfkitbenchmarker/providers/aws/elastic_kubernetes_service.py b/perfkitbenchmarker/providers/aws/elastic_kubernetes_service.py index 1a5b0050e5..d0e729d4bc 100644 --- a/perfkitbenchmarker/providers/aws/elastic_kubernetes_service.py +++ b/perfkitbenchmarker/providers/aws/elastic_kubernetes_service.py @@ -45,8 +45,9 @@ class EksCluster(container_service.KubernetesCluster): def __init__(self, spec): # EKS requires a region and optionally a list of one or zones. # Interpret the zone as a comma separated list of zones or a region. - self.control_plane_zones = ( - spec.vm_spec.zone and spec.vm_spec.zone.split(',')) + self.control_plane_zones = spec.vm_spec.zone and spec.vm_spec.zone.split( + ',' + ) # Do this before super, because commas in zones confuse EC2 virtual machines if len(self.control_plane_zones) > 1: # This will become self.zone @@ -54,7 +55,8 @@ def __init__(self, spec): super().__init__(spec) if not self.control_plane_zones: raise errors.Config.MissingOption( - 'container_cluster.vm_spec.AWS.zone is required.') + 'container_cluster.vm_spec.AWS.zone is required.' + ) elif len(self.control_plane_zones) == 1 and util.IsRegion(self.zone): self.region = self.zone self.control_plane_zones = [] @@ -63,15 +65,15 @@ def __init__(self, spec): self.region = util.GetRegionFromZones(self.control_plane_zones) # control_plane_zones must be a superset of the node zones for nodepool in self.nodepools.values(): - if (nodepool.zone and - nodepool.zone not in self.control_plane_zones): + if nodepool.zone and nodepool.zone not in self.control_plane_zones: self.control_plane_zones.append(nodepool.zone) if len(self.control_plane_zones) == 1: # eksctl essentially requires you pass --zones if you pass --node-zones # and --zones must have at least 2 zones # https://github.com/weaveworks/eksctl/issues/4735 - self.control_plane_zones.append(self.region + - ('b' if self.zone.endswith('a') else 'a')) + self.control_plane_zones.append( + self.region + ('b' if self.zone.endswith('a') else 'a') + ) self.cluster_version = FLAGS.container_cluster_version # TODO(user) support setting boot disk type if EKS does. self.account = util.GetAccount() @@ -123,13 +125,14 @@ def _Create(self): 'nodes-min': self.min_nodes, 'nodes-max': self.max_nodes, }) - eksctl_flags.update( - self._GetNodeFlags(self.default_nodepool)) + eksctl_flags.update(self._GetNodeFlags(self.default_nodepool)) cmd = [FLAGS.eksctl, 'create', 'cluster'] + sorted( - '--{}={}'.format(k, v) for k, v in eksctl_flags.items() if v) + '--{}={}'.format(k, v) for k, v in eksctl_flags.items() if v + ) stdout, _, retcode = vm_util.IssueCommand( - cmd, timeout=1800, raise_on_failure=False) + cmd, timeout=1800, raise_on_failure=False + ) if retcode: # TODO(pclay): add other quota errors if 'The maximum number of VPCs has been reached' in stdout: @@ -147,7 +150,9 @@ def _Create(self): ebs_csi_driver_role = f'AmazonEKS_EBS_CSI_DriverRole_{self.name}' cmd = [ - FLAGS.eksctl, 'create', 'iamserviceaccount', + FLAGS.eksctl, + 'create', + 'iamserviceaccount', '--name=ebs-csi-controller-sa', '--namespace=kube-system', f'--region={self.region}', @@ -160,7 +165,9 @@ def _Create(self): vm_util.IssueCommand(cmd) cmd = [ - FLAGS.eksctl, 'create', 'addon', + FLAGS.eksctl, + 'create', + 'addon', '--name=aws-ebs-csi-driver', f'--region={self.region}', f'--cluster={self.name}', @@ -190,38 +197,41 @@ def _GetNodeFlags( """Get common flags for creating clusters and node_groups.""" tags = util.MakeDefaultTags() return { - 'nodes': - nodepool_config.num_nodes, - 'node-labels': - f'pkb_nodepool={nodepool_config.name}', - 'node-type': - nodepool_config.machine_type, - 'node-volume-size': - nodepool_config.disk_size, + 'nodes': nodepool_config.num_nodes, + 'node-labels': f'pkb_nodepool={nodepool_config.name}', + 'node-type': nodepool_config.machine_type, + 'node-volume-size': nodepool_config.disk_size, # zone may be split a comma separated list - 'node-zones': - nodepool_config.zone, - 'region': - self.region, - 'tags': - ','.join(f'{k}={v}' for k, v in tags.items()), - 'ssh-public-key': - aws_virtual_machine.AwsKeyFileManager.GetKeyNameForRun(), + 'node-zones': nodepool_config.zone, + 'region': self.region, + 'tags': ','.join(f'{k}={v}' for k, v in tags.items()), + 'ssh-public-key': ( + aws_virtual_machine.AwsKeyFileManager.GetKeyNameForRun() + ), } def _Delete(self): """Deletes the control plane and worker nodes.""" super()._Delete() - cmd = [FLAGS.eksctl, 'delete', 'cluster', - '--name', self.name, - '--region', self.region] + cmd = [ + FLAGS.eksctl, + 'delete', + 'cluster', + '--name', + self.name, + '--region', + self.region, + ] vm_util.IssueCommand(cmd, timeout=1800) def _IsReady(self): """Returns True if the workers are ready, else False.""" get_cmd = [ - FLAGS.kubectl, '--kubeconfig', FLAGS.kubeconfig, - 'get', 'nodes', + FLAGS.kubectl, + '--kubeconfig', + FLAGS.kubeconfig, + 'get', + 'nodes', ] stdout, _, _ = vm_util.IssueCommand(get_cmd) ready_nodes = len(re.findall('Ready', stdout)) @@ -237,7 +247,10 @@ def ResizeNodePool( ): """Change the number of nodes in the node group.""" cmd = [ - FLAGS.eksctl, 'scale', 'nodegroup', node_pool, + FLAGS.eksctl, + 'scale', + 'nodegroup', + node_pool, f'--nodes={new_size}', f'--nodes-min={new_size}', f'--nodes-max={new_size}', diff --git a/perfkitbenchmarker/providers/aws/elasticache.py b/perfkitbenchmarker/providers/aws/elasticache.py index 3ef2aa0e57..72c4bde3fa 100644 --- a/perfkitbenchmarker/providers/aws/elasticache.py +++ b/perfkitbenchmarker/providers/aws/elasticache.py @@ -42,47 +42,63 @@ def __init__(self, network, cluster_id, region, node_type, num_servers=1): self.vpc_id = network.subnet.vpc_id self.security_group_id = ( - network.regional_network.vpc.default_security_group_id) + network.regional_network.vpc.default_security_group_id + ) self.subnet_id = network.subnet.id self.subnet_group_name = '%ssubnet' % cluster_id def Create(self): # Open the port memcached needs aws_network.AwsFirewall.GetFirewall().AllowPortInSecurityGroup( - self.region, self.security_group_id, ELASTICACHE_PORT) + self.region, self.security_group_id, ELASTICACHE_PORT + ) # Create a cache subnet group - cmd = ['aws', 'elasticache', 'create-cache-subnet-group', - '--region=%s' % self.region, - '--cache-subnet-group-name=%s' % self.subnet_group_name, - '--cache-subnet-group-description="PKB memcached_ycsb benchmark"', - '--subnet-ids=%s' % self.subnet_id] + cmd = [ + 'aws', + 'elasticache', + 'create-cache-subnet-group', + '--region=%s' % self.region, + '--cache-subnet-group-name=%s' % self.subnet_group_name, + '--cache-subnet-group-description="PKB memcached_ycsb benchmark"', + '--subnet-ids=%s' % self.subnet_id, + ] vm_util.IssueCommand(cmd) # Create the cluster - cmd = ['aws', 'elasticache', 'create-cache-cluster', - '--engine=memcached', - '--cache-subnet-group-name=%s' % self.subnet_group_name, - '--cache-cluster-id=%s' % self.cluster_id, - '--num-cache-nodes=%s' % self.num_servers, - '--region=%s' % self.region, - '--cache-node-type=%s' % self.node_type, - '--tags'] + util.MakeFormattedDefaultTags() + cmd = [ + 'aws', + 'elasticache', + 'create-cache-cluster', + '--engine=memcached', + '--cache-subnet-group-name=%s' % self.subnet_group_name, + '--cache-cluster-id=%s' % self.cluster_id, + '--num-cache-nodes=%s' % self.num_servers, + '--region=%s' % self.region, + '--cache-node-type=%s' % self.node_type, + '--tags', + ] + util.MakeFormattedDefaultTags() vm_util.IssueCommand(cmd) # Wait for the cluster to come up cluster_info = self._WaitForClusterUp() # Parse out the hosts - self.hosts = [(node['Endpoint']['Address'], node['Endpoint']['Port']) - for node in cluster_info['CacheNodes']] + self.hosts = [ + (node['Endpoint']['Address'], node['Endpoint']['Port']) + for node in cluster_info['CacheNodes'] + ] assert len(self.hosts) == self.num_servers def Destroy(self): # Delete the ElastiCache cluster - cmd = ['aws', 'elasticache', 'delete-cache-cluster', - '--cache-cluster-id=%s' % self.cluster_id, - '--region=%s' % self.region] + cmd = [ + 'aws', + 'elasticache', + 'delete-cache-cluster', + '--cache-cluster-id=%s' % self.cluster_id, + '--region=%s' % self.region, + ] vm_util.IssueCommand(cmd, raise_on_failure=False) # Don't have to delete the subnet group. It will be deleted with the subnet. @@ -95,9 +111,11 @@ def GetHosts(self): return ['%s:%s' % (ip, port) for ip, port in self.hosts] def GetMetadata(self): - return {'num_servers': self.num_servers, - 'elasticache_region': self.region, - 'elasticache_node_type': self.node_type} + return { + 'num_servers': self.num_servers, + 'elasticache_region': self.region, + 'elasticache_node_type': self.node_type, + } def _GetClusterInfo(self): cmd = ['aws', 'elasticache', 'describe-cache-clusters'] @@ -107,8 +125,11 @@ def _GetClusterInfo(self): out, _, _ = vm_util.IssueCommand(cmd) return json.loads(out)['CacheClusters'][0] - @vm_util.Retry(poll_interval=15, timeout=300, - retryable_exceptions=(errors.Resource.RetryableCreationError)) + @vm_util.Retry( + poll_interval=15, + timeout=300, + retryable_exceptions=(errors.Resource.RetryableCreationError), + ) def _WaitForClusterUp(self): """Block until the ElastiCache memcached cluster is up. @@ -125,8 +146,9 @@ def _WaitForClusterUp(self): if there is an error connecting to the port or otherwise running the remote check command. """ - logging.info('Trying to get ElastiCache cluster info for %s', - self.cluster_id) + logging.info( + 'Trying to get ElastiCache cluster info for %s', self.cluster_id + ) cluster_status = None try: cluster_info = self._GetClusterInfo() @@ -136,8 +158,10 @@ def _WaitForClusterUp(self): return cluster_info except errors.VirtualMachine.RemoteCommandError as e: raise errors.Resource.RetryableCreationError( - 'ElastiCache memcached cluster not up yet: %s.' % str(e)) + 'ElastiCache memcached cluster not up yet: %s.' % str(e) + ) else: raise errors.Resource.RetryableCreationError( - 'ElastiCache memcached cluster not up yet. Status: %s' % - cluster_status) + 'ElastiCache memcached cluster not up yet. Status: %s' + % cluster_status + ) diff --git a/perfkitbenchmarker/providers/aws/flags.py b/perfkitbenchmarker/providers/aws/flags.py index 75c630410b..4cee801520 100644 --- a/perfkitbenchmarker/providers/aws/flags.py +++ b/perfkitbenchmarker/providers/aws/flags.py @@ -16,46 +16,86 @@ from absl import flags flags.DEFINE_string( - 'aws_user_name', '', 'This determines the user name that Perfkit will ' - 'attempt to use. Defaults are OS specific.') -flags.DEFINE_integer('aws_provisioned_iops', None, - 'IOPS for Provisioned IOPS (SSD) volumes in AWS.') -flags.DEFINE_integer('aws_provisioned_throughput', None, - 'Provisioned throughput (MB/s) for (SSD) volumes in AWS.') + 'aws_user_name', + '', + 'This determines the user name that Perfkit will ' + 'attempt to use. Defaults are OS specific.', +) +flags.DEFINE_integer( + 'aws_provisioned_iops', + None, + 'IOPS for Provisioned IOPS (SSD) volumes in AWS.', +) +flags.DEFINE_integer( + 'aws_provisioned_throughput', + None, + 'Provisioned throughput (MB/s) for (SSD) volumes in AWS.', +) -flags.DEFINE_string('aws_dax_node_type', 'dax.r4.large', - 'The node type used for creating AWS DAX cluster.') -flags.DEFINE_integer('aws_dax_replication_factor', 3, - 'The replication factor of AWS DAX cluster.') -flags.DEFINE_string('aws_emr_loguri', None, - 'The log-uri parameter to pass to AWS when creating a ' - 'cluster. If not set, a bucket will be created.') -flags.DEFINE_integer('aws_emr_job_wait_time', 18000, - 'The time to wait for an EMR job to finish, in seconds') -flags.DEFINE_boolean('aws_spot_instances', False, - 'Whether to use AWS spot instances for any AWS VMs.') -flags.DEFINE_float('aws_spot_price', None, - 'The spot price to bid for AWS spot instances. Defaults ' - 'to on-demand price when left as None.') -flags.DEFINE_enum('aws_spot_block_duration_minutes', None, - ['60', '120', '180', '240', '300', '360'], 'The required ' - 'duration for the Spot Instances (also known as Spot blocks),' - ' in minutes. This value must be a multiple of 60.') -flags.DEFINE_integer('aws_boot_disk_size', None, - 'The boot disk size in GiB for AWS VMs.') -flags.DEFINE_string('kops', 'kops', - 'The path to the kops binary.') -flags.DEFINE_string('aws_image_name_filter', None, - 'The filter to use when searching for an image for a VM. ' - 'See usage details in aws_virtual_machine.py around ' - 'IMAGE_NAME_FILTER.') -flags.DEFINE_string('aws_image_name_regex', None, - 'The Python regex to use to further filter images for a ' - 'VM. This applies after the aws_image_name_filter. See ' - 'usage details in aws_virtual_machine.py around ' - 'IMAGE_NAME_REGEX.') -flags.DEFINE_string('aws_preprovisioned_data_bucket', None, - 'AWS bucket where pre-provisioned data has been copied.') +flags.DEFINE_string( + 'aws_dax_node_type', + 'dax.r4.large', + 'The node type used for creating AWS DAX cluster.', +) +flags.DEFINE_integer( + 'aws_dax_replication_factor', + 3, + 'The replication factor of AWS DAX cluster.', +) +flags.DEFINE_string( + 'aws_emr_loguri', + None, + 'The log-uri parameter to pass to AWS when creating a ' + 'cluster. If not set, a bucket will be created.', +) +flags.DEFINE_integer( + 'aws_emr_job_wait_time', + 18000, + 'The time to wait for an EMR job to finish, in seconds', +) +flags.DEFINE_boolean( + 'aws_spot_instances', + False, + 'Whether to use AWS spot instances for any AWS VMs.', +) +flags.DEFINE_float( + 'aws_spot_price', + None, + 'The spot price to bid for AWS spot instances. Defaults ' + 'to on-demand price when left as None.', +) +flags.DEFINE_enum( + 'aws_spot_block_duration_minutes', + None, + ['60', '120', '180', '240', '300', '360'], + 'The required ' + 'duration for the Spot Instances (also known as Spot blocks),' + ' in minutes. This value must be a multiple of 60.', +) +flags.DEFINE_integer( + 'aws_boot_disk_size', None, 'The boot disk size in GiB for AWS VMs.' +) +flags.DEFINE_string('kops', 'kops', 'The path to the kops binary.') +flags.DEFINE_string( + 'aws_image_name_filter', + None, + 'The filter to use when searching for an image for a VM. ' + 'See usage details in aws_virtual_machine.py around ' + 'IMAGE_NAME_FILTER.', +) +flags.DEFINE_string( + 'aws_image_name_regex', + None, + 'The Python regex to use to further filter images for a ' + 'VM. This applies after the aws_image_name_filter. See ' + 'usage details in aws_virtual_machine.py around ' + 'IMAGE_NAME_REGEX.', +) +flags.DEFINE_string( + 'aws_preprovisioned_data_bucket', + None, + 'AWS bucket where pre-provisioned data has been copied.', +) ELASTICACHE_NODE_TYPE = flags.DEFINE_string( 'elasticache_node_type', 'cache.m4.large', @@ -64,66 +104,107 @@ flags.DEFINE_string( 'elasticache_failover_zone', None, 'AWS elasticache failover zone' ) -flags.DEFINE_string('aws_efs_token', None, - 'The creation token used to create the EFS resource. ' - 'If the file system already exists, it will use that ' - 'instead of creating a new one.') -flags.DEFINE_boolean('aws_delete_file_system', True, - 'Whether to delete the EFS file system.') -flags.DEFINE_enum('efs_throughput_mode', 'provisioned', - ['provisioned', 'bursting'], - 'The throughput mode to use for EFS.') -flags.DEFINE_float('efs_provisioned_throughput', 1024.0, - 'The throughput limit of EFS (in MiB/s) when run in ' - 'provisioned mode.') -flags.DEFINE_boolean('provision_athena', False, - 'Whether to provision the Athena database.') -flags.DEFINE_boolean('teardown_athena', True, - 'Whether to teardown the Athena database.') flags.DEFINE_string( - 'athena_output_location_prefix', 'athena-cli-results', + 'aws_efs_token', + None, + 'The creation token used to create the EFS resource. ' + 'If the file system already exists, it will use that ' + 'instead of creating a new one.', +) +flags.DEFINE_boolean( + 'aws_delete_file_system', True, 'Whether to delete the EFS file system.' +) +flags.DEFINE_enum( + 'efs_throughput_mode', + 'provisioned', + ['provisioned', 'bursting'], + 'The throughput mode to use for EFS.', +) +flags.DEFINE_float( + 'efs_provisioned_throughput', + 1024.0, + 'The throughput limit of EFS (in MiB/s) when run in provisioned mode.', +) +flags.DEFINE_boolean( + 'provision_athena', False, 'Whether to provision the Athena database.' +) +flags.DEFINE_boolean( + 'teardown_athena', True, 'Whether to teardown the Athena database.' +) +flags.DEFINE_string( + 'athena_output_location_prefix', + 'athena-cli-results', 'Prefix of the S3 bucket name for Athena Query Output. Suffix will be the ' 'region and the run URI, and the bucket will be dynamically created and ' - 'deleted during the test.') + 'deleted during the test.', +) flags.DEFINE_string('eksctl', 'eksctl', 'Path to eksctl.') -flags.DEFINE_enum('redshift_client_interface', 'JDBC', ['JDBC'], - 'The Runtime Interface used when interacting with Redshift.') -flags.DEFINE_enum('athena_client_interface', 'JAVA', ['JAVA'], - 'The Runtime Interface used when interacting with Athena.') +flags.DEFINE_enum( + 'redshift_client_interface', + 'JDBC', + ['JDBC'], + 'The Runtime Interface used when interacting with Redshift.', +) +flags.DEFINE_enum( + 'athena_client_interface', + 'JAVA', + ['JAVA'], + 'The Runtime Interface used when interacting with Athena.', +) flags.DEFINE_string('athena_query_timeout', '600', 'Query timeout in seconds.') -flags.DEFINE_string('athena_workgroup', '', - 'Use athena workgroup to separate applications and choose ' - 'execution configuration like the engine version.') +flags.DEFINE_string( + 'athena_workgroup', + '', + 'Use athena workgroup to separate applications and choose ' + 'execution configuration like the engine version.', +) +flags.DEFINE_boolean( + 'athena_metrics_collection', + False, + 'Should the cloud watch metrics be collected for Athena query executions.', +) flags.DEFINE_boolean( - 'athena_metrics_collection', False, - 'Should the cloud watch metrics be collected for Athena query executions.') + 'athena_workgroup_delete', + True, + 'Should the dedicated athena workgroups be deleted or kept alive for' + ' investigations.', +) +flags.DEFINE_enum( + 'aws_credit_specification', + None, + ['CpuCredits=unlimited', 'CpuCredits=standard'], + 'Credit specification for burstable vms.', +) flags.DEFINE_boolean( - 'athena_workgroup_delete', True, - 'Should the dedicated athena workgroups be deleted or kept alive for investigations.' -) -flags.DEFINE_enum('aws_credit_specification', None, - ['CpuCredits=unlimited', 'CpuCredits=standard'], - 'Credit specification for burstable vms.') -flags.DEFINE_boolean('aws_vm_hibernate', False, - 'Whether to hibernate(suspend) an aws vm' - 'instance.') + 'aws_vm_hibernate', + False, + 'Whether to hibernate(suspend) an aws vminstance.', +) flags.DEFINE_string( - 'aws_glue_crawler_role', None, + 'aws_glue_crawler_role', + None, "Role's ARN to be used by the crawler. Must have policies that grant " - 'permission for using AWS Glue and read access to S3.') + 'permission for using AWS Glue and read access to S3.', +) flags.DEFINE_string( - 'aws_glue_job_role', None, + 'aws_glue_job_role', + None, "Role's ARN to be used by Glue Jobs. Must have policies that grant " - 'permission for using AWS Glue and read access to S3.') + 'permission for using AWS Glue and read access to S3.', +) flags.DEFINE_string( - 'aws_emr_serverless_role', None, + 'aws_emr_serverless_role', + None, "Role's ARN to be used by AWS EMR Serverless Jobs. Must have policies that " - 'grant AWS Serverless read & write S3 access.') + 'grant AWS Serverless read & write S3 access.', +) flags.DEFINE_integer( - 'aws_glue_crawler_sample_size', None, + 'aws_glue_crawler_sample_size', + None, 'Sets how many files will be crawled in each leaf directory. If left ' 'unset, all the files will be crawled. May range from 1 to 249.', - 1, 249 + 1, + 249, ) # TODO(user): Create a spec for aurora. AURORA_STORAGE_TYPE = flags.DEFINE_enum( diff --git a/perfkitbenchmarker/providers/aws/provider_info.py b/perfkitbenchmarker/providers/aws/provider_info.py index 24415c58e6..3060292b98 100644 --- a/perfkitbenchmarker/providers/aws/provider_info.py +++ b/perfkitbenchmarker/providers/aws/provider_info.py @@ -17,5 +17,4 @@ class AWSProviderInfo(provider_info.BaseProviderInfo): - CLOUD = provider_info.AWS diff --git a/perfkitbenchmarker/providers/aws/redshift.py b/perfkitbenchmarker/providers/aws/redshift.py index e12f89c46d..8528bc97c2 100644 --- a/perfkitbenchmarker/providers/aws/redshift.py +++ b/perfkitbenchmarker/providers/aws/redshift.py @@ -38,7 +38,9 @@ VALID_EXIST_STATUSES = ['creating', 'available'] DELETION_STATUSES = ['deleting'] READY_STATUSES = ['available'] -ELIMINATE_AUTOMATED_SNAPSHOT_RETENTION = '--automated-snapshot-retention-period=0' +ELIMINATE_AUTOMATED_SNAPSHOT_RETENTION = ( + '--automated-snapshot-retention-period=0' +) DEFAULT_DATABASE_NAME = 'dev' BOOTSTRAP_DB = 'sample' REDSHIFT_JDBC_JAR = 'redshift-jdbc-client-1.1.jar' @@ -52,8 +54,14 @@ def AddTags(resource_arn, region): region: The AWS region resource was created in. """ cmd_prefix = util.AWS_PREFIX - tag_cmd = cmd_prefix + ['redshift', 'create-tags', '--region=%s' % region, - '--resource-name', resource_arn, '--tags'] + tag_cmd = cmd_prefix + [ + 'redshift', + 'create-tags', + '--region=%s' % region, + '--resource-name', + resource_arn, + '--tags', + ] tag_cmd += util.MakeFormattedDefaultTags() vm_util.IssueCommand(tag_cmd) @@ -66,8 +74,9 @@ def GetDefaultRegion(): return stdout -def GetRedshiftClientInterface(database: str, user: str, - password: str) -> edw_service.EdwClientInterface: +def GetRedshiftClientInterface( + database: str, user: str, password: str +) -> edw_service.EdwClientInterface: """Builds and Returns the requested Redshift client Interface. Args: @@ -131,15 +140,21 @@ def Prepare(self, package_name: str) -> None: service_specific_dir = os.path.join('edw', Redshift.SERVICE_TYPE) self.client_vm.PushFile( data.ResourcePath( - os.path.join(service_specific_dir, 'script_runner.sh'))) + os.path.join(service_specific_dir, 'script_runner.sh') + ) + ) runner_permission_update_cmd = 'chmod 755 {}'.format('script_runner.sh') self.client_vm.RemoteCommand(runner_permission_update_cmd) self.client_vm.PushFile( - data.ResourcePath(os.path.join('edw', 'script_driver.py'))) + data.ResourcePath(os.path.join('edw', 'script_driver.py')) + ) self.client_vm.PushFile( data.ResourcePath( - os.path.join(service_specific_dir, - 'provider_specific_script_driver.py'))) + os.path.join( + service_specific_dir, 'provider_specific_script_driver.py' + ) + ) + ) def ExecuteQuery(self, query_name: Text) -> Tuple[float, Dict[str, str]]: """Executes a query and returns performance details. @@ -154,10 +169,10 @@ def ExecuteQuery(self, query_name: Text) -> Tuple[float, Dict[str, str]]: successful query the value is expected to be positive. performance_details: A dictionary of query execution attributes eg. job_id """ - query_command = ('python script_driver.py --script={} --host={} ' - '--database={} --user={} --password={}').format( - query_name, self.host, self.database, self.user, - self.password) + query_command = ( + 'python script_driver.py --script={} --host={} ' + '--database={} --user={} --password={}' + ).format(query_name, self.host, self.database, self.user, self.password) stdout, _ = self.client_vm.RemoteCommand(query_command) performance = json.loads(stdout) details = copy.copy(self.GetMetadata()) @@ -209,8 +224,9 @@ def Prepare(self, package_name: str) -> None: self.client_vm.Install('openjdk') # Push the executable jar to the working directory on client vm - self.client_vm.InstallPreprovisionedPackageData(package_name, - [REDSHIFT_JDBC_JAR], '') + self.client_vm.InstallPreprovisionedPackageData( + package_name, [REDSHIFT_JDBC_JAR], '' + ) def ExecuteQuery(self, query_name: Text) -> Tuple[float, Dict[str, str]]: """Executes a query and returns performance details. @@ -225,9 +241,10 @@ def ExecuteQuery(self, query_name: Text) -> Tuple[float, Dict[str, str]]: successful query the value is expected to be positive. performance_details: A dictionary of query execution attributes eg. job_id """ - query_command = ('java -cp {} com.google.cloud.performance.edw.Single ' - '--endpoint {} --query_file {}').format( - REDSHIFT_JDBC_JAR, self.host, query_name) + query_command = ( + 'java -cp {} com.google.cloud.performance.edw.Single ' + '--endpoint {} --query_file {}' + ).format(REDSHIFT_JDBC_JAR, self.host, query_name) stdout, _ = self.client_vm.RemoteCommand(query_command) performance = json.loads(stdout) details = copy.copy(self.GetMetadata()) @@ -237,8 +254,9 @@ def ExecuteQuery(self, query_name: Text) -> Tuple[float, Dict[str, str]]: details.update(performance['details']) return performance['query_wall_time_in_secs'], details - def ExecuteSimultaneous(self, submission_interval: int, - queries: List[str]) -> str: + def ExecuteSimultaneous( + self, submission_interval: int, queries: List[str] + ) -> str: """Executes queries simultaneously on client and return performance details. Simultaneous app expects queries as white space separated query file names. @@ -251,10 +269,12 @@ def ExecuteSimultaneous(self, submission_interval: int, Returns: A serialized dictionary of execution details. """ - cmd = ('java -cp {} com.google.cloud.performance.edw.Simultaneous ' - '--endpoint {} --submission_interval {} --query_files {}'.format( - REDSHIFT_JDBC_JAR, self.host, submission_interval, - ' '.join(queries))) + cmd = ( + 'java -cp {} com.google.cloud.performance.edw.Simultaneous ' + '--endpoint {} --submission_interval {} --query_files {}'.format( + REDSHIFT_JDBC_JAR, self.host, submission_interval, ' '.join(queries) + ) + ) stdout, _ = self.client_vm.RemoteCommand(cmd) return stdout @@ -268,10 +288,14 @@ def ExecuteThroughput(self, concurrency_streams: List[List[str]]) -> str: Returns: A serialized dictionary of execution details. """ - cmd = ('java -cp {} com.google.cloud.performance.edw.Throughput ' - '--endpoint {} --query_streams {}'.format( - REDSHIFT_JDBC_JAR, self.host, - ' '.join([','.join(stream) for stream in concurrency_streams]))) + cmd = ( + 'java -cp {} com.google.cloud.performance.edw.Throughput ' + '--endpoint {} --query_streams {}'.format( + REDSHIFT_JDBC_JAR, + self.host, + ' '.join([','.join(stream) for stream in concurrency_streams]), + ) + ) stdout, _ = self.client_vm.RemoteCommand(cmd) return stdout @@ -309,15 +333,20 @@ def __init__(self, edw_service_spec): self.cluster_subnet_group = None self.cluster_parameter_group = None self.arn = '' - self.cluster_subnet_group = aws_cluster_subnet_group.RedshiftClusterSubnetGroup( - self.cmd_prefix) - self.cluster_parameter_group = aws_cluster_parameter_group.RedshiftClusterParameterGroup( - self.cmd_prefix) + self.cluster_subnet_group = ( + aws_cluster_subnet_group.RedshiftClusterSubnetGroup(self.cmd_prefix) + ) + self.cluster_parameter_group = ( + aws_cluster_parameter_group.RedshiftClusterParameterGroup( + self.cmd_prefix + ) + ) if self.db is None: self.db = DEFAULT_DATABASE_NAME - self.client_interface = GetRedshiftClientInterface(self.db, self.user, - self.password) + self.client_interface = GetRedshiftClientInterface( + self.db, self.user, self.password + ) def _CreateDependencies(self): self.cluster_subnet_group.Create() @@ -328,12 +357,26 @@ def _Create(self): if self.snapshot: self.Restore(self.snapshot, self.cluster_identifier) else: - self.Initialize(self.cluster_identifier, self.node_type, self.node_count, - self.user, self.password, self.cluster_parameter_group, - self.cluster_subnet_group) - - def Initialize(self, cluster_identifier, node_type, node_count, user, - password, cluster_parameter_group, cluster_subnet_group): + self.Initialize( + self.cluster_identifier, + self.node_type, + self.node_count, + self.user, + self.password, + self.cluster_parameter_group, + self.cluster_subnet_group, + ) + + def Initialize( + self, + cluster_identifier, + node_type, + node_count, + user, + password, + cluster_parameter_group, + cluster_subnet_group, + ): """Method to initialize a Redshift cluster from an configuration parameters. The cluster is initialized in the EC2-VPC platform, that runs it in a @@ -347,9 +390,9 @@ def Initialize(self, cluster_identifier, node_type, node_count, user, Args: cluster_identifier: A unique identifier for the cluster. - node_type: The node type to be provisioned for the cluster. - Valid Values: ds2.xlarge | ds2.8xlarge | ds2.xlarge | ds2.8xlarge | - dc1.large | dc1.8xlarge | dc2.large | dc2.8xlarge + node_type: The node type to be provisioned for the cluster. Valid Values: + ds2.xlarge | ds2.8xlarge | ds2.xlarge | ds2.8xlarge | dc1.large | + dc1.8xlarge | dc2.large | dc2.8xlarge node_count: The number of compute nodes in the cluster. user: The user name associated with the master user account for the cluster that is being created. @@ -367,11 +410,16 @@ def Initialize(self, cluster_identifier, node_type, node_count, user, MissingOption: If any of the required parameters is missing. """ if not (cluster_identifier and node_type and user and password): - raise errors.Config.MissingOption('Need cluster_identifier, user and ' - 'password set for creating a cluster.') + raise errors.Config.MissingOption( + 'Need cluster_identifier, user and ' + 'password set for creating a cluster.' + ) prefix = [ - 'redshift', 'create-cluster', '--cluster-identifier', cluster_identifier + 'redshift', + 'create-cluster', + '--cluster-identifier', + cluster_identifier, ] if node_count == 1: @@ -380,36 +428,53 @@ def Initialize(self, cluster_identifier, node_type, node_count, user, worker_count_cmd = ['--number-of-nodes', str(node_count)] postfix = [ - '--node-type', node_type, '--master-username', user, - '--master-user-password', password, '--cluster-parameter-group-name', - cluster_parameter_group.name, '--cluster-subnet-group-name', - cluster_subnet_group.name, '--publicly-accessible', - ELIMINATE_AUTOMATED_SNAPSHOT_RETENTION + '--node-type', + node_type, + '--master-username', + user, + '--master-user-password', + password, + '--cluster-parameter-group-name', + cluster_parameter_group.name, + '--cluster-subnet-group-name', + cluster_subnet_group.name, + '--publicly-accessible', + ELIMINATE_AUTOMATED_SNAPSHOT_RETENTION, ] cmd = self.cmd_prefix + prefix + worker_count_cmd + postfix stdout, stderr, _ = vm_util.IssueCommand(cmd, raise_on_failure=False) if not stdout: - raise errors.Resource.CreationError('Cluster creation failure: ' - '{}'.format(stderr)) + raise errors.Resource.CreationError( + 'Cluster creation failure: {}'.format(stderr) + ) def _ValidateSnapshot(self, snapshot_identifier): """Validate the presence of a cluster snapshot based on its metadata.""" - cmd = self.cmd_prefix + ['redshift', 'describe-cluster-snapshots', - '--snapshot-identifier', snapshot_identifier] + cmd = self.cmd_prefix + [ + 'redshift', + 'describe-cluster-snapshots', + '--snapshot-identifier', + snapshot_identifier, + ] stdout, _, _ = vm_util.IssueCommand(cmd) if not stdout: - raise errors.Config.InvalidValue('Cluster snapshot indicated by ' - 'edw_service_cluster_snapshot does not' - ' exist: {}.' - .format(snapshot_identifier)) + raise errors.Config.InvalidValue( + 'Cluster snapshot indicated by ' + 'edw_service_cluster_snapshot does not' + ' exist: {}.'.format(snapshot_identifier) + ) result = json.loads(stdout) return result['Snapshots'][0]['Status'] == 'available' def _SnapshotDetails(self, snapshot_identifier): """Delete a redshift cluster and disallow creation of a snapshot.""" - cmd = self.cmd_prefix + ['redshift', 'describe-cluster-snapshots', - '--snapshot-identifier', snapshot_identifier] + cmd = self.cmd_prefix + [ + 'redshift', + 'describe-cluster-snapshots', + '--snapshot-identifier', + snapshot_identifier, + ] stdout, _, _ = vm_util.IssueCommand(cmd) result = json.loads(stdout) node_type = result['Snapshots'][0]['NodeType'] @@ -432,13 +497,15 @@ def Restore(self, snapshot_identifier, cluster_identifier): Args: snapshot_identifier: Identifier of the snapshot to restore cluster_identifier: Identifier of the restored cluster + Returns: None """ if not (self.user and self.password and self.db): raise errors.Config.MissingOption( - 'Need the db, user and password set for restoring a cluster') + 'Need the db, user and password set for restoring a cluster' + ) if self._ValidateSnapshot(snapshot_identifier): node_type, node_count = self._SnapshotDetails(snapshot_identifier) @@ -446,24 +513,34 @@ def Restore(self, snapshot_identifier, cluster_identifier): # snapshot's configuration self.node_type = node_type self.node_count = node_count - cmd = self.cmd_prefix + ['redshift', 'restore-from-cluster-snapshot', - '--cluster-identifier', cluster_identifier, - '--snapshot-identifier', snapshot_identifier, - '--cluster-subnet-group-name', - self.cluster_subnet_group.name, - '--cluster-parameter-group-name', - self.cluster_parameter_group.name, - '--publicly-accessible', - '--automated-snapshot-retention-period=1'] + cmd = self.cmd_prefix + [ + 'redshift', + 'restore-from-cluster-snapshot', + '--cluster-identifier', + cluster_identifier, + '--snapshot-identifier', + snapshot_identifier, + '--cluster-subnet-group-name', + self.cluster_subnet_group.name, + '--cluster-parameter-group-name', + self.cluster_parameter_group.name, + '--publicly-accessible', + '--automated-snapshot-retention-period=1', + ] stdout, stderr, _ = vm_util.IssueCommand(cmd) if not stdout: - raise errors.Resource.CreationError('Cluster creation failure: ' - '{}'.format(stderr)) + raise errors.Resource.CreationError( + 'Cluster creation failure: {}'.format(stderr) + ) def __DescribeCluster(self): """Describe a redshift cluster.""" - cmd = self.cmd_prefix + ['redshift', 'describe-clusters', - '--cluster-identifier', self.cluster_identifier] + cmd = self.cmd_prefix + [ + 'redshift', + 'describe-clusters', + '--cluster-identifier', + self.cluster_identifier, + ] return vm_util.IssueCommand(cmd, raise_on_failure=False) def _Exists(self): @@ -480,8 +557,10 @@ def _Exists(self): Boolean value indicating the existence of a cluster. """ stdout, _, _ = self.__DescribeCluster() - if (not stdout or (json.loads(stdout)['Clusters'][0]['ClusterStatus'] not in - VALID_EXIST_STATUSES)): + if not stdout or ( + json.loads(stdout)['Clusters'][0]['ClusterStatus'] + not in VALID_EXIST_STATUSES + ): return False else: return True @@ -500,16 +579,20 @@ def _PostCreate(self): stdout, _, _ = self.__DescribeCluster() self.endpoint = json.loads(stdout)['Clusters'][0]['Endpoint']['Address'] account = util.GetAccount() - self.arn = 'arn:aws:redshift:{}:{}:cluster:{}'.format(self.region, account, - self. - cluster_identifier) + self.arn = 'arn:aws:redshift:{}:{}:cluster:{}'.format( + self.region, account, self.cluster_identifier + ) AddTags(self.arn, self.region) def _Delete(self): """Delete a redshift cluster and disallow creation of a snapshot.""" - cmd = self.cmd_prefix + ['redshift', 'delete-cluster', - '--cluster-identifier', self.cluster_identifier, - '--skip-final-cluster-snapshot'] + cmd = self.cmd_prefix + [ + 'redshift', + 'delete-cluster', + '--cluster-identifier', + self.cluster_identifier, + '--skip-final-cluster-snapshot', + ] vm_util.IssueCommand(cmd, raise_on_failure=False) def _IsDeleting(self): @@ -518,8 +601,10 @@ def _IsDeleting(self): if not stdout: return False else: - return (json.loads(stdout)['Clusters'][0]['ClusterStatus'] in - DELETION_STATUSES) + return ( + json.loads(stdout)['Clusters'][0]['ClusterStatus'] + in DELETION_STATUSES + ) def _DeleteDependencies(self): """Delete dependencies of a redshift cluster.""" diff --git a/perfkitbenchmarker/providers/aws/s3.py b/perfkitbenchmarker/providers/aws/s3.py index 623c951a0d..c9e8624fac 100644 --- a/perfkitbenchmarker/providers/aws/s3.py +++ b/perfkitbenchmarker/providers/aws/s3.py @@ -48,9 +48,11 @@ def PrepareService(self, location): def MakeBucket(self, bucket_name, raise_on_failure=True, tag_bucket=True): command = [ - 'aws', 's3', 'mb', + 'aws', + 's3', + 'mb', 's3://%s' % bucket_name, - '--region=%s' % self.region + '--region=%s' % self.region, ] _, stderr, ret_code = vm_util.IssueCommand(command, raise_on_failure=False) if ret_code and raise_on_failure: @@ -60,36 +62,49 @@ def MakeBucket(self, bucket_name, raise_on_failure=True, tag_bucket=True): # Tag the bucket with the persistent timeout flag so that buckets can # optionally stick around after PKB runs. default_tags = util.MakeFormattedDefaultTags( - timeout_minutes=max(FLAGS.timeout_minutes, - FLAGS.persistent_timeout_minutes)) + timeout_minutes=max( + FLAGS.timeout_minutes, FLAGS.persistent_timeout_minutes + ) + ) tag_set = ','.join('{%s}' % tag for tag in default_tags) - vm_util.IssueRetryableCommand( - ['aws', 's3api', 'put-bucket-tagging', - '--bucket', bucket_name, - '--tagging', 'TagSet=[%s]' % tag_set, - '--region=%s' % self.region]) + vm_util.IssueRetryableCommand([ + 'aws', + 's3api', + 'put-bucket-tagging', + '--bucket', + bucket_name, + '--tagging', + 'TagSet=[%s]' % tag_set, + '--region=%s' % self.region, + ]) if object_storage_service.OBJECT_TTL_DAYS.value: # NOTE: buckets created with older APIs may have a different configuration # (e.g. Prefix instead of Filter): This needs to stay updated for new # buckets. - config = json.dumps( - { - 'Rules': [{ - 'Expiration': { - 'Days': object_storage_service.OBJECT_TTL_DAYS.value - }, - 'ID': 'PKB_OBJECT_TTL', - 'Filter': {}, - 'Status': 'Enabled', - }] - } + config = json.dumps({ + 'Rules': [{ + 'Expiration': { + 'Days': object_storage_service.OBJECT_TTL_DAYS.value + }, + 'ID': 'PKB_OBJECT_TTL', + 'Filter': {}, + 'Status': 'Enabled', + }] + }) + vm_util.IssueCommand( + util.AWS_PREFIX + + [ + 's3api', + 'put-bucket-lifecycle-configuration', + '--region', + self.region, + '--bucket', + bucket_name, + '--lifecycle-configuration', + config, + ] ) - vm_util.IssueCommand(util.AWS_PREFIX + [ - 's3api', 'put-bucket-lifecycle-configuration', - '--region', self.region, - '--bucket', bucket_name, - '--lifecycle-configuration', config]) def Copy(self, src_url, dst_url, recursive=False): """See base class.""" @@ -104,8 +119,9 @@ def Copy(self, src_url, dst_url, recursive=False): def CopyToBucket(self, src_path, bucket, object_path): """See base class.""" dst_url = self.MakeRemoteCliDownloadUrl(bucket, object_path) - vm_util.IssueCommand(['aws', 's3', 'cp', src_path, dst_url, - '--region', self.region]) + vm_util.IssueCommand( + ['aws', 's3', 'cp', src_path, dst_url, '--region', self.region] + ) def MakeRemoteCliDownloadUrl(self, bucket, object_path): """See base class.""" @@ -115,12 +131,16 @@ def MakeRemoteCliDownloadUrl(self, bucket, object_path): def GenerateCliDownloadFileCommand(self, src_url, local_path): """See base class.""" return 'aws s3 cp "%s" "%s" --region=%s' % ( - src_url, local_path, self.region) + src_url, + local_path, + self.region, + ) def List(self, bucket): """See base class.""" stdout, _, _ = vm_util.IssueCommand( - ['aws', 's3', 'ls', bucket, '--region', self.region]) + ['aws', 's3', 'ls', bucket, '--region', self.region] + ) return stdout def ListTopLevelSubfolders(self, bucket): @@ -155,18 +175,28 @@ def _SuppressFailure(stdout, stderr, retcode): return False vm_util.IssueCommand( - ['aws', 's3', 'rb', - 's3://%s' % bucket, - '--region', self.region, - '--force'], # --force deletes even if bucket contains objects. - suppress_failure=_SuppressFailure) + [ + 'aws', + 's3', + 'rb', + 's3://%s' % bucket, + '--region', + self.region, + '--force', + ], # --force deletes even if bucket contains objects. + suppress_failure=_SuppressFailure, + ) def EmptyBucket(self, bucket): - vm_util.IssueCommand( - ['aws', 's3', 'rm', - 's3://%s' % bucket, - '--region', self.region, - '--recursive']) + vm_util.IssueCommand([ + 'aws', + 's3', + 'rm', + 's3://%s' % bucket, + '--region', + self.region, + '--recursive', + ]) def MakeBucketPubliclyReadable(self, bucket, also_make_writable=False): """See base class.""" @@ -176,9 +206,15 @@ def MakeBucketPubliclyReadable(self, bucket, also_make_writable=False): actions.append(_WRITE) logging.warning('Making bucket %s publicly writable!', bucket) vm_util.IssueCommand([ - 'aws', 's3api', 'put-bucket-policy', '--region', self.region, - '--bucket', bucket, '--policy', - _MakeS3BucketPolicy(bucket, actions) + 'aws', + 's3api', + 'put-bucket-policy', + '--region', + self.region, + '--bucket', + bucket, + '--policy', + _MakeS3BucketPolicy(bucket, actions), ]) def GetDownloadUrl(self, bucket, object_name, use_https=True): @@ -194,27 +230,32 @@ def PrepareVM(self, vm): vm.Install('boto3') vm.PushFile( - object_storage_service.FindCredentialFile('~/' + - AWS_CREDENTIAL_LOCATION), - AWS_CREDENTIAL_LOCATION) - vm.PushFile(object_storage_service.FindBotoFile(), - object_storage_service.DEFAULT_BOTO_LOCATION_USER) + object_storage_service.FindCredentialFile( + '~/' + AWS_CREDENTIAL_LOCATION + ), + AWS_CREDENTIAL_LOCATION, + ) + vm.PushFile( + object_storage_service.FindBotoFile(), + object_storage_service.DEFAULT_BOTO_LOCATION_USER, + ) def CleanupVM(self, vm): vm.Uninstall('awscli') def CLIUploadDirectory(self, vm, directory, file_names, bucket): return vm.RemoteCommand( - 'time aws s3 sync %s s3://%s/' % (directory, bucket)) + 'time aws s3 sync %s s3://%s/' % (directory, bucket) + ) def CLIDownloadBucket(self, vm, bucket, objects, dest): - return vm.RemoteCommand( - 'time aws s3 sync s3://%s/ %s' % (bucket, dest)) + return vm.RemoteCommand('time aws s3 sync s3://%s/ %s' % (bucket, dest)) def Metadata(self, vm): return { - object_storage_service.BOTO_LIB_VERSION: + object_storage_service.BOTO_LIB_VERSION: ( linux_packages.GetPipPackageVersion(vm, 'boto3') + ) } def APIScriptArgs(self): @@ -225,18 +266,17 @@ def APIScriptFiles(cls): return ['s3.py'] -def _MakeS3BucketPolicy(bucket: str, - actions: List[str], - object_prefix='') -> str: +def _MakeS3BucketPolicy( + bucket: str, actions: List[str], object_prefix='' +) -> str: # https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html return json.dumps({ - 'Version': - '2012-10-17', + 'Version': '2012-10-17', 'Statement': [{ 'Principal': '*', 'Sid': 'PkbAcl', 'Effect': 'Allow', 'Action': actions, - 'Resource': [f'arn:aws:s3:::{bucket}/{object_prefix}*'] - }] + 'Resource': [f'arn:aws:s3:::{bucket}/{object_prefix}*'], + }], }) diff --git a/perfkitbenchmarker/providers/aws/snowflake_aws.py b/perfkitbenchmarker/providers/aws/snowflake_aws.py index 84d1b85bae..acf2f0d110 100644 --- a/perfkitbenchmarker/providers/aws/snowflake_aws.py +++ b/perfkitbenchmarker/providers/aws/snowflake_aws.py @@ -28,11 +28,13 @@ class Snowflake(snowflake.Snowflake): """Class representing a Snowflake Data Warehouse Instance hosted on AWS.""" + CLOUD = provider_info.AWS SERVICE_TYPE = 'snowflake_aws' class Snowflakeexternal(snowflake.Snowflake): """Class representing Snowflake External Warehouses on AWS.""" + CLOUD = provider_info.AWS SERVICE_TYPE = 'snowflakeexternal_aws' diff --git a/perfkitbenchmarker/providers/aws/spectrum.py b/perfkitbenchmarker/providers/aws/spectrum.py index 3d7057703d..adf05bb70c 100644 --- a/perfkitbenchmarker/providers/aws/spectrum.py +++ b/perfkitbenchmarker/providers/aws/spectrum.py @@ -40,8 +40,14 @@ def AddTags(resource_arn, region): region: The AWS region resource was created in. """ cmd_prefix = util.AWS_PREFIX - tag_cmd = cmd_prefix + ['redshift', 'create-tags', '--region=%s' % region, - '--resource-name', resource_arn, '--tags'] + tag_cmd = cmd_prefix + [ + 'redshift', + 'create-tags', + '--region=%s' % region, + '--resource-name', + resource_arn, + '--tags', + ] tag_cmd += util.MakeFormattedDefaultTags() vm_util.IssueCommand(tag_cmd) @@ -61,12 +67,14 @@ def __init__(self, cluster_identifier, iam_role_name, cmd_prefix): self.cmd_prefix = cmd_prefix self.cluster_identifier = cluster_identifier self.iam_role_name = iam_role_name - cmd = self.cmd_prefix + ['redshift', - 'modify-cluster-iam-roles', - '--cluster-identifier', - self.cluster_identifier, - '--add-iam-roles', - self.iam_role_name] + cmd = self.cmd_prefix + [ + 'redshift', + 'modify-cluster-iam-roles', + '--cluster-identifier', + self.cluster_identifier, + '--add-iam-roles', + self.iam_role_name, + ] vm_util.IssueCommand(cmd) @@ -96,15 +104,25 @@ def _IsClusterReady(self): def __DescribeCluster(self): """Describe a spectrum cluster.""" - cmd = self.cmd_prefix + ['redshift', 'describe-clusters', - '--cluster-identifier', self.cluster_identifier] + cmd = self.cmd_prefix + [ + 'redshift', + 'describe-clusters', + '--cluster-identifier', + self.cluster_identifier, + ] return vm_util.IssueCommand(cmd) def _IsSnapshotRestored(self): """Method to return if the cluster snapshot is completed restoring.""" - stdout, _, _, = self.__DescribeCluster() - return (json.loads(stdout)['Clusters'][0]['RestoreStatus']['Status'] in - SNAPSHOT_READY_STATUSES) + ( + stdout, + _, + _, + ) = self.__DescribeCluster() + return ( + json.loads(stdout)['Clusters'][0]['RestoreStatus']['Status'] + in SNAPSHOT_READY_STATUSES + ) def _PostCreate(self): """Perform general post create operations on the cluster. @@ -112,10 +130,13 @@ def _PostCreate(self): Get the endpoint to be used for interacting with the cluster and apply tags on the cluster. """ - @vm_util.Retry(poll_interval=self.POLL_INTERVAL, fuzz=0, - timeout=self.READY_TIMEOUT, - retryable_exceptions=( - errors.Resource.RetryableCreationError,)) + + @vm_util.Retry( + poll_interval=self.POLL_INTERVAL, + fuzz=0, + timeout=self.READY_TIMEOUT, + retryable_exceptions=(errors.Resource.RetryableCreationError,), + ) def WaitUntilReady(): if not self._IsReady(): raise errors.Resource.RetryableCreationError('Adding IAM Role') @@ -123,15 +144,15 @@ def WaitUntilReady(): stdout, _, _ = self.__DescribeCluster() self.adding_iam_role = None if self.iam_role is not None: - self.adding_iam_role = AddingIAMRole(self.cluster_identifier, - self.iam_role, - self.cmd_prefix) + self.adding_iam_role = AddingIAMRole( + self.cluster_identifier, self.iam_role, self.cmd_prefix + ) WaitUntilReady() stdout, _, _ = self.__DescribeCluster() self.endpoint = json.loads(stdout)['Clusters'][0]['Endpoint']['Address'] account = util.GetAccount() - self.arn = 'arn:aws:redshift:{}:{}:cluster:{}'.format(self.region, account, - self. - cluster_identifier) + self.arn = 'arn:aws:redshift:{}:{}:cluster:{}'.format( + self.region, account, self.cluster_identifier + ) AddTags(self.arn, self.region) diff --git a/perfkitbenchmarker/providers/aws/util.py b/perfkitbenchmarker/providers/aws/util.py index 20e294be6c..33e9ee64a9 100644 --- a/perfkitbenchmarker/providers/aws/util.py +++ b/perfkitbenchmarker/providers/aws/util.py @@ -29,15 +29,18 @@ AWS_PATH = 'aws' AWS_PREFIX = [AWS_PATH, '--output', 'json'] FLAGS = flags.FLAGS -STOCKOUT_MESSAGE = ('Creation failed due to insufficient capacity indicating a ' - 'potential stockout scenario.') +STOCKOUT_MESSAGE = ( + 'Creation failed due to insufficient capacity indicating a ' + 'potential stockout scenario.' +) def IsRegion(zone_or_region): """Returns whether "zone_or_region" is a region.""" if not re.match(r'[a-z]{2}-[a-z]+-[0-9][a-z]?$', zone_or_region): raise ValueError( - '%s is not a valid AWS zone or region name' % zone_or_region) + '%s is not a valid AWS zone or region name' % zone_or_region + ) return zone_or_region[-1] in string.digits @@ -53,6 +56,7 @@ def GetRegionFromZones(zones): Args: zones: A set of zones. + Raises: Exception: if the zones are in different regions. """ @@ -63,9 +67,10 @@ def GetRegionFromZones(zones): region = current_region else: if region != current_region: - raise Exception('Not All zones are in the same region %s not same as ' - '%s. zones: %s' % - (region, current_region, ','.join(zones))) + raise Exception( + 'Not All zones are in the same region %s not same as %s. zones: %s' + % (region, current_region, ','.join(zones)) + ) return region @@ -74,7 +79,7 @@ def GetZonesInRegion(region: str) -> Set[str]: get_zones_cmd = AWS_PREFIX + [ 'ec2', 'describe-availability-zones', - '--region={0}'.format(region) + '--region={0}'.format(region), ] stdout, _, _ = vm_util.IssueCommand(get_zones_cmd) response = json.loads(stdout) @@ -89,10 +94,16 @@ def GetZonesFromMachineType(machine_type: str) -> Set[str]: """Returns all available zones for a given machine type.""" zones = set() for region in GetAllRegions(): - get_zones_cmd = AWS_PREFIX + [ - 'ec2', 'describe-instance-type-offerings', - '--location-type=availability-zone', f'--region={region}' - ] + AwsFilter({'instance-type': machine_type}) + get_zones_cmd = ( + AWS_PREFIX + + [ + 'ec2', + 'describe-instance-type-offerings', + '--location-type=availability-zone', + f'--region={region}', + ] + + AwsFilter({'instance-type': machine_type}) + ) stdout, _, _ = vm_util.IssueCommand(get_zones_cmd) response = json.loads(stdout) for item in response['InstanceTypeOfferings']: @@ -166,8 +177,9 @@ def FormatTagSpecifications(resource_type, tags_dict): Returns: A list of tags formatted as arguments for 'tag-specifications' parameter. """ - tags = ','.join('{Key=%s,Value=%s}' % - (k, v) for k, v in six.iteritems(tags_dict)) + tags = ','.join( + '{Key=%s,Value=%s}' % (k, v) for k, v in six.iteritems(tags_dict) + ) return 'ResourceType=%s,Tags=[%s]' % (resource_type, tags) @@ -182,12 +194,18 @@ def AddTags(resource_id, region, **kwargs): if not kwargs: return - tag_cmd = AWS_PREFIX + [ - 'ec2', - 'create-tags', - '--region=%s' % region, - '--resources', resource_id, - '--tags'] + FormatTags(kwargs) + tag_cmd = ( + AWS_PREFIX + + [ + 'ec2', + 'create-tags', + '--region=%s' % region, + '--resources', + resource_id, + '--tags', + ] + + FormatTags(kwargs) + ) IssueRetryableCommand(tag_cmd) @@ -259,7 +277,7 @@ def IssueRetryableCommand(cmd, env=None, suppress_failure=None): Args: cmd: A list of strings such as is given to the subprocess.Popen() - constructor. + constructor. env: An alternate environment to pass to the Popen command. suppress_failure: A function to pass to vm_util.IssueCommand() @@ -267,13 +285,16 @@ def IssueRetryableCommand(cmd, env=None, suppress_failure=None): A tuple of stdout and stderr from running the provided command. """ stdout, stderr, retcode = vm_util.IssueCommand( - cmd, env=env, raise_on_failure=False, suppress_failure=suppress_failure) + cmd, env=env, raise_on_failure=False, suppress_failure=suppress_failure + ) if retcode: raise errors.VmUtil.CalledProcessException( - 'Command returned a non-zero exit code.\n') + 'Command returned a non-zero exit code.\n' + ) if stderr: raise errors.VmUtil.CalledProcessException( - 'The command had output on stderr:\n%s' % stderr) + 'The command had output on stderr:\n%s' % stderr + ) return stdout, stderr diff --git a/perfkitbenchmarker/providers/azure/azure_blob_storage.py b/perfkitbenchmarker/providers/azure/azure_blob_storage.py index b5cc4d1b06..08ea1b5efc 100644 --- a/perfkitbenchmarker/providers/azure/azure_blob_storage.py +++ b/perfkitbenchmarker/providers/azure/azure_blob_storage.py @@ -44,10 +44,12 @@ def __init__(self): STORAGE_NAME = provider_info.AZURE - def PrepareService(self, - region, - existing_storage_account_and_resource_group=None, - try_to_create_storage_account_and_resource_group=False): + def PrepareService( + self, + region, + existing_storage_account_and_resource_group=None, + try_to_create_storage_account_and_resource_group=False, + ): """See base class (without additional args). TODO(deitz): We should use the same interface across the clouds without @@ -73,7 +75,8 @@ def PrepareService(self, existing_storage_account, existing_resource_group = None, None if existing_storage_account_and_resource_group: existing_storage_account, existing_resource_group = ( - existing_storage_account_and_resource_group) + existing_storage_account_and_resource_group + ) assert existing_storage_account is not None assert existing_resource_group is not None else: @@ -88,17 +91,20 @@ def PrepareService(self, # is True, however, then we do try to create it. In this case, we shouldn't # raise on a failure since it may already exist. raise_on_create_failure = not ( - existing_storage_account_and_resource_group and - try_to_create_storage_account_and_resource_group) + existing_storage_account_and_resource_group + and try_to_create_storage_account_and_resource_group + ) # We use a separate resource group so that our buckets can optionally stick # around after PKB runs. This is useful for things like cold reads tests self.resource_group = azure_network.AzureResourceGroup( resource_group_name, use_existing=not try_to_create_storage_account_and_resource_group, - timeout_minutes=max(FLAGS.timeout_minutes, - FLAGS.persistent_timeout_minutes), - raise_on_create_failure=raise_on_create_failure) + timeout_minutes=max( + FLAGS.timeout_minutes, FLAGS.persistent_timeout_minutes + ), + raise_on_create_failure=raise_on_create_failure, + ) self.resource_group.Create() # We use a different Azure storage account than the VM account @@ -113,39 +119,41 @@ def PrepareService(self, kind=FLAGS.azure_blob_account_kind, resource_group=self.resource_group, use_existing=not try_to_create_storage_account_and_resource_group, - raise_on_create_failure=raise_on_create_failure) + raise_on_create_failure=raise_on_create_failure, + ) self.storage_account.Create() if object_storage_service.OBJECT_TTL_DAYS.value: if try_to_create_storage_account_and_resource_group: ttl_days = object_storage_service.OBJECT_TTL_DAYS.value - policy = json.dumps( - { - 'rules': [{ - 'enabled': True, - 'name': 'PKB_BLOB_TTL', - 'type': 'Lifecycle', - 'definition': { - 'actions': { - 'version': { - 'delete': { - 'daysAfterCreationGreaterThan': ttl_days - } - } - }, - # A blobtype filter is required. Matches all objects. - 'filters': { - 'blobTypes': ['appendBlob', 'blockBlob'] - }, + policy = json.dumps({ + 'rules': [{ + 'enabled': True, + 'name': 'PKB_BLOB_TTL', + 'type': 'Lifecycle', + 'definition': { + 'actions': { + 'version': { + 'delete': {'daysAfterCreationGreaterThan': ttl_days} + } }, - }] - } - ) + # A blobtype filter is required. Matches all objects. + 'filters': {'blobTypes': ['appendBlob', 'blockBlob']}, + }, + }] + }) vm_util.IssueCommand([ azure.AZURE_PATH, - 'storage', 'account', 'management-policy', 'create', - '--account-name', self.storage_account.name, - '--policy', policy, - '--resource-group', self.resource_group.name]) + 'storage', + 'account', + 'management-policy', + 'create', + '--account-name', + self.storage_account.name, + '--policy', + policy, + '--resource-group', + self.resource_group.name, + ]) else: logging.warning( 'Not setting object TTL, because of existing storage account.' @@ -160,50 +168,78 @@ def CleanupService(self): def MakeBucket(self, bucket, raise_on_failure=True, tag_bucket=True): del tag_bucket _, stderr, ret_code = vm_util.IssueCommand( - [azure.AZURE_PATH, 'storage', 'container', 'create', '--name', bucket] + - self.storage_account.connection_args, - raise_on_failure=False) + [azure.AZURE_PATH, 'storage', 'container', 'create', '--name', bucket] + + self.storage_account.connection_args, + raise_on_failure=False, + ) if ret_code and raise_on_failure: raise errors.Benchmarks.BucketCreationError(stderr) def DeleteBucket(self, bucket): - if (not hasattr(self, 'storage_account') or - not self.storage_account or - not hasattr(self.storage_account, 'connection_args') or - not self.storage_account.connection_args): + if ( + not hasattr(self, 'storage_account') + or not self.storage_account + or not hasattr(self.storage_account, 'connection_args') + or not self.storage_account.connection_args + ): logging.warning( 'storage_account not properly configured. Skipping DeleteBucket %s', - bucket) + bucket, + ) return vm_util.IssueCommand( - [azure.AZURE_PATH, 'storage', 'container', 'delete', '--name', bucket] + - self.storage_account.connection_args, - raise_on_failure=False) + [azure.AZURE_PATH, 'storage', 'container', 'delete', '--name', bucket] + + self.storage_account.connection_args, + raise_on_failure=False, + ) def Copy(self, src_url, dst_url, recursive=False): """See base class.""" raise NotImplementedError() def CopyToBucket(self, src_path, bucket, object_path): - vm_util.IssueCommand(['az', 'storage', 'blob', 'upload', - '--account-name', self.storage_account.name, - '--file', src_path, - '--container', bucket, - '--name', object_path] + - self.storage_account.connection_args) + vm_util.IssueCommand( + [ + 'az', + 'storage', + 'blob', + 'upload', + '--account-name', + self.storage_account.name, + '--file', + src_path, + '--container', + bucket, + '--name', + object_path, + ] + + self.storage_account.connection_args + ) def _GenerateDownloadToken(self, bucket, object_path): blob_store_expiry = datetime.datetime.utcnow() + datetime.timedelta( - days=365) - stdout, _, _ = vm_util.IssueCommand([ - 'az', 'storage', 'blob', 'generate-sas', - '--account-name', self.storage_account.name, - '--container-name', bucket, - '--name', object_path, - '--expiry', blob_store_expiry.strftime('%Y-%m-%dT%H:%M:%SZ'), - '--permissions', 'r' - ] + self.storage_account.connection_args) + days=365 + ) + stdout, _, _ = vm_util.IssueCommand( + [ + 'az', + 'storage', + 'blob', + 'generate-sas', + '--account-name', + self.storage_account.name, + '--container-name', + bucket, + '--name', + object_path, + '--expiry', + blob_store_expiry.strftime('%Y-%m-%dT%H:%M:%SZ'), + '--permissions', + 'r', + ] + + self.storage_account.connection_args + ) token = stdout.strip('\n').strip('"') return token @@ -211,22 +247,27 @@ def MakeRemoteCliDownloadUrl(self, bucket, object_path): """See base class.""" token = self._GenerateDownloadToken(bucket, object_path) url = 'https://{acc}.blob.core.windows.net/{con}/{src}?{tkn}'.format( - acc=self.storage_account.name, - con=bucket, - src=object_path, - tkn=token) + acc=self.storage_account.name, con=bucket, src=object_path, tkn=token + ) return url def GenerateCliDownloadFileCommand(self, src_url, dst_url): """See base class.""" - return 'wget -O {dst_url} "{src_url}"'.format(src_url=src_url, - dst_url=dst_url) + return 'wget -O {dst_url} "{src_url}"'.format( + src_url=src_url, dst_url=dst_url + ) def List(self, bucket): """See base class.""" stdout, _, _ = vm_util.IssueCommand([ - 'az', 'storage', 'blob', 'list', '--container-name', bucket, - '--account-name', self.storage_account.name + 'az', + 'storage', + 'blob', + 'list', + '--container-name', + bucket, + '--account-name', + self.storage_account.name, ]) return [metadata['name'] for metadata in json.loads(str(stdout))] @@ -263,25 +304,33 @@ def PrepareVM(self, vm): def CLIUploadDirectory(self, vm, directory, file_names, bucket): return vm.RemoteCommand( - ('time for file in {files}; ' - 'do azure storage blob upload -q {directory}/$file {bucket} ' - '--connection-string {connection_string}; ' - 'done').format( - files=' '.join(file_names), - directory=directory, - bucket=bucket, - connection_string=self.storage_account.connection_string)) + ( + 'time for file in {files}; ' + 'do azure storage blob upload -q {directory}/$file {bucket} ' + '--connection-string {connection_string}; ' + 'done' + ).format( + files=' '.join(file_names), + directory=directory, + bucket=bucket, + connection_string=self.storage_account.connection_string, + ) + ) def CLIDownloadBucket(self, vm, bucket, objects, dest): return vm.RemoteCommand( - ('time for object in {objects}; ' - 'do azure storage blob download {bucket} $object {dest} ' - '--connection-string {connection_string}; ' - 'done').format( - objects=' '.join(objects), - bucket=bucket, - dest=dest, - connection_string=self.storage_account.connection_string)) + ( + 'time for object in {objects}; ' + 'do azure storage blob download {bucket} $object {dest} ' + '--connection-string {connection_string}; ' + 'done' + ).format( + objects=' '.join(objects), + bucket=bucket, + dest=dest, + connection_string=self.storage_account.connection_string, + ) + ) def Metadata(self, vm): return { @@ -291,7 +340,7 @@ def Metadata(self, vm): def APIScriptArgs(self): return [ '--azure_account=%s' % self.storage_account.name, - '--azure_key=%s' % self.storage_account.key + '--azure_key=%s' % self.storage_account.key, ] @classmethod diff --git a/perfkitbenchmarker/providers/azure/azure_container_instances.py b/perfkitbenchmarker/providers/azure/azure_container_instances.py index 239eaf4516..afcec89896 100644 --- a/perfkitbenchmarker/providers/azure/azure_container_instances.py +++ b/perfkitbenchmarker/providers/azure/azure_container_instances.py @@ -89,7 +89,7 @@ def _Delete(self): @property def ip_address(self): """Container instances don't have private ips yet.""" - raise NotImplementedError('ACI containers don\'t have private ips.') + raise NotImplementedError("ACI containers don't have private ips.") @ip_address.setter def ip_address(self, value): @@ -98,8 +98,13 @@ def ip_address(self, value): def _GetContainerInstance(self): """Gets a representation of the container and returns it.""" - show_cmd = [azure.AZURE_PATH, 'container', 'show', '--name', self.name - ] + self.resource_group.args + show_cmd = [ + azure.AZURE_PATH, + 'container', + 'show', + '--name', + self.name, + ] + self.resource_group.args stdout, _, _ = vm_util.IssueCommand(show_cmd) return json.loads(stdout) @@ -113,21 +118,28 @@ def WaitForExit(self, timeout=None): @vm_util.Retry( timeout=timeout, - retryable_exceptions=(container_service.RetriableContainerException,)) + retryable_exceptions=(container_service.RetriableContainerException,), + ) def _WaitForExit(): container = self._GetContainerInstance()['containers'][0] state = container['instanceView']['currentState']['state'] if state != 'Terminated': raise container_service.RetriableContainerException( - f'Container in ({state}). Not yet in expected state Terminated.') + f'Container in ({state}). Not yet in expected state Terminated.' + ) return container return _WaitForExit() def GetLogs(self): """Returns the logs from the container.""" - logs_cmd = [azure.AZURE_PATH, 'container', 'logs', '--name', self.name - ] + self.resource_group.args + logs_cmd = [ + azure.AZURE_PATH, + 'container', + 'logs', + '--name', + self.name, + ] + self.resource_group.args stdout, _, _ = vm_util.IssueCommand(logs_cmd) return stdout diff --git a/perfkitbenchmarker/providers/azure/azure_disk.py b/perfkitbenchmarker/providers/azure/azure_disk.py index a58e4c2eba..ec3e25da71 100644 --- a/perfkitbenchmarker/providers/azure/azure_disk.py +++ b/perfkitbenchmarker/providers/azure/azure_disk.py @@ -105,7 +105,8 @@ def _GenerateDrivePathSuffixes(): """ character_range = range(ord('a'), ord('z') + 1) products = _ProductWithIncreasingLength( - character_range, MAX_DRIVE_SUFFIX_LENGTH) + character_range, MAX_DRIVE_SUFFIX_LENGTH + ) for p in products: yield ''.join(chr(c) for c in p) @@ -116,6 +117,7 @@ def _GenerateDrivePathSuffixes(): class TooManyAzureDisksError(Exception): """Exception raised when too many disks are attached.""" + pass @@ -129,14 +131,16 @@ def LocalDriveIsNvme(machine_type): """Check if the machine type uses NVMe driver.""" return any( re.search(machine_series, machine_type) - for machine_series in AZURE_NVME_TYPES) + for machine_series in AZURE_NVME_TYPES + ) def HasTempDrive(machine_type): """Check if the machine type has the temp drive (sdb).""" return not any( re.search(machine_series, machine_type) - for machine_series in AZURE_NO_TMP_DISK_TYPES) + for machine_series in AZURE_NO_TMP_DISK_TYPES + ) class AzureDisk(disk.BaseDisk): @@ -144,11 +148,7 @@ class AzureDisk(disk.BaseDisk): _lock = threading.Lock() - def __init__(self, - disk_spec, - vm, - lun, - is_image=False): + def __init__(self, disk_spec, vm, lun, is_image=False): super(AzureDisk, self).__init__(disk_spec) self.host_caching = FLAGS.azure_host_caching self.vm = vm @@ -161,9 +161,11 @@ def __init__(self, self.is_image = is_image self._deleted = False self.machine_type = vm.machine_type - if (self.disk_type == PREMIUM_STORAGE or - self.disk_type == PREMIUM_STORAGE_V2 or - self.disk_type == ULTRA_STORAGE): + if ( + self.disk_type == PREMIUM_STORAGE + or self.disk_type == PREMIUM_STORAGE_V2 + or self.disk_type == ULTRA_STORAGE + ): self.metadata.update({ disk.MEDIA: disk.SSD, disk.REPLICATION: disk.ZONE, @@ -194,43 +196,83 @@ def _Create(self): assert not self.is_image if self.disk_type == ULTRA_STORAGE and not self.vm.availability_zone: - raise Exception(f'Azure Ultradisk is being created in zone "{self.zone}"' - 'which was not specified to have an availability zone. ' - 'Availability zones are specified with zone-\\d e.g. ' - 'eastus1-2 for availability zone 2 in zone eastus1') + raise Exception( + f'Azure Ultradisk is being created in zone "{self.zone}"' + 'which was not specified to have an availability zone. ' + 'Availability zones are specified with zone-\\d e.g. ' + 'eastus1-2 for availability zone 2 in zone eastus1' + ) with self._lock: - _, _, retcode = vm_util.IssueCommand([ - azure.AZURE_PATH, 'vm', 'disk', 'attach', '--new', '--caching', - self.host_caching, '--name', self.name, '--lun', - str(self.lun), '--sku', self.disk_type, '--vm-name', self.vm_name, - '--size-gb', - str(self.disk_size) - ] + self.resource_group.args, raise_on_failure=False, timeout=600) + _, _, retcode = vm_util.IssueCommand( + [ + azure.AZURE_PATH, + 'vm', + 'disk', + 'attach', + '--new', + '--caching', + self.host_caching, + '--name', + self.name, + '--lun', + str(self.lun), + '--sku', + self.disk_type, + '--vm-name', + self.vm_name, + '--size-gb', + str(self.disk_size), + ] + + self.resource_group.args, + raise_on_failure=False, + timeout=600, + ) if retcode: raise errors.Resource.RetryableCreationError( - 'Error creating Azure disk.') - - _, _, retcode = vm_util.IssueCommand([ - azure.AZURE_PATH, 'disk', 'update', '--name', self.name, '--set', - util.GetTagsJson(self.resource_group.timeout_minutes) - ] + self.resource_group.args, raise_on_failure=False) + 'Error creating Azure disk.' + ) + + _, _, retcode = vm_util.IssueCommand( + [ + azure.AZURE_PATH, + 'disk', + 'update', + '--name', + self.name, + '--set', + util.GetTagsJson(self.resource_group.timeout_minutes), + ] + + self.resource_group.args, + raise_on_failure=False, + ) if retcode: raise errors.Resource.RetryableCreationError( - 'Error tagging Azure disk.') - - if (self.disk_type in [ULTRA_STORAGE, PREMIUM_STORAGE_V2] and ( - FLAGS.azure_provisioned_iops or FLAGS.azure_provisioned_throughput)): - args = ([azure.AZURE_PATH, 'disk', 'update', '--name', self.name] + - self.resource_group.args) + 'Error tagging Azure disk.' + ) + + if self.disk_type in [ULTRA_STORAGE, PREMIUM_STORAGE_V2] and ( + FLAGS.azure_provisioned_iops or FLAGS.azure_provisioned_throughput + ): + args = [ + azure.AZURE_PATH, + 'disk', + 'update', + '--name', + self.name, + ] + self.resource_group.args if FLAGS.azure_provisioned_iops: - args = args + ['--disk-iops-read-write', - str(FLAGS.azure_provisioned_iops)] + args = args + [ + '--disk-iops-read-write', + str(FLAGS.azure_provisioned_iops), + ] if FLAGS.azure_provisioned_throughput: - args = args + ['--disk-mbps-read-write', - str(FLAGS.azure_provisioned_throughput)] + args = args + [ + '--disk-mbps-read-write', + str(FLAGS.azure_provisioned_throughput), + ] _, _, _ = vm_util.IssueCommand(args, raise_on_failure=True) @@ -245,10 +287,19 @@ def _Exists(self): if self._deleted: return False - stdout, _, _ = vm_util.IssueCommand([ - azure.AZURE_PATH, 'disk', 'show', '--output', 'json', '--name', - self.name - ] + self.resource_group.args, raise_on_failure=False) + stdout, _, _ = vm_util.IssueCommand( + [ + azure.AZURE_PATH, + 'disk', + 'show', + '--output', + 'json', + '--name', + self.name, + ] + + self.resource_group.args, + raise_on_failure=False, + ) try: json.loads(stdout) return True diff --git a/perfkitbenchmarker/providers/azure/azure_flexible_server.py b/perfkitbenchmarker/providers/azure/azure_flexible_server.py index d6c46ff608..6b748ccbe5 100644 --- a/perfkitbenchmarker/providers/azure/azure_flexible_server.py +++ b/perfkitbenchmarker/providers/azure/azure_flexible_server.py @@ -56,12 +56,14 @@ class AzureFlexibleServer(azure_relational_db.AzureRelationalDb): Note that the client VM's region and the region requested for the database must be the same. - """ + SERVER_TYPE = 'flexible-server' CLOUD = provider_info.AZURE - ENGINE = [sql_engine_utils.FLEXIBLE_SERVER_POSTGRES, - sql_engine_utils.FLEXIBLE_SERVER_MYSQL] + ENGINE = [ + sql_engine_utils.FLEXIBLE_SERVER_POSTGRES, + sql_engine_utils.FLEXIBLE_SERVER_MYSQL, + ] @staticmethod def GetDefaultEngineVersion(engine: str): @@ -129,9 +131,7 @@ def GetAzCommandForEngine(self): return engine def _PostCreate(self): - """Perform general post create operations on the cluster. - - """ + """Perform general post create operations on the cluster.""" # Calling the grand parent class. super(azure_relational_db.AzureRelationalDb, self)._PostCreate() cmd = [ diff --git a/perfkitbenchmarker/providers/azure/azure_kubernetes_service.py b/perfkitbenchmarker/providers/azure/azure_kubernetes_service.py index 1ac926814f..5ec95200be 100644 --- a/perfkitbenchmarker/providers/azure/azure_kubernetes_service.py +++ b/perfkitbenchmarker/providers/azure/azure_kubernetes_service.py @@ -66,11 +66,18 @@ def _Create(self): """Creates the registry.""" if self._Exists(): return - vm_util.IssueCommand([ - azure.AZURE_PATH, 'acr', 'create', - '--name', self.name, - '--sku', self.sku - ] + self.resource_group.args) + vm_util.IssueCommand( + [ + azure.AZURE_PATH, + 'acr', + 'create', + '--name', + self.name, + '--sku', + self.sku, + ] + + self.resource_group.args + ) def _Delete(self): """Deletes the registry.""" @@ -83,10 +90,16 @@ def _PostCreate(self): # has read permission, because it created the repo. if not FLAGS.bootstrap_azure_service_principal: create_role_assignment_cmd = [ - azure.AZURE_PATH, 'role', 'assignment', 'create', - '--assignee', self.service_principal.app_id, - '--role', 'Reader', - '--scope', self.acr_id, + azure.AZURE_PATH, + 'role', + 'assignment', + 'create', + '--assignee', + self.service_principal.app_id, + '--role', + 'Reader', + '--scope', + self.acr_id, ] vm_util.IssueRetryableCommand(create_role_assignment_cmd) @@ -102,14 +115,18 @@ def _DeleteDependencies(self): def Login(self): """Logs in to the registry.""" vm_util.IssueCommand([ - azure.AZURE_PATH, 'acr', 'login', - '--name', self.name, + azure.AZURE_PATH, + 'acr', + 'login', + '--name', + self.name, ]) def GetFullRegistryTag(self, image): """Gets the full tag of the image.""" full_tag = '{login_server}/{name}'.format( - login_server=self.login_server, name=image) + login_server=self.login_server, name=image + ) return full_tag @@ -153,11 +170,17 @@ def GetResourceMetadata(self): def _Create(self): """Creates the AKS cluster.""" cmd = [ - azure.AZURE_PATH, 'aks', 'create', - '--name', self.name, - '--location', self.region, - '--ssh-key-value', vm_util.GetPublicKeyPath(), - '--service-principal', self.service_principal.app_id, + azure.AZURE_PATH, + 'aks', + 'create', + '--name', + self.name, + '--location', + self.region, + '--ssh-key-value', + vm_util.GetPublicKeyPath(), + '--service-principal', + self.service_principal.app_id, # TODO(pclay): avoid logging client secret '--client-secret', self.service_principal.password, @@ -175,7 +198,8 @@ def _Create(self): vm_util.Retry(timeout=300)(vm_util.IssueCommand)( cmd, # Half hour timeout on creating the cluster. - timeout=1800) + timeout=1800, + ) for _, nodepool in self.nodepools.items(): self._CreateNodePool(nodepool) @@ -185,10 +209,16 @@ def _CreateNodePool( ): """Creates a node pool.""" cmd = [ - azure.AZURE_PATH, 'aks', 'nodepool', 'add', - '--cluster-name', self.name, - '--name', nodepool_config.name, - '--labels', f'pkb_nodepool={nodepool_config.name}', + azure.AZURE_PATH, + 'aks', + 'nodepool', + 'add', + '--cluster-name', + self.name, + '--name', + nodepool_config.name, + '--labels', + f'pkb_nodepool={nodepool_config.name}', ] + self._GetNodeFlags(nodepool_config) vm_util.IssueCommand(cmd, timeout=600) @@ -217,9 +247,17 @@ def _Exists(self): """Returns True if the cluster exists.""" if self._deleted: return False - stdout, _, _ = vm_util.IssueCommand([ - azure.AZURE_PATH, 'aks', 'show', '--name', self.name, - ] + self.resource_group.args, raise_on_failure=False) + stdout, _, _ = vm_util.IssueCommand( + [ + azure.AZURE_PATH, + 'aks', + 'show', + '--name', + self.name, + ] + + self.resource_group.args, + raise_on_failure=False, + ) try: cluster = json.loads(stdout) self.node_resource_group = cluster['nodeResourceGroup'] @@ -249,27 +287,42 @@ def _PostCreate(self): """Tags the cluster resource group.""" super(AksCluster, self)._PostCreate() set_tags_cmd = [ - azure.AZURE_PATH, 'group', 'update', '-g', self.node_resource_group, - '--set', util.GetTagsJson(self.resource_group.timeout_minutes) + azure.AZURE_PATH, + 'group', + 'update', + '-g', + self.node_resource_group, + '--set', + util.GetTagsJson(self.resource_group.timeout_minutes), ] vm_util.IssueCommand(set_tags_cmd) def _IsReady(self): """Returns True if the cluster is ready.""" - vm_util.IssueCommand([ - azure.AZURE_PATH, 'aks', 'get-credentials', - '--admin', - '--name', self.name, - '--file', FLAGS.kubeconfig, - ] + self.resource_group.args) + vm_util.IssueCommand( + [ + azure.AZURE_PATH, + 'aks', + 'get-credentials', + '--admin', + '--name', + self.name, + '--file', + FLAGS.kubeconfig, + ] + + self.resource_group.args + ) version_cmd = [FLAGS.kubectl, '--kubeconfig', FLAGS.kubeconfig, 'version'] _, _, retcode = vm_util.IssueCommand(version_cmd, raise_on_failure=False) if retcode: return False # POD creation will fail until the default service account is created. get_cmd = [ - FLAGS.kubectl, '--kubeconfig', FLAGS.kubeconfig, - 'get', 'serviceAccounts' + FLAGS.kubectl, + '--kubeconfig', + FLAGS.kubeconfig, + 'get', + 'serviceAccounts', ] stdout, _, _ = vm_util.IssueCommand(get_cmd) return 'default' in stdout @@ -294,9 +347,14 @@ def ResizeNodePool( ): """Change the number of nodes in the node pool.""" cmd = [ - azure.AZURE_PATH, 'aks', 'nodepool', 'scale', - '--cluster-name', self.name, - '--name', node_pool, - f'--node-count={new_size}' + azure.AZURE_PATH, + 'aks', + 'nodepool', + 'scale', + '--cluster-name', + self.name, + '--name', + node_pool, + f'--node-count={new_size}', ] + self.resource_group.args vm_util.IssueCommand(cmd) diff --git a/perfkitbenchmarker/providers/azure/azure_network.py b/perfkitbenchmarker/providers/azure/azure_network.py index 4c3a5623cc..83c6099ef1 100644 --- a/perfkitbenchmarker/providers/azure/azure_network.py +++ b/perfkitbenchmarker/providers/azure/azure_network.py @@ -41,8 +41,9 @@ FLAGS = flags.FLAGS SSH_PORT = 22 -flags.DEFINE_boolean('azure_infiniband', False, - 'Install Mellanox OpenFabrics drivers') +flags.DEFINE_boolean( + 'azure_infiniband', False, 'Install Mellanox OpenFabrics drivers' +) _AZ_VERSION_LOG = flags.DEFINE_boolean( 'azure_version_log', True, 'Log az --version.' ) @@ -63,22 +64,26 @@ def GetResourceGroup(zone=None): return spec.azure_resource_group except AttributeError: group = AzureResourceGroup( - 'pkb%s-%s' % (FLAGS.run_uri, spec.uid), zone=zone) + 'pkb%s-%s' % (FLAGS.run_uri, spec.uid), zone=zone + ) spec.azure_resource_group = group return group class AzureResourceGroup(resource.BaseResource): """A Resource Group, the basic unit of Azure provisioning.""" + _az_lock = threading.Condition() _az_version: Optional[str] = None - def __init__(self, - name, - zone=None, - use_existing=False, - timeout_minutes=None, - raise_on_create_failure=True): + def __init__( + self, + name, + zone=None, + use_existing=False, + timeout_minutes=None, + raise_on_create_failure=True, + ): super(AzureResourceGroup, self).__init__() AzureResourceGroup._log_az_version() self.name = name @@ -89,7 +94,8 @@ def __init__(self, # actual resources, but we need to choose *some* region for every # benchmark, even if the user doesn't specify one. self.region = util.GetRegionFromZone( - FLAGS.zone[0] if FLAGS.zone else zone or DEFAULT_REGION) + FLAGS.zone[0] if FLAGS.zone else zone or DEFAULT_REGION + ) # Whenever an Azure CLI command needs a resource group, it's # always specified the same way. self.args = ['--resource-group', self.name] @@ -101,19 +107,29 @@ def _Create(self): # FLAGS.zone[0]. _, _, retcode = vm_util.IssueCommand( [ - azure.AZURE_PATH, 'group', 'create', '--name', self.name, - '--location', self.region, '--tags' - ] + util.GetTags(self.timeout_minutes), - raise_on_failure=False) + azure.AZURE_PATH, + 'group', + 'create', + '--name', + self.name, + '--location', + self.region, + '--tags', + ] + + util.GetTags(self.timeout_minutes), + raise_on_failure=False, + ) if retcode and self.raise_on_create_failure: raise errors.Resource.RetryableCreationError( - 'Error creating Azure resource group') + 'Error creating Azure resource group' + ) def _Exists(self): stdout, _, _ = vm_util.IssueCommand( [azure.AZURE_PATH, 'group', 'show', '--name', self.name], - raise_on_failure=False) + raise_on_failure=False, + ) try: json.loads(stdout) return True @@ -127,7 +143,8 @@ def _Delete(self): [azure.AZURE_PATH, 'group', 'delete', '--yes', '--name', self.name], timeout=600, raise_on_failure=False, - raise_on_timeout=False) + raise_on_timeout=False, + ) def AddTag(self, key, value): """Add a single tag to an existing Resource Group. @@ -140,8 +157,13 @@ def AddTag(self, key, value): errors.resource.CreationError on failure. """ tag_cmd = [ - azure.AZURE_PATH, 'group', 'update', '--name', self.name, '--set', - 'tags.' + util.FormatTag(key, value) + azure.AZURE_PATH, + 'group', + 'update', + '--name', + self.name, + '--set', + 'tags.' + util.FormatTag(key, value), ] _, _, retcode = vm_util.IssueCommand(tag_cmd, raise_on_failure=False) if retcode: @@ -155,8 +177,8 @@ def _log_az_version(cls): with cls._az_lock: if not cls._az_version: cls.az_version, _, _ = vm_util.IssueCommand( - [azure.AZURE_PATH, '--version'], - raise_on_failure=True) + [azure.AZURE_PATH, '--version'], raise_on_failure=True + ) class AzureStorageAccount(resource.BaseResource): @@ -164,15 +186,17 @@ class AzureStorageAccount(resource.BaseResource): total_storage_accounts = 0 - def __init__(self, - storage_type, - region, - name, - kind=None, - access_tier=None, - resource_group=None, - use_existing=False, - raise_on_create_failure=True): + def __init__( + self, + storage_type, + region, + name, + kind=None, + access_tier=None, + resource_group=None, + use_existing=False, + raise_on_create_failure=True, + ): super(AzureStorageAccount, self).__init__() self.storage_type = storage_type self.name = name @@ -194,31 +218,51 @@ def __init__(self, def _Create(self): """Creates the storage account.""" if not self.use_existing: - create_cmd = [ - azure.AZURE_PATH, 'storage', 'account', 'create', '--kind', self.kind, - '--sku', self.storage_type, '--name', self.name, '--tags' - ] + util.GetTags( - self.resource_group.timeout_minutes) + self.resource_group.args + create_cmd = ( + [ + azure.AZURE_PATH, + 'storage', + 'account', + 'create', + '--kind', + self.kind, + '--sku', + self.storage_type, + '--name', + self.name, + '--tags', + ] + + util.GetTags(self.resource_group.timeout_minutes) + + self.resource_group.args + ) if self.region: create_cmd.extend(['--location', self.region]) if self.kind == 'BlobStorage': create_cmd.extend(['--access-tier', self.access_tier]) vm_util.IssueCommand( - create_cmd, raise_on_failure=self.raise_on_create_failure) + create_cmd, raise_on_failure=self.raise_on_create_failure + ) def _PostCreate(self): """Get our connection string and our keys.""" self.connection_string = util.GetAzureStorageConnectionString( - self.name, self.resource_group.args) + self.name, self.resource_group.args + ) self.connection_args = ['--connection-string', self.connection_string] - self.key = util.GetAzureStorageAccountKey(self.name, - self.resource_group.args) + self.key = util.GetAzureStorageAccountKey( + self.name, self.resource_group.args + ) def _Delete(self): """Deletes the storage account.""" delete_cmd = [ - azure.AZURE_PATH, 'storage', 'account', 'delete', '--name', self.name, - '--yes' + azure.AZURE_PATH, + 'storage', + 'account', + 'delete', + '--name', + self.name, + '--yes', ] + self.resource_group.args vm_util.IssueCommand(delete_cmd, raise_on_failure=False) @@ -226,10 +270,18 @@ def _Exists(self): """Returns true if the storage account exists.""" stdout, _, _ = vm_util.IssueCommand( [ - azure.AZURE_PATH, 'storage', 'account', 'show', '--output', 'json', - '--name', self.name - ] + self.resource_group.args, - raise_on_failure=False) + azure.AZURE_PATH, + 'storage', + 'account', + 'show', + '--output', + 'json', + '--name', + self.name, + ] + + self.resource_group.args, + raise_on_failure=False, + ) try: json.loads(stdout) @@ -267,12 +319,14 @@ def __init__(self, spec, region, name, number_subnets): self.address_spaces = [] for zone_num in range(number_subnets): self.address_spaces.append( - network.GetCidrBlock(self.regional_index, zone_num)) + network.GetCidrBlock(self.regional_index, zone_num) + ) self.is_created = False @classmethod - def GetForRegion(cls, spec, region, name, number_subnets=1 - ) -> Optional['AzureVirtualNetwork']: + def GetForRegion( + cls, spec, region, name, number_subnets=1 + ) -> Optional['AzureVirtualNetwork']: """Retrieves or creates an AzureVirtualNetwork. Args: @@ -292,8 +346,9 @@ def GetForRegion(cls, spec, region, name, number_subnets=1 """ benchmark_spec = context.GetThreadBenchmarkSpec() if benchmark_spec is None: - raise errors.Error('GetNetwork called in a thread without a ' - 'BenchmarkSpec.') + raise errors.Error( + 'GetNetwork called in a thread without a BenchmarkSpec.' + ) if azure_flags.AZURE_SUBNET_ID.value: # AzureVirtualNetworks are not Resources, so we just return None here as # the network exists but will be accessed via the subnet. @@ -304,8 +359,9 @@ def GetForRegion(cls, spec, region, name, number_subnets=1 # benchmark_spec.networks_lock. number_subnets = max(number_subnets, len(FLAGS.zone)) if key not in benchmark_spec.regional_networks: - benchmark_spec.regional_networks[key] = cls(spec, region, name, - number_subnets) + benchmark_spec.regional_networks[key] = cls( + spec, region, name, number_subnets + ) AzureVirtualNetwork._regional_network_count += 1 return benchmark_spec.regional_networks[key] @@ -313,8 +369,8 @@ def GetNextAddressSpace(self): """Returns the next available address space for next subnet.""" with self.vnet_lock: assert self.address_index < len( - self.address_spaces), 'Only allocated {} addresses'.format( - len(self.address_spaces)) + self.address_spaces + ), 'Only allocated {} addresses'.format(len(self.address_spaces)) next_address_space = self.address_spaces[self.address_index] self.address_index += 1 return next_address_space @@ -325,12 +381,26 @@ def Create(self): if self.is_created: return - logging.info('Creating %d Azure subnets in %s', len(self.address_spaces), - self.region) - vm_util.IssueRetryableCommand([ - azure.AZURE_PATH, 'network', 'vnet', 'create', '--location', self - .region, '--name', self.name, '--address-prefixes' - ] + self.address_spaces + self.resource_group.args) + logging.info( + 'Creating %d Azure subnets in %s', + len(self.address_spaces), + self.region, + ) + vm_util.IssueRetryableCommand( + [ + azure.AZURE_PATH, + 'network', + 'vnet', + 'create', + '--location', + self.region, + '--name', + self.name, + '--address-prefixes', + ] + + self.address_spaces + + self.resource_group.args + ) self.is_created = True @@ -343,10 +413,18 @@ def Exists(self): """Returns true if the virtual network exists.""" stdout, _, _ = vm_util.IssueCommand( [ - azure.AZURE_PATH, 'network', 'vnet', 'show', '--output', 'json', - '--name', self.name - ] + self.resource_group.args, - raise_on_failure=False) + azure.AZURE_PATH, + 'network', + 'vnet', + 'show', + '--output', + 'json', + '--name', + self.name, + ] + + self.resource_group.args, + raise_on_failure=False, + ) return bool(json.loads(stdout)) @@ -378,21 +456,42 @@ def _Create(self): if not self.address_space: self.address_space = self.vnet.GetNextAddressSpace() - vm_util.IssueCommand([ - azure.AZURE_PATH, 'network', 'vnet', 'subnet', 'create', '--vnet-name', - self.vnet.name, '--address-prefix', self.address_space, '--name', - self.name - ] + self.resource_group.args) + vm_util.IssueCommand( + [ + azure.AZURE_PATH, + 'network', + 'vnet', + 'subnet', + 'create', + '--vnet-name', + self.vnet.name, + '--address-prefix', + self.address_space, + '--name', + self.name, + ] + + self.resource_group.args + ) @vm_util.Retry() def _Exists(self): stdout, _, _ = vm_util.IssueCommand( [ - azure.AZURE_PATH, 'network', 'vnet', 'subnet', 'show', - '--vnet-name', self.vnet.name, '--output', 'json', '--name', - self.name - ] + self.resource_group.args, - raise_on_failure=False) + azure.AZURE_PATH, + 'network', + 'vnet', + 'subnet', + 'show', + '--vnet-name', + self.vnet.name, + '--output', + 'json', + '--name', + self.name, + ] + + self.resource_group.args, + raise_on_failure=False, + ) return bool(json.loads(stdout)) @@ -422,19 +521,36 @@ def __init__(self, region, subnet, name): self.have_deny_all_rule = False def _Create(self): - vm_util.IssueCommand([ - azure.AZURE_PATH, 'network', 'nsg', 'create', '--location', - self.region, '--name', self.name - ] + self.resource_group.args) + vm_util.IssueCommand( + [ + azure.AZURE_PATH, + 'network', + 'nsg', + 'create', + '--location', + self.region, + '--name', + self.name, + ] + + self.resource_group.args + ) @vm_util.Retry() def _Exists(self): stdout, _, _ = vm_util.IssueCommand( [ - azure.AZURE_PATH, 'network', 'nsg', 'show', '--output', 'json', - '--name', self.name - ] + self.resource_group.args, - raise_on_failure=False) + azure.AZURE_PATH, + 'network', + 'nsg', + 'show', + '--output', + 'json', + '--name', + self.name, + ] + + self.resource_group.args, + raise_on_failure=False, + ) return bool(json.loads(stdout)) @@ -451,18 +567,25 @@ def _GetRulePriority(self, rule, rule_name): return rule_priority def AttachToSubnet(self): - vm_util.IssueRetryableCommand([ - azure.AZURE_PATH, 'network', 'vnet', 'subnet', 'update', '--name', - self.subnet.name, '--network-security-group', self.name - ] + self.resource_group.args + self.subnet.vnet.args) + vm_util.IssueRetryableCommand( + [ + azure.AZURE_PATH, + 'network', + 'vnet', + 'subnet', + 'update', + '--name', + self.subnet.name, + '--network-security-group', + self.name, + ] + + self.resource_group.args + + self.subnet.vnet.args + ) def AllowPort( - self, - vm, - start_port, - end_port=None, - source_range=None, - protocol='*'): + self, vm, start_port, end_port=None, source_range=None, protocol='*' + ): """Open a port or port range. Args: @@ -492,13 +615,27 @@ def AllowPort( rule_name = 'allow-%s-%s' % (port_range, source_range_str) rule_priority = self._GetRulePriority(rule, rule_name) - network_cmd = [ - azure.AZURE_PATH, 'network', 'nsg', 'rule', 'create', '--name', - rule_name, '--destination-port-range', port_range, - '--protocol', protocol, '--access', 'Allow', - '--priority', - str(rule_priority) - ] + ['--source-address-prefixes'] + source_range + network_cmd = ( + [ + azure.AZURE_PATH, + 'network', + 'nsg', + 'rule', + 'create', + '--name', + rule_name, + '--destination-port-range', + port_range, + '--protocol', + protocol, + '--access', + 'Allow', + '--priority', + str(rule_priority), + ] + + ['--source-address-prefixes'] + + source_range + ) network_cmd.extend(self.resource_group.args + self.args) vm_util.IssueRetryableCommand(network_cmd) @@ -512,11 +649,25 @@ def AllowIcmp(self): return rule_priority = self._GetRulePriority(rule, rule_name) network_cmd = [ - azure.AZURE_PATH, 'network', 'nsg', 'rule', 'create', '--name', - rule_name, '--access', 'Allow', '--source-address-prefixes', - source_address, '--source-port-ranges', '*', - '--destination-port-ranges', '*', '--priority', - str(rule_priority), '--protocol', 'Icmp' + azure.AZURE_PATH, + 'network', + 'nsg', + 'rule', + 'create', + '--name', + rule_name, + '--access', + 'Allow', + '--source-address-prefixes', + source_address, + '--source-port-ranges', + '*', + '--destination-port-ranges', + '*', + '--priority', + str(rule_priority), + '--protocol', + 'Icmp', ] network_cmd.extend(self.resource_group.args + self.args) vm_util.IssueRetryableCommand(network_cmd) @@ -546,7 +697,8 @@ def AllowPort(self, vm, start_port, end_port=None, source_range=None): if vm.network.nsg: vm.network.nsg.AllowPort( - vm, start_port, end_port=end_port, source_range=source_range) + vm, start_port, end_port=end_port, source_range=source_range + ) def DisallowAllPorts(self): """Closes all ports on the firewall.""" @@ -582,10 +734,13 @@ def __init__(self, spec): # Placement Group no_placement_group = ( - not FLAGS.placement_group_style or - FLAGS.placement_group_style == placement_group.PLACEMENT_GROUP_NONE) - has_optional_pg = (FLAGS.placement_group_style == - placement_group.PLACEMENT_GROUP_CLOSEST_SUPPORTED) + not FLAGS.placement_group_style + or FLAGS.placement_group_style == placement_group.PLACEMENT_GROUP_NONE + ) + has_optional_pg = ( + FLAGS.placement_group_style + == placement_group.PLACEMENT_GROUP_CLOSEST_SUPPORTED + ) if no_placement_group: self.placement_group = None elif has_optional_pg and len(set(FLAGS.zone)) > 1: @@ -597,21 +752,26 @@ def __init__(self, spec): elif len(set(FLAGS.zone)) > 1: raise errors.Benchmarks.UnsupportedConfigError( 'inter-zone/inter-region tests do not support placement groups. ' - 'Use placement group style closest_supported.') + 'Use placement group style closest_supported.' + ) # With dedicated hosting and/or an availability zone, an availability set # cannot be created - elif (FLAGS.placement_group_style == azure_placement_group.AVAILABILITY_SET - and (is_dedicated_host or in_availability_zone)): + elif ( + FLAGS.placement_group_style == azure_placement_group.AVAILABILITY_SET + and (is_dedicated_host or in_availability_zone) + ): self.placement_group = None else: placement_group_spec = azure_placement_group.AzurePlacementGroupSpec( 'AzurePlacementGroupSpec', flag_values=FLAGS, zone=self.zone, - resource_group=self.resource_group.name) + resource_group=self.resource_group.name, + ) self.placement_group = azure_placement_group.AzurePlacementGroup( - placement_group_spec) + placement_group_spec + ) # Storage account names can't include separator characters :(. storage_account_prefix = 'pkb%s' % FLAGS.run_uri @@ -621,8 +781,10 @@ def __init__(self, spec): # awful naming scheme. suffix = 'storage%d' % AzureStorageAccount.total_storage_accounts self.storage_account = AzureStorageAccount( - FLAGS.azure_storage_type, self.region, - storage_account_prefix[:24 - len(suffix)] + suffix) + FLAGS.azure_storage_type, + self.region, + storage_account_prefix[: 24 - len(suffix)] + suffix, + ) # Length restriction from https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/resource-name-rules#microsoftnetwork pylint: disable=line-too-long prefix = '%s-%s' % (self.resource_group.name, self.region) @@ -639,8 +801,9 @@ def __init__(self, spec): # usage of an nsg is not currently supported with an existing subnet. self.nsg = None else: - self.nsg = AzureNetworkSecurityGroup(self.region, self.subnet, - self.subnet.name + '-nsg') + self.nsg = AzureNetworkSecurityGroup( + self.region, self.subnet, self.subnet.name + '-nsg' + ) @vm_util.Retry() def Create(self): @@ -684,8 +847,7 @@ def Peer(self, peering_network): if self.vnet is peering_network.vnet: return - spec = network.BaseVPCPeeringSpec(self.vnet, - peering_network.vnet) + spec = network.BaseVPCPeeringSpec(self.vnet, peering_network.vnet) self.vpc_peering = AzureVpcPeering(spec) peering_network.vpc_peering = self.vpc_peering self.vpc_peering.Create() @@ -701,33 +863,52 @@ class AzureVpcPeering(network.BaseVPCPeering): def _Create(self): """Creates the peering object.""" - self.name = '%s-%s-%s' % (self.network_a.resource_group.name, - self.network_a.region, self.network_b.region) + self.name = '%s-%s-%s' % ( + self.network_a.resource_group.name, + self.network_a.region, + self.network_b.region, + ) # Creates Peering Connection create_cmd = [ - azure.AZURE_PATH, 'network', 'vnet', 'peering', 'create', - '--name', self.name, - '--vnet-name', self.network_a.name, - '--remote-vnet', self.network_b.name, - '--allow-vnet-access' + azure.AZURE_PATH, + 'network', + 'vnet', + 'peering', + 'create', + '--name', + self.name, + '--vnet-name', + self.network_a.name, + '--remote-vnet', + self.network_b.name, + '--allow-vnet-access', ] + self.network_a.resource_group.args vm_util.IssueRetryableCommand(create_cmd) # Accepts Peering Connection accept_cmd = [ - azure.AZURE_PATH, 'network', 'vnet', 'peering', 'create', - '--name', self.name, - '--vnet-name', self.network_b.name, - '--remote-vnet', self.network_a.name, - '--allow-vnet-access' + azure.AZURE_PATH, + 'network', + 'vnet', + 'peering', + 'create', + '--name', + self.name, + '--vnet-name', + self.network_b.name, + '--remote-vnet', + self.network_a.name, + '--allow-vnet-access', ] + self.network_b.resource_group.args vm_util.IssueRetryableCommand(accept_cmd) - logging.info('Created VPC peering between %s and %s', - self.network_a.address_spaces[0], - self.network_b.address_spaces[0]) + logging.info( + 'Created VPC peering between %s and %s', + self.network_a.address_spaces[0], + self.network_b.address_spaces[0], + ) def _Delete(self): """Deletes the peering connection.""" diff --git a/perfkitbenchmarker/providers/azure/azure_placement_group.py b/perfkitbenchmarker/providers/azure/azure_placement_group.py index 26b5ba8abc..3db173cd89 100644 --- a/perfkitbenchmarker/providers/azure/azure_placement_group.py +++ b/perfkitbenchmarker/providers/azure/azure_placement_group.py @@ -32,8 +32,8 @@ AVAILABILITY_SET = 'availability-set' _CLI_STRATEGY_ARGS_DICT = { PROXIMITY_PLACEMENT_GROUP: ['ppg'], - AVAILABILITY_SET: ['vm', 'availability-set'] - } + AVAILABILITY_SET: ['vm', 'availability-set'], +} class AzurePlacementGroupSpec(placement_group.BasePlacementGroupSpec): @@ -54,21 +54,24 @@ def _GetOptionDecoderConstructions(cls): The pair specifies a decoder class and its __init__() keyword arguments to construct in order to decode the named option. """ - result = super(AzurePlacementGroupSpec, - cls)._GetOptionDecoderConstructions() + result = super( + AzurePlacementGroupSpec, cls + )._GetOptionDecoderConstructions() result.update({ - 'resource_group': (option_decoders.StringDecoder, { - 'none_ok': False - }), - 'placement_group_style': (option_decoders.EnumDecoder, { - 'valid_values': - set([ - PROXIMITY_PLACEMENT_GROUP, - AVAILABILITY_SET - ] + list(placement_group.PLACEMENT_GROUP_OPTIONS)), - 'default': - placement_group.PLACEMENT_GROUP_NONE, - }) + 'resource_group': ( + option_decoders.StringDecoder, + {'none_ok': False}, + ), + 'placement_group_style': ( + option_decoders.EnumDecoder, + { + 'valid_values': set( + [PROXIMITY_PLACEMENT_GROUP, AVAILABILITY_SET] + + list(placement_group.PLACEMENT_GROUP_OPTIONS) + ), + 'default': placement_group.PLACEMENT_GROUP_NONE, + }, + ), }) return result @@ -95,9 +98,17 @@ def __init__(self, azure_placement_group_spec): def _Create(self): """Create the placement group.""" - create_cmd = [azure.AZURE_PATH] + _CLI_STRATEGY_ARGS_DICT[self.strategy] + [ - 'create', '--resource-group', self.resource_group, '--name', self.name - ] + create_cmd = ( + [azure.AZURE_PATH] + + _CLI_STRATEGY_ARGS_DICT[self.strategy] + + [ + 'create', + '--resource-group', + self.resource_group, + '--name', + self.name, + ] + ) if self.region: create_cmd.extend(['--location', self.region]) vm_util.IssueCommand(create_cmd) @@ -108,10 +119,19 @@ def _Delete(self): @vm_util.Retry() def _Exists(self): """Returns True if the placement group exists.""" - show_cmd = [azure.AZURE_PATH] + _CLI_STRATEGY_ARGS_DICT[self.strategy] + [ - 'show', '--output', 'json', '--resource-group', self.resource_group, - '--name', self.name - ] + show_cmd = ( + [azure.AZURE_PATH] + + _CLI_STRATEGY_ARGS_DICT[self.strategy] + + [ + 'show', + '--output', + 'json', + '--resource-group', + self.resource_group, + '--name', + self.name, + ] + ) stdout, _, _ = vm_util.IssueCommand(show_cmd, raise_on_failure=False) return bool(json.loads(stdout)) diff --git a/perfkitbenchmarker/providers/azure/azure_redis_cache.py b/perfkitbenchmarker/providers/azure/azure_redis_cache.py index b9735601f1..602fd4ff02 100644 --- a/perfkitbenchmarker/providers/azure/azure_redis_cache.py +++ b/perfkitbenchmarker/providers/azure/azure_redis_cache.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Module containing classes for Azure Redis Cache. -""" +"""Module containing classes for Azure Redis Cache.""" import json import time @@ -48,7 +47,10 @@ def __init__(self, spec): self.resource_group = azure_network.GetResourceGroup(self.redis_region) self.azure_redis_size = FLAGS.azure_redis_size self.failover_style = FLAGS.redis_failover_style - if self.failover_style == managed_memory_store.Failover.FAILOVER_SAME_REGION: + if ( + self.failover_style + == managed_memory_store.Failover.FAILOVER_SAME_REGION + ): self.azure_tier = 'Premium' else: self.azure_tier = 'Basic' @@ -61,16 +63,13 @@ def GetResourceMetadata(self): dict mapping string property key to value. """ result = { - 'cloud_redis_failover_style': - self.failover_style, - 'cloud_redis_region': - self.redis_region, - 'cloud_redis_azure_tier': - self.azure_tier, - 'cloud_redis_azure_redis_size': - self.azure_redis_size, - 'cloud_redis_version': - managed_memory_store.ParseReadableVersion(self.redis_version), + 'cloud_redis_failover_style': self.failover_style, + 'cloud_redis_region': self.redis_region, + 'cloud_redis_azure_tier': self.azure_tier, + 'cloud_redis_azure_redis_size': self.azure_redis_size, + 'cloud_redis_version': managed_memory_store.ParseReadableVersion( + self.redis_version + ), } return result @@ -86,21 +85,31 @@ def CheckPrerequisites(benchmark_config): """ if FLAGS.managed_memory_store_version: raise errors.Config.InvalidValue( - 'Custom Redis version not supported on Azure Redis. ') + 'Custom Redis version not supported on Azure Redis. ' + ) if FLAGS.redis_failover_style in [ - managed_memory_store.Failover.FAILOVER_SAME_ZONE]: + managed_memory_store.Failover.FAILOVER_SAME_ZONE + ]: raise errors.Config.InvalidValue( - 'Azure redis with failover in the same zone is not supported.') + 'Azure redis with failover in the same zone is not supported.' + ) def _Create(self): """Creates the cache.""" cmd = [ - azure.AZURE_PATH, 'redis', 'create', - '--resource-group', self.resource_group.name, - '--location', self.redis_region, - '--name', self.name, - '--sku', self.azure_tier, - '--vm-size', self.azure_redis_size, + azure.AZURE_PATH, + 'redis', + 'create', + '--resource-group', + self.resource_group.name, + '--location', + self.redis_region, + '--name', + self.name, + '--sku', + self.azure_tier, + '--vm-size', + self.azure_redis_size, '--enable-non-ssl-port', ] vm_util.IssueCommand(cmd, timeout=TIMEOUT) @@ -108,9 +117,13 @@ def _Create(self): def _Delete(self): """Deletes the cache.""" cmd = [ - azure.AZURE_PATH, 'redis', 'delete', - '--resource-group', self.resource_group.name, - '--name', self.name, + azure.AZURE_PATH, + 'redis', + 'delete', + '--resource-group', + self.resource_group.name, + '--name', + self.name, '--yes', ] vm_util.IssueCommand(cmd, timeout=TIMEOUT) @@ -121,11 +134,18 @@ def DescribeCache(self): Returns: stdout, stderr and retcode. """ - stdout, stderr, retcode = vm_util.IssueCommand([ - azure.AZURE_PATH, 'redis', 'show', - '--resource-group', self.resource_group.name, - '--name', self.name, - ], raise_on_failure=False) + stdout, stderr, retcode = vm_util.IssueCommand( + [ + azure.AZURE_PATH, + 'redis', + 'show', + '--resource-group', + self.resource_group.name, + '--name', + self.name, + ], + raise_on_failure=False, + ) return stdout, stderr, retcode def _Exists(self): @@ -150,8 +170,10 @@ def _IsReady(self): True if cache is ready and false otherwise. """ stdout, _, retcode = self.DescribeCache() - if (retcode == 0 and - json.loads(stdout).get('provisioningState', None) == 'Succeeded'): + if ( + retcode == 0 + and json.loads(stdout).get('provisioningState', None) == 'Succeeded' + ): self.redis_version = json.loads(stdout).get('redisVersion', 'unspecified') return True return False @@ -173,18 +195,27 @@ def _PopulateEndpoint(self): stdout, _, retcode = self.DescribeCache() if retcode != 0: raise errors.Resource.RetryableGetError( - f'Failed to retrieve information on {self.name}.') + f'Failed to retrieve information on {self.name}.' + ) response = json.loads(stdout) self._ip = response['hostName'] self._port = response['port'] - stdout, _, retcode = vm_util.IssueCommand([ - azure.AZURE_PATH, 'redis', 'list-keys', - '--resource-group', self.resource_group.name, - '--name', self.name, - ], raise_on_failure=False) + stdout, _, retcode = vm_util.IssueCommand( + [ + azure.AZURE_PATH, + 'redis', + 'list-keys', + '--resource-group', + self.resource_group.name, + '--name', + self.name, + ], + raise_on_failure=False, + ) if retcode != 0: raise errors.Resource.RetryableGetError( - f'Failed to retrieve information on {self.name}.') + f'Failed to retrieve information on {self.name}.' + ) response = json.loads(stdout) self._password = response['primaryKey'] diff --git a/perfkitbenchmarker/providers/azure/azure_relational_db.py b/perfkitbenchmarker/providers/azure/azure_relational_db.py index f8e353b7c0..4760721dbd 100644 --- a/perfkitbenchmarker/providers/azure/azure_relational_db.py +++ b/perfkitbenchmarker/providers/azure/azure_relational_db.py @@ -100,8 +100,8 @@ class AzureRelationalDb(relational_db.BaseRelationalDb): Note that the client VM's region and the region requested for the database must be the same. - """ + CLOUD = provider_info.AZURE ENGINE = [ sql_engine_utils.POSTGRES, @@ -117,7 +117,8 @@ def __init__(self, relational_db_spec): super(AzureRelationalDb, self).__init__(relational_db_spec) if util.IsZone(self.spec.db_spec.zone): raise errors.Config.InvalidValue( - 'Availability zones are currently not supported by Azure DBs') + 'Availability zones are currently not supported by Azure DBs' + ) self.region = util.GetRegionFromZone(self.spec.db_spec.zone) self.resource_group = azure_network.GetResourceGroup(self.region) @@ -129,7 +130,6 @@ def GetResourceMetadata(self): Returns: metadata: dict of Azure DB metadata. - """ metadata = super(AzureRelationalDb, self).GetResourceMetadata() metadata.update({ @@ -149,6 +149,7 @@ def GetDefaultEngineVersion(engine): Args: engine (string): type of database (my_sql or postgres). + Returns: (string): Default engine version. Raises: @@ -163,7 +164,8 @@ def GetDefaultEngineVersion(engine): return DEFALUT_SQLSERVER_VERSION else: raise relational_db.RelationalDbEngineNotFoundError( - 'Unsupported engine {0}'.format(engine)) + 'Unsupported engine {0}'.format(engine) + ) def GetAzCommandForEngine(self): engine = self.spec.engine @@ -182,6 +184,7 @@ def GetConfigFromMachineType(self, machine_type): Args: machine_type (string): Azure machine type i.e GP_Gen5_4 + Returns: (string, string, string): edition, family, vcore Raises: @@ -190,8 +193,10 @@ def GetConfigFromMachineType(self, machine_type): machine_type = machine_type.split('_') if len(machine_type) != 3: raise relational_db.UnsupportedError( - 'Unsupported machine type {0},' - ' sample machine type GP_Gen5_2'.format(machine_type)) + 'Unsupported machine type {0}, sample machine type GP_Gen5_2'.format( + machine_type + ) + ) edition = machine_type[0] if edition == 'BC': edition = 'BusinessCritical' @@ -199,7 +204,8 @@ def GetConfigFromMachineType(self, machine_type): edition = 'GeneralPurpose' else: raise relational_db.UnsupportedError( - 'Unsupported edition {}. Only supports BC or GP'.format(machine_type)) + 'Unsupported edition {}. Only supports BC or GP'.format(machine_type) + ) family = machine_type[1] vcore = machine_type[2] @@ -248,20 +254,22 @@ def RenameDatabase(self, new_name): '--name', self.database_name, '--new-name', - new_name + new_name, ] vm_util.IssueCommand(cmd) self.database_name = new_name else: raise relational_db.RelationalDbEngineNotFoundError( - 'Unsupported engine {0}'.format(engine)) + 'Unsupported engine {0}'.format(engine) + ) def _ApplyDbFlags(self): """Applies the MySqlFlags to a managed instance.""" for flag in FLAGS.db_flags: name_and_value = flag.split('=') _, stderr, _ = self.SetDbConfiguration( - name_and_value[0], name_and_value[1]) + name_and_value[0], name_and_value[1] + ) if stderr: raise KeyError( 'Invalid MySQL flags: {0}. Error {1}'.format( @@ -284,18 +292,22 @@ def _CreateMySqlOrPostgresInstance(self): # and additional increments of 1024 MB up to maximum of 16777216 MB. azure_disk_size_mb = self.spec.db_disk_spec.disk_size * 1024 if azure_disk_size_mb > AZURE_MAX_DB_DISK_SIZE_MB: - error_msg = ('Azure disk size was specified as in the disk spec as %s,' - 'got rounded to %s which is greater than the ' - 'maximum of 16777216 MB' % ( - self.spec.db_disk_spec.disk_size, azure_disk_size_mb)) + error_msg = ( + 'Azure disk size was specified as in the disk spec as %s,' + 'got rounded to %s which is greater than the ' + 'maximum of 16777216 MB' + % (self.spec.db_disk_spec.disk_size, azure_disk_size_mb) + ) raise errors.Config.InvalidValue(error_msg) elif azure_disk_size_mb < AZURE_MIN_DB_DISK_SIZE_MB: - error_msg = ('Azure disk size was specified ' - 'as in the disk spec as %s, got rounded to %s ' - 'which is smaller than the minimum of 5120 MB' % ( - self.spec.db_disk_spec.disk_size, azure_disk_size_mb)) - raise errors.Config.InvalidValue(error_msg) + error_msg = ( + 'Azure disk size was specified ' + 'as in the disk spec as %s, got rounded to %s ' + 'which is smaller than the minimum of 5120 MB' + % (self.spec.db_disk_spec.disk_size, azure_disk_size_mb) + ) + raise errors.Config.InvalidValue(error_msg) cmd = [ azure.AZURE_PATH, @@ -338,7 +350,7 @@ def _CreateSqlServerInstance(self): '--admin-user', self.spec.database_username, '--admin-password', - self.spec.database_password + self.spec.database_password, ] vm_util.IssueCommand(cmd) @@ -373,8 +385,9 @@ def _CreateSqlServerInstance(self): ] else: # Sample machine_type: GP_Gen5_2 - edition, family, vcore = ( - self.GetConfigFromMachineType(self.spec.db_spec.machine_type)) + edition, family, vcore = self.GetConfigFromMachineType( + self.spec.db_spec.machine_type + ) cmd = [ azure.AZURE_PATH, self.GetAzCommandForEngine(), @@ -393,7 +406,7 @@ def _CreateSqlServerInstance(self): '--capacity', vcore, '--zone-redundant', - 'true' if self.spec.high_availability else 'false' + 'true' if self.spec.high_availability else 'false', ] vm_util.IssueCommand(cmd, timeout=CREATE_AZURE_DB_TIMEOUT) self.database_name = DEFAULT_DATABASE_NAME @@ -407,8 +420,11 @@ def _CreateAzureManagedSqlInstance(self): elif self.engine_type == sql_engine_utils.SQLSERVER: self._CreateSqlServerInstance() else: - raise NotImplementedError('Unknown how to create Azure data base ' - 'engine {0}'.format(self.engine_type)) + raise NotImplementedError( + 'Unknown how to create Azure data base engine {0}'.format( + self.engine_type + ) + ) def _Create(self): """Creates the Azure RDS instance. @@ -416,7 +432,6 @@ def _Create(self): Raises: NotImplementedError: if unknown how to create self.spec.engine. Exception: if attempting to create a non high availability database. - """ self._CreateAzureManagedSqlInstance() @@ -469,9 +484,7 @@ def _IsReady(self, timeout=IS_READY_TIMEOUT): return self._IsInstanceReady(timeout) def _PostCreate(self): - """Perform general post create operations on the cluster. - - """ + """Perform general post create operations on the cluster.""" super()._PostCreate() cmd = [ @@ -495,8 +508,9 @@ def _PostCreate(self): if self.spec.engine == 'mysql' or self.spec.engine == 'postgres': # Azure will add @domainname after the database username - self.spec.database_username = (self.spec.database_username + '@' + - self.endpoint.split('.')[0]) + self.spec.database_username = ( + self.spec.database_username + '@' + self.endpoint.split('.')[0] + ) def _Reboot(self): """Reboot the managed db.""" @@ -546,7 +560,8 @@ def _IsInstanceReady(self, timeout=IS_READY_TIMEOUT): state = server_show_json['state'] else: raise relational_db.RelationalDbEngineNotFoundError( - 'The db engine does not contain a valid state') + 'The db engine does not contain a valid state' + ) if state == 'Ready': break diff --git a/perfkitbenchmarker/providers/azure/azure_service_bus.py b/perfkitbenchmarker/providers/azure/azure_service_bus.py index e77df1d64f..6f78470159 100644 --- a/perfkitbenchmarker/providers/azure/azure_service_bus.py +++ b/perfkitbenchmarker/providers/azure/azure_service_bus.py @@ -19,12 +19,16 @@ FLAGS = flags.FLAGS MESSAGING_SERVICE_SCRIPTS_VM_AZURE_DIR = os.path.join( - msgsvc.MESSAGING_SERVICE_SCRIPTS_VM_LIB_DIR, 'azure') + msgsvc.MESSAGING_SERVICE_SCRIPTS_VM_LIB_DIR, 'azure' +) MESSAGING_SERVICE_SCRIPTS_AZURE_PREFIX = 'messaging_service_scripts/azure' MESSAGING_SERVICE_SCRIPTS_AZURE_FILES = [ - '__init__.py', 'azure_service_bus_client.py' + '__init__.py', + 'azure_service_bus_client.py', ] -MESSAGING_SERVICE_SCRIPTS_AZURE_BIN = 'messaging_service_scripts/azure_benchmark.py' +MESSAGING_SERVICE_SCRIPTS_AZURE_BIN = ( + 'messaging_service_scripts/azure_benchmark.py' +) class AzureServiceBus(msgsvc.BaseMessagingService): @@ -46,8 +50,11 @@ def _Create(self): self._CreateSubscription() def _Exists(self): - return (self._NamespaceExists() and self._TopicExists() and - self._SubscriptionExists()) + return ( + self._NamespaceExists() + and self._TopicExists() + and self._SubscriptionExists() + ) def _Delete(self): self._DeleteSubscription() @@ -63,23 +70,33 @@ def _IsDeleting(self): Returns: A bool. True if the resource is not yet deleted, else False. """ - return (self._NamespaceExists() or self._TopicExists() or - self._SubscriptionExists()) + return ( + self._NamespaceExists() + or self._TopicExists() + or self._SubscriptionExists() + ) - def Run(self, benchmark_scenario: str, number_of_messages: int, - message_size: int, warmup_messages: int, - streaming_pull: bool = False) -> Dict[str, Any]: + def Run( + self, + benchmark_scenario: str, + number_of_messages: int, + message_size: int, + warmup_messages: int, + streaming_pull: bool = False, + ) -> Dict[str, Any]: if streaming_pull: raise ValueError('Unsupported StreamingPull in AWS SQS.') connection_str = self._GetPrimaryConnectionString() - command = (f'python3 -m azure_benchmark ' - f'--topic_name={self.topic_name} ' - f'--subscription_name={self.subscription_name} ' - f'--benchmark_scenario={benchmark_scenario} ' - f'--number_of_messages={number_of_messages} ' - f'--message_size={message_size} ' - f'--warmup_messages={warmup_messages} ' - f'--connection_str="{connection_str}"') + command = ( + 'python3 -m azure_benchmark ' + f'--topic_name={self.topic_name} ' + f'--subscription_name={self.subscription_name} ' + f'--benchmark_scenario={benchmark_scenario} ' + f'--number_of_messages={number_of_messages} ' + f'--message_size={message_size} ' + f'--warmup_messages={warmup_messages} ' + f'--connection_str="{connection_str}"' + ) results = self.client_vm.RemoteCommand(command) results = json.loads(results[0]) return results @@ -87,11 +104,14 @@ def Run(self, benchmark_scenario: str, number_of_messages: int, def _InstallCloudClients(self): # Install/uploads Azure specific modules/files. self.client_vm.RemoteCommand( - 'sudo pip3 install azure-servicebus', ignore_failure=False) + 'sudo pip3 install azure-servicebus', ignore_failure=False + ) - self._CopyFiles(MESSAGING_SERVICE_SCRIPTS_AZURE_PREFIX, - MESSAGING_SERVICE_SCRIPTS_AZURE_FILES, - MESSAGING_SERVICE_SCRIPTS_VM_AZURE_DIR) + self._CopyFiles( + MESSAGING_SERVICE_SCRIPTS_AZURE_PREFIX, + MESSAGING_SERVICE_SCRIPTS_AZURE_FILES, + MESSAGING_SERVICE_SCRIPTS_VM_AZURE_DIR, + ) self.client_vm.PushDataFile(MESSAGING_SERVICE_SCRIPTS_AZURE_BIN) @property @@ -101,16 +121,28 @@ def location(self): def _CreateTopic(self): """Creates Service Bus topic.""" cmd = [ - azure.AZURE_PATH, 'servicebus', 'topic', 'create', '--name', - self.topic_name, '--namespace-name', self.namespace_name + azure.AZURE_PATH, + 'servicebus', + 'topic', + 'create', + '--name', + self.topic_name, + '--namespace-name', + self.namespace_name, ] + self.resource_group.args vm_util.IssueCommand(cmd) def _TopicExists(self) -> bool: """Checks whether Service Bus topic already exists.""" cmd = [ - azure.AZURE_PATH, 'servicebus', 'topic', 'show', '--name', - self.topic_name, '--namespace-name', self.namespace_name + azure.AZURE_PATH, + 'servicebus', + 'topic', + 'show', + '--name', + self.topic_name, + '--namespace-name', + self.namespace_name, ] + self.resource_group.args _, _, retcode = vm_util.IssueCommand(cmd, raise_on_failure=False) return retcode == 0 @@ -118,26 +150,48 @@ def _TopicExists(self) -> bool: def _DeleteTopic(self): """Handle Service Bus topic deletion.""" cmd = [ - azure.AZURE_PATH, 'servicebus', 'topic', 'delete', '--name', - self.topic_name, '--namespace-name', self.namespace_name + azure.AZURE_PATH, + 'servicebus', + 'topic', + 'delete', + '--name', + self.topic_name, + '--namespace-name', + self.namespace_name, ] + self.resource_group.args vm_util.IssueCommand(cmd, raise_on_failure=False) def _CreateSubscription(self): """Creates Service Bus subscription.""" cmd = [ - azure.AZURE_PATH, 'servicebus', 'topic', 'subscription', 'create', - '--name', self.subscription_name, '--topic-name', self.topic_name, - '--namespace-name', self.namespace_name + azure.AZURE_PATH, + 'servicebus', + 'topic', + 'subscription', + 'create', + '--name', + self.subscription_name, + '--topic-name', + self.topic_name, + '--namespace-name', + self.namespace_name, ] + self.resource_group.args vm_util.IssueCommand(cmd) def _SubscriptionExists(self) -> bool: """Checks whether Service Bus subscription already exists.""" cmd = [ - azure.AZURE_PATH, 'servicebus', 'topic', 'subscription', 'show', - '--name', self.subscription_name, '--topic-name', self.topic_name, - '--namespace-name', self.namespace_name + azure.AZURE_PATH, + 'servicebus', + 'topic', + 'subscription', + 'show', + '--name', + self.subscription_name, + '--topic-name', + self.topic_name, + '--namespace-name', + self.namespace_name, ] + self.resource_group.args _, _, retcode = vm_util.IssueCommand(cmd, raise_on_failure=False) return retcode == 0 @@ -145,25 +199,43 @@ def _SubscriptionExists(self) -> bool: def _DeleteSubscription(self): """Handle Service Bus subscription deletion.""" cmd = [ - azure.AZURE_PATH, 'servicebus', 'topic', 'subscription', 'delete', - '--name', self.subscription_name, '--topic-name', self.topic_name, - '--namespace-name', self.namespace_name + azure.AZURE_PATH, + 'servicebus', + 'topic', + 'subscription', + 'delete', + '--name', + self.subscription_name, + '--topic-name', + self.topic_name, + '--namespace-name', + self.namespace_name, ] + self.resource_group.args vm_util.IssueCommand(cmd, raise_on_failure=False) def _CreateNamespace(self): """Creates an Azure Service Bus Namespace.""" cmd = [ - azure.AZURE_PATH, 'servicebus', 'namespace', 'create', '--name', - self.namespace_name, '--location', self.location + azure.AZURE_PATH, + 'servicebus', + 'namespace', + 'create', + '--name', + self.namespace_name, + '--location', + self.location, ] + self.resource_group.args vm_util.IssueCommand(cmd) def _NamespaceExists(self) -> bool: """Checks if our Service Bus Namespace exists.""" cmd = [ - azure.AZURE_PATH, 'servicebus', 'namespace', 'show', '--name', - self.namespace_name + azure.AZURE_PATH, + 'servicebus', + 'namespace', + 'show', + '--name', + self.namespace_name, ] + self.resource_group.args _, _, retcode = vm_util.IssueCommand(cmd, raise_on_failure=False) return retcode == 0 @@ -171,21 +243,35 @@ def _NamespaceExists(self) -> bool: def _DeleteNamespace(self): """Deletes the Azure Service Bus namespace.""" cmd = [ - azure.AZURE_PATH, 'servicebus', 'namespace', 'delete', '--name', - self.namespace_name + azure.AZURE_PATH, + 'servicebus', + 'namespace', + 'delete', + '--name', + self.namespace_name, ] + self.resource_group.args vm_util.IssueCommand(cmd, raise_on_failure=False) def _GetPrimaryConnectionString(self): """Gets Azure Service Bus Namespace connection string.""" cmd = [ - azure.AZURE_PATH, 'servicebus', 'namespace', 'authorization-rule', - 'keys', 'list', '--name=RootManageSharedAccessKey', '--namespace-name', - self.namespace_name, '--query=primaryConnectionString', '-o=tsv' + azure.AZURE_PATH, + 'servicebus', + 'namespace', + 'authorization-rule', + 'keys', + 'list', + '--name=RootManageSharedAccessKey', + '--namespace-name', + self.namespace_name, + '--query=primaryConnectionString', + '-o=tsv', ] + self.resource_group.args output, stderror, retcode = vm_util.IssueCommand( - cmd, raise_on_failure=False) + cmd, raise_on_failure=False + ) if retcode: logging.warning( - 'Failed to get Service Bus Namespace connection string! %s', stderror) + 'Failed to get Service Bus Namespace connection string! %s', stderror + ) return output.strip() diff --git a/perfkitbenchmarker/providers/azure/azure_smb_service.py b/perfkitbenchmarker/providers/azure/azure_smb_service.py index c8189d5dc9..b11f3563e3 100644 --- a/perfkitbenchmarker/providers/azure/azure_smb_service.py +++ b/perfkitbenchmarker/providers/azure/azure_smb_service.py @@ -33,7 +33,6 @@ 4. On teardown the mount point is first deleted. Blocks on that returning. 5. The Azure Files service is then deleted. Does not block as can take some time. - """ import json @@ -94,7 +93,8 @@ def GetRemoteAddress(self): if self.name is None: raise errors.Resource.RetryableGetError('Filer not created') return '//{storage}.file.core.windows.net/{name}'.format( - storage=self.storage_account_name, name=self.name) + storage=self.storage_account_name, name=self.name + ) def GetStorageAccountAndKey(self): logging.debug('Calling GetStorageAccountAndKey on SMB server %s', self.name) @@ -112,14 +112,19 @@ def _Create(self): """ logging.info('Creating SMB server %s', self.name) if FLAGS.smb_tier == 'Standard_LRS': - storage_account_number = azure_network.AzureStorageAccount.total_storage_accounts - 1 - self.storage_account_name = 'pkb%s' % FLAGS.run_uri + 'storage' + str( - storage_account_number) + storage_account_number = ( + azure_network.AzureStorageAccount.total_storage_accounts - 1 + ) + self.storage_account_name = ( + 'pkb%s' % FLAGS.run_uri + 'storage' + str(storage_account_number) + ) elif FLAGS.smb_tier == 'Premium_LRS': storage_account_number = ( - azure_network.AzureStorageAccount.total_storage_accounts) - self.storage_account_name = 'pkb%s' % FLAGS.run_uri + 'filestorage' + str( - storage_account_number) + azure_network.AzureStorageAccount.total_storage_accounts + ) + self.storage_account_name = ( + 'pkb%s' % FLAGS.run_uri + 'filestorage' + str(storage_account_number) + ) # Premium Files uses a different storage account kind from Standard Files. # See links in description for more details. self.storage_account = azure_network.AzureStorageAccount( @@ -128,13 +133,16 @@ def _Create(self): name=self.storage_account_name, kind='FileStorage', resource_group=self.resource_group, - use_existing=False) + use_existing=False, + ) self.storage_account.Create() self.connection_args = util.GetAzureStorageConnectionArgs( - self.storage_account_name, self.resource_group.args) + self.storage_account_name, self.resource_group.args + ) self.storage_account_key = util.GetAzureStorageAccountKey( - self.storage_account_name, self.resource_group.args) + self.storage_account_name, self.resource_group.args + ) self._AzureSmbCommand('create') diff --git a/perfkitbenchmarker/providers/azure/azure_sql_data_warehouse.py b/perfkitbenchmarker/providers/azure/azure_sql_data_warehouse.py index cd6225f3cd..93ec457eb4 100644 --- a/perfkitbenchmarker/providers/azure/azure_sql_data_warehouse.py +++ b/perfkitbenchmarker/providers/azure/azure_sql_data_warehouse.py @@ -39,8 +39,12 @@ def GetSqlDataWarehouseClientInterface( - server_name: str, database: str, user: str, password: str, - resource_group: str) -> edw_service.EdwClientInterface: + server_name: str, + database: str, + user: str, + password: str, + resource_group: str, +) -> edw_service.EdwClientInterface: """Builds and Returns the requested SqlDataWarehouse client Interface. Args: @@ -58,11 +62,13 @@ def GetSqlDataWarehouseClientInterface( requested. """ if FLAGS.sqldatawarehouse_client_interface == 'CLI': - return CliClientInterface(server_name, database, user, password, - resource_group) + return CliClientInterface( + server_name, database, user, password, resource_group + ) if FLAGS.sqldatawarehouse_client_interface == 'JDBC': - return JdbcClientInterface(server_name, database, user, password, - resource_group) + return JdbcClientInterface( + server_name, database, user, password, resource_group + ) raise RuntimeError('Unknown SqlDataWarehouse Client Interface requested.') @@ -80,8 +86,14 @@ class CliClientInterface(edw_service.EdwClientInterface): resource_group: Azure resource group used to whitelist the VM's IP address. """ - def __init__(self, server_name: str, database: str, user: str, password: str, - resource_group: str): + def __init__( + self, + server_name: str, + database: str, + user: str, + password: str, + resource_group: str, + ): self.server_name = server_name self.database = database self.user = user @@ -103,27 +115,45 @@ def Prepare(self, package_name: str) -> None: self.whitelist_ip = self.client_vm.ip_address cmd = [ - azure.AZURE_PATH, 'sql', 'server', 'firewall-rule', 'create', '--name', - self.whitelist_ip, '--resource-group', self.resource_group, '--server', - self.server_name, '--end-ip-address', self.whitelist_ip, - '--start-ip-address', self.whitelist_ip + azure.AZURE_PATH, + 'sql', + 'server', + 'firewall-rule', + 'create', + '--name', + self.whitelist_ip, + '--resource-group', + self.resource_group, + '--server', + self.server_name, + '--end-ip-address', + self.whitelist_ip, + '--start-ip-address', + self.whitelist_ip, ] vm_util.IssueCommand(cmd) # Push the framework to execute a sql query and gather performance details - service_specific_dir = os.path.join('edw', - Azuresqldatawarehouse.SERVICE_TYPE) + service_specific_dir = os.path.join( + 'edw', Azuresqldatawarehouse.SERVICE_TYPE + ) self.client_vm.PushFile( data.ResourcePath( - os.path.join(service_specific_dir, 'script_runner.sh'))) + os.path.join(service_specific_dir, 'script_runner.sh') + ) + ) runner_permission_update_cmd = 'chmod 755 {}'.format('script_runner.sh') self.client_vm.RemoteCommand(runner_permission_update_cmd) self.client_vm.PushFile( - data.ResourcePath(os.path.join('edw', 'script_driver.py'))) + data.ResourcePath(os.path.join('edw', 'script_driver.py')) + ) self.client_vm.PushFile( data.ResourcePath( - os.path.join(service_specific_dir, - 'provider_specific_script_driver.py'))) + os.path.join( + service_specific_dir, 'provider_specific_script_driver.py' + ) + ) + ) def ExecuteQuery(self, query_name: Text) -> Tuple[float, Dict[str, str]]: """Executes a query and returns performance details. @@ -140,9 +170,15 @@ def ExecuteQuery(self, query_name: Text) -> Tuple[float, Dict[str, str]]: """ query_command = ( 'python script_driver.py --script={} --server={} --database={} ' - '--user={} --password={} --query_timeout={}').format( - query_name, self.server_name, self.database, self.user, - self.password, FLAGS.query_timeout) + '--user={} --password={} --query_timeout={}' + ).format( + query_name, + self.server_name, + self.database, + self.user, + self.password, + FLAGS.query_timeout, + ) stdout, _ = self.client_vm.RemoteCommand(query_command) performance = json.loads(stdout) details = copy.copy(self.GetMetadata()) @@ -165,8 +201,14 @@ class JdbcClientInterface(edw_service.EdwClientInterface): resource_group: Azure resource group used to whitelist the VM's IP address. """ - def __init__(self, server_name: str, database: str, user: str, password: str, - resource_group: str): + def __init__( + self, + server_name: str, + database: str, + user: str, + password: str, + resource_group: str, + ): self.server_name = server_name self.database = database self.user = user @@ -188,16 +230,28 @@ def Prepare(self, package_name: str) -> None: self.whitelist_ip = self.client_vm.ip_address cmd = [ - azure.AZURE_PATH, 'sql', 'server', 'firewall-rule', 'create', '--name', - self.whitelist_ip, '--resource-group', self.resource_group, '--server', - self.server_name, '--end-ip-address', self.whitelist_ip, - '--start-ip-address', self.whitelist_ip + azure.AZURE_PATH, + 'sql', + 'server', + 'firewall-rule', + 'create', + '--name', + self.whitelist_ip, + '--resource-group', + self.resource_group, + '--server', + self.server_name, + '--end-ip-address', + self.whitelist_ip, + '--start-ip-address', + self.whitelist_ip, ] vm_util.IssueCommand(cmd) # Push the executable jar to the working directory on client vm - self.client_vm.InstallPreprovisionedPackageData(package_name, - [SYNAPSE_JDBC_JAR], '') + self.client_vm.InstallPreprovisionedPackageData( + package_name, [SYNAPSE_JDBC_JAR], '' + ) def ExecuteQuery(self, query_name: Text) -> Tuple[float, Dict[str, str]]: """Executes a query and returns performance details. @@ -212,11 +266,13 @@ def ExecuteQuery(self, query_name: Text) -> Tuple[float, Dict[str, str]]: successful query the value is expected to be positive. performance_details: A dictionary of query execution attributes eg. job_id """ - query_command = (f'java -cp {SYNAPSE_JDBC_JAR} ' - f'com.google.cloud.performance.edw.Single ' - f'--server {self.server_name} --database {self.database} ' - f'--query_timeout {FLAGS.query_timeout} ' - f'--query_file {query_name}') + query_command = ( + f'java -cp {SYNAPSE_JDBC_JAR} ' + 'com.google.cloud.performance.edw.Single ' + f'--server {self.server_name} --database {self.database} ' + f'--query_timeout {FLAGS.query_timeout} ' + f'--query_file {query_name}' + ) stdout, _ = self.client_vm.RemoteCommand(query_command) performance = json.loads(stdout) details = copy.copy(self.GetMetadata()) @@ -226,8 +282,9 @@ def ExecuteQuery(self, query_name: Text) -> Tuple[float, Dict[str, str]]: details.update(performance['details']) return performance['query_wall_time_in_secs'], details - def ExecuteSimultaneous(self, submission_interval: int, - queries: List[str]) -> str: + def ExecuteSimultaneous( + self, submission_interval: int, queries: List[str] + ) -> str: """Executes queries simultaneously on client and return performance details. Simultaneous app expects queries as white space separated query file names. @@ -241,11 +298,13 @@ def ExecuteSimultaneous(self, submission_interval: int, A serialized dictionary of execution details. """ query_list = ' '.join(queries) - cmd = (f'java -cp {SYNAPSE_JDBC_JAR} ' - f'com.google.cloud.performance.edw.Simultaneous ' - f'--server {self.server_name} --database {self.database} ' - f'--submission_interval {submission_interval} --query_timeout ' - f'{FLAGS.query_timeout} --query_files {query_list}') + cmd = ( + f'java -cp {SYNAPSE_JDBC_JAR} ' + 'com.google.cloud.performance.edw.Simultaneous ' + f'--server {self.server_name} --database {self.database} ' + f'--submission_interval {submission_interval} --query_timeout ' + f'{FLAGS.query_timeout} --query_files {query_list}' + ) stdout, _ = self.client_vm.RemoteCommand(cmd) return stdout @@ -262,9 +321,10 @@ def ExecuteThroughput(self, concurrency_streams: List[List[str]]) -> str: query_list = ' '.join([','.join(stream) for stream in concurrency_streams]) cmd = ( f'java -cp {SYNAPSE_JDBC_JAR} ' - f'com.google.cloud.performance.edw.Throughput ' + 'com.google.cloud.performance.edw.Throughput ' f'--server {self.server_name} --database {self.database} ' - f'--query_timeout {FLAGS.query_timeout} --query_streams {query_list}') + f'--query_timeout {FLAGS.query_timeout} --query_streams {query_list}' + ) stdout, _ = self.client_vm.RemoteCommand(cmd) return stdout @@ -285,42 +345,46 @@ def __init__(self, edw_service_spec): self.resource_group = edw_service_spec.resource_group self.server_name = edw_service_spec.server_name self.client_interface = GetSqlDataWarehouseClientInterface( - self.server_name, self.db, self.user, self.password, - self.resource_group) + self.server_name, self.db, self.user, self.password, self.resource_group + ) def WhitelistIPAddress(self, ip_address): """To whitelist the IP address on the cluster.""" self.whitelist_ip = ip_address - cmd = [azure.AZURE_PATH, - 'sql', - 'server', - 'firewall-rule', - 'create', - '--name', - self.whitelist_ip, - '--resource-group', - self.resource_group, - '--server', - self.server_name, - '--end-ip-address', - self.whitelist_ip, - '--start-ip-address', - self.whitelist_ip] + cmd = [ + azure.AZURE_PATH, + 'sql', + 'server', + 'firewall-rule', + 'create', + '--name', + self.whitelist_ip, + '--resource-group', + self.resource_group, + '--server', + self.server_name, + '--end-ip-address', + self.whitelist_ip, + '--start-ip-address', + self.whitelist_ip, + ] vm_util.IssueCommand(cmd) def __DescribeCluster(self): """Describe cluster.""" - cmd = [azure.AZURE_PATH, - 'sql', - 'dw', - 'show', - '--name', - self.db, - '--resource-group', - self.resource_group, - '--server', - self.server_name] + cmd = [ + azure.AZURE_PATH, + 'sql', + 'dw', + 'show', + '--name', + self.db, + '--resource-group', + self.resource_group, + '--server', + self.server_name, + ] return vm_util.IssueCommand(cmd, raise_on_failure=False) def _Exists(self): @@ -342,16 +406,18 @@ def _IsReady(self): def _Create(self): """Resuming the cluster.""" - cmd = [azure.AZURE_PATH, - 'sql', - 'dw', - 'resume', - '--name', - self.db, - '--resource-group', - self.resource_group, - '--server', - self.server_name] + cmd = [ + azure.AZURE_PATH, + 'sql', + 'dw', + 'resume', + '--name', + self.db, + '--resource-group', + self.resource_group, + '--server', + self.server_name, + ] vm_util.IssueCommand(cmd, timeout=420) def _IsDeleting(self): @@ -364,25 +430,35 @@ def _IsDeleting(self): def _Delete(self): """Pausing cluster.""" - cmd = [azure.AZURE_PATH, - 'sql', - 'dw', - 'pause', - '--name', - self.db, - '--resource-group', - self.resource_group, - '--server', - self.server_name] + cmd = [ + azure.AZURE_PATH, + 'sql', + 'dw', + 'pause', + '--name', + self.db, + '--resource-group', + self.resource_group, + '--server', + self.server_name, + ] vm_util.IssueCommand(cmd, raise_on_failure=False) def _DeleteDependencies(self): """Delete dependencies of the cluster.""" if self.client_interface.whitelist_ip is not None: cmd = [ - azure.AZURE_PATH, 'sql', 'server', 'firewall-rule', 'delete', - '--name', self.client_interface.whitelist_ip, '--resource-group', - self.resource_group, '--server', self.server_name + azure.AZURE_PATH, + 'sql', + 'server', + 'firewall-rule', + 'delete', + '--name', + self.client_interface.whitelist_ip, + '--resource-group', + self.resource_group, + '--server', + self.server_name, ] vm_util.IssueCommand(cmd, raise_on_failure=False) diff --git a/perfkitbenchmarker/providers/azure/azure_virtual_machine.py b/perfkitbenchmarker/providers/azure/azure_virtual_machine.py index e86e77e29d..c82f44522e 100644 --- a/perfkitbenchmarker/providers/azure/azure_virtual_machine.py +++ b/perfkitbenchmarker/providers/azure/azure_virtual_machine.py @@ -76,35 +76,71 @@ 'Standard_L32as_v3': 4, 'Standard_L48as_v3': 6, 'Standard_L64as_v3': 8, - 'Standard_L80as_v3': 10 + 'Standard_L80as_v3': 10, } _MACHINE_TYPES_ONLY_SUPPORT_GEN2_IMAGES = ( - 'Standard_ND96asr_v4', 'Standard_ND96asr_A100_v4', - 'Standard_ND96amsr_A100_v4', 'Standard_M208ms_v2', 'Standard_M208s_v2', - 'Standard_M416ms_v2', 'Standard_M416s_v2', 'Standard_ND40rs_v2', - 'Standard_M32ms_v2', 'Standard_M64s_v2', 'Standard_M64ms_v2', - 'Standard_M128s_v2', 'Standard_M128ms_v2', 'Standard_M192is_v2', - 'Standard_M192ims_v2', 'Standard_M32dms_v2', 'Standard_M64ds_v2', - 'Standard_M128ds_v2', 'Standard_M128dms_v2', 'Standard_M192ids_v2', - 'Standard_M192idms_v2', 'Standard_DC2s_v2', 'Standard_DC2s_v3', - 'Standard_DC32ds_v3', 'Standard_DC32s_v3', 'Standard_DC48ds_v3', - 'Standard_DC48s_v3', 'Standard_DC4ds_v3', 'Standard_DC4s_v2', - 'Standard_DC4s_v3', 'Standard_DC8_v2', 'Standard_DC8ds_v3', - 'Standard_DC8s_v3', 'Standard_FX12mds', 'Standard_FX24mds', - 'Standard_FX36mds', 'Standard_FX48mds', 'Standard_FX4mds', - 'Standard_M64dms_v2', 'Standard_DC16ds_v3', 'Standard_DC16s_v3', - 'Standard_DC1ds_v3', 'Standard_DC1s_v3', 'Standard_DC24ds_v3', - 'Standard_DC24s_v3', 'Standard_DC2ds_v3', 'Standard_DC1s_v2') + 'Standard_ND96asr_v4', + 'Standard_ND96asr_A100_v4', + 'Standard_ND96amsr_A100_v4', + 'Standard_M208ms_v2', + 'Standard_M208s_v2', + 'Standard_M416ms_v2', + 'Standard_M416s_v2', + 'Standard_ND40rs_v2', + 'Standard_M32ms_v2', + 'Standard_M64s_v2', + 'Standard_M64ms_v2', + 'Standard_M128s_v2', + 'Standard_M128ms_v2', + 'Standard_M192is_v2', + 'Standard_M192ims_v2', + 'Standard_M32dms_v2', + 'Standard_M64ds_v2', + 'Standard_M128ds_v2', + 'Standard_M128dms_v2', + 'Standard_M192ids_v2', + 'Standard_M192idms_v2', + 'Standard_DC2s_v2', + 'Standard_DC2s_v3', + 'Standard_DC32ds_v3', + 'Standard_DC32s_v3', + 'Standard_DC48ds_v3', + 'Standard_DC48s_v3', + 'Standard_DC4ds_v3', + 'Standard_DC4s_v2', + 'Standard_DC4s_v3', + 'Standard_DC8_v2', + 'Standard_DC8ds_v3', + 'Standard_DC8s_v3', + 'Standard_FX12mds', + 'Standard_FX24mds', + 'Standard_FX36mds', + 'Standard_FX48mds', + 'Standard_FX4mds', + 'Standard_M64dms_v2', + 'Standard_DC16ds_v3', + 'Standard_DC16s_v3', + 'Standard_DC1ds_v3', + 'Standard_DC1s_v3', + 'Standard_DC24ds_v3', + 'Standard_DC24s_v3', + 'Standard_DC2ds_v3', + 'Standard_DC1s_v2', +) # https://docs.microsoft.com/en-us/azure/virtual-machines/windows/scheduled-events -_SCHEDULED_EVENTS_CMD = ('curl -H Metadata:true http://169.254.169.254/metadata' - '/scheduledevents?api-version=2019-01-01') - -_SCHEDULED_EVENTS_CMD_WIN = ('Invoke-RestMethod -Headers @{"Metadata"="true"} ' - '-Uri http://169.254.169.254/metadata/' - 'scheduledevents?api-version=2019-01-01 | ' - 'ConvertTo-Json') +_SCHEDULED_EVENTS_CMD = ( + 'curl -H Metadata:true http://169.254.169.254/metadata' + '/scheduledevents?api-version=2019-01-01' +) + +_SCHEDULED_EVENTS_CMD_WIN = ( + 'Invoke-RestMethod -Headers @{"Metadata"="true"} ' + '-Uri http://169.254.169.254/metadata/' + 'scheduledevents?api-version=2019-01-01 | ' + 'ConvertTo-Json' +) # Recognized Errors _OS_PROVISIONING_TIMED_OUT = 'OSProvisioningTimedOut' @@ -117,7 +153,7 @@ CONFIDENTIAL_MILAN_TYPES = [ r'(Standard_DC[0-9]+as?d?s_v5)', - r'(Standard_EC[0-9]+as?d?s_v5)' + r'(Standard_EC[0-9]+as?d?s_v5)', ] # Reference - @@ -153,8 +189,10 @@ class AzureVmSpec(virtual_machine.BaseVmSpec): def __init__(self, *args, **kwargs): super(AzureVmSpec, self).__init__(*args, **kwargs) - if isinstance(self.machine_type, - custom_virtual_machine_spec.AzurePerformanceTierDecoder): + if isinstance( + self.machine_type, + custom_virtual_machine_spec.AzurePerformanceTierDecoder, + ): self.tier = self.machine_type.tier self.compute_units = self.machine_type.compute_units self.machine_type = None @@ -179,7 +217,8 @@ def _ApplyFlags(cls, config_values, flag_values): config_values['machine_type'] = yaml.safe_load(flag_values.machine_type) if flag_values['azure_accelerated_networking'].present: config_values['accelerated_networking'] = ( - flag_values.azure_accelerated_networking) + flag_values.azure_accelerated_networking + ) if flag_values['azure_low_priority_vms'].present: config_values['low_priority'] = flag_values.azure_low_priority_vms @@ -194,20 +233,17 @@ def _GetOptionDecoderConstructions(cls): """ result = super(AzureVmSpec, cls)._GetOptionDecoderConstructions() result.update({ - 'machine_type': - (custom_virtual_machine_spec.AzureMachineTypeDecoder, {}), - 'accelerated_networking': (option_decoders.BooleanDecoder, { - 'default': False - }), - 'boot_disk_size': (option_decoders.IntDecoder, { - 'default': None - }), - 'boot_disk_type': (option_decoders.StringDecoder, { - 'default': None - }), - 'low_priority': (option_decoders.BooleanDecoder, { - 'default': False - }), + 'machine_type': ( + custom_virtual_machine_spec.AzureMachineTypeDecoder, + {}, + ), + 'accelerated_networking': ( + option_decoders.BooleanDecoder, + {'default': False}, + ), + 'boot_disk_size': (option_decoders.IntDecoder, {'default': None}), + 'boot_disk_type': (option_decoders.StringDecoder, {'default': None}), + 'low_priority': (option_decoders.BooleanDecoder, {'default': False}), }) return result @@ -227,8 +263,14 @@ def __init__(self, region, availability_zone, name, dns_name=None): def _Create(self): cmd = [ - azure.AZURE_PATH, 'network', 'public-ip', 'create', '--location', - self.region, '--name', self.name + azure.AZURE_PATH, + 'network', + 'public-ip', + 'create', + '--location', + self.region, + '--name', + self.name, ] + self.resource_group.args if self.availability_zone: @@ -241,10 +283,12 @@ def _Create(self): _, stderr, retcode = vm_util.IssueCommand(cmd, raise_on_failure=False) - if retcode and re.search(r'Cannot create more than \d+ public IP addresses', - stderr): + if retcode and re.search( + r'Cannot create more than \d+ public IP addresses', stderr + ): raise errors.Benchmarks.QuotaFailure( - virtual_machine.QUOTA_EXCEEDED_MESSAGE + stderr) + virtual_machine.QUOTA_EXCEEDED_MESSAGE + stderr + ) def _Exists(self): if self._deleted: @@ -252,10 +296,18 @@ def _Exists(self): stdout, _, _ = vm_util.IssueCommand( [ - azure.AZURE_PATH, 'network', 'public-ip', 'show', '--output', - 'json', '--name', self.name - ] + self.resource_group.args, - raise_on_failure=False) + azure.AZURE_PATH, + 'network', + 'public-ip', + 'show', + '--output', + 'json', + '--name', + self.name, + ] + + self.resource_group.args, + raise_on_failure=False, + ) try: json.loads(stdout) return True @@ -263,10 +315,19 @@ def _Exists(self): return False def GetIPAddress(self): - stdout, _ = vm_util.IssueRetryableCommand([ - azure.AZURE_PATH, 'network', 'public-ip', 'show', '--output', 'json', - '--name', self.name - ] + self.resource_group.args) + stdout, _ = vm_util.IssueRetryableCommand( + [ + azure.AZURE_PATH, + 'network', + 'public-ip', + 'show', + '--output', + 'json', + '--name', + self.name, + ] + + self.resource_group.args + ) response = json.loads(stdout) return response['ipAddress'] @@ -278,12 +339,14 @@ def _Delete(self): class AzureNIC(resource.BaseResource): """Class to represent an Azure NIC.""" - def __init__(self, - network: azure_network.AzureNetwork, - name: str, - public_ip: Optional[str], - accelerated_networking: bool, - private_ip=None): + def __init__( + self, + network: azure_network.AzureNetwork, + name: str, + public_ip: Optional[str], + accelerated_networking: bool, + private_ip=None, + ): super(AzureNIC, self).__init__() self.network = network self.name = name @@ -307,8 +370,12 @@ def _Create(self): cmd += ['--subnet', self.network.subnet.id] else: # auto created subnet - cmd += ['--vnet-name', self.network.subnet.vnet.name, - '--subnet', self.network.subnet.name] + cmd += [ + '--vnet-name', + self.network.subnet.vnet.name, + '--subnet', + self.network.subnet.name, + ] if self.public_ip: cmd += ['--public-ip-address', self.public_ip] if self.private_ip: @@ -326,10 +393,18 @@ def _Exists(self): # the resource doesn't exist, but no-op 'set' does. stdout, _, _ = vm_util.IssueCommand( [ - azure.AZURE_PATH, 'network', 'nic', 'show', '--output', 'json', - '--name', self.name - ] + self.resource_group.args, - raise_on_failure=False) + azure.AZURE_PATH, + 'network', + 'nic', + 'show', + '--output', + 'json', + '--name', + self.name, + ] + + self.resource_group.args, + raise_on_failure=False, + ) try: json.loads(stdout) return True @@ -339,10 +414,19 @@ def _Exists(self): def GetInternalIP(self): """Grab some data.""" - stdout, _ = vm_util.IssueRetryableCommand([ - azure.AZURE_PATH, 'network', 'nic', 'show', '--output', 'json', - '--name', self.name - ] + self.resource_group.args) + stdout, _ = vm_util.IssueRetryableCommand( + [ + azure.AZURE_PATH, + 'network', + 'nic', + 'show', + '--output', + 'json', + '--name', + self.name, + ] + + self.resource_group.args + ) response = json.loads(stdout) ip_config = response['ipConfigurations'][0] @@ -376,7 +460,7 @@ def __init__(self, name, region, resource_group, availability_zone): def _Create(self): """See base class.""" - create_cmd = ([ + create_cmd = [ azure.AZURE_PATH, 'vm', 'host', @@ -391,7 +475,7 @@ def _Create(self): # https://docs.microsoft.com/en-us/azure/virtual-machines/windows/dedicated-hosts#high-availability-considerations '--platform-fault-domain-count', '1', - ] + self.resource_group.args) + ] + self.resource_group.args if self.availability_zone: create_cmd.extend(['--zone', self.availability_zone]) @@ -400,7 +484,7 @@ def _Create(self): def _Delete(self): """See base class.""" - delete_cmd = ([ + delete_cmd = [ azure.AZURE_PATH, 'vm', 'host', @@ -408,14 +492,21 @@ def _Delete(self): 'delete', '--host-group', self.name, - ] + self.resource_group.args) + ] + self.resource_group.args vm_util.IssueCommand(delete_cmd) def _Exists(self): """See base class.""" show_cmd = [ - azure.AZURE_PATH, 'vm', 'host', 'group', 'show', '--output', 'json', - '--name', self.name + azure.AZURE_PATH, + 'vm', + 'host', + 'group', + 'show', + '--output', + 'json', + '--name', + self.name, ] + self.resource_group.args stdout, _, _ = vm_util.IssueCommand(show_cmd, raise_on_failure=False) try: @@ -435,8 +526,9 @@ def _GetSkuType(machine_type): elif re.match('Standard_E[0-9]*s_v3', machine_type): sku = 'ESv3-Type1' else: - raise ValueError('Dedicated hosting does not support machine type %s.' % - machine_type) + raise ValueError( + 'Dedicated hosting does not support machine type %s.' % machine_type + ) return sku @@ -449,12 +541,12 @@ class AzureDedicatedHost(resource.BaseResource): region: The region the host will exist in. resource_group: The group of resources for the host. """ + _lock = threading.Lock() # globals guarded by _lock host_group_map = {} - def __init__(self, name, region, resource_group, sku_type, - availability_zone): + def __init__(self, name, region, resource_group, sku_type, availability_zone): super(AzureDedicatedHost, self).__init__() self.name = name + '-Host' self.region = region @@ -468,16 +560,16 @@ def _CreateDependencies(self): """See base class.""" with self._lock: if self.region not in self.host_group_map: - new_host_group = AzureDedicatedHostGroup(self.name, self.region, - self.resource_group, - self.availability_zone) + new_host_group = AzureDedicatedHostGroup( + self.name, self.region, self.resource_group, self.availability_zone + ) new_host_group.Create() self.host_group_map[self.region] = new_host_group.name self.host_group = self.host_group_map[self.region] def _Create(self): """See base class.""" - create_cmd = ([ + create_cmd = [ azure.AZURE_PATH, 'vm', 'host', @@ -496,12 +588,12 @@ def _Create(self): # benchmarks require '--platform-fault-domain', '0', - ] + self.resource_group.args) + ] + self.resource_group.args vm_util.IssueCommand(create_cmd) def _Delete(self): """See base class.""" - delete_cmd = ([ + delete_cmd = [ azure.AZURE_PATH, 'vm', 'host', @@ -511,7 +603,7 @@ def _Delete(self): '--name', self.name, '--yes', - ] + self.resource_group.args) + ] + self.resource_group.args vm_util.IssueCommand(delete_cmd) def _Exists(self): @@ -540,11 +632,13 @@ def _MachineTypeIsArm(machine_type): """Check if the machine type uses ARM.""" return any( re.search(machine_series, machine_type) - for machine_series in AZURE_ARM_TYPES) + for machine_series in AZURE_ARM_TYPES + ) class AzureVirtualMachine(virtual_machine.BaseVirtualMachine): """Object representing an Azure Virtual Machine.""" + CLOUD = provider_info.AZURE _lock = threading.Lock() @@ -599,10 +693,13 @@ def __init__(self, vm_spec): re.search(machine_series, self.machine_type) for machine_series in CONFIDENTIAL_MILAN_TYPES ) - self.trusted_launch_unsupported_type = any( - re.search(machine_series, self.machine_type) - for machine_series in TRUSTED_LAUNCH_UNSUPPORTED_TYPES - ) or self.OS_TYPE in TRUSTED_LAUNCH_UNSUPPORTED_OS_TYPES + self.trusted_launch_unsupported_type = ( + any( + re.search(machine_series, self.machine_type) + for machine_series in TRUSTED_LAUNCH_UNSUPPORTED_TYPES + ) + or self.OS_TYPE in TRUSTED_LAUNCH_UNSUPPORTED_OS_TYPES + ) arm_arch = 'neoverse-n1' if _MachineTypeIsArm(self.machine_type) else None if arm_arch: self.host_arch = arm_arch @@ -635,14 +732,11 @@ def __init__(self, vm_spec): disk_spec = disk.BaseDiskSpec('azure_os_disk') disk_spec.disk_type = ( - vm_spec.boot_disk_type or self.storage_account.storage_type) + vm_spec.boot_disk_type or self.storage_account.storage_type + ) if vm_spec.boot_disk_size: disk_spec.disk_size = vm_spec.boot_disk_size - self.os_disk = azure_disk.AzureDisk( - disk_spec, - self, - None, - is_image=True) + self.os_disk = azure_disk.AzureDisk(disk_spec, self, None, is_image=True) @property @classmethod @@ -659,13 +753,18 @@ def _CreateDependencies(self): if self.use_dedicated_host: with self._lock: self.host_list = self.host_map[(self.host_series_sku, self.region)] - if (not self.host_list or (self.num_vms_per_host and - self.host_list[-1].fill_fraction + - 1.0 / self.num_vms_per_host > 1.0)): - new_host = AzureDedicatedHost(self.name, self.region, - self.resource_group, - self.host_series_sku, - self.availability_zone) + if not self.host_list or ( + self.num_vms_per_host + and self.host_list[-1].fill_fraction + 1.0 / self.num_vms_per_host + > 1.0 + ): + new_host = AzureDedicatedHost( + self.name, + self.region, + self.resource_group, + self.host_series_sku, + self.availability_zone, + ) self.host_list.append(new_host) new_host.Create() self.host = self.host_list[-1] @@ -673,8 +772,10 @@ def _CreateDependencies(self): self.host.fill_fraction += 1.0 / self.num_vms_per_host def _RequiresUltraDisk(self): - return any(disk_spec.disk_type == azure_disk.ULTRA_STORAGE - for disk_spec in self.disk_specs) + return any( + disk_spec.disk_type == azure_disk.ULTRA_STORAGE + for disk_spec in self.disk_specs + ) def _Create(self): """See base class.""" @@ -685,10 +786,14 @@ def _Create(self): confidential_args = [] if self.machine_type_is_confidential: confidential_args = [ - '--enable-vtpm', 'true', - '--enable-secure-boot', 'true', - '--security-type', 'ConfidentialVM', - '--os-disk-security-encryption-type', 'VMGuestStateOnly', + '--enable-vtpm', + 'true', + '--enable-secure-boot', + 'true', + '--security-type', + 'ConfidentialVM', + '--os-disk-security-encryption-type', + 'VMGuestStateOnly', ] tags = {} @@ -738,7 +843,8 @@ def _Create(self): # deployed to particular hosts. if self.use_dedicated_host: create_cmd.extend( - ['--host-group', self.host.host_group, '--host', self.host.name]) + ['--host-group', self.host.host_group, '--host', self.host.name] + ) num_hosts = len(self.host_list) if self.network.placement_group: @@ -755,14 +861,16 @@ def _Create(self): # Uses a custom default because create has a very long tail. azure_vm_create_timeout = 1800 _, stderr, retcode = vm_util.IssueCommand( - create_cmd, timeout=azure_vm_create_timeout, raise_on_failure=False) + create_cmd, timeout=azure_vm_create_timeout, raise_on_failure=False + ) if retcode: if 'quota' in stderr.lower(): raise errors.Benchmarks.QuotaFailure( - virtual_machine.QUOTA_EXCEEDED_MESSAGE + stderr) + virtual_machine.QUOTA_EXCEEDED_MESSAGE + stderr + ) elif re.search( - r'requested VM size \S+ is not available', stderr) or re.search( - r'not available in location .+ for subscription', stderr): + r'requested VM size \S+ is not available', stderr + ) or re.search(r'not available in location .+ for subscription', stderr): raise errors.Benchmarks.UnsupportedConfigError(stderr) elif self.low_priority and 'OverconstrainedAllocationRequest' in stderr: raise errors.Benchmarks.InsufficientCapacityCloudFailure(stderr) @@ -772,50 +880,66 @@ def _Create(self): raise errors.Resource.CreationError( f'Failed to create VM: {self.machine_type} is likely a confidential' ' machine, which PKB does not support at this time.\n\n' - f' Full error: {stderr} return code: {retcode}') + f' Full error: {stderr} return code: {retcode}' + ) elif "cannot boot Hypervisor Generation '1'" in stderr: raise errors.Resource.CreationError( f'Failed to create VM: {self.machine_type} is unable to support V1 ' 'Hypervision. Please update _MACHINE_TYPES_ONLY_SUPPORT_GEN2_IMAGES' ' in azure_virtual_machine.py.\n\n' - f' Full error: {stderr} return code: {retcode}') - elif (self.use_dedicated_host and 'AllocationFailed' in stderr): + f' Full error: {stderr} return code: {retcode}' + ) + elif self.use_dedicated_host and 'AllocationFailed' in stderr: if self.num_vms_per_host: raise errors.Resource.CreationError( 'Failed to create host: %d vms of type %s per host exceeds ' - 'memory capacity limits of the host' % - (self.num_vms_per_host, self.machine_type)) + 'memory capacity limits of the host' + % (self.num_vms_per_host, self.machine_type) + ) else: logging.warning( 'Creation failed due to insufficient host capacity. A new host ' - 'will be created and instance creation will be retried.') + 'will be created and instance creation will be retried.' + ) with self._lock: if num_hosts == len(self.host_list): - new_host = AzureDedicatedHost(self.name, self.region, - self.resource_group, - self.host_series_sku, - self.availability_zone) + new_host = AzureDedicatedHost( + self.name, + self.region, + self.resource_group, + self.host_series_sku, + self.availability_zone, + ) self.host_list.append(new_host) new_host.Create() self.host = self.host_list[-1] raise errors.Resource.RetryableCreationError() - elif (not self.use_dedicated_host and 'AllocationFailed' in stderr): + elif not self.use_dedicated_host and 'AllocationFailed' in stderr: raise errors.Benchmarks.InsufficientCapacityCloudFailure(stderr) - elif (not self.use_dedicated_host and - 'OverconstrainedZonalAllocationRequest' in stderr): + elif ( + not self.use_dedicated_host + and 'OverconstrainedZonalAllocationRequest' in stderr + ): raise errors.Benchmarks.UnsupportedConfigError(stderr) - elif 'SkuNotAvailable' in stderr: + elif 'SkuNotAvailable' in stderr: raise errors.Benchmarks.UnsupportedConfigError(stderr) else: raise errors.Resource.CreationError( - 'Failed to create VM: %s return code: %s' % (stderr, retcode)) + 'Failed to create VM: %s return code: %s' % (stderr, retcode) + ) def _Exists(self): """Returns True if the VM exists.""" if self._deleted: return False show_cmd = [ - azure.AZURE_PATH, 'vm', 'show', '--output', 'json', '--name', self.name + azure.AZURE_PATH, + 'vm', + 'show', + '--output', + 'json', + '--name', + self.name, ] + self.resource_group.args stdout, _, _ = vm_util.IssueCommand(show_cmd, raise_on_failure=False) try: @@ -830,21 +954,35 @@ def _Delete(self): def _Start(self): """Starts the VM.""" - start_cmd = ([azure.AZURE_PATH, 'vm', 'start', '--name', self.name] + - self.resource_group.args) + start_cmd = [ + azure.AZURE_PATH, + 'vm', + 'start', + '--name', + self.name, + ] + self.resource_group.args vm_util.IssueCommand(start_cmd) if self.public_ip: self.ip_address = self.public_ip.GetIPAddress() def _Stop(self): """Stops the VM.""" - stop_cmd = ([azure.AZURE_PATH, 'vm', 'stop', '--name', self.name] + - self.resource_group.args) + stop_cmd = [ + azure.AZURE_PATH, + 'vm', + 'stop', + '--name', + self.name, + ] + self.resource_group.args vm_util.IssueCommand(stop_cmd) # remove resources, similar to GCE stop - deallocate_cmd = ( - [azure.AZURE_PATH, 'vm', 'deallocate', '--name', self.name] + - self.resource_group.args) + deallocate_cmd = [ + azure.AZURE_PATH, + 'vm', + 'deallocate', + '--name', + self.name, + ] + self.resource_group.args vm_util.IssueCommand(deallocate_cmd) def _Suspend(self): @@ -858,17 +996,33 @@ def _Resume(self): @vm_util.Retry() def _PostCreate(self): """Get VM data.""" - stdout, _ = vm_util.IssueRetryableCommand([ - azure.AZURE_PATH, 'vm', 'show', '--output', 'json', '--name', self.name - ] + self.resource_group.args) + stdout, _ = vm_util.IssueRetryableCommand( + [ + azure.AZURE_PATH, + 'vm', + 'show', + '--output', + 'json', + '--name', + self.name, + ] + + self.resource_group.args + ) response = json.loads(stdout) self.os_disk.name = response['storageProfile']['osDisk']['name'] self.os_disk.created = True - vm_util.IssueCommand([ - azure.AZURE_PATH, 'disk', 'update', '--name', self.os_disk.name, - '--set', - util.GetTagsJson(self.resource_group.timeout_minutes) - ] + self.resource_group.args) + vm_util.IssueCommand( + [ + azure.AZURE_PATH, + 'disk', + 'update', + '--name', + self.os_disk.name, + '--set', + util.GetTagsJson(self.resource_group.timeout_minutes), + ] + + self.resource_group.args + ) self.internal_ip = self.nic.GetInternalIP() if self.public_ip: self.ip_address = self.public_ip.GetIPAddress() @@ -932,14 +1086,17 @@ def DownloadPreprovisionedData( install_path: The install path on this VM. module_name: Name of the module associated with this data file. filename: The name of the file that was downloaded. - timeout: Timeout value for downloading preprovisionedData, Five minutes - by default. + timeout: Timeout value for downloading preprovisionedData, Five minutes by + default. """ # N.B. Should already be installed by ShouldDownloadPreprovisionedData self.Install('azure_cli') self.RobustRemoteCommand( - GenerateDownloadPreprovisionedDataCommand(install_path, module_name, - filename), timeout=timeout) + GenerateDownloadPreprovisionedDataCommand( + install_path, module_name, filename + ), + timeout=timeout, + ) def ShouldDownloadPreprovisionedData(self, module_name, filename): """Returns whether or not preprovisioned data is available.""" @@ -947,7 +1104,8 @@ def ShouldDownloadPreprovisionedData(self, module_name, filename): # connection strings and do not use credentials on the VM. self.Install('azure_cli') return FLAGS.azure_preprovisioned_data_bucket and self.TryRemoteCommand( - GenerateStatPreprovisionedDataCommand(module_name, filename)) + GenerateStatPreprovisionedDataCommand(module_name, filename) + ) def GetResourceMetadata(self): result = super(AzureVirtualMachine, self).GetResourceMetadata() @@ -965,13 +1123,15 @@ def GetResourceMetadata(self): def _UpdateInterruptibleVmStatusThroughMetadataService(self): stdout, stderr, return_code = self.RemoteCommandWithReturnCode( - _SCHEDULED_EVENTS_CMD) + _SCHEDULED_EVENTS_CMD + ) if return_code: logging.error('Checking Interrupt Error: %s', stderr) else: events = json.loads(stdout).get('Events', []) self.spot_early_termination = any( - event.get('EventType') == 'Preempt' for event in events) + event.get('EventType') == 'Preempt' for event in events + ) if self.spot_early_termination: logging.info('Spotted early termination on %s', self) @@ -998,37 +1158,42 @@ def GetVmStatusCode(self): return self.low_priority_status_code -class Debian9BasedAzureVirtualMachine(AzureVirtualMachine, - linux_virtual_machine.Debian9Mixin): +class Debian9BasedAzureVirtualMachine( + AzureVirtualMachine, linux_virtual_machine.Debian9Mixin +): # From https://wiki.debian.org/Cloud/MicrosoftAzure IMAGE_URN = 'credativ:Debian:9:latest' -class Debian10BasedAzureVirtualMachine(AzureVirtualMachine, - linux_virtual_machine.Debian10Mixin): +class Debian10BasedAzureVirtualMachine( + AzureVirtualMachine, linux_virtual_machine.Debian10Mixin +): # From https://wiki.debian.org/Cloud/MicrosoftAzure GEN2_IMAGE_URN = 'Debian:debian-10:10-gen2:latest' IMAGE_URN = 'Debian:debian-10:10:latest' -class Debian11BasedAzureVirtualMachine(AzureVirtualMachine, - linux_virtual_machine.Debian11Mixin): +class Debian11BasedAzureVirtualMachine( + AzureVirtualMachine, linux_virtual_machine.Debian11Mixin +): # From https://wiki.debian.org/Cloud/MicrosoftAzure GEN2_IMAGE_URN = 'Debian:debian-11:11-gen2:latest' IMAGE_URN = 'Debian:debian-11:11:latest' ARM_IMAGE_URN = 'Debian:debian-11:11-backports-arm64:latest' -class Debian12BasedAzureVirtualMachine(AzureVirtualMachine, - linux_virtual_machine.Debian12Mixin): +class Debian12BasedAzureVirtualMachine( + AzureVirtualMachine, linux_virtual_machine.Debian12Mixin +): # From https://wiki.debian.org/Cloud/MicrosoftAzure GEN2_IMAGE_URN = 'Debian:debian-12:12-gen2:latest' IMAGE_URN = 'Debian:debian-12:12:latest' ARM_IMAGE_URN = 'Debian:debian-12:12-arm64:latest' -class Ubuntu1604BasedAzureVirtualMachine(AzureVirtualMachine, - linux_virtual_machine.Ubuntu1604Mixin): +class Ubuntu1604BasedAzureVirtualMachine( + AzureVirtualMachine, linux_virtual_machine.Ubuntu1604Mixin +): GEN2_IMAGE_URN = 'Canonical:UbuntuServer:16_04-lts-gen2:latest' IMAGE_URN = 'Canonical:UbuntuServer:16.04-LTS:latest' # No ARM image when running @@ -1036,27 +1201,42 @@ class Ubuntu1604BasedAzureVirtualMachine(AzureVirtualMachine, # | grep 16_04 -class Ubuntu1804BasedAzureVirtualMachine(AzureVirtualMachine, - linux_virtual_machine.Ubuntu1804Mixin): +class Ubuntu1804BasedAzureVirtualMachine( + AzureVirtualMachine, linux_virtual_machine.Ubuntu1804Mixin +): GEN2_IMAGE_URN = 'Canonical:UbuntuServer:18_04-lts-gen2:latest' IMAGE_URN = 'Canonical:UbuntuServer:18.04-LTS:latest' ARM_IMAGE_URN = 'Canonical:UbuntuServer:18_04-lts-arm64:latest' -class Ubuntu2004BasedAzureVirtualMachine(AzureVirtualMachine, - linux_virtual_machine.Ubuntu2004Mixin): - GEN2_IMAGE_URN = 'Canonical:0001-com-ubuntu-server-focal:20_04-lts-gen2:latest' +class Ubuntu2004BasedAzureVirtualMachine( + AzureVirtualMachine, linux_virtual_machine.Ubuntu2004Mixin +): + GEN2_IMAGE_URN = ( + 'Canonical:0001-com-ubuntu-server-focal:20_04-lts-gen2:latest' + ) IMAGE_URN = 'Canonical:0001-com-ubuntu-server-focal:20_04-lts:latest' - CONFIDENTIAL_IMAGE_URN = 'Canonical:0001-com-ubuntu-confidential-vm-focal:20_04-lts-cvm:latest' - ARM_IMAGE_URN = 'Canonical:0001-com-ubuntu-server-focal:20_04-lts-arm64:latest' + CONFIDENTIAL_IMAGE_URN = ( + 'Canonical:0001-com-ubuntu-confidential-vm-focal:20_04-lts-cvm:latest' + ) + ARM_IMAGE_URN = ( + 'Canonical:0001-com-ubuntu-server-focal:20_04-lts-arm64:latest' + ) -class Ubuntu2204BasedAzureVirtualMachine(AzureVirtualMachine, - linux_virtual_machine.Ubuntu2204Mixin): - GEN2_IMAGE_URN = 'Canonical:0001-com-ubuntu-server-jammy:22_04-lts-gen2:latest' +class Ubuntu2204BasedAzureVirtualMachine( + AzureVirtualMachine, linux_virtual_machine.Ubuntu2204Mixin +): + GEN2_IMAGE_URN = ( + 'Canonical:0001-com-ubuntu-server-jammy:22_04-lts-gen2:latest' + ) IMAGE_URN = 'Canonical:0001-com-ubuntu-server-jammy:22_04-lts:latest' - CONFIDENTIAL_IMAGE_URN = 'Canonical:0001-com-ubuntu-confidential-vm-jammy:22_04-lts-cvm:latest' - ARM_IMAGE_URN = 'Canonical:0001-com-ubuntu-server-jammy:22_04-lts-arm64:latest' + CONFIDENTIAL_IMAGE_URN = ( + 'Canonical:0001-com-ubuntu-confidential-vm-jammy:22_04-lts-cvm:latest' + ) + ARM_IMAGE_URN = ( + 'Canonical:0001-com-ubuntu-server-jammy:22_04-lts-arm64:latest' + ) class Ubuntu2304BasedAzureVirtualMachine( @@ -1070,58 +1250,67 @@ class Ubuntu2304BasedAzureVirtualMachine( ARM_IMAGE_URN = 'Canonical:0001-com-ubuntu-server-lunar:23_04-arm64:latest' -class Rhel7BasedAzureVirtualMachine(AzureVirtualMachine, - linux_virtual_machine.Rhel7Mixin): +class Rhel7BasedAzureVirtualMachine( + AzureVirtualMachine, linux_virtual_machine.Rhel7Mixin +): GEN2_IMAGE_URN = 'RedHat:RHEL:7lvm-gen2:latest' IMAGE_URN = 'RedHat:RHEL:7-LVM:latest' -class Rhel8BasedAzureVirtualMachine(AzureVirtualMachine, - linux_virtual_machine.Rhel8Mixin): +class Rhel8BasedAzureVirtualMachine( + AzureVirtualMachine, linux_virtual_machine.Rhel8Mixin +): GEN2_IMAGE_URN = 'RedHat:RHEL:8-lvm-gen2:latest' IMAGE_URN = 'RedHat:RHEL:8-LVM:latest' -class Rhel9BasedAzureVirtualMachine(AzureVirtualMachine, - linux_virtual_machine.Rhel9Mixin): +class Rhel9BasedAzureVirtualMachine( + AzureVirtualMachine, linux_virtual_machine.Rhel9Mixin +): GEN2_IMAGE_URN = 'RedHat:RHEL:9-lvm-gen2:latest' IMAGE_URN = 'RedHat:RHEL:9-lvm:latest' -class CentOs7BasedAzureVirtualMachine(AzureVirtualMachine, - linux_virtual_machine.CentOs7Mixin): +class CentOs7BasedAzureVirtualMachine( + AzureVirtualMachine, linux_virtual_machine.CentOs7Mixin +): GEN2_IMAGE_URN = 'OpenLogic:CentOS-LVM:7-lvm-gen2:latest' IMAGE_URN = 'OpenLogic:CentOS-LVM:7-lvm:latest' -class CentOs8BasedAzureVirtualMachine(AzureVirtualMachine, - linux_virtual_machine.CentOs8Mixin): +class CentOs8BasedAzureVirtualMachine( + AzureVirtualMachine, linux_virtual_machine.CentOs8Mixin +): GEN2_IMAGE_URN = 'OpenLogic:CentOS-LVM:8-lvm-gen2:latest' IMAGE_URN = 'OpenLogic:CentOS-LVM:8-lvm:latest' # Requires enabling for your subscription: # https://portal.azure.com/#view/Microsoft_Azure_Marketplace/LegalTermsSkuProgrammaticAccessBlade/legalTermsSkuProgrammaticAccessData~/%7B%22product%22%3A%7B%22publisherId%22%3A%22erockyenterprisesoftwarefoundationinc1653071250513%22%2C%22offerId%22%3A%22rockylinux%22%2C%22planId%22%3A%22free%22%2C%22standardContractAmendmentsRevisionId%22%3Anull%2C%22isCspEnabled%22%3Atrue%7D%7D -class RockyLinux8BasedAzureVirtualMachine(AzureVirtualMachine, - linux_virtual_machine.RockyLinux8Mixin - ): +class RockyLinux8BasedAzureVirtualMachine( + AzureVirtualMachine, linux_virtual_machine.RockyLinux8Mixin +): # TODO(pclay): Update to rockylinux-8:latest if that is ever so-named. - IMAGE_URN = 'erockyenterprisesoftwarefoundationinc1653071250513:rockylinux:free:8.6.0' + IMAGE_URN = ( + 'erockyenterprisesoftwarefoundationinc1653071250513:rockylinux:free:8.6.0' + ) # Requires enabling for your subscription: # https://portal.azure.com/#view/Microsoft_Azure_Marketplace/LegalTermsSkuProgrammaticAccessBlade/legalTermsSkuProgrammaticAccessData~/%7B%22product%22%3A%7B%22publisherId%22%3A%22erockyenterprisesoftwarefoundationinc1653071250513%22%2C%22offerId%22%3A%22rockylinux%22%2C%22planId%22%3A%22free%22%2C%22standardContractAmendmentsRevisionId%22%3Anull%2C%22isCspEnabled%22%3Atrue%7D%7D -class RockyLinux9BasedAzureVirtualMachine(AzureVirtualMachine, - linux_virtual_machine.RockyLinux9Mixin - ): +class RockyLinux9BasedAzureVirtualMachine( + AzureVirtualMachine, linux_virtual_machine.RockyLinux9Mixin +): IMAGE_URN = 'erockyenterprisesoftwarefoundationinc1653071250513:rockylinux-9:rockylinux-9:latest' + # TODO(user): Add Fedora CoreOS when available: # https://docs.fedoraproject.org/en-US/fedora-coreos/provisioning-azure/ -class BaseWindowsAzureVirtualMachine(AzureVirtualMachine, - windows_virtual_machine.BaseWindowsMixin): +class BaseWindowsAzureVirtualMachine( + AzureVirtualMachine, windows_virtual_machine.BaseWindowsMixin +): """Class supporting Windows Azure virtual machines.""" # This ia a required attribute, but this is a base class. @@ -1140,24 +1329,40 @@ def _PostCreate(self): super(BaseWindowsAzureVirtualMachine, self)._PostCreate() config_dict = {'commandToExecute': windows_virtual_machine.STARTUP_SCRIPT} config = json.dumps(config_dict) - vm_util.IssueRetryableCommand([ - azure.AZURE_PATH, 'vm', 'extension', 'set', '--vm-name', self.name, - '--name', 'CustomScriptExtension', '--publisher', 'Microsoft.Compute', - '--version', '1.4', - '--protected-settings=%s' % config - ] + self.resource_group.args) + vm_util.IssueRetryableCommand( + [ + azure.AZURE_PATH, + 'vm', + 'extension', + 'set', + '--vm-name', + self.name, + '--name', + 'CustomScriptExtension', + '--publisher', + 'Microsoft.Compute', + '--version', + '1.4', + '--protected-settings=%s' % config, + ] + + self.resource_group.args + ) def _UpdateInterruptibleVmStatusThroughMetadataService(self): stdout, _ = self.RemoteCommand(_SCHEDULED_EVENTS_CMD_WIN) events = json.loads(stdout).get('Events', []) self.spot_early_termination = any( - event.get('EventType') == 'Preempt' for event in events) + event.get('EventType') == 'Preempt' for event in events + ) if self.spot_early_termination: logging.info('Spotted early termination on %s', self) def DownloadPreprovisionedData( - self, install_path, module_name, filename, - timeout=virtual_machine.PREPROVISIONED_DATA_TIMEOUT + self, + install_path, + module_name, + filename, + timeout=virtual_machine.PREPROVISIONED_DATA_TIMEOUT, ): """Downloads a data file from Azure blob storage with pre-provisioned data. @@ -1173,28 +1378,36 @@ def DownloadPreprovisionedData( install_path: The install path on this VM. module_name: Name of the module associated with this data file. filename: The name of the file that was downloaded. - timeout: Timeout value for downloading preprovisionedData, Five minutes - by default. + timeout: Timeout value for downloading preprovisionedData, Five minutes by + default. """ # TODO(deitz): Add retry logic. temp_local_path = vm_util.GetTempDir() vm_util.IssueCommand( GenerateDownloadPreprovisionedDataCommand( temp_local_path, module_name, filename - ).split('&')[-1].split(), + ) + .split('&')[-1] + .split(), timeout=timeout, ) self.PushFile( - vm_util.PrependTempDir(filename), - ntpath.join(install_path, filename) + vm_util.PrependTempDir(filename), ntpath.join(install_path, filename) ) vm_util.IssueCommand(['rm', vm_util.PrependTempDir(filename)]) def ShouldDownloadPreprovisionedData(self, module_name, filename): """Returns whether or not preprovisioned data is available.""" - return FLAGS.azure_preprovisioned_data_bucket and vm_util.IssueCommand( - GenerateStatPreprovisionedDataCommand( - module_name, filename).split(' '), raise_on_failure=False)[-1] == 0 + return ( + FLAGS.azure_preprovisioned_data_bucket + and vm_util.IssueCommand( + GenerateStatPreprovisionedDataCommand(module_name, filename).split( + ' ' + ), + raise_on_failure=False, + )[-1] + == 0 + ) def DiskDriveIsLocal(self, device, model): """Helper method to determine if a disk drive is a local ssd to stripe.""" @@ -1206,73 +1419,83 @@ def DiskDriveIsLocal(self, device, model): class Windows2016CoreAzureVirtualMachine( - BaseWindowsAzureVirtualMachine, - windows_virtual_machine.Windows2016CoreMixin): + BaseWindowsAzureVirtualMachine, windows_virtual_machine.Windows2016CoreMixin +): GEN2_IMAGE_URN = 'MicrosoftWindowsServer:windowsserver-gen2preview:2016-datacenter-gen2:latest' - IMAGE_URN = 'MicrosoftWindowsServer:WindowsServer:2016-Datacenter-Server-Core:latest' + IMAGE_URN = ( + 'MicrosoftWindowsServer:WindowsServer:2016-Datacenter-Server-Core:latest' + ) class Windows2019CoreAzureVirtualMachine( - BaseWindowsAzureVirtualMachine, - windows_virtual_machine.Windows2019CoreMixin): + BaseWindowsAzureVirtualMachine, windows_virtual_machine.Windows2019CoreMixin +): GEN2_IMAGE_URN = 'MicrosoftWindowsServer:windowsserver-gen2preview:2019-datacenter-gen2:latest' IMAGE_URN = 'MicrosoftWindowsServer:WindowsServer:2019-Datacenter-Core:latest' class Windows2022CoreAzureVirtualMachine( - BaseWindowsAzureVirtualMachine, - windows_virtual_machine.Windows2022CoreMixin): + BaseWindowsAzureVirtualMachine, windows_virtual_machine.Windows2022CoreMixin +): IMAGE_URN = 'MicrosoftWindowsServer:WindowsServer:2022-Datacenter-Core:latest' class Windows2016DesktopAzureVirtualMachine( BaseWindowsAzureVirtualMachine, - windows_virtual_machine.Windows2016DesktopMixin): + windows_virtual_machine.Windows2016DesktopMixin, +): GEN2_IMAGE_URN = 'MicrosoftWindowsServer:windowsserver-gen2preview:2016-datacenter-gen2:latest' IMAGE_URN = 'MicrosoftWindowsServer:WindowsServer:2016-Datacenter:latest' class Windows2019DesktopAzureVirtualMachine( BaseWindowsAzureVirtualMachine, - windows_virtual_machine.Windows2019DesktopMixin): + windows_virtual_machine.Windows2019DesktopMixin, +): GEN2_IMAGE_URN = 'MicrosoftWindowsServer:windowsserver-gen2preview:2019-datacenter-gen2:latest' IMAGE_URN = 'MicrosoftWindowsServer:WindowsServer:2019-Datacenter:latest' class Windows2022DesktopAzureVirtualMachine( BaseWindowsAzureVirtualMachine, - windows_virtual_machine.Windows2022DesktopMixin): + windows_virtual_machine.Windows2022DesktopMixin, +): IMAGE_URN = 'MicrosoftWindowsServer:WindowsServer:2022-Datacenter:latest' class Windows2019DesktopSQLServer2019StandardAzureVirtualMachine( BaseWindowsAzureVirtualMachine, - windows_virtual_machine.Windows2019SQLServer2019Standard): + windows_virtual_machine.Windows2019SQLServer2019Standard, +): GEN2_IMAGE_URN = 'MicrosoftSQLServer:sql2019-ws2019:standard-gen2:latest' IMAGE_URN = 'MicrosoftSQLServer:sql2019-ws2019:standard:latest' class Windows2019DesktopSQLServer2019EnterpriseAzureVirtualMachine( BaseWindowsAzureVirtualMachine, - windows_virtual_machine.Windows2019SQLServer2019Enterprise): + windows_virtual_machine.Windows2019SQLServer2019Enterprise, +): GEN2_IMAGE_URN = 'MicrosoftSQLServer:sql2019-ws2019:enterprise-gen2:latest' IMAGE_URN = 'MicrosoftSQLServer:sql2019-ws2019:enterprise:latest' class Windows2022DesktopSQLServer2019StandardAzureVirtualMachine( BaseWindowsAzureVirtualMachine, - windows_virtual_machine.Windows2022SQLServer2019Standard): + windows_virtual_machine.Windows2022SQLServer2019Standard, +): IMAGE_URN = 'MicrosoftSQLServer:sql2019-ws2022:standard:latest' class Windows2022DesktopSQLServer2019EnterpriseAzureVirtualMachine( BaseWindowsAzureVirtualMachine, - windows_virtual_machine.Windows2022SQLServer2019Enterprise): + windows_virtual_machine.Windows2022SQLServer2019Enterprise, +): IMAGE_URN = 'MicrosoftSQLServer:sql2019-ws2022:enterprise:latest' -def GenerateDownloadPreprovisionedDataCommand(install_path, module_name, - filename): +def GenerateDownloadPreprovisionedDataCommand( + install_path, module_name, filename +): """Returns a string used to download preprovisioned data.""" module_name_with_underscores_removed = module_name.replace('_', '-') destpath = posixpath.join(install_path, filename) @@ -1295,7 +1518,9 @@ def GenerateDownloadPreprovisionedDataCommand(install_path, module_name, container_name=module_name_with_underscores_removed, name=filename, file=destpath, - connection_string=connection_string)) + connection_string=connection_string, + ) + ) if install_path: return '{0} && {1}'.format(mkdir_command, download_command) return download_command @@ -1306,12 +1531,15 @@ def GenerateStatPreprovisionedDataCommand(module_name, filename): module_name_with_underscores_removed = module_name.replace('_', '-') account_name = FLAGS.azure_preprovisioned_data_bucket connection_string = util.GetAzureStorageConnectionString(account_name, []) - return ('az storage blob show ' - '--account-name {account_name} ' - '--container-name {container_name} ' - '--name {name} ' - '--connection-string "{connection_string}"'.format( - account_name=account_name, - container_name=module_name_with_underscores_removed, - name=filename, - connection_string=connection_string)) + return ( + 'az storage blob show ' + '--account-name {account_name} ' + '--container-name {container_name} ' + '--name {name} ' + '--connection-string "{connection_string}"'.format( + account_name=account_name, + container_name=module_name_with_underscores_removed, + name=filename, + connection_string=connection_string, + ) + ) diff --git a/perfkitbenchmarker/providers/azure/flags.py b/perfkitbenchmarker/providers/azure/flags.py index 410700f21f..b98c804b9c 100644 --- a/perfkitbenchmarker/providers/azure/flags.py +++ b/perfkitbenchmarker/providers/azure/flags.py @@ -20,9 +20,11 @@ READ_ONLY = 'ReadOnly' READ_WRITE = 'ReadWrite' flags.DEFINE_enum( - 'azure_host_caching', NONE, + 'azure_host_caching', + NONE, [NONE, READ_ONLY, READ_WRITE], - 'The type of host caching to use on Azure data disks.') + 'The type of host caching to use on Azure data disks.', +) # Azure Storage Account types. See # http://azure.microsoft.com/en-us/pricing/details/storage/ for more information # about the different types. @@ -39,66 +41,113 @@ # Azure redis cache tiers. See # https://docs.microsoft.com/en-us/azure/redis-cache/cache-faq for information. -VALID_CACHE_SIZES = ['C0', 'C1', 'C2', 'C3', 'C4', 'C5', 'C6', - 'P1', 'P2', 'P3', 'P4', 'P5'] +VALID_CACHE_SIZES = [ + 'C0', + 'C1', + 'C2', + 'C3', + 'C4', + 'C5', + 'C6', + 'P1', + 'P2', + 'P3', + 'P4', + 'P5', +] flags.DEFINE_enum( - 'azure_storage_type', LRS, + 'azure_storage_type', + LRS, [LRS, PLRS, ULRS, ZRS, GRS, RAGRS], 'The type of storage account to create. See ' 'http://azure.microsoft.com/en-us/pricing/details/storage/ for more ' 'information. To use remote ssd scratch disks, you must use Premium_LRS. ' 'If you use Premium_LRS, you must use the DS series of machines, or else ' - 'VM creation will fail.') + 'VM creation will fail.', +) flags.DEFINE_enum( - 'azure_blob_account_kind', BLOB_STORAGE, + 'azure_blob_account_kind', + BLOB_STORAGE, [STORAGE, BLOB_STORAGE], 'The type of storage account to use for blob storage. Choosing Storage ' 'will let you use ZRS storage. Choosing BlobStorage will give you access ' - 'to Hot and Cold storage tiers.') + 'to Hot and Cold storage tiers.', +) -flags.DEFINE_integer('azure_provisioned_iops', None, - 'IOPS for Provisioned IOPS volumes in Azure.') -flags.DEFINE_integer('azure_provisioned_throughput', None, - 'Provisioned throughput (MB/s) for volumes in Azure.') - -flags.DEFINE_string('azure_preprovisioned_data_bucket', None, - 'Azure blob storage account where pre-provisioned data ' - 'has been copied.') - -flags.DEFINE_boolean('azure_accelerated_networking', False, - 'Enable Azure Accelerated Networking. See ' - 'https://docs.microsoft.com/en-us/azure/virtual-network/' - 'create-vm-accelerated-networking-cli' - 'for more information.') +flags.DEFINE_integer( + 'azure_provisioned_iops', + None, + 'IOPS for Provisioned IOPS volumes in Azure.', +) +flags.DEFINE_integer( + 'azure_provisioned_throughput', + None, + 'Provisioned throughput (MB/s) for volumes in Azure.', +) + +flags.DEFINE_string( + 'azure_preprovisioned_data_bucket', + None, + 'Azure blob storage account where pre-provisioned data has been copied.', +) + +flags.DEFINE_boolean( + 'azure_accelerated_networking', + False, + 'Enable Azure Accelerated Networking. See ' + 'https://docs.microsoft.com/en-us/azure/virtual-network/' + 'create-vm-accelerated-networking-cli' + 'for more information.', +) AZURE_SUBNET_ID = flags.DEFINE_string( - 'azure_subnet_id', None, - 'The id of an already created subnet to use instead of creating a new one.') + 'azure_subnet_id', + None, + 'The id of an already created subnet to use instead of creating a new one.', +) -flags.DEFINE_enum('azure_tier', 'Basic', VALID_TIERS, - 'Performance tier to use for the machine type. Defaults to ' - 'Basic.') +flags.DEFINE_enum( + 'azure_tier', + 'Basic', + VALID_TIERS, + 'Performance tier to use for the machine type. Defaults to Basic.', +) flags.DEFINE_integer( - 'azure_compute_units', None, - 'Number of compute units to allocate for the machine type') - -flags.DEFINE_enum('azure_redis_size', - 'C3', VALID_CACHE_SIZES, - 'Azure redis cache size to use.') - -flags.DEFINE_boolean('azure_low_priority_vms', False, - 'Whether to set the priority to low for Azure VMs') - -flags.DEFINE_boolean('bootstrap_azure_service_principal', True, - 'Whether to use the current service principal credentials ' - 'when passing a service principal to a service. This has ' - 'no effect if the logged in user is not a service ' - 'principal. This is useful, because service principals ' - "usually lack the 'User Authentication Admin' role that " - 'allows creation of new service principals.') -flags.DEFINE_enum('sqldatawarehouse_client_interface', 'CLI', ['CLI', 'JDBC'], - 'The Runtime Interface used when interacting with Synapse.') + 'azure_compute_units', + None, + 'Number of compute units to allocate for the machine type', +) + +flags.DEFINE_enum( + 'azure_redis_size', + 'C3', + VALID_CACHE_SIZES, + 'Azure redis cache size to use.', +) + +flags.DEFINE_boolean( + 'azure_low_priority_vms', + False, + 'Whether to set the priority to low for Azure VMs', +) + +flags.DEFINE_boolean( + 'bootstrap_azure_service_principal', + True, + 'Whether to use the current service principal credentials ' + 'when passing a service principal to a service. This has ' + 'no effect if the logged in user is not a service ' + 'principal. This is useful, because service principals ' + "usually lack the 'User Authentication Admin' role that " + 'allows creation of new service principals.', +) +flags.DEFINE_enum( + 'sqldatawarehouse_client_interface', + 'CLI', + ['CLI', 'JDBC'], + 'The Runtime Interface used when interacting with Synapse.', +) flags.DEFINE_string('query_timeout', '600', 'Query timeout in seconds.') diff --git a/perfkitbenchmarker/providers/azure/provider_info.py b/perfkitbenchmarker/providers/azure/provider_info.py index e400ac4dd8..fa7c8cac08 100644 --- a/perfkitbenchmarker/providers/azure/provider_info.py +++ b/perfkitbenchmarker/providers/azure/provider_info.py @@ -18,6 +18,5 @@ class AzureProviderInfo(provider_info.BaseProviderInfo): - UNSUPPORTED_BENCHMARKS = ['mysql_service'] CLOUD = provider_info.AZURE diff --git a/perfkitbenchmarker/providers/azure/service_principal.py b/perfkitbenchmarker/providers/azure/service_principal.py index b84d1ee549..431c315287 100644 --- a/perfkitbenchmarker/providers/azure/service_principal.py +++ b/perfkitbenchmarker/providers/azure/service_principal.py @@ -48,17 +48,21 @@ def LoadFromFile(cls): """Loads a service principal from a file.""" with open( object_storage_service.FindCredentialFile( - azure_credentials.PROFILE_FILE), - encoding='utf-8-sig') as profile_fp: + azure_credentials.PROFILE_FILE + ), + encoding='utf-8-sig', + ) as profile_fp: subscriptions = json.load(profile_fp)['subscriptions'] subscription = [sub for sub in subscriptions if sub['isDefault']][0] subscription_type = subscription['user']['type'] if subscription_type != 'servicePrincipal': # We are using user auth, and will probably have permission to create a # service principal. - logging.info("Azure credentials are of type '%s'. " - 'Will try to create a new service principal.', - subscription_type) + logging.info( + "Azure credentials are of type '%s'. " + 'Will try to create a new service principal.', + subscription_type, + ) return cls() # name and id are backwards name = subscription['id'] @@ -68,16 +72,22 @@ def LoadFromFile(cls): # will exist. with open( object_storage_service.FindCredentialFile( - azure_credentials.SERVICE_PRINCIPAL_FILE) + azure_credentials.SERVICE_PRINCIPAL_FILE + ) ) as service_principals_fp: for sp in json.load(service_principals_fp): if sp['client_id'] == app_id: - logging.info("Azure credentials are of type 'servicePrincipal'. " - 'Will reuse them for benchmarking.') + logging.info( + "Azure credentials are of type 'servicePrincipal'. " + 'Will reuse them for benchmarking.' + ) return cls( - name, app_id, password=sp['client_secret'], user_managed=True) - logging.warning('No access tokens found matching Azure defaultProfile ' - 'Will try to create a new service principal.') + name, app_id, password=sp['client_secret'], user_managed=True + ) + logging.warning( + 'No access tokens found matching Azure defaultProfile ' + 'Will try to create a new service principal.' + ) return cls() def __init__(self, name=None, app_id=None, password=None, user_managed=False): @@ -93,8 +103,13 @@ def __init__(self, name=None, app_id=None, password=None, user_managed=False): def _Create(self): """Creates the service principal.""" cmd = [ - azure.AZURE_PATH, 'ad', 'sp', 'create-for-rbac', '--name', self.name, - '--skip-assignment' + azure.AZURE_PATH, + 'ad', + 'sp', + 'create-for-rbac', + '--name', + self.name, + '--skip-assignment', ] stdout, _, _ = vm_util.IssueCommand(cmd) response = json.loads(stdout) @@ -104,7 +119,9 @@ def _Create(self): if not self.app_id or not self.password: raise errors.Resource.CreationError( 'Invalid creation response when creating service principal. ' - 'Expected appId and password. Received:\n' + stdout) + 'Expected appId and password. Received:\n' + + stdout + ) return True return False diff --git a/perfkitbenchmarker/providers/azure/snowflake_azure.py b/perfkitbenchmarker/providers/azure/snowflake_azure.py index 80daeafd7a..5759517bf2 100644 --- a/perfkitbenchmarker/providers/azure/snowflake_azure.py +++ b/perfkitbenchmarker/providers/azure/snowflake_azure.py @@ -28,11 +28,13 @@ class Snowflake(snowflake.Snowflake): """Class representing a Snowflake Data Warehouse Instance hosted on Azure.""" + CLOUD = provider_info.AZURE SERVICE_TYPE = 'snowflake_azure' class Snowflakeexternal(snowflake.Snowflake): """Class representing Snowflake External Warehouses on Azure.""" + CLOUD = provider_info.AZURE SERVICE_TYPE = 'snowflakeexternal_azure' diff --git a/perfkitbenchmarker/providers/azure/util.py b/perfkitbenchmarker/providers/azure/util.py index c4ffda174e..9fc1ae9ac6 100644 --- a/perfkitbenchmarker/providers/azure/util.py +++ b/perfkitbenchmarker/providers/azure/util.py @@ -33,25 +33,46 @@ def GetAzureStorageConnectionString(storage_account_name, resource_group_args): """Get connection string.""" stdout, _ = vm_util.IssueRetryableCommand( - [AZURE_PATH, 'storage', 'account', 'show-connection-string', - '--name', storage_account_name] + resource_group_args + AZURE_SUFFIX) + [ + AZURE_PATH, + 'storage', + 'account', + 'show-connection-string', + '--name', + storage_account_name, + ] + + resource_group_args + + AZURE_SUFFIX + ) response = json.loads(stdout) return response['connectionString'] def GetAzureStorageConnectionArgs(storage_account_name, resource_group_args): """Get connection CLI arguments.""" - return ['--connection-string', - GetAzureStorageConnectionString(storage_account_name, - resource_group_args)] + return [ + '--connection-string', + GetAzureStorageConnectionString( + storage_account_name, resource_group_args + ), + ] def GetAzureStorageAccountKey(storage_account_name, resource_group_args): """Get storage account key.""" stdout, _ = vm_util.IssueRetryableCommand( - [AZURE_PATH, 'storage', 'account', 'keys', 'list', - '--account-name', storage_account_name] + - resource_group_args + AZURE_SUFFIX) + [ + AZURE_PATH, + 'storage', + 'account', + 'keys', + 'list', + '--account-name', + storage_account_name, + ] + + resource_group_args + + AZURE_SUFFIX + ) response = json.loads(stdout) # A new storage account comes with two keys, but we only need one. @@ -126,10 +147,9 @@ def IsZone(zone_or_region): """Returns whether "zone_or_region" is a zone. Args: - zone_or_region: string, Azure zone or region. Format for Azure - availability + zone_or_region: string, Azure zone or region. Format for Azure availability zone support is "region-availability_zone". Example: eastus2-1 specifies - Azure region eastus2 with availability zone 1. + Azure region eastus2 with availability zone 1. """ return re.match(r'[a-z]+[0-9]?-[0-9]$', zone_or_region, re.IGNORECASE) @@ -142,8 +162,9 @@ def GetRegionFromZone(zone_or_region: str) -> str: if IsZone(zone_or_region): return zone_or_region[:-2] - raise ValueError('%s is not a valid Azure zone or region name' % - zone_or_region) + raise ValueError( + '%s is not a valid Azure zone or region name' % zone_or_region + ) def GetZonesInRegion(region: str) -> Set[str]: @@ -162,7 +183,8 @@ def ShouldKeepZoneFromCLI(zone: str) -> bool: def GetZonesFromMachineType(machine_type: str) -> Set[str]: """Returns a set of zones for a machine type.""" stdout, _ = vm_util.IssueRetryableCommand( - [AZURE_PATH, 'vm', 'list-skus', '--size', machine_type]) + [AZURE_PATH, 'vm', 'list-skus', '--size', machine_type] + ) zones = set() for item in json.loads(stdout): for location_info in item['locationInfo']: @@ -175,9 +197,9 @@ def GetZonesFromMachineType(machine_type: str) -> Set[str]: def GetAllRegions() -> Set[str]: """Returns all valid regions.""" - stdout, _ = vm_util.IssueRetryableCommand([ - AZURE_PATH, 'account', 'list-locations', '--output', 'json' - ]) + stdout, _ = vm_util.IssueRetryableCommand( + [AZURE_PATH, 'account', 'list-locations', '--output', 'json'] + ) # Filter out staging regions from the output. return set([ item['name'] for item in json.loads(stdout) if _IsRecommendedRegion(item) @@ -195,9 +217,13 @@ def GetAllZones() -> Set[str]: def GetGeoFromRegion(region: str) -> str: """Gets valid geo from the region, i.e. region westus2 returns US.""" stdout, _ = vm_util.IssueRetryableCommand([ - AZURE_PATH, 'account', 'list-locations', - '--output', 'json', - '--query', f"[?name == '{region}'].metadata.geographyGroup" + AZURE_PATH, + 'account', + 'list-locations', + '--output', + 'json', + '--query', + f"[?name == '{region}'].metadata.geographyGroup", ]) return stdout.splitlines()[1].strip('" ') @@ -205,9 +231,13 @@ def GetGeoFromRegion(region: str) -> str: def GetRegionsInGeo(geo: str) -> Set[str]: """Gets valid regions in the geo.""" stdout, _ = vm_util.IssueRetryableCommand([ - AZURE_PATH, 'account', 'list-locations', - '--output', 'json', - '--query', f"[?metadata.geographyGroup == '{geo}']" + AZURE_PATH, + 'account', + 'list-locations', + '--output', + 'json', + '--query', + f"[?metadata.geographyGroup == '{geo}']", ]) return set([ item['name'] for item in json.loads(stdout) if _IsRecommendedRegion(item) diff --git a/perfkitbenchmarker/providers/cloudstack/cloudstack_disk.py b/perfkitbenchmarker/providers/cloudstack/cloudstack_disk.py index 6bd5e24ecb..49ead60056 100644 --- a/perfkitbenchmarker/providers/cloudstack/cloudstack_disk.py +++ b/perfkitbenchmarker/providers/cloudstack/cloudstack_disk.py @@ -29,14 +29,11 @@ class CloudStackDisk(disk.BaseDisk): """Object representing a Cloudstack Disk.""" - def __init__(self, disk_spec, name, zone_id, project_id=None): super(CloudStackDisk, self).__init__(disk_spec) self.cs = util.CsClient( - FLAGS.CS_API_URL, - FLAGS.CS_API_KEY, - FLAGS.CS_API_SECRET + FLAGS.CS_API_URL, FLAGS.CS_API_KEY, FLAGS.CS_API_SECRET ) self.attached_vm_name = None @@ -47,28 +44,24 @@ def __init__(self, disk_spec, name, zone_id, project_id=None): self.project_id = project_id self.disk_offering_id = self._GetBestOfferingId(self.disk_size) - assert self.disk_offering_id, "Unable get disk offering of given size" + assert self.disk_offering_id, 'Unable get disk offering of given size' if disk_spec.disk_type: - logging.warn("Cloudstack does not support disk types") - + logging.warn('Cloudstack does not support disk types') @vm_util.Retry(max_retries=3) def _Create(self): """Creates the disk.""" + volume = self.cs.create_volume( + self.name, self.disk_offering_id, self.zone_id, self.project_id + ) - volume = self.cs.create_volume(self.name, - self.disk_offering_id, - self.zone_id, - self.project_id) - - assert volume, "Unable to create volume" + assert volume, 'Unable to create volume' self.volume_id = volume['id'] self.disk_type = volume['type'] - self.actual_disk_size = int(volume['size']) / (2 ** 30) # In GB - + self.actual_disk_size = int(volume['size']) / (2**30) # In GB def _Delete(self): """Deletes the disk.""" @@ -76,7 +69,6 @@ def _Delete(self): if vol: self.cs.delete_volume(self.volume_id) - def _Exists(self): """Returns true if the disk exists.""" vol = self.cs.get_volume(self.name, self.project_id) @@ -84,35 +76,30 @@ def _Exists(self): return True return False - @vm_util.Retry(max_retries=3) def Attach(self, vm): """Attaches the disk to a VM. Args: - vm: The CloudStackVirtualMachine instance to which - the disk will be attached. - + vm: The CloudStackVirtualMachine instance to which the disk will be + attached. """ res = self.cs.attach_volume(self.volume_id, vm.id) - assert res, "Unable to attach volume" + assert res, 'Unable to attach volume' self.device_id = res['deviceid'] - self.device_path = "/dev/xvd" + \ - str(string.ascii_lowercase[self.device_id]) - - + self.device_path = '/dev/xvd' + str(string.ascii_lowercase[self.device_id]) def Detach(self): """Detaches the disk from a VM.""" self.cs.detach_volume(self.volume_id) - def _GetBestOfferingId(self, disk_size): - """ Given a disk_size (in GB), try to find a disk + """Given a disk_size (in GB), try to find a disk + offering that is atleast as big as the requested one. """ diff --git a/perfkitbenchmarker/providers/cloudstack/cloudstack_network.py b/perfkitbenchmarker/providers/cloudstack/cloudstack_network.py index e4def5ec75..1352f1287e 100644 --- a/perfkitbenchmarker/providers/cloudstack/cloudstack_network.py +++ b/perfkitbenchmarker/providers/cloudstack/cloudstack_network.py @@ -27,19 +27,16 @@ class CloudStackNetwork(network.BaseNetwork): CLOUD = provider_info.CLOUDSTACK def __init__(self, spec): - super(CloudStackNetwork, self).__init__(spec) - self.cs = util.CsClient(FLAGS.CS_API_URL, - FLAGS.CS_API_KEY, - FLAGS.CS_API_SECRET) + self.cs = util.CsClient( + FLAGS.CS_API_URL, FLAGS.CS_API_KEY, FLAGS.CS_API_SECRET + ) self.project_id = None self.network_id = None - def _AcquireNetworkDetails(self): - if FLAGS.project: project = self.cs.get_project(FLAGS.project) if project: @@ -51,14 +48,15 @@ def _AcquireNetworkDetails(self): if zone: self.zone_id = zone['id'] - assert self.zone_id, "Zone required to create a network" + assert self.zone_id, 'Zone required to create a network' self.network_name = None - nw_off = self.cs.get_network_offering(FLAGS.cs_network_offering, - self.project_id) + nw_off = self.cs.get_network_offering( + FLAGS.cs_network_offering, self.project_id + ) - assert nw_off, "Network offering not found" + assert nw_off, 'Network offering not found' self.network_offering_id = nw_off['id'] self.network_name = 'perfkit-network-%s' % FLAGS.run_uri @@ -67,15 +65,13 @@ def _AcquireNetworkDetails(self): self.vpc_id = None if FLAGS.cs_use_vpc: - - assert FLAGS.cs_vpc_offering, "VPC flag should specify the VPC offering" + assert FLAGS.cs_vpc_offering, 'VPC flag should specify the VPC offering' vpc_off = self.cs.get_vpc_offering(FLAGS.cs_vpc_offering) - assert vpc_off, "Use VPC specified but VPC offering not found" + assert vpc_off, 'Use VPC specified but VPC offering not found' self.vpc_offering_id = vpc_off['id'] self.vpc_name = 'perfkit-vpc-%s' % FLAGS.run_uri - @vm_util.Retry(max_retries=3) def Create(self): """Creates the actual network.""" @@ -89,32 +85,33 @@ def Create(self): # Create a VPC first cidr = '10.0.0.0/16' - vpc = self.cs.create_vpc(self.vpc_name, - self.zone_id, - cidr, - self.vpc_offering_id, - self.project_id) + vpc = self.cs.create_vpc( + self.vpc_name, + self.zone_id, + cidr, + self.vpc_offering_id, + self.project_id, + ) self.vpc_id = vpc['id'] gateway = '10.0.0.1' netmask = '255.255.255.0' acl = self.cs.get_network_acl('default_allow', self.project_id) - assert acl, "Default allow ACL not found" - + assert acl, 'Default allow ACL not found' # Create the network - network = self.cs.create_network(self.network_name, - self.network_offering_id, - self.zone_id, - self.project_id, - self.vpc_id, - gateway, - netmask, - acl['id']) - - - - assert network, "No network could be created" + network = self.cs.create_network( + self.network_name, + self.network_offering_id, + self.zone_id, + self.project_id, + self.vpc_id, + gateway, + netmask, + acl['id'], + ) + + assert network, 'No network could be created' self.network_id = network['id'] self.id = self.network_id diff --git a/perfkitbenchmarker/providers/cloudstack/cloudstack_virtual_machine.py b/perfkitbenchmarker/providers/cloudstack/cloudstack_virtual_machine.py index d79cdabfa5..5ca4a8fdf6 100644 --- a/perfkitbenchmarker/providers/cloudstack/cloudstack_virtual_machine.py +++ b/perfkitbenchmarker/providers/cloudstack/cloudstack_virtual_machine.py @@ -11,10 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" -Class representing a Cloudstack instance. This module uses the csapi -library which calls the cloudstack API. For more information refer to -the Cloudstack documentation at https://github.com/syed/PerfKitBenchmarker.git +"""Class representing a Cloudstack instance. + +This module uses the csapi library which calls the cloudstack API. For more +information refer to the Cloudstack documentation at +https://github.com/syed/PerfKitBenchmarker.git """ @@ -53,18 +54,18 @@ def __init__(self, vm_spec): super(CloudStackVirtualMachine, self).__init__(vm_spec) self.network = cloudstack_network.CloudStackNetwork.GetNetwork(self) - self.cs = util.CsClient(FLAGS.CS_API_URL, - FLAGS.CS_API_KEY, - FLAGS.CS_API_SECRET) + self.cs = util.CsClient( + FLAGS.CS_API_URL, FLAGS.CS_API_KEY, FLAGS.CS_API_SECRET + ) self.project_id = None if FLAGS.project: project = self.cs.get_project(FLAGS.project) - assert project, "Project not found" + assert project, 'Project not found' self.project_id = project['id'] zone = self.cs.get_zone(self.zone) - assert zone, "Zone not found" + assert zone, 'Zone not found' self.zone_id = zone['id'] self.user_name = self.DEFAULT_USER_NAME @@ -81,12 +82,11 @@ def _CreateDependencies(self): pub_key = keyfd.read() if not self.cs.get_ssh_keypair(self.ssh_keypair_name, self.project_id): + res = self.cs.register_ssh_keypair( + self.ssh_keypair_name, pub_key, self.project_id + ) - res = self.cs.register_ssh_keypair(self.ssh_keypair_name, - pub_key, - self.project_id) - - assert res, "Unable to create ssh keypair" + assert res, 'Unable to create ssh keypair' # Allocate a public ip network_id = self.network.id @@ -99,7 +99,7 @@ def _CreateDependencies(self): self.ip_address = public_ip['ipaddress'] self.ip_address_id = public_ip['id'] else: - logging.warn("Unable to allocate public IP") + logging.warn('Unable to allocate public IP') def _DeleteDependencies(self): """Delete VM dependencies.""" @@ -116,23 +116,25 @@ def _Create(self): """Create a Cloudstack VM instance.""" service_offering = self.cs.get_serviceoffering(self.machine_type) - assert service_offering, "No service offering found" + assert service_offering, 'No service offering found' template = self.cs.get_template(self.image, self.project_id) - assert template, "No template found" + assert template, 'No template found' network_id = self.network.id vm = None - vm = self.cs.create_vm(self.name, - self.zone_id, - service_offering['id'], - template['id'], - [network_id], - self.ssh_keypair_name, - self.project_id) + vm = self.cs.create_vm( + self.name, + self.zone_id, + service_offering['id'], + template['id'], + [network_id], + self.ssh_keypair_name, + self.project_id, + ) - assert vm, "Unable to create VM" + assert vm, 'Unable to create VM' self._vm = vm self.id = vm['virtualmachine']['id'] @@ -147,12 +149,11 @@ def _PostCreate(self): # Create a Static NAT rule if not self.cs.snat_rule_exists(self.ip_address_id, self.id): + snat_rule = self.cs.enable_static_nat( + self.ip_address_id, self.id, self.network.id + ) - snat_rule = self.cs.enable_static_nat(self.ip_address_id, - self.id, - self.network.id) - - assert snat_rule, "Unable to create static NAT" + assert snat_rule, 'Unable to create static NAT' def _Delete(self): """Delete the VM instance.""" @@ -183,12 +184,10 @@ def CreateScratchDisk(self, _, disk_spec): self.disks = [] for i in range(disk_spec.num_striped_disks): - name = 'disk-%s-%d-%d' % (self.name, i + 1, self.disk_counter) - scratch_disk = cloudstack_disk.CloudStackDisk(disk_spec, - name, - self.zone_id, - self.project_id) + scratch_disk = cloudstack_disk.CloudStackDisk( + disk_spec, name, self.zone_id, self.project_id + ) self.disks.append(scratch_disk) self.disk_counter += 1 @@ -199,6 +198,7 @@ def CreateScratchDisk(self, _, disk_spec): ) -class CentOs7BasedCloudStackVirtualMachine(CloudStackVirtualMachine, - linux_vm.CentOs7Mixin): +class CentOs7BasedCloudStackVirtualMachine( + CloudStackVirtualMachine, linux_vm.CentOs7Mixin +): DEFAULT_IMAGE = 'CentOS 7 HVM base (64bit)' diff --git a/perfkitbenchmarker/providers/cloudstack/flags.py b/perfkitbenchmarker/providers/cloudstack/flags.py index f53f16b3cc..f7c2204243 100644 --- a/perfkitbenchmarker/providers/cloudstack/flags.py +++ b/perfkitbenchmarker/providers/cloudstack/flags.py @@ -16,25 +16,28 @@ from absl import flags -flags.DEFINE_string('CS_API_URL', - os.environ.get('CS_API_URL'), - 'API endpoint for Cloudstack.') - -flags.DEFINE_string('CS_API_KEY', - os.environ.get('CS_API_KEY'), - 'Key for API authentication') - -flags.DEFINE_string('CS_API_SECRET', - os.environ.get('CS_API_SECRET'), - 'Secret for API authentication') - -flags.DEFINE_string('cs_network_offering', - 'DefaultIsolatedNetworkOfferingForVpcNetworksNoLB', - 'Name of the network offering') - -flags.DEFINE_string('cs_vpc_offering', - 'Default VPC offering', - 'Name of the VPC offering') - -flags.DEFINE_boolean('cs_use_vpc', True, - 'Use VPC to create networks') +flags.DEFINE_string( + 'CS_API_URL', os.environ.get('CS_API_URL'), 'API endpoint for Cloudstack.' +) + +flags.DEFINE_string( + 'CS_API_KEY', os.environ.get('CS_API_KEY'), 'Key for API authentication' +) + +flags.DEFINE_string( + 'CS_API_SECRET', + os.environ.get('CS_API_SECRET'), + 'Secret for API authentication', +) + +flags.DEFINE_string( + 'cs_network_offering', + 'DefaultIsolatedNetworkOfferingForVpcNetworksNoLB', + 'Name of the network offering', +) + +flags.DEFINE_string( + 'cs_vpc_offering', 'Default VPC offering', 'Name of the VPC offering' +) + +flags.DEFINE_boolean('cs_use_vpc', True, 'Use VPC to create networks') diff --git a/perfkitbenchmarker/providers/cloudstack/provider_info.py b/perfkitbenchmarker/providers/cloudstack/provider_info.py index 7d72287a6d..e0aa4d0fe6 100644 --- a/perfkitbenchmarker/providers/cloudstack/provider_info.py +++ b/perfkitbenchmarker/providers/cloudstack/provider_info.py @@ -12,14 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" Provider info for CloudStack - -""" +"""Provider info for CloudStack""" from perfkitbenchmarker import provider_info class CloudStackProviderInfo(provider_info.BaseProviderInfo): - UNSUPPORTED_BENCHMARKS = ['mysql_service'] CLOUD = provider_info.CLOUDSTACK diff --git a/perfkitbenchmarker/providers/cloudstack/util.py b/perfkitbenchmarker/providers/cloudstack/util.py index 5ef0ef2b53..6735cc33e9 100644 --- a/perfkitbenchmarker/providers/cloudstack/util.py +++ b/perfkitbenchmarker/providers/cloudstack/util.py @@ -18,6 +18,7 @@ try: from requests.packages import urllib3 + urllib3.disable_warnings() except ImportError: pass @@ -28,17 +29,10 @@ class CsClient(object): def __init__(self, url, api_key, secret): - - self._cs = API(api_key, - secret, - url, - logging=False) + self._cs = API(api_key, secret, url, logging=False) def get_zone(self, zone_name): - - cs_args = { - 'command': 'listZones' - } + cs_args = {'command': 'listZones'} zones = self._cs.request(cs_args) @@ -50,16 +44,11 @@ def get_zone(self, zone_name): return None def get_template(self, template_name, project_id=None): - - cs_args = { - 'command': 'listTemplates', - 'templatefilter': 'executable' - } + cs_args = {'command': 'listTemplates', 'templatefilter': 'executable'} if project_id: cs_args.update({'projectid': project_id}) - templates = self._cs.request(cs_args) if templates and 'template' in templates: @@ -70,7 +59,6 @@ def get_template(self, template_name, project_id=None): return None def get_serviceoffering(self, service_offering_name): - cs_args = { 'command': 'listServiceOfferings', } @@ -84,12 +72,8 @@ def get_serviceoffering(self, service_offering_name): return None - def get_project(self, project_name): - - cs_args = { - 'command': 'listProjects' - } + cs_args = {'command': 'listProjects'} projects = self._cs.request(cs_args) @@ -101,16 +85,15 @@ def get_project(self, project_name): return None def get_network(self, network_name, project_id=None, vpc_id=None): - cs_args = { 'command': 'listNetworks', } if project_id: - cs_args.update({"projectid": project_id}) + cs_args.update({'projectid': project_id}) if vpc_id: - cs_args.update({"vpcid": vpc_id}) + cs_args.update({'vpcid': vpc_id}) networks = self._cs.request(cs_args) @@ -136,13 +119,12 @@ def get_network_offering(self, network_offering_name, project_id): return None def get_vpc(self, vpc_name, project_id=None): - cs_args = { 'command': 'listVPCs', } if project_id: - cs_args.update({"projectid": project_id}) + cs_args.update({'projectid': project_id}) vpcs = self._cs.request(cs_args) @@ -154,7 +136,6 @@ def get_vpc(self, vpc_name, project_id=None): return None def get_vpc_offering(self, vpc_offering_name): - cs_args = { 'command': 'listVPCOfferings', } @@ -169,13 +150,12 @@ def get_vpc_offering(self, vpc_offering_name): return None def get_virtual_machine(self, vm_name, project_id=None): - cs_args = { 'command': 'listVirtualMachines', } if project_id: - cs_args.update({"projectid": project_id}) + cs_args.update({'projectid': project_id}) vms = self._cs.request(cs_args) @@ -186,15 +166,16 @@ def get_virtual_machine(self, vm_name, project_id=None): return None - def create_vm(self, - name, - zone_id, - service_offering_id, - template_id, - network_ids=None, - keypair=None, - project_id=None): - + def create_vm( + self, + name, + zone_id, + service_offering_id, + template_id, + network_ids=None, + keypair=None, + project_id=None, + ): create_vm_args = { 'command': 'deployVirtualMachine', 'serviceofferingid': service_offering_id, @@ -204,31 +185,29 @@ def create_vm(self, } if network_ids: - create_vm_args.update({"networkids": network_ids}) + create_vm_args.update({'networkids': network_ids}) if keypair: create_vm_args.update({'keypair': keypair}) if project_id: - create_vm_args.update({"projectid": project_id}) + create_vm_args.update({'projectid': project_id}) vm = self._cs.request(create_vm_args) return vm def delete_vm(self, vm_id): - cs_args = { 'command': 'destroyVirtualMachine', 'id': vm_id, - 'expunge': 'true' # Requres root/domain admin + 'expunge': 'true', # Requres root/domain admin } res = self._cs.request(cs_args) return res def create_vpc(self, name, zone_id, cidr, vpc_offering_id, project_id=None): - cs_args = { 'command': 'createVPC', 'name': name, @@ -239,7 +218,7 @@ def create_vpc(self, name, zone_id, cidr, vpc_offering_id, project_id=None): } if project_id: - cs_args.update({"projectid": project_id}) + cs_args.update({'projectid': project_id}) vpc = self._cs.request(cs_args) @@ -249,26 +228,23 @@ def create_vpc(self, name, zone_id, cidr, vpc_offering_id, project_id=None): return None def delete_vpc(self, vpc_id): - - cs_args = { - 'command': 'deleteVPC', - 'id': vpc_id - } + cs_args = {'command': 'deleteVPC', 'id': vpc_id} res = self._cs.request(cs_args) return res - def create_network(self, - name, - network_offering_id, - zone_id, - project_id=None, - vpc_id=None, - gateway=None, - netmask=None, - acl_id=None): - + def create_network( + self, + name, + network_offering_id, + zone_id, + project_id=None, + vpc_id=None, + gateway=None, + netmask=None, + acl_id=None, + ): cs_args = { 'command': 'createNetwork', 'name': name, @@ -278,14 +254,14 @@ def create_network(self, } if project_id: - cs_args.update({"projectid": project_id}) + cs_args.update({'projectid': project_id}) if vpc_id: cs_args.update({ 'vpcid': vpc_id, 'gateway': gateway, 'netmask': netmask, - 'aclid': acl_id + 'aclid': acl_id, }) nw = self._cs.request(cs_args) @@ -296,7 +272,6 @@ def create_network(self, return nw def delete_network(self, network_id): - cs_args = { 'command': 'deleteNetwork', 'id': network_id, @@ -306,7 +281,6 @@ def delete_network(self, network_id): return res def alloc_public_ip(self, network_id, is_vpc=False): - cs_args = { 'command': 'associateIpAddress', } @@ -324,22 +298,17 @@ def alloc_public_ip(self, network_id, is_vpc=False): return None def release_public_ip(self, ipaddress_id): - - cs_args = { - 'command': 'disassociateIpAddress', - 'id': ipaddress_id - } + cs_args = {'command': 'disassociateIpAddress', 'id': ipaddress_id} res = self._cs.request(cs_args) return res def enable_static_nat(self, ip_address_id, vm_id, network_id): - cs_args = { 'command': 'enableStaticNat', 'ipaddressid': ip_address_id, - 'virtualmachineid': vm_id + 'virtualmachineid': vm_id, } if network_id: @@ -353,29 +322,23 @@ def enable_static_nat(self, ip_address_id, vm_id, network_id): return None def snat_rule_exists(self, ip_address_id, vm_id): - - cs_args = { - 'command': 'listPublicIpAddresses', - 'id': ip_address_id - } + cs_args = {'command': 'listPublicIpAddresses', 'id': ip_address_id} res = self._cs.request(cs_args) - assert 'publicipaddress' in res, "No public IP address found" - assert len(res['publicipaddress']) == 1, "More than One\ - Public IP address" + assert 'publicipaddress' in res, 'No public IP address found' + assert ( + len(res['publicipaddress']) == 1 + ), 'More than One Public IP address' res = res['publicipaddress'][0] - if res and 'virtualmachineid' in res and \ - res['virtualmachineid'] == vm_id: + if res and 'virtualmachineid' in res and res['virtualmachineid'] == vm_id: return True return False - def register_ssh_keypair(self, name, public_key, project_id=None): - cs_args = { 'command': 'registerSSHKeyPair', 'name': name, @@ -383,34 +346,31 @@ def register_ssh_keypair(self, name, public_key, project_id=None): } if project_id: - cs_args.update({"projectid": project_id}) + cs_args.update({'projectid': project_id}) res = self._cs.request(cs_args, method='post') return res - def unregister_ssh_keypair(self, name, project_id=None): - cs_args = { 'command': 'deleteSSHKeyPair', 'name': name, } if project_id: - cs_args.update({"projectid": project_id}) + cs_args.update({'projectid': project_id}) res = self._cs.request(cs_args) return res def get_ssh_keypair(self, name, project_id=None): - cs_args = { 'command': 'listSSHKeyPairs', 'name': name, } if project_id: - cs_args.update({"projectid": project_id}) + cs_args.update({'projectid': project_id}) kps = self._cs.request(cs_args) @@ -422,13 +382,12 @@ def get_ssh_keypair(self, name, project_id=None): return None def get_network_acl(self, name, project_id=None): - cs_args = { 'command': 'listNetworkACLLists', } if project_id: - cs_args.update({"projectid": project_id}) + cs_args.update({'projectid': project_id}) acllist = self._cs.request(cs_args) @@ -439,12 +398,11 @@ def get_network_acl(self, name, project_id=None): return None def create_volume(self, name, diskoffering_id, zone_id, project_id=None): - cs_args = { 'command': 'createVolume', 'diskofferingid': diskoffering_id, 'zoneid': zone_id, - 'name': name + 'name': name, } if project_id: @@ -457,15 +415,13 @@ def create_volume(self, name, diskoffering_id, zone_id, project_id=None): return None - def get_volume(self, name, project_id=None): - cs_args = { 'command': 'listVolumes', } if project_id: - cs_args.update({"projectid": project_id}) + cs_args.update({'projectid': project_id}) vols = self._cs.request(cs_args) @@ -477,21 +433,16 @@ def get_volume(self, name, project_id=None): return None def delete_volume(self, volume_id): - - cs_args = { - 'command': 'deleteVolume', - 'id': volume_id - } + cs_args = {'command': 'deleteVolume', 'id': volume_id} res = self._cs.request(cs_args) return res def attach_volume(self, vol_id, vm_id): - cs_args = { 'command': 'attachVolume', 'id': vol_id, - 'virtualmachineid': vm_id + 'virtualmachineid': vm_id, } res = self._cs.request(cs_args) @@ -501,9 +452,7 @@ def attach_volume(self, vol_id, vm_id): return None - def detach_volume(self, vol_id): - cs_args = { 'command': 'detachVolume', 'id': vol_id, @@ -513,7 +462,6 @@ def detach_volume(self, vol_id): return res def list_disk_offerings(self): - cs_args = { 'command': 'listDiskOfferings', } diff --git a/perfkitbenchmarker/providers/digitalocean/digitalocean_disk.py b/perfkitbenchmarker/providers/digitalocean/digitalocean_disk.py index aafaa40301..63c9f577ea 100644 --- a/perfkitbenchmarker/providers/digitalocean/digitalocean_disk.py +++ b/perfkitbenchmarker/providers/digitalocean/digitalocean_disk.py @@ -77,18 +77,16 @@ def __init__(self, disk_spec, zone): def _Create(self): self.volume_name = 'pkb-%s-%s' % (FLAGS.run_uri, self.disk_number) - response, retcode = util.DoctlAndParse( - [ - 'compute', - 'volume', - 'create', - self.volume_name, - '--region', - self.zone, - '--size', - str(self.disk_size) + 'gb', - ] - ) + response, retcode = util.DoctlAndParse([ + 'compute', + 'volume', + 'create', + self.volume_name, + '--region', + self.zone, + '--size', + str(self.disk_size) + 'gb', + ]) if retcode: raise errors.Resource.RetryableCreationError( 'Error creating disk: %s' % (response,) diff --git a/perfkitbenchmarker/providers/digitalocean/digitalocean_virtual_machine.py b/perfkitbenchmarker/providers/digitalocean/digitalocean_virtual_machine.py index 48a4c9b260..33b7e0ba89 100644 --- a/perfkitbenchmarker/providers/digitalocean/digitalocean_virtual_machine.py +++ b/perfkitbenchmarker/providers/digitalocean/digitalocean_virtual_machine.py @@ -11,8 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Class to represent a DigitalOcean Virtual Machine object (Droplet). -""" +"""Class to represent a DigitalOcean Virtual Machine object (Droplet).""" from perfkitbenchmarker import disk from perfkitbenchmarker import disk_strategies @@ -35,7 +34,7 @@ # # To avoid this, just disable the root password (we don't need it), # and remove the forced expiration. -CLOUD_CONFIG_TEMPLATE = '''#cloud-config +CLOUD_CONFIG_TEMPLATE = """#cloud-config users: - name: {0} ssh-authorized-keys: @@ -46,7 +45,7 @@ runcmd: - [ passwd, -l, root ] - [ chage, -d, -1, -I, -1, -E, -1, -M, 999999, root ] -''' +""" class DigitalOceanVirtualMachine(virtual_machine.BaseVirtualMachine): @@ -73,26 +72,34 @@ def _Create(self): with open(self.ssh_public_key) as f: public_key = f.read().rstrip('\n') - response, retcode = util.DoctlAndParse( - ['compute', 'droplet', 'create', - self.name, - '--region', self.zone, - '--size', self.machine_type, - '--image', self.image, - '--user-data', CLOUD_CONFIG_TEMPLATE.format( - self.user_name, public_key), - '--enable-private-networking', - '--wait']) + response, retcode = util.DoctlAndParse([ + 'compute', + 'droplet', + 'create', + self.name, + '--region', + self.zone, + '--size', + self.machine_type, + '--image', + self.image, + '--user-data', + CLOUD_CONFIG_TEMPLATE.format(self.user_name, public_key), + '--enable-private-networking', + '--wait', + ]) if retcode: - raise errors.Resource.RetryableCreationError('Creation failed: %s' % - (response,)) + raise errors.Resource.RetryableCreationError( + 'Creation failed: %s' % (response,) + ) self.droplet_id = response[0]['id'] @vm_util.Retry() def _PostCreate(self): """Get the instance's data.""" response, retcode = util.DoctlAndParse( - ['compute', 'droplet', 'get', self.droplet_id]) + ['compute', 'droplet', 'get', self.droplet_id] + ) for interface in response[0]['networks']['v4']: if interface['type'] == 'public': self.ip_address = interface['ip_address'] @@ -103,21 +110,24 @@ def _Delete(self): """Delete a DigitalOcean VM instance.""" response, retcode = util.DoctlAndParse( - ['compute', 'droplet', 'delete', self.droplet_id, '--force']) + ['compute', 'droplet', 'delete', self.droplet_id, '--force'] + ) # The command doesn't return the HTTP status code, and the error # format is very difficult to parse, so we string # search. TODO(user): parse the error message. if retcode and '404' in response['errors'][0]['detail']: return elif retcode: - raise errors.Resource.RetryableDeletionError('Deletion failed: %s' % - (response,)) + raise errors.Resource.RetryableDeletionError( + 'Deletion failed: %s' % (response,) + ) def _Exists(self): """Returns true if the VM exists.""" response, retcode = util.DoctlAndParse( - ['compute', 'droplet', 'get', self.droplet_id]) + ['compute', 'droplet', 'get', self.droplet_id] + ) return retcode == 0 @@ -130,27 +140,36 @@ def CreateScratchDisk(self, _, disk_spec): if disk_spec.disk_type == disk.LOCAL: if self.scratch_disks and self.scratch_disks[0].disk_type == disk.LOCAL: - raise errors.Error('DigitalOcean does not support multiple local ' - 'disks.') + raise errors.Error( + 'DigitalOcean does not support multiple local disks.' + ) if disk_spec.num_striped_disks != 1: - raise ValueError('num_striped_disks=%s, but DigitalOcean VMs can only ' - 'have one local disk.' % disk_spec.num_striped_disks) + raise ValueError( + 'num_striped_disks=%s, but DigitalOcean VMs can only ' + 'have one local disk.' + % disk_spec.num_striped_disks + ) # The single unique local disk on DigitalOcean is also the boot # disk, so we can't follow the normal procedure of formatting # and mounting. Instead, create a folder at the "mount point" so # the rest of PKB will work as expected and deliberately skip # self._CreateScratchDiskFromDisks. - self.RemoteCommand('sudo mkdir -p {0} && sudo chown -R $USER:$USER {0}' - .format(disk_spec.mount_point)) + self.RemoteCommand( + 'sudo mkdir -p {0} && sudo chown -R $USER:$USER {0}'.format( + disk_spec.mount_point + ) + ) self.scratch_disks.append( - digitalocean_disk.DigitalOceanLocalDisk(disk_spec)) + digitalocean_disk.DigitalOceanLocalDisk(disk_spec) + ) else: disks = [] for _ in range(disk_spec.num_striped_disks): # Disk 0 is the local disk. data_disk = digitalocean_disk.DigitalOceanBlockStorageDisk( - disk_spec, self.zone) + disk_spec, self.zone + ) data_disk.disk_number = self.remote_disk_counter + 1 self.remote_disk_counter += 1 disks.append(data_disk) diff --git a/perfkitbenchmarker/providers/digitalocean/provider_info.py b/perfkitbenchmarker/providers/digitalocean/provider_info.py index 92872c1767..dd961265de 100644 --- a/perfkitbenchmarker/providers/digitalocean/provider_info.py +++ b/perfkitbenchmarker/providers/digitalocean/provider_info.py @@ -12,15 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" Provider info for Digitial Ocean - -""" +"""Provider info for Digitial Ocean""" from perfkitbenchmarker import provider_info class DigitalOceanProviderInfo(provider_info.BaseProviderInfo): - UNSUPPORTED_BENCHMARKS = ['mysql_service'] CLOUD = provider_info.DIGITALOCEAN diff --git a/perfkitbenchmarker/providers/digitalocean/util.py b/perfkitbenchmarker/providers/digitalocean/util.py index 2b52dc87fe..ea830721aa 100644 --- a/perfkitbenchmarker/providers/digitalocean/util.py +++ b/perfkitbenchmarker/providers/digitalocean/util.py @@ -27,8 +27,8 @@ def DoctlAndParse(arg_list): """Run a doctl command and parse the output. Args: - arg_list: a list of arguments for doctl. Will be formated with - str() before being sent to the process. + arg_list: a list of arguments for doctl. Will be formated with str() before + being sent to the process. Returns: A tuple of @@ -40,10 +40,9 @@ def DoctlAndParse(arg_list): """ stdout, _, retcode = vm_util.IssueCommand( - ['doctl'] + - [str(arg) for arg in arg_list] + - ['--output=json'], - raise_on_failure=False) + ['doctl'] + [str(arg) for arg in arg_list] + ['--output=json'], + raise_on_failure=False, + ) # In case of error, doctl sometimes prints "null" before printing a # JSON error string to stdout. TODO(user): improve parsing of @@ -61,26 +60,30 @@ def DoctlAndParse(arg_list): class ActionInProgressException(Exception): """Exception to indicate that a DigitalOcean action is in-progress.""" + pass class ActionFailedError(Exception): """Exception raised when a DigitalOcean action fails.""" + pass class UnknownStatusError(Exception): """Exception raised for an unknown status message.""" + pass -@vm_util.Retry(poll_interval=DEFAULT_ACTION_WAIT_SECONDS, - max_retries=DEFAULT_ACTION_MAX_TRIES, - retryable_exceptions=ActionInProgressException) +@vm_util.Retry( + poll_interval=DEFAULT_ACTION_WAIT_SECONDS, + max_retries=DEFAULT_ACTION_MAX_TRIES, + retryable_exceptions=ActionInProgressException, +) def WaitForAction(action_id): """Wait until a VM action completes.""" - response, retcode = DoctlAndParse( - ['compute', 'action', 'get', action_id]) + response, retcode = DoctlAndParse(['compute', 'action', 'get', action_id]) if retcode: logging.warn('Unexpected action lookup failure.') raise ActionFailedError('Failed to get action %s' % action_id) @@ -94,5 +97,6 @@ def WaitForAction(action_id): elif status == 'errored': raise ActionFailedError('Action %s errored' % action_id) else: - raise UnknownStatusError('Action %s had unknown status "%s"' % - (action_id, status)) + raise UnknownStatusError( + 'Action %s had unknown status "%s"' % (action_id, status) + ) diff --git a/perfkitbenchmarker/providers/docker/docker_container_instance.py b/perfkitbenchmarker/providers/docker/docker_container_instance.py index 25af62bf6d..f197fbc9fe 100644 --- a/perfkitbenchmarker/providers/docker/docker_container_instance.py +++ b/perfkitbenchmarker/providers/docker/docker_container_instance.py @@ -80,8 +80,9 @@ def _Create(self): self._BuildImageLocally() create_command = self._FormatCreateCommand() - container_info, _, _ = vm_util.IssueCommand(create_command, - raise_on_failure=False) + container_info, _, _ = vm_util.IssueCommand( + create_command, raise_on_failure=False + ) self.container_id = container_info.encode('ascii') def _FormatCreateCommand(self): @@ -127,13 +128,23 @@ def _PostCreate(self): self._GetIpAddresses() # Copy ssh key to container to enable ssh login - copy_ssh_command = ['docker', 'cp', self.ssh_public_key, - '%s:/root/.ssh/authorized_keys' % self.name] + copy_ssh_command = [ + 'docker', + 'cp', + self.ssh_public_key, + '%s:/root/.ssh/authorized_keys' % self.name, + ] vm_util.IssueCommand(copy_ssh_command, raise_on_failure=False) # change ownership of authorized_key file to root in container - chown_command = ['docker', 'exec', self.name, 'chown', - 'root:root', '/root/.ssh/authorized_keys'] + chown_command = [ + 'docker', + 'exec', + self.name, + 'chown', + 'root:root', + '/root/.ssh/authorized_keys', + ] vm_util.IssueCommand(chown_command, raise_on_failure=False) self._ConfigureProxy() @@ -209,8 +220,9 @@ def _GetContainerInfo(self): """ logging.info('Checking Container Information') inspect_cmd = ['docker', 'inspect', self.name] - info, _, return_code = vm_util.IssueCommand(inspect_cmd, - raise_on_failure=False) + info, _, return_code = vm_util.IssueCommand( + inspect_cmd, raise_on_failure=False + ) info = json.loads(info) return info, return_code @@ -247,8 +259,9 @@ def _BuildVolumesBody(self): def _LocalImageExists(self, docker_image_name): """Returns whether a Docker image exists locally.""" inspect_cmd = ['docker', 'image', 'inspect', docker_image_name] - info, _, return_code = vm_util.IssueCommand(inspect_cmd, - raise_on_failure=False) + info, _, return_code = vm_util.IssueCommand( + inspect_cmd, raise_on_failure=False + ) info = json.loads(info) logging.info('Checking if Docker Image Exists') if info and return_code == 0: @@ -264,12 +277,20 @@ def _BuildImageLocally(self): PerfKitBenchmarker/data/docker/pkb//Dockerfile """ directory = os.path.dirname( - data.ResourcePath(os.path.join(DOCKERFILE_DIRECTORY, - self.container_image, - 'Dockerfile'))) + data.ResourcePath( + os.path.join( + DOCKERFILE_DIRECTORY, self.container_image, 'Dockerfile' + ) + ) + ) build_cmd = [ - 'docker', 'build', '--no-cache', - '-t', self.container_image, directory] + 'docker', + 'build', + '--no-cache', + '-t', + self.container_image, + directory, + ] vm_util.IssueCommand(build_cmd, raise_on_failure=False) @@ -285,8 +306,9 @@ def GetResourceMetadata(self): return result -class DebianBasedDockerContainer(DockerContainer, - linux_virtual_machine.BaseDebianMixin): +class DebianBasedDockerContainer( + DockerContainer, linux_virtual_machine.BaseDebianMixin +): def _GetNumCpus(self): return self.cpus @@ -298,19 +320,23 @@ def ApplySysctlPersistent(self, sysctl_params): Args: sysctl_params: dict - the keys and values to write """ - logging.warn('sysctl flags are applied when container is created. ' - 'Not all sysctl flags work with Docker. It does not ' - 'support flags that modify the host system') + logging.warn( + 'sysctl flags are applied when container is created. ' + 'Not all sysctl flags work with Docker. It does not ' + 'support flags that modify the host system' + ) def _RebootIfNecessary(self): """Override RebootIfNecessary for Docker Provider.""" logging.warn('Docker Containers cannot be rebooted to apply flags') -class Ubuntu1604BasedDockerContainer(DebianBasedDockerContainer, - linux_virtual_machine.Ubuntu1604Mixin): +class Ubuntu1604BasedDockerContainer( + DebianBasedDockerContainer, linux_virtual_machine.Ubuntu1604Mixin +): DEFAULT_IMAGE = UBUNTU16_IMAGE + # Note: to add support for ubuntu 14 and ubuntu 18, we simply need to # create/test Dockerfiles for those distros. This should be # fairly simple, but may require a few changes from the diff --git a/perfkitbenchmarker/providers/docker/docker_container_spec.py b/perfkitbenchmarker/providers/docker/docker_container_spec.py index a4c812e01e..93c2551d41 100644 --- a/perfkitbenchmarker/providers/docker/docker_container_spec.py +++ b/perfkitbenchmarker/providers/docker/docker_container_spec.py @@ -26,8 +26,8 @@ class DockerContainerSpec(virtual_machine.BaseVmSpec): """Object containing the information needed to create a Docker Instance. Attributes: - privileged_docker: None of boolean. Indicates if Docker container - should be run in privileged mode. + privileged_docker: None of boolean. Indicates if Docker container should be + run in privileged mode. """ CLOUD = provider_info.DOCKER @@ -36,11 +36,14 @@ class DockerContainerSpec(virtual_machine.BaseVmSpec): def _GetOptionDecoderConstructions(cls): result = super(DockerContainerSpec, cls)._GetOptionDecoderConstructions() result.update({ - 'privileged_docker': (option_decoders.BooleanDecoder, - {'default': False}), - 'machine_type': (custom_virtual_machine_spec.MachineTypeDecoder, { - 'default': None - }) + 'privileged_docker': ( + option_decoders.BooleanDecoder, + {'default': False}, + ), + 'machine_type': ( + custom_virtual_machine_spec.MachineTypeDecoder, + {'default': None}, + ), }) logging.warn(result) return result diff --git a/perfkitbenchmarker/providers/docker/docker_disk.py b/perfkitbenchmarker/providers/docker/docker_disk.py index d43fe9e953..b736d1621a 100644 --- a/perfkitbenchmarker/providers/docker/docker_disk.py +++ b/perfkitbenchmarker/providers/docker/docker_disk.py @@ -28,7 +28,6 @@ def CreateDisks(disk_specs, vm_name): scratch_disks = [] for disk_num, disk_spec in enumerate(disk_specs): - logging.info('Creating Disk number: %d', disk_num) volume_disk = DockerDisk(disk_spec, disk_num, vm_name) diff --git a/perfkitbenchmarker/providers/docker/flags.py b/perfkitbenchmarker/providers/docker/flags.py index bfbd8e6507..8ad7e5d9d0 100644 --- a/perfkitbenchmarker/providers/docker/flags.py +++ b/perfkitbenchmarker/providers/docker/flags.py @@ -14,14 +14,19 @@ from absl import flags -flags.DEFINE_string('docker_custom_image', None, - 'Custom docker image location') +flags.DEFINE_string('docker_custom_image', None, 'Custom docker image location') -flags.DEFINE_boolean('privileged_docker', False, - 'If True, will attempt to create Docker containers ' - 'in a privileged mode. Note that some benchmarks execute ' - 'commands which are only allowed in privileged mode.') +flags.DEFINE_boolean( + 'privileged_docker', + False, + 'If True, will attempt to create Docker containers ' + 'in a privileged mode. Note that some benchmarks execute ' + 'commands which are only allowed in privileged mode.', +) -flags.DEFINE_string('docker_cli', 'docker', - 'Path to docker cli. You can set it here if it is' - 'not in your system PATH or not at a default location') +flags.DEFINE_string( + 'docker_cli', + 'docker', + 'Path to docker cli. You can set it here if it is' + 'not in your system PATH or not at a default location', +) diff --git a/perfkitbenchmarker/providers/gcp/gce_disk.py b/perfkitbenchmarker/providers/gcp/gce_disk.py index de0cb7ad3e..40d1a7b7fd 100644 --- a/perfkitbenchmarker/providers/gcp/gce_disk.py +++ b/perfkitbenchmarker/providers/gcp/gce_disk.py @@ -143,6 +143,7 @@ def AddLabels(gcp_resource: resource.BaseResource, disk_name: str): class GceDiskSpec(disk.BaseDiskSpec): """Object holding the information needed to create a GCPDisk.""" + interface: str num_local_ssds: int create_with_vm: bool diff --git a/perfkitbenchmarker/providers/gcp/gcp_relational_db.py b/perfkitbenchmarker/providers/gcp/gcp_relational_db.py index 4e1bc1a661..acba0556fd 100644 --- a/perfkitbenchmarker/providers/gcp/gcp_relational_db.py +++ b/perfkitbenchmarker/providers/gcp/gcp_relational_db.py @@ -121,7 +121,8 @@ def MoveSQLServerTempDb(self): # Moves the SQL Server temporary database to LocalSSD. for vm in vms: stdout, _ = vm.RemoteCommand( - 'Get-PSDrive -PSProvider FileSystem | Select Name') + 'Get-PSDrive -PSProvider FileSystem | Select Name' + ) drive_list = [ str(drive.strip().replace('\r', '')) diff --git a/perfkitbenchmarker/providers/gcp/gcs.py b/perfkitbenchmarker/providers/gcp/gcs.py index 5ddf3a23a6..658a5f0b54 100644 --- a/perfkitbenchmarker/providers/gcp/gcs.py +++ b/perfkitbenchmarker/providers/gcp/gcs.py @@ -388,13 +388,11 @@ def Metadata(self, vm): 'gcs_client': str(FLAGS.gcs_client), } if FLAGS.gcs_client == GCS_CLIENT_BOTO: - metadata.update( - { - object_storage_service.BOTO_LIB_VERSION: ( - linux_packages.GetPipPackageVersion(vm, 'boto') - ) - } - ) + metadata.update({ + object_storage_service.BOTO_LIB_VERSION: ( + linux_packages.GetPipPackageVersion(vm, 'boto') + ) + }) return metadata def APIScriptArgs(self): diff --git a/perfkitbenchmarker/providers/gcp/google_kubernetes_engine.py b/perfkitbenchmarker/providers/gcp/google_kubernetes_engine.py index 5fa834ab1a..c80acf8039 100644 --- a/perfkitbenchmarker/providers/gcp/google_kubernetes_engine.py +++ b/perfkitbenchmarker/providers/gcp/google_kubernetes_engine.py @@ -273,9 +273,7 @@ def _IssueResourceCreationCommand(self, cmd): util.CheckGcloudResponseKnownFailures(stderr, retcode) raise errors.Resource.CreationError(stderr) - def _AddNodeParamsToCmd( - self, nodepool_config, cmd - ): + def _AddNodeParamsToCmd(self, nodepool_config, cmd): """Modifies cmd to include node specific command arguments.""" # Apply labels to all nodepools. cmd.flags['labels'] = util.MakeFormattedDefaultTags() diff --git a/perfkitbenchmarker/providers/ibmcloud/flags.py b/perfkitbenchmarker/providers/ibmcloud/flags.py index ed201586fc..ccedd029f6 100644 --- a/perfkitbenchmarker/providers/ibmcloud/flags.py +++ b/perfkitbenchmarker/providers/ibmcloud/flags.py @@ -15,80 +15,72 @@ from absl import flags -flags.DEFINE_string('ibmcloud_region', None, - 'IBMCloud internal DC name') +flags.DEFINE_string('ibmcloud_region', None, 'IBMCloud internal DC name') -flags.DEFINE_integer('ibmcloud_volume_iops', 20000, - 'Desired volume IOPS.') +flags.DEFINE_integer('ibmcloud_volume_iops', 20000, 'Desired volume IOPS.') -flags.DEFINE_integer('ibmcloud_volume_bandwidth', None, - 'Desired volume bandwidth in Mbps.') +flags.DEFINE_integer( + 'ibmcloud_volume_bandwidth', None, 'Desired volume bandwidth in Mbps.' +) -flags.DEFINE_boolean('ibmcloud_volume_encrypted', False, - 'Enable encryption on volume creates.') +flags.DEFINE_boolean( + 'ibmcloud_volume_encrypted', False, 'Enable encryption on volume creates.' +) -flags.DEFINE_string('ibmcloud_image_username', 'root', - 'Ssh username for cloud image.') +flags.DEFINE_string( + 'ibmcloud_image_username', 'root', 'Ssh username for cloud image.' +) -flags.DEFINE_integer('ibmcloud_polling_delay', 2, - 'Delay between polling attempts in seconds.') +flags.DEFINE_integer( + 'ibmcloud_polling_delay', 2, 'Delay between polling attempts in seconds.' +) -flags.DEFINE_integer('ibmcloud_timeout', 600, - 'timeout in secs.') +flags.DEFINE_integer('ibmcloud_timeout', 600, 'timeout in secs.') -flags.DEFINE_integer('ibmcloud_boot_disk_size', 10, - 'boot volume disk size.') +flags.DEFINE_integer('ibmcloud_boot_disk_size', 10, 'boot volume disk size.') -flags.DEFINE_boolean('ibmcloud_debug', False, - 'debug flag.') +flags.DEFINE_boolean('ibmcloud_debug', False, 'debug flag.') -flags.DEFINE_boolean('ibmcloud_resources_keep', False, - 'keep resources.') +flags.DEFINE_boolean('ibmcloud_resources_keep', False, 'keep resources.') -flags.DEFINE_string('ibmcloud_volume_profile', 'custom', - 'volume profile') +flags.DEFINE_string('ibmcloud_volume_profile', 'custom', 'volume profile') -flags.DEFINE_string('ibmcloud_bootvol_encryption_key', None, - 'boot volume encryption key crn') +flags.DEFINE_string( + 'ibmcloud_bootvol_encryption_key', None, 'boot volume encryption key crn' +) -flags.DEFINE_string('ibmcloud_datavol_encryption_key', None, - 'data volume encryption key crn') +flags.DEFINE_string( + 'ibmcloud_datavol_encryption_key', None, 'data volume encryption key crn' +) -flags.DEFINE_string('ibmcloud_vpcid', None, - 'IBM Cloud vpc id') +flags.DEFINE_string('ibmcloud_vpcid', None, 'IBM Cloud vpc id') -flags.DEFINE_string('ibmcloud_subnet', None, - 'primary subnet id') +flags.DEFINE_string('ibmcloud_subnet', None, 'primary subnet id') -flags.DEFINE_string('ibmcloud_networks', None, - 'additional network ids, comma separated') +flags.DEFINE_string( + 'ibmcloud_networks', None, 'additional network ids, comma separated' +) -flags.DEFINE_string('ibmcloud_prefix', 'perfkit', - 'resource name prefix') +flags.DEFINE_string('ibmcloud_prefix', 'perfkit', 'resource name prefix') -flags.DEFINE_string('ibmcloud_rgid', None, - 'Resource Group id for the account.') +flags.DEFINE_string('ibmcloud_rgid', None, 'Resource Group id for the account.') -flags.DEFINE_integer('ibmcloud_boot_volume_iops', 3000, - 'boot voume iops') +flags.DEFINE_integer('ibmcloud_boot_volume_iops', 3000, 'boot voume iops') -flags.DEFINE_integer('ibmcloud_boot_volume_size', 0, - 'boot voume size in GB') +flags.DEFINE_integer('ibmcloud_boot_volume_size', 0, 'boot voume size in GB') -flags.DEFINE_string('ibmcloud_pub_keyid', None, - 'rias public sshkey id') +flags.DEFINE_string('ibmcloud_pub_keyid', None, 'rias public sshkey id') -flags.DEFINE_integer('ibmcloud_network_mtu', 9000, - 'MTU size on network interfaces.') +flags.DEFINE_integer( + 'ibmcloud_network_mtu', 9000, 'MTU size on network interfaces.' +) -flags.DEFINE_integer('ibmcloud_subnets_extra', 0, - 'extra subnets to lookup') +flags.DEFINE_integer('ibmcloud_subnets_extra', 0, 'extra subnets to lookup') -flags.DEFINE_integer('ibmcloud_vdisks_extra', 0, - 'extra disks to create') +flags.DEFINE_integer('ibmcloud_vdisks_extra', 0, 'extra disks to create') -flags.DEFINE_string('ibmcloud_image_info', None, - 'image info in json formatted file') +flags.DEFINE_string( + 'ibmcloud_image_info', None, 'image info in json formatted file' +) -flags.DEFINE_boolean('ibmcloud_encrypted_image', False, - 'encrypted image.') +flags.DEFINE_boolean('ibmcloud_encrypted_image', False, 'encrypted image.') diff --git a/perfkitbenchmarker/providers/ibmcloud/ibm_api.py b/perfkitbenchmarker/providers/ibmcloud/ibm_api.py index e478d730d7..15a09f6c37 100644 --- a/perfkitbenchmarker/providers/ibmcloud/ibm_api.py +++ b/perfkitbenchmarker/providers/ibmcloud/ibm_api.py @@ -105,25 +105,32 @@ def __init__(self, *args, **kwargs): **kwargs: resource - A IBM Cloud resource of type BaseResource. """ self._CheckEnvironment() - if (self.gen is None or - time.time() - self.ibmcloud_auth_token_time >= ONE_HOUR): + if ( + self.gen is None + or time.time() - self.ibmcloud_auth_token_time >= ONE_HOUR + ): IbmAPICommand.gen = ibmcloud_manager.IbmCloud( account=self.ibmcloud_account_id, apikey=self.ibmcloud_apikey, verbose=False, version='v1', silent=True, - force=False) + force=False, + ) IbmAPICommand.gen_instmgr = ibmcloud_manager.InstanceManager( - IbmAPICommand.gen) + IbmAPICommand.gen + ) IbmAPICommand.gen_imgmgr = ibmcloud_manager.ImageManager( - IbmAPICommand.gen) + IbmAPICommand.gen + ) IbmAPICommand.gen_fipmgr = ibmcloud_manager.FipManager(IbmAPICommand.gen) IbmAPICommand.gen_keymgr = ibmcloud_manager.KeyManager(IbmAPICommand.gen) IbmAPICommand.gen_subnetmgr = ibmcloud_manager.SubnetManager( - IbmAPICommand.gen) + IbmAPICommand.gen + ) IbmAPICommand.gen_volumemgr = ibmcloud_manager.VolumeManager( - IbmAPICommand.gen) + IbmAPICommand.gen + ) IbmAPICommand.gen_vpcmgr = ibmcloud_manager.VPCManager(IbmAPICommand.gen) IbmAPICommand.gen_sgmgr = ibmcloud_manager.SGManager(IbmAPICommand.gen) self.ibmcloud_auth_token = self.GetToken() @@ -133,8 +140,7 @@ def __init__(self, *args, **kwargs): self.flags = kwargs self.user_data = None - -# self.sgid = None + # self.sgid = None @property def vpc_id(self): @@ -232,10 +238,13 @@ def _CheckEnvironment(self): if not os.environ.get('IBMCLOUD_ENDPOINT'): raise errors.Config.InvalidValue( 'PerfKit Benchmarker on IBM Cloud requires that the ' - 'environment variable IBMCLOUD_ENDPOINT is set.') + 'environment variable IBMCLOUD_ENDPOINT is set.' + ) account = util.Account( os.environ.get('IBMCLOUD_ACCOUNT_ID'), - os.environ.get('IBMCLOUD_APIKEY'), None) + os.environ.get('IBMCLOUD_APIKEY'), + None, + ) if not account.name or not account.apikey: accounts = os.environ.get('IBMCLOUD_ACCOUNTS') # read from config file if accounts and accounts: @@ -255,7 +264,8 @@ def _CheckEnvironment(self): raise errors.Config.InvalidValue( 'PerfKit Benchmarker on IBM Cloud requires that the ' 'environment variables IBMCLOUD_ACCOUNT_ID and IBMCLOUD_APIKEY ' - 'are correctly set.') + 'are correctly set.' + ) def GetToken(self): """Returns user token.""" @@ -263,10 +273,12 @@ def GetToken(self): def CreateSgRules(self): """Creates default security group rules needed for vms to communicate.""" - self.gen_sgmgr.CreateRule(self.sg_id, 'inbound', 'ipv4', '0.0.0.0/0', - 'icmp', None) - self.gen_sgmgr.CreateRule(self.sg_id, 'inbound', 'ipv4', '0.0.0.0/0', 'tcp', - 22) + self.gen_sgmgr.CreateRule( + self.sg_id, 'inbound', 'ipv4', '0.0.0.0/0', 'icmp', None + ) + self.gen_sgmgr.CreateRule( + self.sg_id, 'inbound', 'ipv4', '0.0.0.0/0', 'tcp', 22 + ) self.gen_sgmgr.CreateRule( self.sg_id, 'inbound', @@ -275,7 +287,8 @@ def CreateSgRules(self): 'tcp', None, port_min=1024, - port_max=50000) + port_max=50000, + ) self.gen_sgmgr.CreateRule( self.sg_id, 'inbound', @@ -284,9 +297,11 @@ def CreateSgRules(self): 'udp', None, port_min=1024, - port_max=50000) - self.gen_sgmgr.CreateRule(self.sg_id, 'inbound', 'ipv4', '0.0.0.0/0', 'tcp', - 443) + port_max=50000, + ) + self.gen_sgmgr.CreateRule( + self.sg_id, 'inbound', 'ipv4', '0.0.0.0/0', 'tcp', 443 + ) def GetSecurityGroupId(self): """Returns default security group id.""" @@ -315,7 +330,8 @@ def CreatePrefix(self): The json representation of the created address prefix. """ return self.gen_vpcmgr.CreatePrefix( - self.vpc_id, self.zone, self.cidr, name=self.name) + self.vpc_id, self.zone, self.cidr, name=self.name + ) def CreateSubnet(self): """Creates a subnet on the vpc. @@ -340,7 +356,6 @@ def DeleteResource(self): Flags: items: type of resource to delete. id: matching id to delete. - """ data_mgr = self.gen_subnetmgr if self.items == 'vpcs': @@ -371,8 +386,12 @@ def GetResource(self): Returns: The id of the first matching resource. """ - logging.info('Looking up existing %s matching prefix: %s, zone: %s', - self.items, self.prefix, self.zone) + logging.info( + 'Looking up existing %s matching prefix: %s, zone: %s', + self.items, + self.prefix, + self.zone, + ) data_mgr = self.gen_subnetmgr if self.items == 'vpcs': data_mgr = self.gen_vpcmgr @@ -516,8 +535,14 @@ def CreateInstance(self): """ kwargs = {} for arg in [ - 'key', 'zone', 'subnet', 'networks', 'encryption_key', 'iops', - 'capacity', 'resource_group' + 'key', + 'zone', + 'subnet', + 'networks', + 'encryption_key', + 'iops', + 'capacity', + 'resource_group', ]: if arg in self.flags: kwargs[arg] = self.flags[arg] @@ -528,7 +553,9 @@ def CreateInstance(self): self.profile, self.vpc_id, user_data=self.user_data, - **kwargs)) + **kwargs + ) + ) def InstanceGetPrimaryVnic(self): """Finds the primary network vnic on vm. @@ -540,8 +567,10 @@ def InstanceGetPrimaryVnic(self): The found vnic id. """ resp = self.gen_instmgr.Show(self.instance_id) - if 'primary_network_interface' in resp and 'href' in resp[ - 'primary_network_interface']: + if ( + 'primary_network_interface' in resp + and 'href' in resp['primary_network_interface'] + ): items = resp['primary_network_interface']['href'].split('/') logging.info('vnic found: %s', items[len(items) - 1]) return items[len(items) - 1] @@ -588,25 +617,31 @@ def InstanceStatus(self): while status != States.RUNNING and time.time() < endtime: time.sleep(FLAGS.ibmcloud_polling_delay) status = self._GetStatus(self.gen_instmgr.Show(self.instance_id)) - logging.info('Checking instance status: %s, %s', self.instance_id, - status) + logging.info( + 'Checking instance status: %s, %s', self.instance_id, status + ) return status def InstanceStart(self): """Starts the instance matching instance id and returns final status.""" resp = self.gen_instmgr.Start(self.flags['instanceid']) status = self._GetStatus(resp) - logging.info('Instance start issued: %s, %s', self.flags['instanceid'], - status) + logging.info( + 'Instance start issued: %s, %s', self.flags['instanceid'], status + ) if status != States.RUNNING: endtime = time.time() + FLAGS.ibmcloud_timeout status = self._GetStatus(self.gen_instmgr.Show(self.flags['instanceid'])) while status != States.RUNNING and time.time() < endtime: time.sleep(FLAGS.ibmcloud_polling_delay) status = self._GetStatus( - self.gen_instmgr.Show(self.flags['instanceid'])) - logging.info('Checking instance start status: %s, %s', - self.flags['instanceid'], status) + self.gen_instmgr.Show(self.flags['instanceid']) + ) + logging.info( + 'Checking instance start status: %s, %s', + self.flags['instanceid'], + status, + ) return status def InstanceStop(self): @@ -623,8 +658,9 @@ def InstanceStop(self): while status != States.STOPPED and time.time() < endtime: time.sleep(FLAGS.ibmcloud_polling_delay * 2) status = self._GetStatus(self.gen_instmgr.Show(self.instance_id)) - logging.info('Checking instance stop status: %s, %s', self.instance_id, - status) + logging.info( + 'Checking instance stop status: %s, %s', self.instance_id, status + ) return status def InstanceDelete(self): @@ -659,8 +695,12 @@ def InstanceVnicCreate(self): status = self._GetStatus(resp) vnicid = resp['id'] port_speed = resp['port_speed'] - logging.info('Vnic created, vnicid: %s, port speed: %s, vnic status: %s', - vnicid, port_speed, status) + logging.info( + 'Vnic created, vnicid: %s, port speed: %s, vnic status: %s', + vnicid, + port_speed, + status, + ) if status != States.AVAILABLE: endtime = time.time() + FLAGS.ibmcloud_timeout resp = self.gen_instmgr.ShowVnic(self.instance_id, vnicid) @@ -687,7 +727,8 @@ def InstanceVnicShow(self): json representation of the found vnic. """ return json.dumps( - self.gen_instmgr.ShowVnic(self.instance_id, self.vnic_id), indent=2) + self.gen_instmgr.ShowVnic(self.instance_id, self.vnic_id), indent=2 + ) def InstanceVnicList(self): """Returns a list of all vnics on instance. @@ -713,23 +754,28 @@ def InstanceCreateVolume(self): json representation of the attached volume on the instance. """ return json.dumps( - self.gen_instmgr.CreateVolume(self.instance_id, self.name, self.volume, - True), - indent=2) + self.gen_instmgr.CreateVolume( + self.instance_id, self.name, self.volume, True + ), + indent=2, + ) def InstanceDeleteVolume(self): """Deletes the volume on the vm matching instanceid and volume.""" return json.dumps( - self.gen_instmgr.DeleteVolume(self.instance_id, self.volume), indent=2) + self.gen_instmgr.DeleteVolume(self.instance_id, self.volume), indent=2 + ) def InstanceShowVolume(self): """Returns json representation of the volume matching instanceid.""" - for item in self.gen_instmgr.ListVolumes( - self.instance_id)['volume_attachments']: + for item in self.gen_instmgr.ListVolumes(self.instance_id)[ + 'volume_attachments' + ]: if item['volume'].get('id') == self.volume: return json.dumps( self.gen_instmgr.ShowVolume(self.instance_id, item.get('id')), - indent=2) + indent=2, + ) def ImageList(self): """Returns json objects of all available images.""" @@ -740,8 +786,10 @@ def GetImageId(self): resp = self.gen_imgmgr.List()['images'] if resp is not None: for image in resp: - if image['name'].startswith( - self.image_name) and image['status'] == States.AVAILABLE: + if ( + image['name'].startswith(self.image_name) + and image['status'] == States.AVAILABLE + ): return image['id'] return None diff --git a/perfkitbenchmarker/providers/ibmcloud/ibmcloud_disk.py b/perfkitbenchmarker/providers/ibmcloud/ibmcloud_disk.py index 72daf0adf6..db5aef0a98 100644 --- a/perfkitbenchmarker/providers/ibmcloud/ibmcloud_disk.py +++ b/perfkitbenchmarker/providers/ibmcloud/ibmcloud_disk.py @@ -17,7 +17,6 @@ import logging import re import time - from typing import Any, Dict, Optional from absl import flags @@ -72,7 +71,7 @@ def _Create(self): 'zone': self.zone, 'iops': FLAGS.ibmcloud_volume_iops, 'profile': FLAGS.ibmcloud_volume_profile, - 'capacity': self.data_disk_size + 'capacity': self.data_disk_size, }) if self.encryption_key is not None: volcmd.flags['encryption_key'] = self.encryption_key @@ -116,15 +115,17 @@ def Attach(self, vm): if self.vol_id is None: raise errors.Error(f'Cannot attach remote volume {self.name}') if vm.vmid is None: - raise errors.Error(f'Cannot attach remote volume {self.name} ' - 'to non-existing {vm.name} VM') + raise errors.Error( + f'Cannot attach remote volume {self.name} ' + 'to non-existing {vm.name} VM' + ) volcmd = ibm.IbmAPICommand(self) volcmd.flags.update({ 'name': self.name, 'instanceid': vm.vmid, 'volume': self.vol_id, - 'delete': True + 'delete': True, }) # wait till volume is PROVISIONED before attach @@ -145,8 +146,12 @@ def Attach(self, vm): resp = json.loads(volcmd.InstanceCreateVolume()) logging.info('Attached volume, volcmd.flags %s', volcmd.flags) volume = resp['id'] - logging.info('VDISK ID: %s, vdiskname: %s, instanceid: %s', volume, - self.name, vm.vmid) + logging.info( + 'VDISK ID: %s, vdiskname: %s, instanceid: %s', + volume, + self.name, + vm.vmid, + ) self.attached_vdisk_uri = volume self.attached_vm = vm self._WaitForVolumeAttachment(vm) @@ -157,18 +162,19 @@ def Attach(self, vm): max_retries=-1, timeout=_MAX_DISK_ATTACH_SECONDS, log_errors=False, - retryable_exceptions=errors.Resource.RetryableCreationError) + retryable_exceptions=errors.Resource.RetryableCreationError, + ) def _WaitForVolumeAttachment(self, vm): """Waits and checks until the volume status is attached.""" if self.vol_id is None: return volcmd = ibm.IbmAPICommand(self) - volcmd.flags.update({ - 'instanceid': self.attached_vm.vmid, - 'volume': self.vol_id - }) - logging.info('Checking volume on instance %s, volume: %s', vm.vmid, - self.vol_id) + volcmd.flags.update( + {'instanceid': self.attached_vm.vmid, 'volume': self.vol_id} + ) + logging.info( + 'Checking volume on instance %s, volume: %s', vm.vmid, self.vol_id + ) status = None endtime = time.time() + _MAX_DISK_ATTACH_SECONDS while time.time() < endtime: @@ -178,8 +184,9 @@ def _WaitForVolumeAttachment(self, vm): status = resp.get('status', 'unknown') logging.info('Checking instance volume status: %s', status) if status == ibm.States.ATTACHED: - logging.info('Remote volume %s has been attached to %s', self.name, - vm.name) + logging.info( + 'Remote volume %s has been attached to %s', self.name, vm.name + ) break time.sleep(2) if status != ibm.States.ATTACHED: diff --git a/perfkitbenchmarker/providers/ibmcloud/ibmcloud_manager.py b/perfkitbenchmarker/providers/ibmcloud/ibmcloud_manager.py index c47f67a3fb..377525536f 100644 --- a/perfkitbenchmarker/providers/ibmcloud/ibmcloud_manager.py +++ b/perfkitbenchmarker/providers/ibmcloud/ibmcloud_manager.py @@ -29,12 +29,12 @@ 'public_gateways', 'security_groups', 'floating_ips', - 'network_acls' + 'network_acls', ] PLURALS = { 'bgpconf': 'bgpconf', 'edgerouter': 'edgerouter', - 'policy': 'policies' + 'policy': 'policies', } API_VERSION_DATE = '2022-03-29' @@ -66,10 +66,20 @@ class IbmCloudRequestError(Exception): class IbmCloud: """Base object that handles IBM Cloud REST api call requests.""" - def __init__(self, endpoint=None, url=None, account=None, apikey=None, - vm_creator=False, token=None, verbose=False, version='v1', - silent=False, force=False, trace=False): - + def __init__( + self, + endpoint=None, + url=None, + account=None, + apikey=None, + vm_creator=False, + token=None, + verbose=False, + version='v1', + silent=False, + force=False, + trace=False, + ): # For Token self._endpoint = endpoint or os.environ.get('IBMCLOUD_AUTH_ENDPOINT') self._url = url or os.environ.get('IBMCLOUD_ENDPOINT') @@ -83,23 +93,30 @@ def __init__(self, endpoint=None, url=None, account=None, apikey=None, self._force = force self._trace = trace self._vm_creator = vm_creator - self._generation = ('version=' + API_VERSION_DATE + '&' + - GENERATION + - (os.environ.get('IBMCLOUD_GENERATION') or '2')) + self._generation = ( + 'version=' + + API_VERSION_DATE + + '&' + + GENERATION + + (os.environ.get('IBMCLOUD_GENERATION') or '2') + ) if not self._url: raise IbmCloudRequestError( 'url has to specified either in the initialization of the ' - 'client or via an environment variable (\"IBMCLOUD_ENDPOINT\")') + 'client or via an environment variable ("IBMCLOUD_ENDPOINT")' + ) if not self._acct: raise IbmCloudRequestError( 'acct has to specified either in the initialization of the ' - 'client or via an environment variable (\"IBMCLOUD_ACCOUNT_ID\")') + 'client or via an environment variable ("IBMCLOUD_ACCOUNT_ID")' + ) if not self._apikey: raise IbmCloudRequestError( 'apikey has to specified either in the initialization of the ' - 'client or via an environment variable (\"IBMCLOUD_APIKEY\")') + 'client or via an environment variable ("IBMCLOUD_APIKEY")' + ) parsed_url = urllib.parse.urlparse(self._url) self._netloc = parsed_url.netloc or parsed_url.path.split('/', 1)[0] @@ -107,8 +124,11 @@ def __init__(self, endpoint=None, url=None, account=None, apikey=None, self._search_accts = [self._acct, None, 'system'] # new token if not set, force set or if running for more than 50 min - if (not self._token or self._force or self._token_time < - (time.time() - 3000)): + if ( + not self._token + or self._force + or self._token_time < (time.time() - 3000) + ): self.SetToken() @vm_util.Retry(max_retries=3, timeout=vm_util.DEFAULT_TIMEOUT) @@ -118,19 +138,21 @@ def GetToken(self): req_data = { 'grant_type': 'urn:ibm:params:oauth:grant-type:apikey', 'response_type': 'cloud_iam', - 'apikey': self._apikey + 'apikey': self._apikey, } count = 1 while token is None: if not self._silent: - logging.info('Sending a POST request to get a token: %s', - self._endpoint) + logging.info( + 'Sending a POST request to get a token: %s', self._endpoint + ) resp = requests.request( 'POST', self._endpoint, data=req_data, headers=None, - timeout=(HTTP_TIMEOUT_CONNECT, HTTP_TIMEOUT)) + timeout=(HTTP_TIMEOUT_CONNECT, HTTP_TIMEOUT), + ) if resp is not None: if self._verbose: logging.info('response=%s', str(resp)) @@ -144,8 +166,12 @@ def GetToken(self): logging.info('token: %s', token) break else: - logging.error('POST: %s, response status_code: %s, resp: %s', - self._endpoint, resp.status_code, str(resp)) + logging.error( + 'POST: %s, response status_code: %s, resp: %s', + self._endpoint, + resp.status_code, + str(resp), + ) else: logging.info('Request POST: %s, no response returned', self._endpoint) count += 1 @@ -166,13 +192,9 @@ def SetToken(self): except Exception as err: raise IbmCloudRequestError('Authorization failed: %s' % err) - def Request(self, - method, - path, - data=None, - headers=None, - timeout=None, - session=None): + def Request( + self, method, path, data=None, headers=None, timeout=None, session=None + ): """Constructs base rest api calls to run against IBM Cloud regional api server. Args: @@ -226,33 +248,46 @@ def Request(self, self._url + path, data=data, headers=h, - timeout=(HTTP_TIMEOUT_CONNECT, data_timeout)) # verify=False + timeout=(HTTP_TIMEOUT_CONNECT, data_timeout), + ) # verify=False else: res = session.request( method, self._url + path, data=data, headers=h, - timeout=(HTTP_TIMEOUT_CONNECT, data_timeout)) + timeout=(HTTP_TIMEOUT_CONNECT, data_timeout), + ) if res is not None: request_id = ( res.headers['X-Request-Id'] - if 'X-Request-Id' in res.headers else None) + if 'X-Request-Id' in res.headers + else None + ) if self._verbose: - logging.info('Request %s:, %s, request id: %s ', method, path, - request_id) + logging.info( + 'Request %s:, %s, request id: %s ', method, path, request_id + ) logging.info('response=%s', str(res)) if res.text: logging.info('response text=%s', res.text) response_code = res.status_code if res.status_code < 200 or res.status_code > 299: - logging.error('Request %s:, %s, response status_code: %s, res: %s', - method, path, res.status_code, str(res)) + logging.error( + 'Request %s:, %s, response status_code: %s, res: %s', + method, + path, + res.status_code, + str(res), + ) try: if 'errors' in res.text: restext = json.loads(res.text) - logging.info(' *** %s -- %s\n\n', restext['errors'][0]['code'], - restext['errors'][0]['message']) + logging.info( + ' *** %s -- %s\n\n', + restext['errors'][0]['code'], + restext['errors'][0]['message'], + ) except Exception: # pylint: disable=broad-except pass response_text = str(res.text) @@ -325,8 +360,9 @@ def Delete(*args, **kwargs): class BaseManager: """Base class that handles IBM Cloud REST api call requests. - Use the derived classes for each resource type. + Use the derived classes for each resource type. """ + # This value should be overwritten by derived classes. _type = 'base' @@ -370,10 +406,23 @@ class InstanceManager(BaseManager): _type = 'instance' - def Create(self, name, imageid, profile, vpcid, zone=None, key=None, # pytype: disable=signature-mismatch # overriding-parameter-count-checks - subnet=None, networks=None, resource_group=None, - iops=None, capacity=None, user_data=None, - session=None, **kwargs) -> Dict[str, Any]: + def Create( + self, + name, + imageid, + profile, + vpcid, + zone=None, + key=None, # pytype: disable=signature-mismatch # overriding-parameter-count-checks + subnet=None, + networks=None, + resource_group=None, + iops=None, + capacity=None, + user_data=None, + session=None, + **kwargs, + ) -> Dict[str, Any]: """Construct and send a vm create request. Args: @@ -400,15 +449,9 @@ def Create(self, name, imageid, profile, vpcid, zone=None, key=None, # pytype: """ req_data = { 'name': name, - 'image': { - 'id': imageid - }, - 'vpc': { - 'id': vpcid - }, - 'profile': { - 'name': profile - } + 'image': {'id': imageid}, + 'vpc': {'id': vpcid}, + 'profile': {'name': profile}, } if zone: req_data['zone'] = {'name': zone} @@ -457,14 +500,14 @@ def Show(self, instance) -> Dict[str, Any]: inst_uri = self.GetUri(instance) return self._g.Request('GET', inst_uri) - def ShowPolling(self, - instance, - timeout_polling=HTTP_TIMEOUT_CONNECT, - session=None) -> Dict[str, Any]: + def ShowPolling( + self, instance, timeout_polling=HTTP_TIMEOUT_CONNECT, session=None + ) -> Dict[str, Any]: """Send a vm get request.""" inst_uri = self.GetUri(instance) return self._g.Request( - 'GET', inst_uri, timeout=timeout_polling, session=session) + 'GET', inst_uri, timeout=timeout_polling, session=session + ) def ShowInitialization(self, instance) -> Dict[str, Any]: """Send a vm get initialization request.""" @@ -496,12 +539,9 @@ def ShowVnic(self, instance, vnicid) -> Dict[str, Any]: def CreateVnic(self, instance, name, subnet) -> Dict[str, Any]: """Send a vm vnic create request.""" inst_uri = self.GetUri(instance) + '/network_interfaces' - return self._g.Request('POST', inst_uri, { - 'name': name, - 'subnet': { - 'id': subnet - } - }) + return self._g.Request( + 'POST', inst_uri, {'name': name, 'subnet': {'id': subnet}} + ) def ListVnics(self, instance) -> Dict[str, Any]: """Send a vm vnic list request.""" @@ -517,13 +557,14 @@ def CreateVolume(self, instance, name, volume, delete) -> Dict[str, Any]: """Send a volume create request on a vm.""" inst_uri = self.GetUri(instance) + '/volume_attachments' return self._g.Request( - 'POST', '%s' % inst_uri, { + 'POST', + '%s' % inst_uri, + { 'delete_volume_on_instance_delete': delete, 'name': name, - 'volume': { - 'id': volume - } - }) + 'volume': {'id': volume}, + }, + ) def ListVolumes(self, instance) -> Dict[str, Any]: """Send a volume list request on a vm.""" @@ -533,13 +574,15 @@ def ListVolumes(self, instance) -> Dict[str, Any]: def ShowVolume(self, instance, volume_attachment) -> Dict[str, Any]: """Send a volume get request on a vm.""" inst_uri = ( - self.GetUri(instance) + '/volume_attachments/' + volume_attachment) + self.GetUri(instance) + '/volume_attachments/' + volume_attachment + ) return self._g.Request('GET', inst_uri) def DeleteVolume(self, instance, volume_attachment): """Send a volume delete request on a vm.""" inst_uri = ( - self.GetUri(instance) + '/volume_attachments/' + volume_attachment) + self.GetUri(instance) + '/volume_attachments/' + volume_attachment + ) return self._g.Request('DELETE', inst_uri) def ListProfiles(self) -> Dict[str, Any]: @@ -564,24 +607,16 @@ def Create(self, zone, **kwargs) -> Dict[str, Any]: # pytype: disable=signature Args: zone: zone name. - **kwargs: - name - volume name. - profile - volume profile. - capacity - boot volume size. - iops - iops on the volume. - resource_group - optional. + **kwargs: name - volume name. profile - volume profile. capacity - boot + volume size. iops - iops on the volume. resource_group - optional. encryption_key - key to encrypt, optional. Returns: created volume. """ req_data = { - 'zone': { - 'name': zone - }, - 'profile': { - 'name': kwargs.get('profile', 'general-purpose') - } + 'zone': {'name': zone}, + 'profile': {'name': kwargs.get('profile', 'general-purpose')}, } if kwargs.get('capacity', None): req_data['capacity'] = kwargs.get('capacity') @@ -654,27 +689,24 @@ def ListPrefix(self, vpc) -> Dict[str, Any]: def DeletePrefix(self, vpc, prefix): """Send a vpc address prefix delete request.""" - return self._g.Request('DELETE', - 'vpcs/%s/address_prefixes/%s' % (vpc, prefix)) - - def PatchPrefix(self, - vpc, - prefix, - default=False, - name=None) -> Dict[str, Any]: + return self._g.Request( + 'DELETE', 'vpcs/%s/address_prefixes/%s' % (vpc, prefix) + ) + + def PatchPrefix( + self, vpc, prefix, default=False, name=None + ) -> Dict[str, Any]: """Send a vpc address prefix patch request.""" req_data = {'is_default': default} if name: req_data['name'] = name - return self._g.Request('PATCH', - 'vpcs/%s/address_prefixes/%s' % (vpc, prefix), - req_data) - - def CreateRoutingTable(self, - vpc, - name, - routes=None, - session=None) -> Dict[str, Any]: + return self._g.Request( + 'PATCH', 'vpcs/%s/address_prefixes/%s' % (vpc, prefix), req_data + ) + + def CreateRoutingTable( + self, vpc, name, routes=None, session=None + ) -> Dict[str, Any]: """Send a vpc routing table create request.""" req_data = {'name': name} if routes: @@ -685,14 +717,12 @@ def CreateRoutingTable(self, def ShowRoutingTable(self, vpc, routing, session=None) -> Dict[str, Any]: """Send a vpc routing table get request.""" return self._g.Request( - 'GET', 'vpcs/%s/routing_tables/%s' % (vpc, routing), session=session) - - def PatchRoutingTable(self, - vpc, - routing, - ingress, - flag, - session=None) -> Dict[str, Any]: + 'GET', 'vpcs/%s/routing_tables/%s' % (vpc, routing), session=session + ) + + def PatchRoutingTable( + self, vpc, routing, ingress, flag, session=None + ) -> Dict[str, Any]: """Send a vpc routing table patch request.""" vpc_uri = self.GetUri(vpc) + '/routing_tables/' + routing return self._g.Request('PATCH', vpc_uri, {ingress: flag}, session=session) @@ -705,25 +735,26 @@ def DeleteRoutingTable(self, vpc, routing, session=None): def ListRoutingTable(self, vpc, session=None) -> Dict[str, Any]: """Send a vpc routing table list request.""" return self._g.Request( - 'GET', 'vpcs/%s/routing_tables?limit=100' % vpc, session=session) - - def CreateRoute(self, - vpc, - routing, - name, - zone, - action, - destination, - nexthop=None, - session=None) -> Dict[str, Any]: + 'GET', 'vpcs/%s/routing_tables?limit=100' % vpc, session=session + ) + + def CreateRoute( + self, + vpc, + routing, + name, + zone, + action, + destination, + nexthop=None, + session=None, + ) -> Dict[str, Any]: """Send a vpc route create request.""" req_data = { 'name': name, 'action': action, 'destination': destination, - 'zone': { - 'name': zone - } + 'zone': {'name': zone}, } if nexthop: req_data['next_hop'] = {'address': nexthop} @@ -732,8 +763,9 @@ def CreateRoute(self, def DeleteRoute(self, vpc, routing, route, session=None): """Send a vpc route delete request.""" - vpc_uri = self.GetUri( - vpc) + '/routing_tables/' + routing + '/routes/' + route + vpc_uri = ( + self.GetUri(vpc) + '/routing_tables/' + routing + '/routes/' + route + ) return self._g.Request('DELETE', '%s' % vpc_uri, session=session) def ListRoute(self, vpc, routing, session=None) -> Dict[str, Any]: @@ -741,7 +773,8 @@ def ListRoute(self, vpc, routing, session=None) -> Dict[str, Any]: return self._g.Request( 'GET', 'vpcs/%s/routing_tables/%s/routes?limit=100' % (vpc, routing), - session=session) + session=session, + ) class SGManager(BaseManager): @@ -755,17 +788,14 @@ def Create(self, resource_group, vpcid, **kwargs) -> Dict[str, Any]: # pytype: Args: resource_group: optional. vpcid: vpc id. - **kwargs: - name - name of the vm. + **kwargs: name - name of the vm. Returns: returns created security groups. """ req_data = {'vpc': {'id': vpcid}} if resource_group: - req_data['resource_group'] = { - 'id': resource_group - } + req_data['resource_group'] = {'id': resource_group} if kwargs.get('name', None): req_data['name'] = kwargs.get('name', None) @@ -795,15 +825,17 @@ def ListRules(self, sg) -> Dict[str, Any]: sg_uri = self.GetUris(sg) + '/rules' return self._g.Request('GET', sg_uri) - def CreateRule(self, - sg, - direction, - ip_version, - cidr_block, - protocol, - port, - port_min=None, - port_max=None) -> Dict[str, Any]: + def CreateRule( + self, + sg, + direction, + ip_version, + cidr_block, + protocol, + port, + port_min=None, + port_max=None, + ) -> Dict[str, Any]: """Send a security group rule create request. Args: @@ -823,10 +855,8 @@ def CreateRule(self, req_data = { 'direction': direction, 'ip_version': ip_version, - 'remote': { - 'cidr_block': cidr_block - }, - 'protocol': protocol + 'remote': {'cidr_block': cidr_block}, + 'protocol': protocol, } if port: req_data['port_min'] = port @@ -854,18 +884,14 @@ def Create(self, resource_group, vpcid, zone, **kwargs) -> Dict[str, Any]: # py resource_group: optional. vpcid: vpc id. zone: zone name. - **kwargs: - name - name of the vm. - floating_ip - optional, floating ip id. + **kwargs: name - name of the vm. floating_ip - optional, floating ip id. Returns: returns created public gateways. """ req_data = {'vpc': {'id': vpcid}, 'zone': {'name': zone}} if resource_group: - req_data['resource_group'] = { - 'id': resource_group - } + req_data['resource_group'] = {'id': resource_group} if kwargs.get('name', None): req_data['name'] = kwargs.get('name', None) if kwargs.get('floating_ip', None): @@ -908,9 +934,7 @@ def Create(self, key, key_type, **kwargs) -> Dict[str, Any]: # pytype: disable= Args: key: public key string. key_type: rsa. - **kwargs: - name - name of the key: - resource_group - optional. + **kwargs: name - name of the key: resource_group - optional. Returns: returns created key. @@ -980,20 +1004,15 @@ def Create(self, subnet, vpcid, **kwargs) -> Dict[str, Any]: # pytype: disable= Args: subnet: subnet vpcid: vpc id. - **kwargs: - name - name of the subnet. - zone - zone name. - ip_version - ipv4 or 6. - ipv4_cidr_block - cidr for subnet. + **kwargs: name - name of the subnet. zone - zone name. ip_version - ipv4 + or 6. ipv4_cidr_block - cidr for subnet. Returns: created subnet. """ req_data = { 'ipv4_cidr_block': subnet, - 'vpc': { - 'id': vpcid - }, + 'vpc': {'id': vpcid}, 'ip_version': 'ipv4', } if kwargs.get('name'): @@ -1079,9 +1098,7 @@ def Create(self, resource_group, target, **kwargs) -> Dict[str, Any]: # pytype: Args: resource_group: optional. target: id of the vm network interface. - **kwargs: - name - name of the fip. - zone - zone name. + **kwargs: name - name of the fip. zone - zone name. Returns: returns floating IP addresses. diff --git a/perfkitbenchmarker/providers/ibmcloud/ibmcloud_network.py b/perfkitbenchmarker/providers/ibmcloud/ibmcloud_network.py index 44b214bbaf..d91bdc0c4f 100644 --- a/perfkitbenchmarker/providers/ibmcloud/ibmcloud_network.py +++ b/perfkitbenchmarker/providers/ibmcloud/ibmcloud_network.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""""Module containing classes related to IBM Cloud Networking.""" +""" "Module containing classes related to IBM Cloud Networking.""" import json import logging @@ -30,10 +30,20 @@ FLAGS = flags.FLAGS VPC_NAME = 'vpc' -VPC_PREFIX_RANGES = ['10.101.0.0/16', '10.102.0.0/16', '10.103.0.0/16', - '10.104.0.0/16', '10.105.0.0/16'] -VPC_SUBNETS = ['10.101.0.0/20', '10.102.0.0/20', '10.103.0.0/20', - '10.104.0.0/20', '10.105.0.0/20'] +VPC_PREFIX_RANGES = [ + '10.101.0.0/16', + '10.102.0.0/16', + '10.103.0.0/16', + '10.104.0.0/16', + '10.105.0.0/16', +] +VPC_SUBNETS = [ + '10.101.0.0/20', + '10.102.0.0/20', + '10.103.0.0/20', + '10.104.0.0/20', + '10.105.0.0/20', +] """These constants are used to create extra subnets to support multi vnics on a vm, extra subnets are created using predefined cidr's as below. @@ -49,40 +59,64 @@ SUBNETS_EXTRA = { SUBNETX1: [ - '10.101.20.0/24', '10.102.20.0/24', '10.103.20.0/24', '10.104.20.0/24', - '10.105.20.0/24' + '10.101.20.0/24', + '10.102.20.0/24', + '10.103.20.0/24', + '10.104.20.0/24', + '10.105.20.0/24', ], SUBNETX2: [ - '10.101.30.0/24', '10.102.30.0/24', '10.103.30.0/24', '10.104.30.0/24', - '10.105.30.0/24' + '10.101.30.0/24', + '10.102.30.0/24', + '10.103.30.0/24', + '10.104.30.0/24', + '10.105.30.0/24', ], SUBNETX3: [ - '10.101.40.0/24', '10.102.40.0/24', '10.103.40.0/24', '10.104.40.0/24', - '10.105.40.0/24' + '10.101.40.0/24', + '10.102.40.0/24', + '10.103.40.0/24', + '10.104.40.0/24', + '10.105.40.0/24', ], SUBNETX4: [ - '10.101.50.0/24', '10.102.50.0/24', '10.103.50.0/24', '10.104.50.0/24', - '10.105.50.0/24' - ] + '10.101.50.0/24', + '10.102.50.0/24', + '10.103.50.0/24', + '10.104.50.0/24', + '10.105.50.0/24', + ], } SUBNETS_EXTRA_GATEWAY = { SUBNETX1: [ - '10.101.20.1', '10.102.20.1', '10.103.20.1', '10.104.20.1', - '10.105.20.1' + '10.101.20.1', + '10.102.20.1', + '10.103.20.1', + '10.104.20.1', + '10.105.20.1', ], SUBNETX2: [ - '10.101.30.1', '10.102.30.1', '10.103.30.1', '10.104.30.1', - '10.105.30.1' + '10.101.30.1', + '10.102.30.1', + '10.103.30.1', + '10.104.30.1', + '10.105.30.1', ], SUBNETX3: [ - '10.101.40.1', '10.102.40.1', '10.103.40.1', '10.104.40.1', - '10.105.40.1' + '10.101.40.1', + '10.102.40.1', + '10.103.40.1', + '10.104.40.1', + '10.105.40.1', ], SUBNETX4: [ - '10.101.50.1', '10.102.50.1', '10.103.50.1', '10.104.50.1', - '10.105.50.1' - ] + '10.101.50.1', + '10.102.50.1', + '10.103.50.1', + '10.104.50.1', + '10.105.50.1', + ], } _DEFAULT_TIMEOUT = 300 @@ -133,8 +167,10 @@ def GetRouteCommands(data: str, index: int, target_index: int) -> List[str]: items = line.split() if len(items) > 6 and items[0] == route_entry: interface = items[7] - route_cmds.append(f'ip route add {target_cidr_block} via ' - f'{subnet_gateway} dev {interface}') + route_cmds.append( + f'ip route add {target_cidr_block} via ' + f'{subnet_gateway} dev {interface}' + ) return route_cmds @@ -146,6 +182,7 @@ class IbmCloudNetwork(resource.BaseResource): zone: zone name. vpcid: vpc id. """ + _lock = threading.Lock() _lock_vpc = threading.Lock() @@ -173,8 +210,9 @@ def _Create(self): if resp: logging.info('Created vpc prefix range: %s', resp.get('id')) else: - raise errors.Error('IBM Cloud ERROR: Failed to create ' - 'vpc address prefix') + raise errors.Error( + 'IBM Cloud ERROR: Failed to create vpc address prefix' + ) else: raise errors.Error('IBM Cloud ERROR: Failed to create vpc') @@ -186,9 +224,15 @@ def CreateSubnet(self, vpcid: str): cmd.flags.update({ 'vpcid': self.vpcid, 'zone': self.zone, - 'name': (self.prefix + VPC_NAME + util.SUBNET_SUFFIX + - str(subnet_index) + util.DELIMITER + self.zone), - 'cidr': VPC_SUBNETS[subnet_index - 1] + 'name': ( + self.prefix + + VPC_NAME + + util.SUBNET_SUFFIX + + str(subnet_index) + + util.DELIMITER + + self.zone + ), + 'cidr': VPC_SUBNETS[subnet_index - 1], }) logging.info('Creating subnet: %s', cmd.flags) resp = cmd.CreateSubnet() @@ -214,12 +258,17 @@ def CreateFip(self, name: str, vmid: str): return resp['address'], resp['id'] else: raise errors.Error( - f'IBM Cloud ERROR: Failed to create fip for instance {vmid}') + f'IBM Cloud ERROR: Failed to create fip for instance {vmid}' + ) def DeleteFip(self, vmid: str, fip_address: str, fipid: str): """Deletes fip in a IBM Cloud VM instance.""" - logging.info('Deleting FIP, instanceid: %s, fip address: %s, fip id: %s', - vmid, fip_address, fipid) + logging.info( + 'Deleting FIP, instanceid: %s, fip address: %s, fip id: %s', + vmid, + fip_address, + fipid, + ) cmd = ibm.IbmAPICommand(self) cmd.flags['fipid'] = fipid cmd.InstanceFipDelete() @@ -227,11 +276,9 @@ def DeleteFip(self, vmid: str, fip_address: str, fipid: str): def _Exists(self): """Returns true if the VPC exists.""" cmd = ibm.IbmAPICommand(self) - cmd.flags.update({ - 'prefix': self.prefix, - 'zone': self.zone, - 'items': 'vpcs' - }) + cmd.flags.update( + {'prefix': self.prefix, 'zone': self.zone, 'items': 'vpcs'} + ) self.vpcid = cmd.GetResource() if self.vpcid: return True @@ -240,33 +287,42 @@ def _Exists(self): def _Delete(self): """Deletes the vpc and subnets.""" with self._lock: - if (ibmcloud_virtual_machine.IbmCloudVirtualMachine.validated_subnets > 0 - and self.subnet): + if ( + ibmcloud_virtual_machine.IbmCloudVirtualMachine.validated_subnets > 0 + and self.subnet + ): cmd = ibm.IbmAPICommand(self) cmd.flags.update({ 'prefix': self.prefix, 'zone': self.zone, 'id': self.subnet, - 'items': 'subnets' + 'items': 'subnets', }) logging.info('Deleting subnet: %s', self.subnet) cmd.DeleteResource() time_to_end = time.time() + _DEFAULT_TIMEOUT while cmd.GetResource() is not None and time_to_end < time.time(): - logging.info('Subnet still exists, waiting to delete subnet: %s', - self.subnet) + logging.info( + 'Subnet still exists, waiting to delete subnet: %s', self.subnet + ) time.sleep(5) cmd.DeleteResource() ibmcloud_virtual_machine.IbmCloudVirtualMachine.validated_subnets -= 1 # different lock so all threads get a chance to delete its subnet first with self._lock_vpc: - if (self.subnet and self.vpcid not in ibmcloud_virtual_machine - .IbmCloudVirtualMachine.validated_resources_set): + if ( + self.subnet + and self.vpcid + not in ibmcloud_virtual_machine.IbmCloudVirtualMachine.validated_resources_set + ): cmd = ibm.IbmAPICommand(self) # check till all subnets are gone time_to_end = time.time() + _DEFAULT_TIMEOUT - while ibmcloud_virtual_machine.IbmCloudVirtualMachine.validated_subnets > 0: + while ( + ibmcloud_virtual_machine.IbmCloudVirtualMachine.validated_subnets + > 0 + ): logging.info('Subnets not empty yet') if time_to_end < time.time(): break @@ -275,7 +331,10 @@ def _Delete(self): cmd.flags['id'] = self.vpcid logging.info('Waiting to delete vpc') time.sleep(10) - (ibmcloud_virtual_machine.IbmCloudVirtualMachine.validated_resources_set - .add(self.vpcid)) + ( + ibmcloud_virtual_machine.IbmCloudVirtualMachine.validated_resources_set.add( + self.vpcid + ) + ) cmd.DeleteResource() self.vpcid = None diff --git a/perfkitbenchmarker/providers/ibmcloud/ibmcloud_virtual_machine.py b/perfkitbenchmarker/providers/ibmcloud/ibmcloud_virtual_machine.py index 6db6687845..064dee54bf 100644 --- a/perfkitbenchmarker/providers/ibmcloud/ibmcloud_virtual_machine.py +++ b/perfkitbenchmarker/providers/ibmcloud/ibmcloud_virtual_machine.py @@ -93,8 +93,11 @@ def _CreateRiasKey(self): logging.info('Creating rias key') with open(vm_util.GetPublicKeyPath(), 'r') as keyfile: pubkey = keyfile.read() - logging.info('ssh private key file: %s, public key file: %s', - vm_util.GetPrivateKeyPath(), vm_util.GetPublicKeyPath()) + logging.info( + 'ssh private key file: %s, public key file: %s', + vm_util.GetPrivateKeyPath(), + vm_util.GetPublicKeyPath(), + ) cmd = ibm.IbmAPICommand(self) cmd.flags['name'] = self.prefix + str(flags.FLAGS.run_uri) + 'key' cmd.flags['pubkey'] = pubkey @@ -127,11 +130,9 @@ def _SetupResources(self): """Looks up the resources needed, if not found, creates new.""" logging.info('Checking resources') cmd = ibm.IbmAPICommand(self) - cmd.flags.update({ - 'prefix': self.prefix, - 'zone': self.zone, - 'items': 'vpcs' - }) + cmd.flags.update( + {'prefix': self.prefix, 'zone': self.zone, 'items': 'vpcs'} + ) self.vpcid = cmd.GetResource() logging.info('Vpc found: %s', self.vpcid) cmd.flags['items'] = 'subnets' @@ -190,7 +191,8 @@ def _Create(self): self._CreateInstance() if self.subnet: # this is for the primary vnic and fip self.fip_address, self.fip_id = self.network.CreateFip( - self.name + 'fip', self.vmid) + self.name + 'fip', self.vmid + ) self.ip_address = self.fip_address self.internal_ip = self._WaitForIPAssignment(self.subnet) logging.info('Fip: %s, ip: %s', self.ip_address, self.internal_ip) @@ -202,15 +204,25 @@ def _Create(self): for subnet_name in self.subnets.keys(): cmd.flags['name'] = subnet_name cmd.flags['subnet'] = self.subnets[subnet_name]['id'] # subnet id - logging.info('Creating extra vnic for vmid: %s, subnet: %s', - self.vmid, cmd.flags['subnet']) + logging.info( + 'Creating extra vnic for vmid: %s, subnet: %s', + self.vmid, + cmd.flags['subnet'], + ) vnicid, ip_addr = cmd.InstanceVnicCreate() - logging.info('Extra vnic created for vmid: %s, vnicid: %s, ip_addr: %s', - self.vmid, vnicid, ip_addr) + logging.info( + 'Extra vnic created for vmid: %s, vnicid: %s, ip_addr: %s', + self.vmid, + vnicid, + ip_addr, + ) self.subnets[subnet_name]['vnicid'] = vnicid self.subnets[subnet_name]['ip_addr'] = ip_addr - logging.info('Extra vnics created for vmid: %s, subnets: %s', - self.vmid, self.subnets) + logging.info( + 'Extra vnics created for vmid: %s, subnets: %s', + self.vmid, + self.subnets, + ) def _Delete(self): """Delete all the resources that were created.""" @@ -265,7 +277,7 @@ def _CreateInstance(self): 'vpcid': self.vpcid, 'subnet': self.subnet, 'key': self.key, - 'zone': self.zone + 'zone': self.zone, }) cmd.user_data = self.user_data if self.boot_volume_size > 0: @@ -301,14 +313,17 @@ def _WaitForIPAssignment(self, networkid: str): """Finds the IP address assigned to the vm.""" ip_v4_address = '0.0.0.0' count = 0 - while (ip_v4_address == '0.0.0.0' and - count * FLAGS.ibmcloud_polling_delay < 240): + while ( + ip_v4_address == '0.0.0.0' + and count * FLAGS.ibmcloud_polling_delay < 240 + ): time.sleep(FLAGS.ibmcloud_polling_delay) count += 1 cmd = ibm.IbmAPICommand(self) cmd.flags['instanceid'] = self.vmid - logging.info('Looking for IP for instance %s, networkid: %s', - self.vmid, networkid) + logging.info( + 'Looking for IP for instance %s, networkid: %s', self.vmid, networkid + ) resp = cmd.InstanceShow() for network in resp['network_interfaces']: @@ -343,12 +358,16 @@ def CreateScratchDisk(self, _, disk_spec: disk.BaseDiskSpec): Args: disk_spec: virtual_machine.BaseDiskSpec object of the disk. """ - disks_names = ('%s-data-%d-%d' - % (self.name, len(self.scratch_disks), i) - for i in range(disk_spec.num_striped_disks)) - disks = [ibmcloud_disk.IbmCloudDisk(disk_spec, name, self.zone, - encryption_key=self.data_encryption_key) - for name in disks_names] + disks_names = ( + '%s-data-%d-%d' % (self.name, len(self.scratch_disks), i) + for i in range(disk_spec.num_striped_disks) + ) + disks = [ + ibmcloud_disk.IbmCloudDisk( + disk_spec, name, self.zone, encryption_key=self.data_encryption_key + ) + for name in disks_names + ] scratch_disk = self._CreateScratchDiskFromDisks(disk_spec, disks) disk_strategies.PrepareScratchDiskStrategy().PrepareScratchDisk( @@ -373,8 +392,9 @@ def ShouldDownloadPreprovisionedData(self, module_name, filename): return False -class DebianBasedIbmCloudVirtualMachine(IbmCloudVirtualMachine, - linux_virtual_machine.BaseDebianMixin): +class DebianBasedIbmCloudVirtualMachine( + IbmCloudVirtualMachine, linux_virtual_machine.BaseDebianMixin +): def PrepareVMEnvironment(self): time.sleep(_WAIT_TIME_DEBIAN) @@ -383,23 +403,27 @@ def PrepareVMEnvironment(self): super(DebianBasedIbmCloudVirtualMachine, self).PrepareVMEnvironment() -class Debian9BasedIbmCloudVirtualMachine(DebianBasedIbmCloudVirtualMachine, - linux_virtual_machine.Debian9Mixin): +class Debian9BasedIbmCloudVirtualMachine( + DebianBasedIbmCloudVirtualMachine, linux_virtual_machine.Debian9Mixin +): IMAGE_NAME_PREFIX = 'ibm-debian-9-' -class Debian10BasedIbmCloudVirtualMachine(DebianBasedIbmCloudVirtualMachine, - linux_virtual_machine.Debian10Mixin): +class Debian10BasedIbmCloudVirtualMachine( + DebianBasedIbmCloudVirtualMachine, linux_virtual_machine.Debian10Mixin +): IMAGE_NAME_PREFIX = 'ibm-debian-10-' -class Debian11BasedIbmCloudVirtualMachine(DebianBasedIbmCloudVirtualMachine, - linux_virtual_machine.Debian11Mixin): +class Debian11BasedIbmCloudVirtualMachine( + DebianBasedIbmCloudVirtualMachine, linux_virtual_machine.Debian11Mixin +): IMAGE_NAME_PREFIX = 'ibm-debian-11-' class Ubuntu1604BasedIbmCloudVirtualMachine( - IbmCloudVirtualMachine, linux_virtual_machine.Ubuntu1604Mixin): + IbmCloudVirtualMachine, linux_virtual_machine.Ubuntu1604Mixin +): IMAGE_NAME_PREFIX = 'ibm-ubuntu-16-04-' def PrepareVMEnvironment(self): @@ -408,7 +432,8 @@ def PrepareVMEnvironment(self): class Ubuntu1804BasedIbmCloudVirtualMachine( - IbmCloudVirtualMachine, linux_virtual_machine.Ubuntu1804Mixin): + IbmCloudVirtualMachine, linux_virtual_machine.Ubuntu1804Mixin +): IMAGE_NAME_PREFIX = 'ibm-ubuntu-18-04-' def PrepareVMEnvironment(self): @@ -482,8 +507,13 @@ def _PostCreate(self): tf.write(decoded_password_data) tf.close() decrypt_cmd = [ - 'openssl', 'rsautl', '-decrypt', '-in', tf.name, '-inkey', - vm_util.GetPrivateKeyPath() + 'openssl', + 'rsautl', + '-decrypt', + '-in', + tf.name, + '-inkey', + vm_util.GetPrivateKeyPath(), ] password, _ = vm_util.IssueRetryableCommand(decrypt_cmd) self.password = password @@ -491,12 +521,12 @@ def _PostCreate(self): class Windows2016CoreIbmCloudVirtualMachine( - WindowsIbmCloudVirtualMachine, - windows_virtual_machine.Windows2016CoreMixin): + WindowsIbmCloudVirtualMachine, windows_virtual_machine.Windows2016CoreMixin +): IMAGE_NAME_PREFIX = 'ibm-windows-server-2016-full' class Windows2019CoreIbmCloudVirtualMachine( - WindowsIbmCloudVirtualMachine, - windows_virtual_machine.Windows2019CoreMixin): + WindowsIbmCloudVirtualMachine, windows_virtual_machine.Windows2019CoreMixin +): IMAGE_NAME_PREFIX = 'ibm-windows-server-2019-full' diff --git a/perfkitbenchmarker/providers/ibmcloud/provider_info.py b/perfkitbenchmarker/providers/ibmcloud/provider_info.py index 638e2d0719..3fe7e33e6d 100644 --- a/perfkitbenchmarker/providers/ibmcloud/provider_info.py +++ b/perfkitbenchmarker/providers/ibmcloud/provider_info.py @@ -18,6 +18,5 @@ class IBMCloudProviderInfo(provider_info.BaseProviderInfo): - UNSUPPORTED_BENCHMARKS = [] CLOUD = provider_info.IBMCLOUD diff --git a/perfkitbenchmarker/providers/ibmcloud/util.py b/perfkitbenchmarker/providers/ibmcloud/util.py index 06b890f552..91c21ce01c 100644 --- a/perfkitbenchmarker/providers/ibmcloud/util.py +++ b/perfkitbenchmarker/providers/ibmcloud/util.py @@ -35,29 +35,30 @@ # for windows USER_DATA = ( 'Content-Type: text/x-shellscript; ' - "charset=\"us-ascii\"\nContent-Transfer-Encoding: " + 'charset="us-ascii"\nContent-Transfer-Encoding: ' '7bit\nContent-Disposition: attachment; ' - "filename=\"set-content.ps1\"\n#ps1_sysnative\nfunction " + 'filename="set-content.ps1"\n#ps1_sysnative\nfunction ' 'Setup-Remote-Desktop () {\nSet-ItemProperty ' - "\"HKLM:\\SYSTEM\\CurrentControlSet\\Control\\Terminal Server\" -Name " + '"HKLM:\\SYSTEM\\CurrentControlSet\\Control\\Terminal Server" -Name ' 'fDenyTSConnections -Value 0\nSet-ItemProperty ' - "\"HKLM:\\SYSTEM\\CurrentControlSet\\Control\\Terminal " - "Server\\WinStations\\RDP-Tcp\" -Name \"UserAuthentication\" -Value " - "1\nEnable-NetFireWallRule -DisplayGroup \"Remote Desktop\"\n}\nfunction " - "Setup-Ping () {\nSet-NetFirewallRule -DisplayName \"File and Printer " - "Sharing (Echo Request - ICMPv4-In)\" -enabled True\nSet-NetFirewallRule " - "-DisplayName \"File and Printer Sharing (Echo Request - ICMPv6-In)\" " + '"HKLM:\\SYSTEM\\CurrentControlSet\\Control\\Terminal ' + 'Server\\WinStations\\RDP-Tcp" -Name "UserAuthentication" -Value ' + '1\nEnable-NetFireWallRule -DisplayGroup "Remote Desktop"\n}\nfunction ' + 'Setup-Ping () {\nSet-NetFirewallRule -DisplayName "File and Printer ' + 'Sharing (Echo Request - ICMPv4-In)" -enabled True\nSet-NetFirewallRule ' + '-DisplayName "File and Printer Sharing (Echo Request - ICMPv6-In)" ' '-enabled True\n}\nSetup-Remote-Desktop\nSetup-Ping\nNew-NetFirewallRule ' - "-DisplayName \"Allow winrm https 5986\" -Direction Inbound -Action Allow " + '-DisplayName "Allow winrm https 5986" -Direction Inbound -Action Allow ' '-Protocol TCP -LocalPort 5986\nwinrm set winrm/config/service/auth ' - "'@{Basic=\"true\";Certificate=\"true\"}'\n$cert=New-SelfSignedCertificate" + '\'@{Basic="true";Certificate="true"}\'\n$cert=New-SelfSignedCertificate' ' -certstorelocation cert:\\localmachine\\my -dnsname ' '*\n$thumb=($cert).Thumbprint\nNew-WSManInstance -ResourceURI ' - "winrm/config/Listener -SelectorSet @{Address=\"*\";Transport=\"HTTPS\"} " - "-ValueSet @{CertificateThumbprint=\"$thumb\"}\npowercfg /SetActive " - "(powercfg /List | %{if ($_.Contains(\"High " - "performance\")){$_.Split()[3]}})\nSet-NetAdapterAdvancedProperty -Name " - 'Ethernet -RegistryKeyword MTU -RegistryValue 9000\n') + 'winrm/config/Listener -SelectorSet @{Address="*";Transport="HTTPS"} ' + '-ValueSet @{CertificateThumbprint="$thumb"}\npowercfg /SetActive ' + '(powercfg /List | %{if ($_.Contains("High ' + 'performance")){$_.Split()[3]}})\nSet-NetAdapterAdvancedProperty -Name ' + 'Ethernet -RegistryKeyword MTU -RegistryValue 9000\n' +) @dataclasses.dataclass(frozen=True) @@ -115,7 +116,8 @@ def GetGen(account: Account): verbose=False, version='v1', silent=True, - force=True) + force=True, + ) if not gen.Token(): gen.SetToken() # one more try return gen @@ -153,8 +155,9 @@ def GetOsInfo(image: Dict[Any, Any]) -> Dict[str, Any]: data = image['operating_system'] if 'href' in data: del data['href'] # delete this, does not seem necessary - if 'custom' in image[ - 'name']: # this name is not the name in operating_system + if ( + 'custom' in image['name'] + ): # this name is not the name in operating_system custom = True else: # if lookup failed, try read from env if any is set diff --git a/perfkitbenchmarker/providers/kubernetes/flags.py b/perfkitbenchmarker/providers/kubernetes/flags.py index 5c5c9bb5db..bed7835493 100644 --- a/perfkitbenchmarker/providers/kubernetes/flags.py +++ b/perfkitbenchmarker/providers/kubernetes/flags.py @@ -14,59 +14,89 @@ from absl import flags -flags.DEFINE_string('ceph_secret', None, - 'Name of the Ceph Secret used by Kubernetes in order to ' - 'authenticate with Ceph. If provided, overrides keyring.') +flags.DEFINE_string( + 'ceph_secret', + None, + 'Name of the Ceph Secret used by Kubernetes in order to ' + 'authenticate with Ceph. If provided, overrides keyring.', +) -flags.DEFINE_string('ceph_keyring', '/etc/ceph/keyring', - 'Path to the Ceph keyring file.') +flags.DEFINE_string( + 'ceph_keyring', '/etc/ceph/keyring', 'Path to the Ceph keyring file.' +) -flags.DEFINE_string('rbd_pool', 'rbd', - 'Name of RBD pool for Ceph volumes.') +flags.DEFINE_string('rbd_pool', 'rbd', 'Name of RBD pool for Ceph volumes.') -flags.DEFINE_string('rbd_user', 'admin', - 'Name of RADOS user.') +flags.DEFINE_string('rbd_user', 'admin', 'Name of RADOS user.') -flags.DEFINE_list('ceph_monitors', [], - 'IP addresses and ports of Ceph Monitors. ' - 'Must be provided when Ceph scratch disk is required. ' - 'Example: "127.0.0.1:6789,192.168.1.1:6789"') +flags.DEFINE_list( + 'ceph_monitors', + [], + 'IP addresses and ports of Ceph Monitors. ' + 'Must be provided when Ceph scratch disk is required. ' + 'Example: "127.0.0.1:6789,192.168.1.1:6789"', +) -flags.DEFINE_string('username', 'root', - 'User name that Perfkit will attempt to use in order to ' - 'SSH into Docker instance.') +flags.DEFINE_string( + 'username', + 'root', + 'User name that Perfkit will attempt to use in order to ' + 'SSH into Docker instance.', +) -flags.DEFINE_boolean('docker_in_privileged_mode', True, - 'If set to True, will attempt to create Docker containers ' - 'in a privileged mode. Note that some benchmarks execute ' - 'commands which are only allowed in privileged mode.') -flags.DEFINE_boolean('kubernetes_anti_affinity', True, - 'If set to True, PKB pods will not be scheduled on the ' - 'same nodes as other PKB pods.') -flags.DEFINE_multi_string('k8s_volume_parameters', None, - 'A colon separated key-value pair that will be ' - 'added to Kubernetes storage class parameters.') +flags.DEFINE_boolean( + 'docker_in_privileged_mode', + True, + 'If set to True, will attempt to create Docker containers ' + 'in a privileged mode. Note that some benchmarks execute ' + 'commands which are only allowed in privileged mode.', +) +flags.DEFINE_boolean( + 'kubernetes_anti_affinity', + True, + 'If set to True, PKB pods will not be scheduled on the ' + 'same nodes as other PKB pods.', +) +flags.DEFINE_multi_string( + 'k8s_volume_parameters', + None, + 'A colon separated key-value pair that will be ' + 'added to Kubernetes storage class parameters.', +) _K8S_PROVISIONERS = [ - 'kubernetes.io/azure-disk', 'kubernetes.io/gce-pd', 'kubernetes.io/aws-ebs', - 'kubernetes.io/glusterfs' + 'kubernetes.io/azure-disk', + 'kubernetes.io/gce-pd', + 'kubernetes.io/aws-ebs', + 'kubernetes.io/glusterfs', ] -flags.DEFINE_enum('k8s_volume_provisioner', None, _K8S_PROVISIONERS, - 'The name of the provisioner to use for K8s storage ' - 'classes.') -flags.DEFINE_string('k8s_sriov_network', None, - 'The name of the network with sriov enabled interfaces') +flags.DEFINE_enum( + 'k8s_volume_provisioner', + None, + _K8S_PROVISIONERS, + 'The name of the provisioner to use for K8s storage classes.', +) +flags.DEFINE_string( + 'k8s_sriov_network', + None, + 'The name of the network with sriov enabled interfaces', +) SETUP_SSH = flags.DEFINE_boolean( - 'kubernetes_vm_setup_ssh', False, - 'Set up SSH on Kubernetes VMs. Probably not needed any more?') + 'kubernetes_vm_setup_ssh', + False, + 'Set up SSH on Kubernetes VMs. Probably not needed any more?', +) DEFAULT_VM_GROUP_NODEPOOL = flags.DEFINE_string( # default is container_service.DEFAULT_NODEPOOL, but don't import because it # causes a runtime circular dependency. - 'kubernetes_vm_default_nodepool', 'default', + 'kubernetes_vm_default_nodepool', + 'default', 'The nodepool to place VMs in the default VM group into. Defaults to the ' - 'default nodepool, but can be changed for better isolation.') + 'default nodepool, but can be changed for better isolation.', +) USE_NODE_SELECTORS = flags.DEFINE_bool( - 'use_k8s_vm_node_selectors', True, + 'use_k8s_vm_node_selectors', + True, 'Whether to require node selectors to be present when creating K8s VMs. ' - 'Disable this if you are using a pre-existing k8s cluster without labels.') - + 'Disable this if you are using a pre-existing k8s cluster without labels.', +) diff --git a/perfkitbenchmarker/providers/kubernetes/kubernetes_disk.py b/perfkitbenchmarker/providers/kubernetes/kubernetes_disk.py index 1efcae703b..df5e5e9f2b 100644 --- a/perfkitbenchmarker/providers/kubernetes/kubernetes_disk.py +++ b/perfkitbenchmarker/providers/kubernetes/kubernetes_disk.py @@ -44,16 +44,21 @@ def CreateDisks(disk_specs, vm_name): class KubernetesDiskSpec(disk.BaseDiskSpec): """Kubernetes disk Spec class.""" + CLOUD = provider_info.KUBERNETES @classmethod def _GetOptionDecoderConstructions(cls): result = super(KubernetesDiskSpec, cls)._GetOptionDecoderConstructions() result.update({ - 'provisioner': (option_decoders.StringDecoder, - {'default': None, 'none_ok': True}), - 'parameters': (option_decoders.TypeVerifier, - {'default': {}, 'valid_types': (dict,)}) + 'provisioner': ( + option_decoders.StringDecoder, + {'default': None, 'none_ok': True}, + ), + 'parameters': ( + option_decoders.TypeVerifier, + {'default': {}, 'valid_types': (dict,)}, + ), }) return result @@ -65,9 +70,9 @@ def _ApplyFlags(cls, config_values, flag_values): Args: config_values: dict mapping config option names to provided values. Is - modified by this function. + modified by this function. flag_values: flags.FlagValues. Runtime flags that may override the - provided config values. + provided config values. Returns: dict mapping config option names to values derived from the config @@ -79,7 +84,8 @@ def _ApplyFlags(cls, config_values, flag_values): if flag_values['k8s_volume_parameters'].present: config_values['parameters'] = config_values.get('parameters', {}) config_values['parameters'].update( - flag_util.ParseKeyValuePairs(flag_values.k8s_volume_parameters)) + flag_util.ParseKeyValuePairs(flag_values.k8s_volume_parameters) + ) def GetKubernetesDiskClass(volume_type): @@ -112,10 +118,7 @@ def SetDevicePath(self, vm): return def AttachVolumeMountInfo(self, volume_mounts): - volume_mount = { - 'mountPath': self.mount_point, - 'name': self.name - } + volume_mount = {'mountPath': self.mount_point, 'name': self.name} volume_mounts.append(volume_mount) @@ -132,10 +135,7 @@ def GetDevicePath(self): raise errors.Error('GetDevicePath not supported for Kubernetes local disk') def AttachVolumeInfo(self, volumes): - local_volume = { - 'name': self.name, - 'emptyDir': {} - } + local_volume = {'name': self.name, 'emptyDir': {}} volumes.append(local_volume) @@ -150,8 +150,15 @@ def __init__(self, disk_num, disk_spec, name): def _Create(self): """Creates Rados Block Device volumes and installs filesystem on them.""" - cmd = ['rbd', '-p', FLAGS.rbd_pool, 'create', self.name, '--size', - str(1024 * self.disk_size)] + cmd = [ + 'rbd', + '-p', + FLAGS.rbd_pool, + 'create', + self.name, + '--size', + str(1024 * self.disk_size), + ] _, stderr, retcode = vm_util.IssueCommand(cmd, raise_on_failure=False) if retcode != 0: raise Exception('Creating RBD image failed: %s' % stderr) @@ -202,8 +209,8 @@ def AttachVolumeInfo(self, volumes): 'keyring': FLAGS.ceph_keyring, 'user': FLAGS.rbd_user, 'fsType': 'ext4', - 'readOnly': False - } + 'readOnly': False, + }, } if FLAGS.ceph_secret: ceph_volume['rbd']['secretRef'] = {'name': FLAGS.ceph_secret} @@ -226,8 +233,14 @@ class PersistentVolumeClaim(resource.BaseResource): def _WaitForPVCBoundCompletion(self): """Need to wait for the PVC to get up.""" # PVC may take some time to be ready(Bound). - exists_cmd = [FLAGS.kubectl, '--kubeconfig=%s' % FLAGS.kubeconfig, 'get', - 'pvc', '-o=json', self.name] + exists_cmd = [ + FLAGS.kubectl, + '--kubeconfig=%s' % FLAGS.kubeconfig, + 'get', + 'pvc', + '-o=json', + self.name, + ] logging.info('Waiting for PVC %s', self.name) pvc_info, _, _ = vm_util.IssueCommand(exists_cmd, raise_on_failure=False) if pvc_info: @@ -236,8 +249,9 @@ def _WaitForPVCBoundCompletion(self): if pvc == 'Bound': logging.info('PVC is ready.') return - raise Exception('PVC %s is not ready. Retrying to check status.' % - self.name) + raise Exception( + 'PVC %s is not ready. Retrying to check status.' % self.name + ) def __init__(self, name, storage_class, size): super(PersistentVolumeClaim, self).__init__() @@ -261,18 +275,12 @@ def _BuildBody(self): body = { 'kind': 'PersistentVolumeClaim', 'apiVersion': 'v1', - 'metadata': { - 'name': self.name - }, + 'metadata': {'name': self.name}, 'spec': { 'accessModes': ['ReadWriteOnce'], - 'resources': { - 'requests': { - 'storage': '%sGi' % self.size - } - }, + 'resources': {'requests': {'storage': '%sGi' % self.size}}, 'storageClassName': self.storage_class, - } + }, } return json.dumps(body) @@ -291,8 +299,14 @@ def _CheckStorageClassExists(self): # If the StorageClass with the same name and parameters exists # :return: True or False - exists_cmd = [FLAGS.kubectl, '--kubeconfig=%s' % FLAGS.kubeconfig, 'get', - 'sc', '-o=json', self.name] + exists_cmd = [ + FLAGS.kubectl, + '--kubeconfig=%s' % FLAGS.kubeconfig, + 'get', + 'sc', + '-o=json', + self.name, + ] sc_info, _, _ = vm_util.IssueCommand(exists_cmd, raise_on_failure=False) if sc_info: @@ -321,11 +335,9 @@ def _BuildBody(self): body = { 'kind': 'StorageClass', 'apiVersion': 'storage.k8s.io/v1', - 'metadata': { - 'name': self.name - }, + 'metadata': {'name': self.name}, 'provisioner': self.provisioner, - 'parameters': self.parameters + 'parameters': self.parameters, } return json.dumps(body) @@ -339,9 +351,11 @@ class PvcVolume(KubernetesDisk): def __init__(self, disk_num, spec, name): super(PvcVolume, self).__init__(disk_num, spec, name) self.storage_class = StorageClass( - name, self.PROVISIONER or spec.provisioner, spec.parameters) + name, self.PROVISIONER or spec.provisioner, spec.parameters + ) self.pvc = PersistentVolumeClaim( - self.name, self.storage_class.name, spec.disk_size) + self.name, self.storage_class.name, spec.disk_size + ) def _Create(self): self.storage_class.Create() @@ -354,8 +368,6 @@ def _Delete(self): def AttachVolumeInfo(self, volumes): pvc_volume = { 'name': self.name, - 'persistentVolumeClaim': { - 'claimName': self.name - } + 'persistentVolumeClaim': {'claimName': self.name}, } volumes.append(pvc_volume) diff --git a/perfkitbenchmarker/providers/kubernetes/kubernetes_resources_spec.py b/perfkitbenchmarker/providers/kubernetes/kubernetes_resources_spec.py index 14b37499ac..6fc93f25cc 100644 --- a/perfkitbenchmarker/providers/kubernetes/kubernetes_resources_spec.py +++ b/perfkitbenchmarker/providers/kubernetes/kubernetes_resources_spec.py @@ -11,8 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Classes relating to decoding a Kubernetes resource limits or requests. -""" +"""Classes relating to decoding a Kubernetes resource limits or requests.""" from typing import Optional @@ -27,7 +26,7 @@ class KubernetesResourcesSpec(spec.BaseSpec): Attributes: cpus: float. Number of vCPUs. memory: string. Representation of the size of memory, expressed in MiB or - GiB. Must be an integer number of MiB (e.g. "1280MiB", "7.5GiB"). + GiB. Must be an integer number of MiB (e.g. "1280MiB", "7.5GiB"). """ def __init__(self, *args, **kwargs): @@ -45,9 +44,12 @@ def _GetOptionDecoderConstructions(cls): arguments to construct in order to decode the named option. """ result = super( - KubernetesResourcesSpec, cls)._GetOptionDecoderConstructions() - result.update({'cpus': (option_decoders.FloatDecoder, {'min': 0.1}), - 'memory': (custom_virtual_machine_spec.MemoryDecoder, {})}) + KubernetesResourcesSpec, cls + )._GetOptionDecoderConstructions() + result.update({ + 'cpus': (option_decoders.FloatDecoder, {'min': 0.1}), + 'memory': (custom_virtual_machine_spec.MemoryDecoder, {}), + }) return result @@ -63,14 +65,18 @@ def Decode(self, value, component_full_name, flag_values): Args: value: a dict containing 'cpus' and 'memory' keys. component_full_name: string. Fully qualified name of the configurable - component containing the config option. + component containing the config option. flag_values: flags.FlagValues. Runtime flag values to be propagated to - BaseSpec constructors. + BaseSpec constructors. Returns: The decoded KubernetesResourcesSpec. """ - super(KubernetesResourcesDecoder, self).Decode(value, component_full_name, - flag_values) - return KubernetesResourcesSpec(self._GetOptionFullName(component_full_name), - flag_values=flag_values, **value) + super(KubernetesResourcesDecoder, self).Decode( + value, component_full_name, flag_values + ) + return KubernetesResourcesSpec( + self._GetOptionFullName(component_full_name), + flag_values=flag_values, + **value + ) diff --git a/perfkitbenchmarker/providers/kubernetes/provider_info.py b/perfkitbenchmarker/providers/kubernetes/provider_info.py index 1d85b97a04..efb62704ae 100644 --- a/perfkitbenchmarker/providers/kubernetes/provider_info.py +++ b/perfkitbenchmarker/providers/kubernetes/provider_info.py @@ -12,19 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" Provider info for Kubernetes - -""" +"""Provider info for Kubernetes""" from perfkitbenchmarker import provider_info class KubernetesProviderInfo(provider_info.BaseProviderInfo): - - SUPPORTED_BENCHMARKS = ['block_storage_workload', 'cassandra_ycsb', - 'cassandra_stress', 'cluster_boot', 'fio', - 'iperf', 'mesh_network', 'mongodb_ycsb', - 'netperf', 'redis'] + SUPPORTED_BENCHMARKS = [ + 'block_storage_workload', + 'cassandra_ycsb', + 'cassandra_stress', + 'cluster_boot', + 'fio', + 'iperf', + 'mesh_network', + 'mongodb_ycsb', + 'netperf', + 'redis', + ] UNSUPPORTED_BENCHMARKS = ['bonnieplusplus', 'sysbench'] CLOUD = provider_info.KUBERNETES diff --git a/perfkitbenchmarker/providers/mesos/flags.py b/perfkitbenchmarker/providers/mesos/flags.py index e41a8b275b..cdf4d67502 100644 --- a/perfkitbenchmarker/providers/mesos/flags.py +++ b/perfkitbenchmarker/providers/mesos/flags.py @@ -15,19 +15,24 @@ from absl import flags -flags.DEFINE_boolean('mesos_privileged_docker', False, - 'If set to True, will attempt to create Docker containers ' - 'in a privileged mode. Note that some benchmarks execute ' - 'commands which are only allowed in privileged mode.') +flags.DEFINE_boolean( + 'mesos_privileged_docker', + False, + 'If set to True, will attempt to create Docker containers ' + 'in a privileged mode. Note that some benchmarks execute ' + 'commands which are only allowed in privileged mode.', +) -flags.DEFINE_integer('docker_memory_mb', 2048, - 'Memory limit for docker containers.') +flags.DEFINE_integer( + 'docker_memory_mb', 2048, 'Memory limit for docker containers.' +) -flags.DEFINE_float('docker_cpus', 1, - 'CPU limit for docker containers.') +flags.DEFINE_float('docker_cpus', 1, 'CPU limit for docker containers.') -flags.DEFINE_string('marathon_address', 'localhost:8080', - 'Marathon IP address and port.') +flags.DEFINE_string( + 'marathon_address', 'localhost:8080', 'Marathon IP address and port.' +) -flags.DEFINE_string('marathon_auth', 'root:password', - 'Marathon server basic authentication.') +flags.DEFINE_string( + 'marathon_auth', 'root:password', 'Marathon server basic authentication.' +) diff --git a/perfkitbenchmarker/providers/mesos/mesos_disk.py b/perfkitbenchmarker/providers/mesos/mesos_disk.py index 5fba812e8d..4c1a063aac 100644 --- a/perfkitbenchmarker/providers/mesos/mesos_disk.py +++ b/perfkitbenchmarker/providers/mesos/mesos_disk.py @@ -19,9 +19,7 @@ class MesosDisk(disk.BaseDisk): - """ - Base class for Mesos Disks. - """ + """Base class for Mesos Disks.""" def __init__(self, disk_spec): super(MesosDisk, self).__init__(disk_spec) @@ -44,9 +42,10 @@ def GetDevicePath(self): class LocalDisk(MesosDisk): - """ - Represents DAS (direct-attached storage). For each disk, a new directory - on a host is created and then mounted as a volume to a Docker instance. + """Represents DAS (direct-attached storage). + + For each disk, a new directory on a host is created and then mounted as a + volume to a Docker instance. """ def __init__(self, disk_num, disk_spec, name): @@ -58,8 +57,5 @@ def AttachVolumeInfo(self, container_body): # create local volumes automatically - when empty hostPath is passed to # Marathon's API then the sandbox path is used as a host path. However, # new directory should be created and used for this specific purpose. - volume_param = { - 'key': 'volume', - 'value': self.mount_point - } + volume_param = {'key': 'volume', 'value': self.mount_point} container_body['docker']['parameters'].append(volume_param) diff --git a/perfkitbenchmarker/providers/mesos/mesos_docker_instance.py b/perfkitbenchmarker/providers/mesos/mesos_docker_instance.py index f58c8d5383..662c622b6b 100644 --- a/perfkitbenchmarker/providers/mesos/mesos_docker_instance.py +++ b/perfkitbenchmarker/providers/mesos/mesos_docker_instance.py @@ -38,7 +38,7 @@ class MesosDockerSpec(virtual_machine.BaseVmSpec): docker_cpus: None or float. Number of CPUs for Docker instances. docker_memory_mb: None or int. Memory limit (in MB) for Docker instances. mesos_privileged_docker: None of boolean. Indicates if Docker container - should be run in privileged mode. + should be run in privileged mode. """ CLOUD = provider_info.MESOS @@ -49,8 +49,11 @@ def _GetOptionDecoderConstructions(cls): result.update({ 'docker_cpus': (option_decoders.FloatDecoder, {'default': 1}), 'docker_memory_mb': (option_decoders.IntDecoder, {'default': 2048}), - 'mesos_privileged_docker': (option_decoders.BooleanDecoder, - {'default': False})}) + 'mesos_privileged_docker': ( + option_decoders.BooleanDecoder, + {'default': False}, + ), + }) return result def _ApplyFlags(self, config_values, flag_values): @@ -60,14 +63,13 @@ def _ApplyFlags(self, config_values, flag_values): if flag_values['docker_memory_mb'].present: config_values['docker_memory_mb'] = flag_values.docker_memory_mb if flag_values['mesos_privileged_docker'].present: - config_values['mesos_privileged_docker'] =\ + config_values['mesos_privileged_docker'] = ( flag_values.mesos_privileged_docker + ) class MesosDockerInstance(virtual_machine.BaseVirtualMachine): - """ - Represents a Docker instance spawned by Marathon framework on a Mesos cluster - """ + """Represents a Docker instance spawned by Marathon framework on a Mesos cluster""" CLOUD = provider_info.MESOS @@ -77,10 +79,11 @@ def __init__(self, vm_spec): self.cpus = vm_spec.docker_cpus self.memory_mb = vm_spec.docker_memory_mb self.privileged = vm_spec.mesos_privileged_docker - self.api_url = six.moves.urllib.parse.urljoin(FLAGS.marathon_address, - MARATHON_API_PREFIX) + self.api_url = six.moves.urllib.parse.urljoin( + FLAGS.marathon_address, MARATHON_API_PREFIX + ) self.app_url = six.moves.urllib.parse.urljoin(self.api_url, self.name) - auth = FLAGS.marathon_auth.split(":") + auth = FLAGS.marathon_auth.split(':') if len(auth) == 2: self.auth = HTTPBasicAuth(auth[0], auth[1]) else: @@ -102,60 +105,61 @@ def _Delete(self): self._DeleteApp() def _CheckPrerequisites(self): - """ - Exits if any of the prerequisites is not met. - """ + """Exits if any of the prerequisites is not met.""" if self.disk_specs and self.disk_specs[0].disk_type == disk.STANDARD: - raise Exception('Currently only local disks are supported. Please ' - 're-run the benchmark with "--data_disk_type=local".') + raise Exception( + 'Currently only local disks are supported. Please ' + 're-run the benchmark with "--data_disk_type=local".' + ) if not FLAGS.marathon_address: - raise Exception('Please provide the address and port of Marathon ' - 'framework. Example: 10:20:30:40:8080') + raise Exception( + 'Please provide the address and port of Marathon ' + 'framework. Example: 10:20:30:40:8080' + ) def _CreateVolumes(self): - """ - Creates volumes for scratch disks. - """ + """Creates volumes for scratch disks.""" for disk_num, disk_spec in enumerate(self.disk_specs): if disk_spec.disk_type == disk.LOCAL: scratch_disk = LocalDisk(disk_num, disk_spec, self.name) else: - raise Exception('Currently only local disks are supported. Please ' - 're-run the benchmark with "--data_disk_type=local"') + raise Exception( + 'Currently only local disks are supported. Please ' + 're-run the benchmark with "--data_disk_type=local"' + ) scratch_disk._Create() self.scratch_disks.append(scratch_disk) def _CreateApp(self): - """ - Creates Marathon's App (Docker instance). - """ - logging.info("Attempting to create App: %s" % self.name) + """Creates Marathon's App (Docker instance).""" + logging.info('Attempting to create App: %s' % self.name) body = self._BuildAppBody() headers = {'content-type': 'application/json'} - output = requests.post(self.api_url, data=body, headers=headers, - auth=self.auth) + output = requests.post( + self.api_url, data=body, headers=headers, auth=self.auth + ) if output.status_code != requests.codes.CREATED: - raise Exception("Unable to create App: %s" % output.text) - logging.info("App %s created successfully." % self.name) + raise Exception('Unable to create App: %s' % output.text) + logging.info('App %s created successfully.' % self.name) @vm_util.Retry(poll_interval=10, max_retries=600, log_errors=False) def _WaitForBootCompletion(self): - """ - Periodically asks Marathon if the instance is already running. - """ - logging.info("Waiting for App %s to get up and running. It may take a while" - " if a Docker image is being downloaded for the first time." - % self.name) + """Periodically asks Marathon if the instance is already running.""" + logging.info( + 'Waiting for App %s to get up and running. It may take a while' + ' if a Docker image is being downloaded for the first time.' + % self.name + ) output = requests.get(self.app_url, auth=self.auth) output = json.loads(output.text) tasks_running = output['app']['tasksRunning'] if not tasks_running: - raise Exception("Container is not booted yet. Retrying.") + raise Exception('Container is not booted yet. Retrying.') @vm_util.Retry(poll_interval=10, max_retries=100, log_errors=True) def _SetupSSH(self): - """ - Setup SSH connection details for each instance: + """Setup SSH connection details for each instance: + - IP address of the instance is the address of a host which instance is running on, - SSH port is drawn by Marathon and is unique for each instance. @@ -164,18 +168,22 @@ def _SetupSSH(self): output = json.loads(output.text) tasks = output['app']['tasks'] if not tasks or not tasks[0]['ports']: - raise Exception("Unable to figure out where the container is running." - "Retrying to retrieve host and port.") + raise Exception( + 'Unable to figure out where the container is running.' + 'Retrying to retrieve host and port.' + ) self.ip_address = tasks[0]['host'] self.ssh_port = tasks[0]['ports'][0] - internal_ip, _ = self.RemoteCommand("ifconfig eth0 | grep 'inet addr' | awk" - " -F: '{print $2}' | awk '{print $1}'") + internal_ip, _ = self.RemoteCommand( + "ifconfig eth0 | grep 'inet addr' | awk" + " -F: '{print $2}' | awk '{print $1}'" + ) self.internal_ip = internal_ip.rstrip() @vm_util.Retry(poll_interval=10, max_retries=100, log_errors=True) def _ConfigureProxy(self): - """ - In Docker containers environment variables from /etc/environment + """In Docker containers environment variables from /etc/environment + are not sourced - this results in connection problems when running behind proxy. Prepending proxy environment variables to bashrc solves the problem. Note: APPENDING to bashrc will not work because @@ -193,26 +201,26 @@ def _ConfigureProxy(self): @vm_util.Retry(poll_interval=10, max_retries=100, log_errors=True) def _DeleteApp(self): - """ - Deletes an App. - """ + """Deletes an App.""" logging.info('Attempting to delete App: %s' % self.name) output = requests.delete(self.app_url, auth=self.auth) if output.status_code == requests.codes.NOT_FOUND: logging.info('App %s has been already deleted.' % self.name) return if output.status_code != requests.codes.OK: - raise Exception("Deleting App: %s failed. Reattempting." % self.name) + raise Exception('Deleting App: %s failed. Reattempting.' % self.name) def _BuildAppBody(self): - """ - Builds JSON which will be passed as a body of POST request to Marathon + """Builds JSON which will be passed as a body of POST request to Marathon + API in order to create App. """ cat_cmd = ['cat', vm_util.GetPublicKeyPath()] key_file, _ = vm_util.IssueRetryableCommand(cat_cmd) - cmd = "/bin/mkdir /root/.ssh; echo '%s' >> /root/.ssh/authorized_keys; " \ - "/usr/sbin/sshd -D" % key_file + cmd = ( + "/bin/mkdir /root/.ssh; echo '%s' >> /root/.ssh/authorized_keys; " + '/usr/sbin/sshd -D' % key_file + ) body = { 'id': self.name, 'mem': self.memory_mb, @@ -223,17 +231,15 @@ def _BuildAppBody(self): 'docker': { 'image': self.image, 'network': 'BRIDGE', - 'portMappings': [ - { - 'containerPort': 22, - 'hostPort': 0, - 'protocol': 'tcp' - } - ], + 'portMappings': [{ + 'containerPort': 22, + 'hostPort': 0, + 'protocol': 'tcp', + }], 'privileged': self.privileged, - 'parameters': [{'key': 'hostname', 'value': self.name}] - } - } + 'parameters': [{'key': 'hostname', 'value': self.name}], + }, + }, } for scratch_disk in self.scratch_disks: diff --git a/perfkitbenchmarker/providers/mesos/provider_info.py b/perfkitbenchmarker/providers/mesos/provider_info.py index 14445f4381..2c24a45ba6 100644 --- a/perfkitbenchmarker/providers/mesos/provider_info.py +++ b/perfkitbenchmarker/providers/mesos/provider_info.py @@ -12,21 +12,32 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Provider info for Mesos. - -""" +"""Provider info for Mesos.""" from perfkitbenchmarker import provider_info class MesosProviderInfo(provider_info.BaseProviderInfo): - - SUPPORTED_BENCHMARKS = ['block_storage_workload', - 'cassandra_stress', 'cassandra_ycsb', 'cluster_boot', - 'copy_throughput', 'fio', 'hbase_ycsb', 'hpcc', - 'iperf', 'mongodb_ycsb', 'netperf', 'oldisim', 'ping', - 'redis', 'redis_ycsb', 'scimark2', 'silo', - 'sysbench_oltp'] + SUPPORTED_BENCHMARKS = [ + 'block_storage_workload', + 'cassandra_stress', + 'cassandra_ycsb', + 'cluster_boot', + 'copy_throughput', + 'fio', + 'hbase_ycsb', + 'hpcc', + 'iperf', + 'mongodb_ycsb', + 'netperf', + 'oldisim', + 'ping', + 'redis', + 'redis_ycsb', + 'scimark2', + 'silo', + 'sysbench_oltp', + ] UNSUPPORTED_BENCHMARKS = ['bonnieplusplus', 'mysql_service'] CLOUD = provider_info.MESOS diff --git a/perfkitbenchmarker/providers/openstack/flags.py b/perfkitbenchmarker/providers/openstack/flags.py index f9785502d2..5509ec9100 100644 --- a/perfkitbenchmarker/providers/openstack/flags.py +++ b/perfkitbenchmarker/providers/openstack/flags.py @@ -14,61 +14,93 @@ from absl import flags -flags.DEFINE_string('openstack_cli_path', - default='openstack', - help='The path to the OpenStack CLI binary.') +flags.DEFINE_string( + 'openstack_cli_path', + default='openstack', + help='The path to the OpenStack CLI binary.', +) -flags.DEFINE_list('openstack_additional_flags', - default=[], - help='Additional comma separated flags to pass to every ' - 'OpenStack CLI command. See "openstack --help" for ' - 'more.') +flags.DEFINE_list( + 'openstack_additional_flags', + default=[], + help=( + 'Additional comma separated flags to pass to every ' + 'OpenStack CLI command. See "openstack --help" for ' + 'more.' + ), +) -flags.DEFINE_string('openstack_public_network', None, - '(DEPRECATED: Use openstack_floating_ip_pool) ' - 'Name of OpenStack public network.') +flags.DEFINE_string( + 'openstack_public_network', + None, + '(DEPRECATED: Use openstack_floating_ip_pool) ' + 'Name of OpenStack public network.', +) -flags.DEFINE_string('openstack_private_network', 'private', - '(DEPRECATED: Use openstack_network) ' - 'Name of OpenStack private network.') +flags.DEFINE_string( + 'openstack_private_network', + 'private', + '(DEPRECATED: Use openstack_network) Name of OpenStack private network.', +) -flags.DEFINE_string('openstack_network', 'private', - 'Name of OpenStack network. This network provides ' - 'automatically allocated fixed-IP addresses to attached ' - 'instances. Typically, this network is used for internal ' - 'communication between instances. ' - 'If openstack_floating_ip_pool is not ' - 'set then this network will be used to communicate with ' - 'the instance.') +flags.DEFINE_string( + 'openstack_network', + 'private', + 'Name of OpenStack network. This network provides ' + 'automatically allocated fixed-IP addresses to attached ' + 'instances. Typically, this network is used for internal ' + 'communication between instances. ' + 'If openstack_floating_ip_pool is not ' + 'set then this network will be used to communicate with ' + 'the instance.', +) -flags.DEFINE_string('openstack_floating_ip_pool', None, - 'Name of OpenStack floating IP-address pool. If set, ' - 'a floating-ip address from this pool will be associated' - 'to each instance and will be used for communicating ' - 'with it. To use this flag, an internally routable network ' - 'must also be specified via the openstack_network flag.') +flags.DEFINE_string( + 'openstack_floating_ip_pool', + None, + 'Name of OpenStack floating IP-address pool. If set, ' + 'a floating-ip address from this pool will be associated' + 'to each instance and will be used for communicating ' + 'with it. To use this flag, an internally routable network ' + 'must also be specified via the openstack_network flag.', +) -flags.DEFINE_boolean('openstack_config_drive', False, - 'Add possibilities to get metadata from external drive') +flags.DEFINE_boolean( + 'openstack_config_drive', + False, + 'Add possibilities to get metadata from external drive', +) -flags.DEFINE_boolean('openstack_boot_from_volume', False, - 'Boot from volume instead of an image') +flags.DEFINE_boolean( + 'openstack_boot_from_volume', False, 'Boot from volume instead of an image' +) -flags.DEFINE_integer('openstack_volume_size', None, - '(DEPRECATED: Use data_disk_size) ' - 'Size of the volume (GB).') +flags.DEFINE_integer( + 'openstack_volume_size', + None, + '(DEPRECATED: Use data_disk_size) Size of the volume (GB).', +) -flags.DEFINE_string('openstack_volume_type', - default=None, - help='Optional Cinder volume type to use.') +flags.DEFINE_string( + 'openstack_volume_type', + default=None, + help='Optional Cinder volume type to use.', +) -flags.DEFINE_string('openstack_image_username', 'ubuntu', - 'Ssh username for cloud image') +flags.DEFINE_string( + 'openstack_image_username', 'ubuntu', 'Ssh username for cloud image' +) NONE = 'None' -flags.DEFINE_enum('openstack_scheduler_policy', NONE, - [NONE, 'affinity', 'anti-affinity'], - 'Add possibility to use affinity or anti-affinity ' - 'policy in scheduling process') -flags.DEFINE_string('openstack_post_provisioning_script', None, - 'post provisioning script file path') +flags.DEFINE_enum( + 'openstack_scheduler_policy', + NONE, + [NONE, 'affinity', 'anti-affinity'], + 'Add possibility to use affinity or anti-affinity ' + 'policy in scheduling process', +) +flags.DEFINE_string( + 'openstack_post_provisioning_script', + None, + 'post provisioning script file path', +) diff --git a/perfkitbenchmarker/providers/openstack/os_disk.py b/perfkitbenchmarker/providers/openstack/os_disk.py index f6c10597e1..9c3ebb6d20 100644 --- a/perfkitbenchmarker/providers/openstack/os_disk.py +++ b/perfkitbenchmarker/providers/openstack/os_disk.py @@ -44,8 +44,9 @@ def CreateBootVolume(resource, name, image): vol_cmd = os_utils.OpenStackCLICommand(resource, 'volume', 'create', name) vol_cmd.flags['availability-zone'] = resource.zone vol_cmd.flags['image'] = image - vol_cmd.flags['size'] = (resource.disk_size or - GetImageMinDiskSize(resource, image)) + vol_cmd.flags['size'] = resource.disk_size or GetImageMinDiskSize( + resource, image + ) stdout, _, _ = vol_cmd.Issue() vol_resp = json.loads(stdout) return vol_resp @@ -56,21 +57,29 @@ def GetImageMinDiskSize(resource, image): image_cmd = os_utils.OpenStackCLICommand(resource, 'image', 'show', image) stdout, _, _ = image_cmd.Issue() image_resp = json.loads(stdout) - volume_size = max((int(image_resp['min_disk']), - resource.disk_size,)) + volume_size = max(( + int(image_resp['min_disk']), + resource.disk_size, + )) return volume_size def DeleteVolume(resource, volume_id): """Deletes a remote (Cinder) block volume.""" - vol_cmd = os_utils.OpenStackCLICommand(resource, 'volume', 'delete', - volume_id) + vol_cmd = os_utils.OpenStackCLICommand( + resource, 'volume', 'delete', volume_id + ) del vol_cmd.flags['format'] # volume delete does not support json output vol_cmd.Issue() -@vm_util.Retry(poll_interval=5, max_retries=-1, timeout=300, log_errors=False, - retryable_exceptions=errors.Resource.RetryableCreationError) +@vm_util.Retry( + poll_interval=5, + max_retries=-1, + timeout=300, + log_errors=False, + retryable_exceptions=errors.Resource.RetryableCreationError, +) def WaitForVolumeCreation(resource, volume_id): """Waits until volume is available""" vol_cmd = os_utils.OpenStackCLICommand(resource, 'volume', 'show', volume_id) @@ -88,9 +97,8 @@ class OpenStackDiskSpec(disk.BaseDiskSpec): Attributes: disk_size: None or int. Size of the disk in GB. - volume_type: None or string. Volume type to be used to create a - block storage volume. - + volume_type: None or string. Volume type to be used to create a block + storage volume. """ CLOUD = provider_info.OPENSTACK @@ -102,14 +110,16 @@ def _ApplyFlags(cls, config_values, flag_values): Can be overridden by derived classes to add support for specific flags. Args: - config_values: dict mapping config option names to provided values. May - be modified by this function. + config_values: dict mapping config option names to provided values. May be + modified by this function. flag_values: flags.FlagValues. Runtime flags that may override the - provided config values. + provided config values. """ super(OpenStackDiskSpec, cls)._ApplyFlags(config_values, flag_values) - if (flag_values['openstack_volume_size'].present - and not flag_values['data_disk_size'].present): + if ( + flag_values['openstack_volume_size'].present + and not flag_values['data_disk_size'].present + ): config_values['disk_size'] = flag_values.openstack_volume_size else: config_values['disk_size'] = flag_values.data_disk_size @@ -119,12 +129,12 @@ def _ApplyFlags(cls, config_values, flag_values): @classmethod def _GetOptionDecoderConstructions(cls): decoders = super(OpenStackDiskSpec, cls)._GetOptionDecoderConstructions() - decoders.update( - { - 'volume_type': (option_decoders.StringDecoder, - {'default': None, 'none_ok': True},) - } - ) + decoders.update({ + 'volume_type': ( + option_decoders.StringDecoder, + {'default': None, 'none_ok': True}, + ) + }) return decoders @@ -173,18 +183,26 @@ def _AttachVolume(self, vm): if self.id is None: raise errors.Error('Cannot attach remote volume %s' % self.name) if vm.id is None: - msg = 'Cannot attach remote volume %s to non-existing %s VM' % (self.name, - vm.name) + msg = 'Cannot attach remote volume %s to non-existing %s VM' % ( + self.name, + vm.name, + ) raise errors.Error(msg) cmd = os_utils.OpenStackCLICommand( - self, 'server', 'add', 'volume', vm.id, self.id) + self, 'server', 'add', 'volume', vm.id, self.id + ) del cmd.flags['format'] _, stderr, _ = cmd.Issue() if stderr: raise errors.Error(stderr) - @vm_util.Retry(poll_interval=1, max_retries=-1, timeout=300, log_errors=False, - retryable_exceptions=errors.Resource.RetryableCreationError) + @vm_util.Retry( + poll_interval=1, + max_retries=-1, + timeout=300, + log_errors=False, + retryable_exceptions=errors.Resource.RetryableCreationError, + ) def _WaitForVolumeAttachment(self, vm): if self.id is None: return @@ -214,14 +232,20 @@ def _DetachVolume(self): if self.attached_vm_id is None: raise errors.Error('Cannot detach remote volume from a non-existing VM.') cmd = os_utils.OpenStackCLICommand( - self, 'server', 'remove', 'volume', self.attached_vm_id, self.id) + self, 'server', 'remove', 'volume', self.attached_vm_id, self.id + ) del cmd.flags['format'] _, stderr, _ = cmd.Issue() if stderr: raise errors.Error(stderr) - @vm_util.Retry(poll_interval=1, max_retries=-1, timeout=300, log_errors=False, - retryable_exceptions=errors.Resource.RetryableDeletionError) + @vm_util.Retry( + poll_interval=1, + max_retries=-1, + timeout=300, + log_errors=False, + retryable_exceptions=errors.Resource.RetryableDeletionError, + ) def _WaitForVolumeDeletion(self): if self.id is None: return @@ -230,7 +254,14 @@ def _WaitForVolumeDeletion(self): if stderr.strip(): return # Volume could not be found, inferred that has been deleted. resp = json.loads(stdout) - if resp['status'] in ('building', 'available', 'in-use', 'deleting',): - msg = ('Volume %s has not yet been deleted. Retrying to check status.' - % self.id) + if resp['status'] in ( + 'building', + 'available', + 'in-use', + 'deleting', + ): + msg = ( + 'Volume %s has not yet been deleted. Retrying to check status.' + % self.id + ) raise errors.Resource.RetryableDeletionError(msg) diff --git a/perfkitbenchmarker/providers/openstack/os_network.py b/perfkitbenchmarker/providers/openstack/os_network.py index d7de572994..ec117efbc0 100644 --- a/perfkitbenchmarker/providers/openstack/os_network.py +++ b/perfkitbenchmarker/providers/openstack/os_network.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""""Module containing classes related to OpenStack Networking.""" +""" "Module containing classes related to OpenStack Networking.""" from collections import namedtuple import json @@ -37,7 +37,11 @@ FLOATING_IP_ID = 'id' FLOATING_NETWORK_ID = 'floating_network_id' -FLOATING_IP_KEYS = (FLOATING_IP_ADDRESS, FLOATING_IP_ID, FLOATING_NETWORK_ID,) +FLOATING_IP_KEYS = ( + FLOATING_IP_ADDRESS, + FLOATING_IP_ID, + FLOATING_NETWORK_ID, +) FLAGS = flags.FLAGS @@ -49,9 +53,7 @@ class OpenStackFirewall(network.BaseFirewall): - """ - An object representing OpenStack Firewall based on Secure Groups. - """ + """An object representing OpenStack Firewall based on Secure Groups.""" CLOUD = provider_info.OPENSTACK @@ -60,17 +62,20 @@ def __init__(self): self.sec_group_rules_set = set() with self._lock: - cmd = utils.OpenStackCLICommand(self, OSC_SEC_GROUP_CMD, 'show', - SC_GROUP_NAME) + cmd = utils.OpenStackCLICommand( + self, OSC_SEC_GROUP_CMD, 'show', SC_GROUP_NAME + ) stdout, stderr, _ = cmd.Issue() if stderr: - cmd = utils.OpenStackCLICommand(self, OSC_SEC_GROUP_CMD, 'create', - SC_GROUP_NAME) + cmd = utils.OpenStackCLICommand( + self, OSC_SEC_GROUP_CMD, 'create', SC_GROUP_NAME + ) del cmd.flags['format'] # Command does not support json output cmd.Issue() def AllowICMP(self, vm, icmp_type=-1, icmp_code=-1, source_range=None): """Creates a Security Group Rule on the Firewall to allow/disallow + ICMP traffic. Args: @@ -86,8 +91,9 @@ def AllowICMP(self, vm, icmp_type=-1, icmp_code=-1, source_range=None): with self._lock: if sec_group_rule in self.sec_group_rules_set: return - cmd = utils.OpenStackCLICommand(vm, OSC_SEC_GROUP_RULE_CMD, 'create', - vm.group_id) + cmd = utils.OpenStackCLICommand( + vm, OSC_SEC_GROUP_RULE_CMD, 'create', vm.group_id + ) if source_range: cmd.flags['src-ip'] = source_range cmd.flags['dst-port'] = str(icmp_type) @@ -97,6 +103,7 @@ def AllowICMP(self, vm, icmp_type=-1, icmp_code=-1, source_range=None): def AllowPort(self, vm, start_port, end_port=None, source_range=None): """Creates a Security Group Rule on the Firewall to allow for both TCP + and UDP network traffic on given port, or port range. Args: @@ -117,12 +124,16 @@ def AllowPort(self, vm, start_port, end_port=None, source_range=None): with self._lock: if sec_group_rule in self.sec_group_rules_set: return - cmd = utils.OpenStackCLICommand(vm, OSC_SEC_GROUP_RULE_CMD, 'create', - vm.group_id) + cmd = utils.OpenStackCLICommand( + vm, OSC_SEC_GROUP_RULE_CMD, 'create', vm.group_id + ) if source_range: cmd.flags['src-ip'] = source_range cmd.flags['dst-port'] = '%d:%d' % (start_port, end_port) - for prot in (TCP, UDP,): + for prot in ( + TCP, + UDP, + ): cmd.flags['proto'] = prot cmd.Issue() self.sec_group_rules_set.add(sec_group_rule) @@ -133,7 +144,6 @@ def DisallowAllPorts(self): class OpenStackFloatingIPPool(object): - _floating_ip_lock = threading.Lock() # Guards floating IP allocation/release def __init__(self, floating_network_id): @@ -142,8 +152,12 @@ def __init__(self, floating_network_id): def associate(self, vm): with self._floating_ip_lock: floating_ip_obj = self._get_or_create(vm) - cmd = utils.OpenStackCLICommand(vm, 'server add floating ip', vm.id, - floating_ip_obj.floating_ip_address) + cmd = utils.OpenStackCLICommand( + vm, + 'server add floating ip', + vm.id, + floating_ip_obj.floating_ip_address, + ) del cmd.flags['format'] # Command does not support json output format _, stderr, _ = cmd.Issue() if stderr: @@ -158,43 +172,49 @@ def _get_or_create(self, vm): floating_ip_dict_list = json.loads(stdout) for floating_ip_dict in floating_ip_dict_list: - if (floating_ip_dict['Floating Network'] == self.floating_network_id - and floating_ip_dict['Port'] is None): + if ( + floating_ip_dict['Floating Network'] == self.floating_network_id + and floating_ip_dict['Port'] is None + ): # Due to inconsistent output, we need to convert the keys floating_ip_obj = OpenStackFloatingIP( floating_ip_address=floating_ip_dict['Floating IP Address'], floating_network_id=floating_ip_dict['Floating Network'], - id=floating_ip_dict['ID'] + id=floating_ip_dict['ID'], ) return floating_ip_obj return self._allocate(vm) def _allocate(self, vm): - cmd = utils.OpenStackCLICommand(vm, OSC_FLOATING_IP_CMD, 'create', - self.floating_network_id) + cmd = utils.OpenStackCLICommand( + vm, OSC_FLOATING_IP_CMD, 'create', self.floating_network_id + ) stdout, stderr, _ = cmd.Issue() if stderr.strip(): # Strip spaces raise errors.Config.InvalidValue( 'Could not allocate a floating ip from the floating network "%s".' - % self.floating_network_id) + % self.floating_network_id + ) floating_ip_dict = json.loads(stdout) # Extract subset of returned keys floating_ip_obj = OpenStackFloatingIP( floating_ip_address=floating_ip_dict['floating_ip_address'], floating_network_id=floating_ip_dict['floating_network_id'], - id=floating_ip_dict['id'] + id=floating_ip_dict['id'], ) return floating_ip_obj def release(self, vm, floating_ip_obj): - cmd = utils.OpenStackCLICommand(vm, OSC_FLOATING_IP_CMD, 'show', - floating_ip_obj.id) + cmd = utils.OpenStackCLICommand( + vm, OSC_FLOATING_IP_CMD, 'show', floating_ip_obj.id + ) stdout, stderr, _ = cmd.Issue() if stderr: return # Not found, moving on floating_ip_dict = json.loads(stdout) with self._floating_ip_lock: - delete_cmd = utils.OpenStackCLICommand(vm, OSC_FLOATING_IP_CMD, 'delete', - floating_ip_dict['id']) + delete_cmd = utils.OpenStackCLICommand( + vm, OSC_FLOATING_IP_CMD, 'delete', floating_ip_dict['id'] + ) del delete_cmd.flags['format'] # Command not support json output format stdout, stderr, _ = delete_cmd.Issue() diff --git a/perfkitbenchmarker/providers/openstack/os_virtual_machine.py b/perfkitbenchmarker/providers/openstack/os_virtual_machine.py index 2fee693304..ae873d598e 100644 --- a/perfkitbenchmarker/providers/openstack/os_virtual_machine.py +++ b/perfkitbenchmarker/providers/openstack/os_virtual_machine.py @@ -72,10 +72,12 @@ def __init__(self, vm_spec): self.image = self.image or self.DEFAULT_IMAGE # FIXME(meteorfox): Remove --openstack_public_network and # --openstack_private_network once depreciation time has expired - self.network_name = (FLAGS.openstack_network or - FLAGS.openstack_private_network) - self.floating_ip_pool_name = (FLAGS.openstack_floating_ip_pool or - FLAGS.openstack_public_network) + self.network_name = ( + FLAGS.openstack_network or FLAGS.openstack_private_network + ) + self.floating_ip_pool_name = ( + FLAGS.openstack_floating_ip_pool or FLAGS.openstack_public_network + ) self.id = None self.boot_volume_id = None self.server_group_id = None @@ -95,11 +97,13 @@ def _CreateDependencies(self): self._CheckPrerequisites() self.firewall = os_network.OpenStackFirewall.GetFirewall() self.public_network = os_network.OpenStackFloatingIPPool( - OpenStackVirtualMachine.floating_network_id) + OpenStackVirtualMachine.floating_network_id + ) self._UploadSSHPublicKey() source_range = self._GetInternalNetworkCIDR() - self.firewall.AllowPort(self, os_network.MIN_PORT, os_network.MAX_PORT, - source_range) + self.firewall.AllowPort( + self, os_network.MIN_PORT, os_network.MAX_PORT, source_range + ) self.firewall.AllowICMP(self) # Allowing ICMP traffic (i.e. ping) self.AllowRemoteAccessPorts() @@ -130,7 +134,9 @@ def _Delete(self): def _DeleteDependencies(self): """Delete dependencies that were needed for the VM after the VM has been - deleted.""" + + deleted. + """ self._DeleteSSHPublicKey() def _Exists(self): @@ -165,7 +171,8 @@ def _CheckCanaryCommand(self): if stderr: raise errors.Config.InvalidValue( 'OpenStack CLI test command failed. Please make sure the OpenStack ' - 'CLI client is installed and properly configured') + 'CLI client is installed and properly configured' + ) OpenStackVirtualMachine.command_works = True def _CheckPrerequisites(self): @@ -191,8 +198,9 @@ def _CheckImage(self): def _CheckFlavor(self): """Tries to get flavor, if found continues execution otherwise aborts.""" - cmd = os_utils.OpenStackCLICommand(self, 'flavor', 'show', - self.machine_type) + cmd = os_utils.OpenStackCLICommand( + self, 'flavor', 'show', self.machine_type + ) err_msg = VALIDATION_ERROR_MESSAGE.format('Machine type', self.machine_type) self._IssueCommandCheck(cmd, err_msg) @@ -200,32 +208,39 @@ def _CheckNetworks(self): """Tries to get network, if found continues execution otherwise aborts.""" if not self.network_name: if self.floating_ip_pool_name: - msg = ('Cannot associate floating-ip address from pool %s without ' - 'an internally routable network. Make sure ' - '--openstack_network flag is set.') + msg = ( + 'Cannot associate floating-ip address from pool %s without ' + 'an internally routable network. Make sure ' + '--openstack_network flag is set.' + ) else: - msg = ('Cannot build instance without a network. Make sure to set ' - 'either just --openstack_network or both ' - '--openstack_network and --openstack_floating_ip_pool flags.') + msg = ( + 'Cannot build instance without a network. Make sure to set ' + 'either just --openstack_network or both ' + '--openstack_network and --openstack_floating_ip_pool flags.' + ) raise errors.Error(msg) self._CheckNetworkExists(self.network_name) if self.floating_ip_pool_name: floating_network_dict = self._CheckFloatingIPNetworkExists( - self.floating_ip_pool_name) + self.floating_ip_pool_name + ) OpenStackVirtualMachine.floating_network_id = floating_network_dict['id'] def _CheckFloatingIPNetworkExists(self, floating_network_name_or_id): network = self._CheckNetworkExists(floating_network_name_or_id) if network['router:external'] not in ('External', True): - raise errors.Config.InvalidValue('Network "%s" is not External' - % self.floating_ip_pool_name) + raise errors.Config.InvalidValue( + 'Network "%s" is not External' % self.floating_ip_pool_name + ) return network def _CheckNetworkExists(self, network_name_or_id): - cmd = os_utils.OpenStackCLICommand(self, 'network', 'show', - network_name_or_id) + cmd = os_utils.OpenStackCLICommand( + self, 'network', 'show', network_name_or_id + ) err_msg = VALIDATION_ERROR_MESSAGE.format('Network', network_name_or_id) stdout = self._IssueCommandCheck(cmd, err_msg) network = json.loads(stdout) @@ -233,6 +248,7 @@ def _CheckNetworkExists(self, network_name_or_id): def _IssueCommandCheck(self, cmd, err_msg=None): """Issues command and, if stderr is non-empty, raises an error message + Args: cmd: The command to be issued. err_msg: string. Error message if command fails. @@ -249,8 +265,9 @@ def _UploadSSHPublicKey(self): with self._lock: if self.zone in self.uploaded_keypair_set: return - cmd = os_utils.OpenStackCLICommand(self, 'keypair', 'create', - self.key_name) + cmd = os_utils.OpenStackCLICommand( + self, 'keypair', 'create', self.key_name + ) cmd.flags['public-key'] = self.ssh_public_key cmd.IssueRetryable() self.uploaded_keypair_set.add(self.zone) @@ -262,8 +279,9 @@ def _DeleteSSHPublicKey(self): with self._lock: if self.zone in self.deleted_keypair_set: return - cmd = os_utils.OpenStackCLICommand(self, 'keypair', 'delete', - self.key_name) + cmd = os_utils.OpenStackCLICommand( + self, 'keypair', 'delete', self.key_name + ) del cmd.flags['format'] # keypair delete does not support json output cmd.Issue() self.deleted_keypair_set.add(self.zone) @@ -323,8 +341,9 @@ def _GetSchedulerHints(self): return hint_temp % server_group['id'] def _CreateServerGroup(self, group_name): - cmd = os_utils.OpenStackCLICommand(self, 'server group', 'create', - group_name) + cmd = os_utils.OpenStackCLICommand( + self, 'server group', 'create', group_name + ) cmd.flags['policy'] = FLAGS.openstack_scheduler_policy stdout, stderr, _ = cmd.Issue() if stderr: @@ -336,8 +355,9 @@ def _DeleteServerGroup(self): with self._lock: if self.zone in self.deleted_server_group_set: return - cmd = os_utils.OpenStackCLICommand(self, 'server group', 'delete', - self.server_group_id) + cmd = os_utils.OpenStackCLICommand( + self, 'server group', 'delete', self.server_group_id + ) del cmd.flags['format'] # delete does not support json output cmd.Issue() self.deleted_server_group_set.add(self.zone) @@ -370,8 +390,9 @@ def _GetNetworkIPAddress(self, server_dict, network_name): def _GetInternalNetworkCIDR(self): """Returns IP addresses source range of internal network.""" - net_cmd = os_utils.OpenStackCLICommand(self, 'network', 'show', - self.network_name) + net_cmd = os_utils.OpenStackCLICommand( + self, 'network', 'show', self.network_name + ) net_stdout, _, _ = net_cmd.Issue() network = json.loads(net_stdout) @@ -380,24 +401,29 @@ def _GetInternalNetworkCIDR(self): else: self.subnet_id = network['subnets'] - subnet_cmd = os_utils.OpenStackCLICommand(self, 'subnet', 'show', - self.subnet_id) + subnet_cmd = os_utils.OpenStackCLICommand( + self, 'subnet', 'show', self.subnet_id + ) stdout, _, _ = subnet_cmd.Issue() subnet_dict = json.loads(stdout) return subnet_dict['cidr'] def _AllocateFloatingIP(self): floating_ip = self.public_network.associate(self) - logging.info('floating-ip associated: {}'.format( - floating_ip.floating_ip_address)) + logging.info( + 'floating-ip associated: {}'.format(floating_ip.floating_ip_address) + ) return floating_ip def CreateScratchDisk(self, _, disk_spec): - disks_names = ('%s_data_%d_%d' - % (self.name, len(self.scratch_disks), i) - for i in range(disk_spec.num_striped_disks)) - disks = [os_disk.OpenStackDisk(disk_spec, name, self.zone) - for name in disks_names] + disks_names = ( + '%s_data_%d_%d' % (self.name, len(self.scratch_disks), i) + for i in range(disk_spec.num_striped_disks) + ) + disks = [ + os_disk.OpenStackDisk(disk_spec, name, self.zone) + for name in disks_names + ] scratch_disk = self._CreateScratchDiskFromDisks(disk_spec, disks) disk_strategies.PrepareScratchDiskStrategy().PrepareScratchDisk( @@ -416,16 +442,19 @@ def GetResourceMetadata(self): return result -class Rhel7BasedOpenStackVirtualMachine(OpenStackVirtualMachine, - linux_virtual_machine.Rhel7Mixin): +class Rhel7BasedOpenStackVirtualMachine( + OpenStackVirtualMachine, linux_virtual_machine.Rhel7Mixin +): DEFAULT_IMAGE = 'rhel-7.2' -class CentOs7BasedOpenStackVirtualMachine(OpenStackVirtualMachine, - linux_virtual_machine.CentOs7Mixin): +class CentOs7BasedOpenStackVirtualMachine( + OpenStackVirtualMachine, linux_virtual_machine.CentOs7Mixin +): DEFAULT_IMAGE = 'centos7' -class ClearBasedOpenStackVirtualMachine(OpenStackVirtualMachine, - linux_virtual_machine.ClearMixin): +class ClearBasedOpenStackVirtualMachine( + OpenStackVirtualMachine, linux_virtual_machine.ClearMixin +): DEFAULT_IMAGE = 'upstream-clear' diff --git a/perfkitbenchmarker/providers/openstack/provider_info.py b/perfkitbenchmarker/providers/openstack/provider_info.py index aa22dd226b..5ef7c6e97d 100644 --- a/perfkitbenchmarker/providers/openstack/provider_info.py +++ b/perfkitbenchmarker/providers/openstack/provider_info.py @@ -12,14 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Provider info for Open Stack Platform. - -""" +"""Provider info for Open Stack Platform.""" from perfkitbenchmarker import provider_info class OpenStackProviderInfo(provider_info.BaseProviderInfo): - UNSUPPORTED_BENCHMARKS = ['mysql_service'] CLOUD = provider_info.OPENSTACK diff --git a/perfkitbenchmarker/providers/openstack/swift.py b/perfkitbenchmarker/providers/openstack/swift.py index 7638d99c5b..e4bfe1ce4a 100644 --- a/perfkitbenchmarker/providers/openstack/swift.py +++ b/perfkitbenchmarker/providers/openstack/swift.py @@ -21,9 +21,12 @@ from perfkitbenchmarker import provider_info from perfkitbenchmarker import vm_util -flags.DEFINE_boolean('openstack_swift_insecure', False, - 'Allow swiftclient to access Swift service without \n' - 'having to verify the SSL certificate') +flags.DEFINE_boolean( + 'openstack_swift_insecure', + False, + 'Allow swiftclient to access Swift service without \n' + 'having to verify the SSL certificate', +) FLAGS = flags.FLAGS @@ -39,19 +42,27 @@ def __init__(self): self.swift_command_prefix = '' def PrepareService(self, location): - openstack_creds_set = ('OS_AUTH_URL' in os.environ, - 'OS_TENANT_NAME' in os.environ, - 'OS_USERNAME' in os.environ, - 'OS_PASSWORD' in os.environ,) + openstack_creds_set = ( + 'OS_AUTH_URL' in os.environ, + 'OS_TENANT_NAME' in os.environ, + 'OS_USERNAME' in os.environ, + 'OS_PASSWORD' in os.environ, + ) if not all(openstack_creds_set): raise errors.Benchmarks.MissingObjectCredentialException( - 'OpenStack credentials not found in environment variables') + 'OpenStack credentials not found in environment variables' + ) self.swift_command_parts = [ - '--os-auth-url', os.environ['OS_AUTH_URL'], - '--os-tenant-name', os.environ['OS_TENANT_NAME'], - '--os-username', os.environ['OS_USERNAME'], - '--os-password', os.environ['OS_PASSWORD']] + '--os-auth-url', + os.environ['OS_AUTH_URL'], + '--os-tenant-name', + os.environ['OS_TENANT_NAME'], + '--os-username', + os.environ['OS_USERNAME'], + '--os-password', + os.environ['OS_PASSWORD'], + ] if FLAGS.openstack_swift_insecure: self.swift_command_parts.append('--insecure') @@ -61,7 +72,8 @@ def MakeBucket(self, bucket, raise_on_failure=True, tag_bucket=True): del tag_bucket _, stderr, ret_code = vm_util.IssueCommand( ['swift'] + self.swift_command_parts + ['post', bucket], - raise_on_failure=False) + raise_on_failure=False, + ) if ret_code and raise_on_failure: raise errors.Benchmarks.BucketCreationError(stderr) @@ -70,7 +82,8 @@ def DeleteBucket(self, bucket): vm_util.IssueCommand( ['swift'] + self.swift_command_parts + ['delete', bucket], - raise_on_failure=False) + raise_on_failure=False, + ) def Copy(self, src_url, dst_url): """See base class.""" @@ -95,7 +108,8 @@ def List(self, buckets): def EmptyBucket(self, bucket): vm_util.IssueCommand( ['swift'] + self.swift_command_parts + ['delete', bucket], - raise_on_failure=False) + raise_on_failure=False, + ) def PrepareVM(self, vm): vm.Install('swift_client') @@ -107,13 +121,18 @@ def CleanupVM(self, vm): def CLIUploadDirectory(self, vm, directory, file_names, bucket): return vm.RemoteCommand( 'time swift %s upload %s %s' - % (self.swift_command_prefix, bucket, directory)) + % (self.swift_command_prefix, bucket, directory) + ) def CLIDownloadBucket(self, vm, bucket, objects, dest): return vm.RemoteCommand( 'time swift %s download %s -D %s' - % (self.swift_command_prefix, bucket, dest)) + % (self.swift_command_prefix, bucket, dest) + ) def Metadata(self, vm): - return {SWIFTCLIENT_LIB_VERSION: - linux_packages.GetPipPackageVersion(vm, 'python-swiftclient')} + return { + SWIFTCLIENT_LIB_VERSION: linux_packages.GetPipPackageVersion( + vm, 'python-swiftclient' + ) + } diff --git a/perfkitbenchmarker/providers/openstack/utils.py b/perfkitbenchmarker/providers/openstack/utils.py index 04d9f0ed9d..5e13e4929e 100644 --- a/perfkitbenchmarker/providers/openstack/utils.py +++ b/perfkitbenchmarker/providers/openstack/utils.py @@ -28,26 +28,27 @@ class OpenStackCLICommand(object): Attributes: args: list of strings. Positional args to pass to openstack, typically - specifying an operation to perform (e.g. ['image', 'list'] to list - available images). + specifying an operation to perform (e.g. ['image', 'list'] to list + available images). flags: OrderedDict mapping flag name string to flag value. Flags to pass to - openstack cli (e.g. {'os-compute-api-version': '2'}). If a provided - value is True, the flag is passed to openstack cli without a value. If - a provided value is a list, the flag is passed to openstack cli multiple - times, once with each value in the list. + openstack cli (e.g. {'os-compute-api-version': '2'}). If a provided value + is True, the flag is passed to openstack cli without a value. If a + provided value is a list, the flag is passed to openstack cli multiple + times, once with each value in the list. additional_flags: list of strings. Additional flags to append unmodified to - the end of the openstack cli command. + the end of the openstack cli command. """ def __init__(self, resource, *args): """Initializes an OpenStackCLICommand with the provided args and common + flags. Args: resource: An OpenStack resource of type BaseResource. *args: sequence of strings. Positional args to pass to openstack cli, - typically specifying an operation to perform (e.g. ['image', 'list'] - to list available images). + typically specifying an operation to perform (e.g. ['image', 'list'] to + list available images). """ self.args = list(args) self.flags = OrderedDict() @@ -82,7 +83,7 @@ def Issue(self, **kwargs): Args: **kwargs: Keyword arguments to forward to vm_util.IssueCommand when - issuing the openstack cli command. + issuing the openstack cli command. Returns: A tuple of stdout, stderr, and retcode from running the openstack command. @@ -96,7 +97,7 @@ def IssueRetryable(self, **kwargs): Args: **kwargs: Keyword arguments to forward to vm_util.IssueRetryableCommand - when issuing the openstack cli command. + when issuing the openstack cli command. Returns: (stdout, stderr) pair of strings from running the openstack command. diff --git a/perfkitbenchmarker/providers/profitbricks/flags.py b/perfkitbenchmarker/providers/profitbricks/flags.py index f258beb2a0..941029c13d 100644 --- a/perfkitbenchmarker/providers/profitbricks/flags.py +++ b/perfkitbenchmarker/providers/profitbricks/flags.py @@ -33,37 +33,53 @@ ZONE_2 = 'ZONE_2' ZONE_3 = 'ZONE_3' -flags.DEFINE_string('profitbricks_config', - os.getenv('PROFITBRICKS_CONFIG', - '~/.config/profitbricks-auth.cfg'), - ('Path to config file containing your email and password. ' - 'Can also be set via $PROFITBRICKS_CONFIG environment ' - 'variable.\n(File format: email:password)')) +flags.DEFINE_string( + 'profitbricks_config', + os.getenv('PROFITBRICKS_CONFIG', '~/.config/profitbricks-auth.cfg'), + ( + 'Path to config file containing your email and password. ' + 'Can also be set via $PROFITBRICKS_CONFIG environment ' + 'variable.\n(File format: email:password)' + ), +) -flags.DEFINE_string('profitbricks_image_alias', None, - 'An alias to a ProfitBricks public image. If given, ' - 'it will be used instead of a default Ubuntu 14 image. ' - 'E.g., "ubuntu:latest" indicates the latest version of ' - 'Ubuntu is to be used to provision a volume.') +flags.DEFINE_string( + 'profitbricks_image_alias', + None, + 'An alias to a ProfitBricks public image. If given, ' + 'it will be used instead of a default Ubuntu 14 image. ' + 'E.g., "ubuntu:latest" indicates the latest version of ' + 'Ubuntu is to be used to provision a volume.', +) -flags.DEFINE_enum('profitbricks_location', - US_LAS, - [US_LAS, US_EWR, DE_FKB, DE_FRA], - ('Location of data center to be provisioned (us/las, ' - 'us/ewr, de/fkb, de/fra)')) +flags.DEFINE_enum( + 'profitbricks_location', + US_LAS, + [US_LAS, US_EWR, DE_FKB, DE_FRA], + ( + 'Location of data center to be provisioned (us/las, ' + 'us/ewr, de/fkb, de/fra)' + ), +) -flags.DEFINE_enum('profitbricks_boot_volume_type', - HDD, - [HDD, SSD], - ('Choose between HDD or SSD boot volume types.')) +flags.DEFINE_enum( + 'profitbricks_boot_volume_type', + HDD, + [HDD, SSD], + 'Choose between HDD or SSD boot volume types.', +) -flags.DEFINE_integer('profitbricks_boot_volume_size', - 10, - ('Choose the boot volume size in GB.')) +flags.DEFINE_integer( + 'profitbricks_boot_volume_size', 10, 'Choose the boot volume size in GB.' +) -flags.DEFINE_enum('availability_zone', - AUTO, - [AUTO, ZONE_1, ZONE_2, ZONE_3], - ('Direct a storage volume to be created in one of three ' - 'zones per data center (AUTO, ' - 'ZONE_1, ZONE_2, ZONE_3)')) +flags.DEFINE_enum( + 'availability_zone', + AUTO, + [AUTO, ZONE_1, ZONE_2, ZONE_3], + ( + 'Direct a storage volume to be created in one of three ' + 'zones per data center (AUTO, ' + 'ZONE_1, ZONE_2, ZONE_3)' + ), +) diff --git a/perfkitbenchmarker/providers/profitbricks/profitbricks_disk.py b/perfkitbenchmarker/providers/profitbricks/profitbricks_disk.py index ee1eedaadf..4fcf9362fe 100644 --- a/perfkitbenchmarker/providers/profitbricks/profitbricks_disk.py +++ b/perfkitbenchmarker/providers/profitbricks/profitbricks_disk.py @@ -11,33 +11,32 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Module containing classes related to ProfitBricks disks. -""" +"""Module containing classes related to ProfitBricks disks.""" from perfkitbenchmarker import disk from perfkitbenchmarker import errors class ProfitBricksDisk(disk.BaseDisk): - """Dummy Object representing a ProfitBricks Disk.""" + """Dummy Object representing a ProfitBricks Disk.""" - def __init__(self, disk_spec): - super(ProfitBricksDisk, self).__init__(disk_spec) + def __init__(self, disk_spec): + super(ProfitBricksDisk, self).__init__(disk_spec) - def Attach(self, vm): - pass + def Attach(self, vm): + pass - def Detach(self): - pass + def Detach(self): + pass - def GetDevicePath(self): - # ProfitBricks VMs only have a single disk block device which is - # in use for the live filesystem, so it's not usable as a scratch - # disk device. - raise errors.Error('GetDevicePath not supported for ProfitBricks.') + def GetDevicePath(self): + # ProfitBricks VMs only have a single disk block device which is + # in use for the live filesystem, so it's not usable as a scratch + # disk device. + raise errors.Error('GetDevicePath not supported for ProfitBricks.') - def _Create(self): - pass + def _Create(self): + pass - def _Delete(self): - pass + def _Delete(self): + pass diff --git a/perfkitbenchmarker/providers/profitbricks/profitbricks_machine_types.py b/perfkitbenchmarker/providers/profitbricks/profitbricks_machine_types.py index 0f6c8b8b0a..78baacd3f2 100644 --- a/perfkitbenchmarker/providers/profitbricks/profitbricks_machine_types.py +++ b/perfkitbenchmarker/providers/profitbricks/profitbricks_machine_types.py @@ -12,44 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. FLAVORS = [ - { - 'name': 'Micro', - 'ram': 1024, - 'cores': 1 - }, - { - 'name': 'Small', - 'ram': 2048, - 'cores': 1 - }, - { - 'name': 'Medium', - 'ram': 4096, - 'cores': 2 - }, - { - 'name': 'Large', - 'ram': 7168, - 'cores': 4 - }, - { - 'name': 'ExtraLarge', - 'ram': 14336, - 'cores': 8 - }, - { - 'name': 'MemoryIntensiveSmall', - 'ram': 16384, - 'cores': 2 - }, - { - 'name': 'MemoryIntensiveMedium', - 'ram': 28672, - 'cores': 2 - }, - { - 'name': 'MemoryIntensiveLarge', - 'ram': 57344, - 'cores': 2 - }, + {'name': 'Micro', 'ram': 1024, 'cores': 1}, + {'name': 'Small', 'ram': 2048, 'cores': 1}, + {'name': 'Medium', 'ram': 4096, 'cores': 2}, + {'name': 'Large', 'ram': 7168, 'cores': 4}, + {'name': 'ExtraLarge', 'ram': 14336, 'cores': 8}, + {'name': 'MemoryIntensiveSmall', 'ram': 16384, 'cores': 2}, + {'name': 'MemoryIntensiveMedium', 'ram': 28672, 'cores': 2}, + {'name': 'MemoryIntensiveLarge', 'ram': 57344, 'cores': 2}, ] diff --git a/perfkitbenchmarker/providers/profitbricks/profitbricks_virtual_machine.py b/perfkitbenchmarker/providers/profitbricks/profitbricks_virtual_machine.py index 71c0bf959c..7aad66bcdf 100644 --- a/perfkitbenchmarker/providers/profitbricks/profitbricks_virtual_machine.py +++ b/perfkitbenchmarker/providers/profitbricks/profitbricks_virtual_machine.py @@ -11,8 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Class to represent a ProfitBricks Virtual Machine object. -""" +"""Class to represent a ProfitBricks Virtual Machine object.""" import base64 import logging @@ -54,8 +53,10 @@ def _GetOptionDecoderConstructions(cls): arguments to construct in order to decode the named option. """ result = super(CustomMachineTypeSpec, cls)._GetOptionDecoderConstructions() - result.update({'cores': (option_decoders.IntDecoder, {'min': 1}), - 'ram': (option_decoders.IntDecoder, {'min': 1024})}) + result.update({ + 'cores': (option_decoders.IntDecoder, {'min': 1}), + 'ram': (option_decoders.IntDecoder, {'min': 1024}), + }) return result @@ -70,11 +71,11 @@ def Decode(self, value, component_full_name, flag_values): Args: value: Either a string name of a PB machine type or a dict containing - 'cores' and 'ram' keys describing a custom VM. + 'cores' and 'ram' keys describing a custom VM. component_full_name: string. Fully qualified name of the configurable - component containing the config option. + component containing the config option. flag_values: flags.FlagValues. Runtime flag values to be propagated to - BaseSpec constructors. + BaseSpec constructors. Returns: If value is a string, returns it unmodified. Otherwise, returns the @@ -83,16 +84,21 @@ def Decode(self, value, component_full_name, flag_values): Raises: errors.Config.InvalidValue upon invalid input value. """ - super(MachineTypeDecoder, self).Decode(value, component_full_name, - flag_values) + super(MachineTypeDecoder, self).Decode( + value, component_full_name, flag_values + ) if isinstance(value, six.string_types): return value - return CustomMachineTypeSpec(self._GetOptionFullName(component_full_name), - flag_values=flag_values, **value) + return CustomMachineTypeSpec( + self._GetOptionFullName(component_full_name), + flag_values=flag_values, + **value + ) class ProfitBricksVmSpec(virtual_machine.BaseVmSpec): """Object containing the information needed to create a + ProfitBricksVirtualMachine. Attributes: @@ -108,22 +114,22 @@ def __init__(self, *args, **kwargs): logging.info('Using custom hardware configuration.') self.cores = self.machine_type.cores self.ram = self.machine_type.ram - self.machine_type = 'Custom (RAM: {}, Cores: {})'.format(self.ram, - self.cores) + self.machine_type = 'Custom (RAM: {}, Cores: {})'.format( + self.ram, self.cores + ) else: logging.info('Using preset hardware configuration.') self.ram, self.cores = util.ReturnFlavor(self.machine_type) - @classmethod def _ApplyFlags(cls, config_values, flag_values): """Modifies config options based on runtime flag values. Args: - config_values: dict mapping config option names to provided values. May - be modified by this function. + config_values: dict mapping config option names to provided values. May be + modified by this function. flag_values: flags.FlagValues. Runtime flags that may override the - provided config values. + provided config values. """ super(ProfitBricksVmSpec, cls)._ApplyFlags(config_values, flag_values) if flag_values['machine_type'].present: @@ -131,11 +137,13 @@ def _ApplyFlags(cls, config_values, flag_values): if flag_values['profitbricks_location'].present: config_values['location'] = flag_values.profitbricks_location if flag_values['profitbricks_boot_volume_type'].present: - config_values['boot_volume_type'] = \ + config_values['boot_volume_type'] = ( flag_values.profitbricks_boot_volume_type + ) if flag_values['profitbricks_boot_volume_size'].present: - config_values['boot_volume_size'] = \ + config_values['boot_volume_size'] = ( flag_values.profitbricks_boot_volume_size + ) if flag_values['availability_zone'].present: config_values['availability_zone'] = flag_values.availability_zone if flag_values['profitbricks_image_alias'].present: @@ -155,11 +163,19 @@ def _GetOptionDecoderConstructions(cls): 'machine_type': (MachineTypeDecoder, {}), 'location': (option_decoders.StringDecoder, {'default': 'us/las'}), 'image_alias': (option_decoders.StringDecoder, {'default': None}), - 'boot_volume_type': (option_decoders.StringDecoder, {'default': 'HDD'}), - 'boot_volume_size': (option_decoders.IntDecoder, {'default': 10, - 'min': 10}), - 'availability_zone': (option_decoders.StringDecoder, - {'default': 'AUTO'})}) + 'boot_volume_type': ( + option_decoders.StringDecoder, + {'default': 'HDD'}, + ), + 'boot_volume_size': ( + option_decoders.IntDecoder, + {'default': 10, 'min': 10}, + ), + 'availability_zone': ( + option_decoders.StringDecoder, + {'default': 'AUTO'}, + ), + }) return result @@ -173,6 +189,7 @@ def __init__(self, vm_spec): """Initialize a ProfitBricks virtual machine. Args: + vm_spec: virtual_machine.BaseVirtualMachineSpec object of the vm. """ super(ProfitBricksVirtualMachine, self).__init__(vm_spec) @@ -229,29 +246,20 @@ def _Create(self): }, 'entities': { 'volumes': { - 'items': [ - { - 'properties': { - 'size': self.boot_volume_size, - 'name': 'boot volume', - 'image': self.image, - 'imageAlias': self.image_alias, - 'type': self.boot_volume_type, - 'sshKeys': [public_key], - 'availabilityZone': self.availability_zone, - } + 'items': [{ + 'properties': { + 'size': self.boot_volume_size, + 'name': 'boot volume', + 'image': self.image, + 'imageAlias': self.image_alias, + 'type': self.boot_volume_type, + 'sshKeys': [public_key], + 'availabilityZone': self.availability_zone, } - ] + }] }, 'nics': { - 'items': [ - { - 'properties': { - 'name': 'nic1', - 'lan': self.lan_id - } - } - ] + 'items': [{'properties': {'name': 'nic1', 'lan': self.lan_id}}] }, }, } @@ -280,9 +288,11 @@ def _PostCreate(self): """Get the instance's public IP address.""" # Build URL - url = '%s/datacenters/%s/servers/%s?depth=5' % (PROFITBRICKS_API, - self.dc_id, - self.server_id) + url = '%s/datacenters/%s/servers/%s?depth=5' % ( + PROFITBRICKS_API, + self.dc_id, + self.server_id, + ) # Perform Request r = util.PerformRequest('get', url, self.header) @@ -294,8 +304,11 @@ def _Delete(self): """Delete a ProfitBricks VM.""" # Build URL - url = '%s/datacenters/%s/servers/%s' % (PROFITBRICKS_API, self.dc_id, - self.server_id) + url = '%s/datacenters/%s/servers/%s' % ( + PROFITBRICKS_API, + self.dc_id, + self.server_id, + ) # Make call logging.info('Deleting VM: %s', self.server_id) @@ -310,14 +323,14 @@ def _CreateDependencies(self): """Create a data center and LAN prior to creating VM.""" # Create data center - self.dc_id, self.dc_status = util.CreateDatacenter(self.header, - self.location) + self.dc_id, self.dc_status = util.CreateDatacenter( + self.header, self.location + ) if not self._WaitUntilReady(self.dc_status): raise errors.Error('Data center creation failed, see log.') # Create LAN - self.lan_id, self.lan_status = util.CreateLan(self.header, - self.dc_id) + self.lan_id, self.lan_status = util.CreateLan(self.header, self.dc_id) if not self._WaitUntilReady(self.lan_status): raise errors.Error('LAN creation failed, see log.') @@ -359,16 +372,19 @@ def CreateScratchDisk(self, disk_spec): disk_spec: virtual_machine.BaseDiskSpec object of the disk. """ if disk_spec.disk_type != disk.STANDARD: - raise errors.Error('ProfitBricks does not support disk type %s.' % - disk_spec.disk_type) + raise errors.Error( + 'ProfitBricks does not support disk type %s.' % disk_spec.disk_type + ) if self.scratch_disks: # We have a "disk" already, don't add more. - raise errors.Error('ProfitBricks does not require ' - 'a separate disk.') + raise errors.Error('ProfitBricks does not require a separate disk.') # Just create a local directory at the specified path, don't mount # anything. - self.RemoteCommand('sudo mkdir -p {0} && sudo chown -R $USER:$USER {0}' - .format(disk_spec.mount_point)) + self.RemoteCommand( + 'sudo mkdir -p {0} && sudo chown -R $USER:$USER {0}'.format( + disk_spec.mount_point + ) + ) self.scratch_disks.append(profitbricks_disk.ProfitBricksDisk(disk_spec)) diff --git a/perfkitbenchmarker/providers/profitbricks/provider_info.py b/perfkitbenchmarker/providers/profitbricks/provider_info.py index cc109e4ff8..a2343247d2 100644 --- a/perfkitbenchmarker/providers/profitbricks/provider_info.py +++ b/perfkitbenchmarker/providers/profitbricks/provider_info.py @@ -12,14 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Provider info for ProfitBricks. -""" +"""Provider info for ProfitBricks.""" from perfkitbenchmarker import provider_info class ProfitBricksProviderInfo(provider_info.BaseProviderInfo): - UNSUPPORTED_BENCHMARKS = ['mysql_service'] CLOUD = provider_info.PROFITBRICKS diff --git a/perfkitbenchmarker/providers/profitbricks/util.py b/perfkitbenchmarker/providers/profitbricks/util.py index 36c398d0f5..2fe5ad5cfe 100644 --- a/perfkitbenchmarker/providers/profitbricks/util.py +++ b/perfkitbenchmarker/providers/profitbricks/util.py @@ -16,8 +16,7 @@ import logging from perfkitbenchmarker import errors from perfkitbenchmarker.providers import profitbricks -from perfkitbenchmarker.providers.profitbricks import \ - profitbricks_machine_types +from perfkitbenchmarker.providers.profitbricks import profitbricks_machine_types import requests # Global Values @@ -40,8 +39,7 @@ def PerformRequest(action, url, header, json=None): if r.status_code >= 300: action = action.upper() logging.info(r.text) - raise errors.Error('%s call to %s failed, see log.' % (action, - url)) + raise errors.Error('%s call to %s failed, see log.' % (action, url)) return r @@ -57,8 +55,10 @@ def ReturnImage(header, location): # Search for Ubuntu image in preferred zone for image in response['items']: - if('Ubuntu-14' in image['properties']['name'] and - image['properties']['location'] == location): + if ( + 'Ubuntu-14' in image['properties']['name'] + and image['properties']['location'] == location + ): return image['id'] @@ -67,7 +67,7 @@ def ReturnFlavor(machine_type): logging.info('Fetching flavor specs for new VM.') for flavor in FLAVORS: - if(machine_type == flavor['name']): + if machine_type == flavor['name']: return flavor['ram'], flavor['cores'] @@ -83,8 +83,10 @@ def CreateDatacenter(header, location): } # Make call - logging.info('Creating Datacenter: %s in Location: %s' % - (new_dc['properties']['name'], location)) + logging.info( + 'Creating Datacenter: %s in Location: %s' + % (new_dc['properties']['name'], location) + ) url = '%s/datacenters' % PROFITBRICKS_API r = PerformRequest('post', url, header, json=new_dc) diff --git a/perfkitbenchmarker/providers/rackspace/flags.py b/perfkitbenchmarker/providers/rackspace/flags.py index fc20fdb119..50986c89ea 100644 --- a/perfkitbenchmarker/providers/rackspace/flags.py +++ b/perfkitbenchmarker/providers/rackspace/flags.py @@ -14,37 +14,55 @@ from absl import flags -flags.DEFINE_string('rack_path', - default='rack', - help='The path for the rack CLI binary.') +flags.DEFINE_string( + 'rack_path', default='rack', help='The path for the rack CLI binary.' +) -flags.DEFINE_string('rackspace_region', default='IAD', - help='A string indicating which Rackspace region to use.') +flags.DEFINE_string( + 'rackspace_region', + default='IAD', + help='A string indicating which Rackspace region to use.', +) -flags.DEFINE_string('rack_profile', default=None, - help='A string indicating which RackCLI profile to use. ' - 'If none is specified default profile is used ' - '(see https://developer.rackspace.com/docs/' - 'rack-cli/configuration/#config-file)') +flags.DEFINE_string( + 'rack_profile', + default=None, + help=( + 'A string indicating which RackCLI profile to use. ' + 'If none is specified default profile is used ' + '(see https://developer.rackspace.com/docs/' + 'rack-cli/configuration/#config-file)' + ), +) -flags.DEFINE_boolean('rackspace_boot_from_cbs_volume', 'False', - 'When flag is included the instance will use a remote disk' - ' as its boot disk, if machine_type supports it.') +flags.DEFINE_boolean( + 'rackspace_boot_from_cbs_volume', + 'False', + 'When flag is included the instance will use a remote disk' + ' as its boot disk, if machine_type supports it.', +) flags.DEFINE_boolean( - 'rackspace_use_security_group', False, + 'rackspace_use_security_group', + False, '(EXPERIMENTAL) A boolean indicating whether or not to create a security' ' group for the new instance. Applies default security group rules' ' (e.g. allow ingress TCP, and UDP traffic through port 22). If no security' ' group is used, all incoming and outgoing traffic through TCP, UDP and' - ' ICMP is allowed, this is the default.') + ' ICMP is allowed, this is the default.', +) -flags.DEFINE_string('rackspace_network_id', None, - '(EXPERIMENTAL) The ID of an already ' - 'created network to use instead of creating a new one. ' - 'Must have a subnet already associated with the network.') +flags.DEFINE_string( + 'rackspace_network_id', + None, + '(EXPERIMENTAL) The ID of an already ' + 'created network to use instead of creating a new one. ' + 'Must have a subnet already associated with the network.', +) -flags.DEFINE_list('additional_rackspace_flags', - [], - 'Additional global flags to pass to every ' - 'RackCLI command. See "rack --help" for more.') +flags.DEFINE_list( + 'additional_rackspace_flags', + [], + 'Additional global flags to pass to every ' + 'RackCLI command. See "rack --help" for more.', +) diff --git a/perfkitbenchmarker/providers/rackspace/provider_info.py b/perfkitbenchmarker/providers/rackspace/provider_info.py index 390f9acdc5..562ca2d7da 100644 --- a/perfkitbenchmarker/providers/rackspace/provider_info.py +++ b/perfkitbenchmarker/providers/rackspace/provider_info.py @@ -12,14 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" Provider info for Rackspace - -""" +"""Provider info for Rackspace""" from perfkitbenchmarker import provider_info class RackspaceProviderInfo(provider_info.BaseProviderInfo): - UNSUPPORTED_BENCHMARKS = ['mysql_service'] CLOUD = provider_info.RACKSPACE diff --git a/perfkitbenchmarker/providers/rackspace/rackspace_disk.py b/perfkitbenchmarker/providers/rackspace/rackspace_disk.py index 7c489a51f9..ecea133f40 100644 --- a/perfkitbenchmarker/providers/rackspace/rackspace_disk.py +++ b/perfkitbenchmarker/providers/rackspace/rackspace_disk.py @@ -40,42 +40,38 @@ LOCAL = 'local' CBS_SATA = 'cbs-sata' CBS_SSD = 'cbs-ssd' -REMOTE_TYPES = (CBS_SSD, CBS_SATA,) -REMOTE_TYPES_TRANSLATION = { - CBS_SATA: 'SATA', - CBS_SSD: 'SSD' -} +REMOTE_TYPES = ( + CBS_SSD, + CBS_SATA, +) +REMOTE_TYPES_TRANSLATION = {CBS_SATA: 'SATA', CBS_SSD: 'SSD'} DISK_METADATA = { - BOOT: { - disk.REPLICATION: RAID10, - DURABILITY: EPHEMERAL - }, - LOCAL: { - disk.REPLICATION: RAID10, - DURABILITY: EPHEMERAL - }, + BOOT: {disk.REPLICATION: RAID10, DURABILITY: EPHEMERAL}, + LOCAL: {disk.REPLICATION: RAID10, DURABILITY: EPHEMERAL}, CBS_SSD: { disk.REPLICATION: disk.REGION, DURABILITY: PERSISTENT, - disk.MEDIA: disk.SSD + disk.MEDIA: disk.SSD, }, CBS_SATA: { disk.REPLICATION: disk.REGION, DURABILITY: PERSISTENT, - disk.MEDIA: disk.HDD - } + disk.MEDIA: disk.HDD, + }, } class RackspaceDiskSpec(disk.BaseDiskSpec): """Object containing the information needed to create a + RackspaceDisk. Attributes: rackspace_region: None or string. Rackspace region to build VM resources. rack_profile: None or string. Rack CLI profile configuration. """ + CLOUD = provider_info.RACKSPACE @classmethod @@ -90,8 +86,12 @@ def _ApplyFlags(cls, config_values, flag_values): def _GetOptionDecoderConstructions(cls): result = super(RackspaceDiskSpec, cls)._GetOptionDecoderConstructions() result.update({ - 'rackspace_region': (option_decoders.StringDecoder, {'default': 'IAD'}), - 'rack_profile': (option_decoders.StringDecoder, {'default': None})}) + 'rackspace_region': ( + option_decoders.StringDecoder, + {'default': 'IAD'}, + ), + 'rack_profile': (option_decoders.StringDecoder, {'default': None}), + }) return result @@ -128,12 +128,14 @@ def Detach(self): class RackspaceLocalDisk(RackspaceDisk): """RackspaceLocalDisk is a disk object to represent an ephemeral storage disk + locally attached to an instance. """ def __init__(self, disk_spec, name, region, project, device_path, image=None): - super(RackspaceLocalDisk, self).__init__(disk_spec, name, region, project, - image) + super(RackspaceLocalDisk, self).__init__( + disk_spec, name, region, project, image + ) self.exists = False self.device_path = device_path self.name = name @@ -150,25 +152,29 @@ def _Exists(self): class RackspaceBootDisk(RackspaceLocalDisk): """RackspaceBootDisk is a disk object to represent the root (boot) disk of an + instance. Boot disk provides a directory path as a scratch disk space for a benchmark, but does not allow its backing block device to be formatted, or its mount point to be changed. """ def __init__(self, disk_spec, zone, project, device_path, image): - super(RackspaceBootDisk, self).__init__(disk_spec, 'boot-disk', zone, - project, device_path, image) + super(RackspaceBootDisk, self).__init__( + disk_spec, 'boot-disk', zone, project, device_path, image + ) self.mount_point = disk_spec.mount_point class RackspaceRemoteDisk(RackspaceDisk): """RackspaceRemoteDisk is a RackspaceDisk object to represent a remotely + attached Cloud Block Storage Volume. """ def __init__(self, disk_spec, name, region, project, image=None, media=None): - super(RackspaceRemoteDisk, self).__init__(disk_spec, name, region, project, - image) + super(RackspaceRemoteDisk, self).__init__( + disk_spec, name, region, project, image + ) self.media = media self.id = None @@ -213,20 +219,27 @@ def _AttachRemoteDisk(self, vm): raise errors.Error('Cannot attach remote disk %s' % self.name) if vm.id is None: raise errors.VirtualMachine.VmStateError( - 'Cannot attach remote disk %s to non-existing %s VM' % (self.name, - vm.name)) + 'Cannot attach remote disk %s to non-existing %s VM' + % (self.name, vm.name) + ) cmd = util.RackCLICommand(self, 'servers', 'volume-attachment', 'create') cmd.flags['volume-id'] = self.id cmd.flags['server-id'] = vm.id stdout, stderr, _ = cmd.Issue() if stderr: raise errors.Error( - 'Failed to attach remote disk %s to %s' % (self.name, vm.name)) + 'Failed to attach remote disk %s to %s' % (self.name, vm.name) + ) resp = json.loads(stdout) self.device_path = resp['Device'] - @vm_util.Retry(poll_interval=1, max_retries=-1, timeout=300, log_errors=False, - retryable_exceptions=(errors.Resource.RetryableCreationError,)) + @vm_util.Retry( + poll_interval=1, + max_retries=-1, + timeout=300, + log_errors=False, + retryable_exceptions=(errors.Resource.RetryableCreationError,), + ) def _WaitForRemoteDiskAttached(self, vm): cmd = util.RackCLICommand(self, 'block-storage', 'volume', 'get') cmd.flags['id'] = self.id @@ -238,14 +251,16 @@ def _WaitForRemoteDiskAttached(self, vm): logging.info('Disk: %s has been attached to %s.' % (self.name, vm.id)) return raise errors.Resource.RetryableCreationError( - 'Disk: %s is not yet attached. Retrying to check status.' % self.name) + 'Disk: %s is not yet attached. Retrying to check status.' % self.name + ) def _DetachRemoteDisk(self): if self.id is None: raise errors.Error('Cannot detach remote disk %s' % self.name) if self.attached_vm_id is None: raise errors.VirtualMachine.VmStateError( - 'Cannot detach remote disk %s from a non-existing VM' % self.name) + 'Cannot detach remote disk %s from a non-existing VM' % self.name + ) cmd = util.RackCLICommand(self, 'servers', 'volume-attachment', 'delete') cmd.flags['id'] = self.id cmd.flags['server-id'] = self.attached_vm_id @@ -256,8 +271,13 @@ def _DetachRemoteDisk(self): return raise errors.Resource.RetryableDeletionError(stderr) - @vm_util.Retry(poll_interval=1, max_retries=-1, timeout=300, log_errors=False, - retryable_exceptions=(errors.Resource.RetryableDeletionError,)) + @vm_util.Retry( + poll_interval=1, + max_retries=-1, + timeout=300, + log_errors=False, + retryable_exceptions=(errors.Resource.RetryableDeletionError,), + ) def _WaitForRemoteDiskDeletion(self): cmd = util.RackCLICommand(self, 'block-storage', 'volume', 'get') cmd.flags['id'] = self.id @@ -266,4 +286,5 @@ def _WaitForRemoteDiskDeletion(self): logging.info('Disk: %s has been successfully deleted.' % self.name) return raise errors.Resource.RetryableDeletionError( - 'Disk: %s has not been deleted. Retrying to check status.' % self.name) + 'Disk: %s has not been deleted. Retrying to check status.' % self.name + ) diff --git a/perfkitbenchmarker/providers/rackspace/rackspace_network.py b/perfkitbenchmarker/providers/rackspace/rackspace_network.py index 8e77ead27e..b8c2519144 100644 --- a/perfkitbenchmarker/providers/rackspace/rackspace_network.py +++ b/perfkitbenchmarker/providers/rackspace/rackspace_network.py @@ -84,9 +84,17 @@ def _Exists(self): class RackspaceSecurityGroupRule(resource.BaseResource): """An object representing a Security Group Rule.""" - def __init__(self, sec_group_rule_name, sec_group_id, direction=INGRESS, - ip_ver=IPV4, protocol=TCP, port_range_min=PORT_RANGE_MIN, - port_range_max=PORT_RANGE_MAX, source_cidr=None): + def __init__( + self, + sec_group_rule_name, + sec_group_id, + direction=INGRESS, + ip_ver=IPV4, + protocol=TCP, + port_range_min=PORT_RANGE_MIN, + port_range_max=PORT_RANGE_MAX, + source_cidr=None, + ): super(RackspaceSecurityGroupRule, self).__init__() self.id = None self.name = sec_group_rule_name @@ -97,20 +105,22 @@ def __init__(self, sec_group_rule_name, sec_group_id, direction=INGRESS, self.ip_ver = ip_ver assert protocol in SEC_GROUP_PROTOCOLS self.protocol = protocol - assert (int(PORT_RANGE_MIN) <= int(port_range_min) <= int(PORT_RANGE_MAX)) + assert int(PORT_RANGE_MIN) <= int(port_range_min) <= int(PORT_RANGE_MAX) self.port_range_min = port_range_min - assert (int(PORT_RANGE_MIN) <= int(port_range_max) <= int(PORT_RANGE_MAX)) + assert int(PORT_RANGE_MIN) <= int(port_range_max) <= int(PORT_RANGE_MAX) self.port_range_max = port_range_max assert int(port_range_min) <= int(port_range_max) self.source_cidr = source_cidr def __eq__(self, other): # Name does not matter - return (self.sec_group_id == other.sec_group_id and - self.direction == other.direction and - self.ip_ver == other.ip_ver and - self.protocol == other.protocol and - self.source_cidr == other.source_cidr) + return ( + self.sec_group_id == other.sec_group_id + and self.direction == other.direction + and self.ip_ver == other.ip_ver + and self.protocol == other.protocol + and self.source_cidr == other.source_cidr + ) def _Create(self): cmd = util.RackCLICommand(self, 'networks', 'security-group-rule', 'create') @@ -237,9 +247,13 @@ def __init__(self, network_spec): self.tenant_id = network_spec.tenant_id name = FLAGS.rackspace_network_name or 'pkb-network-%s' % FLAGS.run_uri self.network_resource = RackspaceNetworkResource(name, self.tenant_id) - self.subnet = RackspaceSubnet(self.network_resource.id, DEFAULT_SUBNET_CIDR, - ip_ver='4', name='subnet-%s' % name, - tenant_id=self.tenant_id) + self.subnet = RackspaceSubnet( + self.network_resource.id, + DEFAULT_SUBNET_CIDR, + ip_ver='4', + name='subnet-%s' % name, + tenant_id=self.tenant_id, + ) self.security_group = RackspaceSecurityGroup('default-internal-%s' % name) self.default_firewall_rules = [] @@ -257,7 +271,8 @@ def Create(self): self.subnet.Create() self.security_group.Create() self.default_firewall_rules = self._GenerateDefaultRules( - self.security_group.id, self.network_resource.name) + self.security_group.id, self.network_resource.name + ) for rule in self.default_firewall_rules: rule.Create() @@ -276,23 +291,31 @@ def _GenerateDefaultRules(self, sec_group_id, network_name): sec_group_id=sec_group_id, direction=INGRESS, ip_ver=IPV4, - protocol=TCP), + protocol=TCP, + ), RackspaceSecurityGroupRule( sec_group_rule_name='udp-default-internal-%s' % network_name, sec_group_id=sec_group_id, direction=INGRESS, - ip_ver=IPV4, protocol=UDP), + ip_ver=IPV4, + protocol=UDP, + ), RackspaceSecurityGroupRule( sec_group_rule_name='icmp-default-internal-%s' % network_name, sec_group_id=sec_group_id, direction=INGRESS, - ip_ver=IPV4, protocol=ICMP)] + ip_ver=IPV4, + protocol=ICMP, + ), + ] return firewall_rules class RackspaceFirewall(network.BaseFirewall): """An object representing a Rackspace Security Group applied to PublicNet and - ServiceNet.""" + + ServiceNet. + """ CLOUD = provider_info.RACKSPACE diff --git a/perfkitbenchmarker/providers/rackspace/rackspace_virtual_machine.py b/perfkitbenchmarker/providers/rackspace/rackspace_virtual_machine.py index 873ebc7a20..b9725ae1cc 100644 --- a/perfkitbenchmarker/providers/rackspace/rackspace_virtual_machine.py +++ b/perfkitbenchmarker/providers/rackspace/rackspace_virtual_machine.py @@ -51,7 +51,7 @@ FLAGS = flags.FLAGS -CLOUD_CONFIG_TEMPLATE = '''#cloud-config +CLOUD_CONFIG_TEMPLATE = """#cloud-config users: - name: {0} ssh-authorized-keys: @@ -60,25 +60,27 @@ groups: sudo shell: /bin/bash -''' +""" -BLOCK_DEVICE_TEMPLATE = ''' +BLOCK_DEVICE_TEMPLATE = """ source-type=image, source-id={0}, dest=volume, size={1}, shutdown=remove, bootindex=0 -''' +""" -LSBLK_REGEX = (r'NAME="(.*)"\s+MODEL="(.*)"\s+SIZE="(.*)"' - r'\s+TYPE="(.*)"\s+MOUNTPOINT="(.*)"\s+LABEL="(.*)"') +LSBLK_REGEX = ( + r'NAME="(.*)"\s+MODEL="(.*)"\s+SIZE="(.*)"' + r'\s+TYPE="(.*)"\s+MOUNTPOINT="(.*)"\s+LABEL="(.*)"' +) LSBLK_PATTERN = re.compile(LSBLK_REGEX) INSTANCE_EXISTS_STATUSES = frozenset( - ['BUILD', 'ACTIVE', 'PAUSED', 'SHUTOFF', 'ERROR']) -INSTANCE_DELETED_STATUSES = frozenset( - ['DELETED']) + ['BUILD', 'ACTIVE', 'PAUSED', 'SHUTOFF', 'ERROR'] +) +INSTANCE_DELETED_STATUSES = frozenset(['DELETED']) INSTANCE_KNOWN_STATUSES = INSTANCE_EXISTS_STATUSES | INSTANCE_DELETED_STATUSES REMOTE_BOOT_DISK_SIZE_GB = 50 @@ -95,12 +97,14 @@ def RenderBlockDeviceTemplate(image, volume_size): string value for block-device parameter used when creating a VM. """ blk_params = BLOCK_DEVICE_TEMPLATE.replace('\n', '').format( - image, str(volume_size)) + image, str(volume_size) + ) return blk_params class RackspaceVmSpec(virtual_machine.BaseVmSpec): """Object containing the information needed to create a + RackspaceVirtualMachine. Attributes: @@ -116,10 +120,10 @@ def _ApplyFlags(cls, config_values, flag_values): """Modifies config options based on runtime flag values. Args: - config_values: dict mapping config option names to provided values. May - be modified by this function. + config_values: dict mapping config option names to provided values. May be + modified by this function. flag_values: flags.FlagValues. Runtime flags that may override the - provided config values. + provided config values. """ super(RackspaceVmSpec, cls)._ApplyFlags(config_values, flag_values) if flag_values['project'].present: @@ -141,8 +145,12 @@ def _GetOptionDecoderConstructions(cls): result = super(RackspaceVmSpec, cls)._GetOptionDecoderConstructions() result.update({ 'project': (option_decoders.StringDecoder, {'default': None}), - 'rackspace_region': (option_decoders.StringDecoder, {'default': 'IAD'}), - 'rack_profile': (option_decoders.StringDecoder, {'default': None})}) + 'rackspace_region': ( + option_decoders.StringDecoder, + {'default': 'IAD'}, + ), + 'rack_profile': (option_decoders.StringDecoder, {'default': None}), + }) return result @@ -217,15 +225,18 @@ def _Delete(self): def _DeleteDependencies(self): """Deletes dependencies that were need for the VM after the VM has been - deleted.""" + + deleted. + """ # TODO(meteorfox) Delete security group (if applies) self._DeleteSSHPublicKey() def _UploadSSHPublicKey(self): """Uploads SSH public key to the VM's region. 1 key per VM per Region.""" cmd = util.RackCLICommand(self, 'servers', 'keypair', 'upload') - cmd.flags = OrderedDict([ - ('name', self.name), ('file', self.ssh_public_key)]) + cmd.flags = OrderedDict( + [('name', self.name), ('file', self.ssh_public_key)] + ) cmd.Issue() def _DeleteSSHPublicKey(self): @@ -236,8 +247,9 @@ def _DeleteSSHPublicKey(self): def _CreateInstance(self): """Generates and execute command for creating a Rackspace VM.""" - with tempfile.NamedTemporaryFile(dir=vm_util.GetTempDir(), - prefix='user-data') as tf: + with tempfile.NamedTemporaryFile( + dir=vm_util.GetTempDir(), prefix='user-data' + ) as tf: with open(self.ssh_public_key) as f: public_key = f.read().rstrip('\n') tf.write(CLOUD_CONFIG_TEMPLATE.format(self.user_name, public_key)) @@ -246,9 +258,12 @@ def _CreateInstance(self): stdout, stderr, _ = create_cmd.Issue() if stderr: resp = json.loads(stderr) - raise errors.Error(''.join( - ('Non-recoverable error has occurred: %s\n' % str(resp), - 'Following command caused the error: %s' % repr(create_cmd),))) + raise errors.Error( + ''.join(( + 'Non-recoverable error has occurred: %s\n' % str(resp), + 'Following command caused the error: %s' % repr(create_cmd), + )) + ) resp = json.loads(stdout) self.id = resp['ID'] @@ -272,8 +287,10 @@ def _GetCreateCommand(self, tf): create_cmd.flags['image-id'] = self.image if FLAGS.rackspace_network_id is not None: create_cmd.flags['networks'] = ','.join([ - rackspace_network.PUBLIC_NET_ID, rackspace_network.SERVICE_NET_ID, - FLAGS.rackspace_network_id]) + rackspace_network.PUBLIC_NET_ID, + rackspace_network.SERVICE_NET_ID, + FLAGS.rackspace_network_id, + ]) create_cmd.flags['user-data'] = tf.name metadata = ['owner=%s' % FLAGS.owner] for key, value in six.iteritems(self.boot_metadata): @@ -281,8 +298,12 @@ def _GetCreateCommand(self, tf): create_cmd.flags['metadata'] = ','.join(metadata) return create_cmd - @vm_util.Retry(poll_interval=5, max_retries=720, log_errors=False, - retryable_exceptions=(errors.Resource.RetryableCreationError,)) + @vm_util.Retry( + poll_interval=5, + max_retries=720, + log_errors=False, + retryable_exceptions=(errors.Resource.RetryableCreationError,), + ) def _WaitForInstanceUntilActive(self): """Waits until instance achieves non-transient state.""" get_cmd = util.RackCLICommand(self, 'servers', 'instance', 'get') @@ -297,7 +318,8 @@ def _WaitForInstanceUntilActive(self): logging.error('VM: %s failed to boot.' % self.name) raise errors.VirtualMachine.VmStateError() raise errors.Resource.RetryableCreationError( - 'VM: %s is not running. Retrying to check status.' % self.name) + 'VM: %s is not running. Retrying to check status.' % self.name + ) def _DeleteInstance(self): """Executes delete command for removing a Rackspace VM.""" @@ -308,9 +330,13 @@ def _DeleteInstance(self): if 'result' not in resp or 'Deleting' not in resp['result']: raise errors.Resource.RetryableDeletionError() - @vm_util.Retry(poll_interval=5, max_retries=-1, timeout=300, - log_errors=False, - retryable_exceptions=(errors.Resource.RetryableDeletionError,)) + @vm_util.Retry( + poll_interval=5, + max_retries=-1, + timeout=300, + log_errors=False, + retryable_exceptions=(errors.Resource.RetryableDeletionError,), + ) def _WaitForInstanceUntilDeleted(self): """Waits until instance has been fully removed, or deleted.""" get_cmd = util.RackCLICommand(self, 'servers', 'instance', 'get') @@ -330,7 +356,8 @@ def _WaitForInstanceUntilDeleted(self): logging.info('VM: %s has been successfully deleted.', self.name) else: raise errors.Resource.RetryableDeletionError( - 'VM: %s has not been deleted. Retrying to check status.' % self.name) + 'VM: %s has not been deleted. Retrying to check status.' % self.name + ) def AddMetadata(self, **kwargs): """Adds metadata to the VM via RackCLI update-metadata command.""" @@ -338,8 +365,9 @@ def AddMetadata(self, **kwargs): return cmd = util.RackCLICommand(self, 'servers', 'instance', 'update-metadata') cmd.flags['id'] = self.id - cmd.flags['metadata'] = ','.join('{0}={1}'.format(key, value) - for key, value in six.iteritems(kwargs)) + cmd.flags['metadata'] = ','.join( + '{0}={1}'.format(key, value) for key, value in six.iteritems(kwargs) + ) cmd.Issue() def OnStartup(self): @@ -385,7 +413,8 @@ def _AllocateBootDisk(self, disk_spec): raise errors.Error('Only one boot disk can be created per VM') device_path = '/dev/%s' % self.boot_device['name'] scratch_disk = rackspace_disk.RackspaceBootDisk( - disk_spec, self.zone, self.project, device_path, self.image) + disk_spec, self.zone, self.project, device_path, self.image + ) self.boot_disk_allocated = True self.scratch_disks.append(scratch_disk) scratch_disk.Create() @@ -409,13 +438,15 @@ def _AllocateLocalDisks(self, disk_spec): disk_name = '%s-local-disk-%d' % (self.name, i) device_path = '/dev/%s' % local_device['name'] local_disk = rackspace_disk.RackspaceLocalDisk( - disk_spec, disk_name, self.zone, self.project, device_path) + disk_spec, disk_name, self.zone, self.project, device_path + ) self.allocated_disks.add(local_disk) disks.append(local_disk) self._CreateScratchDiskFromDisks(disk_spec, disks) def _AllocateRemoteDisks(self, disk_spec): """Creates and allocates Rackspace Cloud Block Storage volumes as + as data disks. Args: @@ -425,8 +456,12 @@ def _AllocateRemoteDisks(self, disk_spec): for disk_num in range(disk_spec.num_striped_disks): volume_name = '%s-volume-%d' % (self.name, disk_num) scratch_disk = rackspace_disk.RackspaceRemoteDisk( - disk_spec, volume_name, self.zone, self.project, - media=disk_spec.disk_type) + disk_spec, + volume_name, + self.zone, + self.project, + media=disk_spec.disk_type, + ) scratch_disks.append(scratch_disk) scratch_disk = self._CreateScratchDiskFromDisks(disk_spec, scratch_disks) @@ -436,11 +471,12 @@ def _AllocateRemoteDisks(self, disk_spec): def _GetFreeBlockDevices(self, block_devices, disk_spec): """Returns available block devices that are not in used as data disk or as + a boot disk. Args: block_devices: list of dict containing information about all block devices - in the VM. + in the VM. disk_spec: virtual_machine.BaseDiskSpec of the disk. Returns: @@ -455,13 +491,18 @@ def _GetFreeBlockDevices(self, block_devices, disk_spec): free_blk_devices.append(dev) if not free_blk_devices: raise errors.Error( - ''.join(('Machine type %s does not include' % self.machine_type, - ' local disks. Please use a different disk_type,', - ' or a machine_type that provides local disks.'))) + ''.join(( + 'Machine type %s does not include' % self.machine_type, + ' local disks. Please use a different disk_type,', + ' or a machine_type that provides local disks.', + )) + ) elif len(free_blk_devices) < disk_spec.num_striped_disks: - raise errors.Error('Not enough local data disks. ' - 'Requesting %d disk(s) but only %d available.' - % (disk_spec.num_striped_disks, len(free_blk_devices))) + raise errors.Error( + 'Not enough local data disks. ' + 'Requesting %d disk(s) but only %d available.' + % (disk_spec.num_striped_disks, len(free_blk_devices)) + ) return free_blk_devices def _GetBlockDevices(self): @@ -471,11 +512,19 @@ def _GetBlockDevices(self): list of dicts block devices in the VM. """ stdout, _ = self.RemoteCommand( - 'sudo lsblk -o NAME,MODEL,SIZE,TYPE,MOUNTPOINT,LABEL -n -b -P') + 'sudo lsblk -o NAME,MODEL,SIZE,TYPE,MOUNTPOINT,LABEL -n -b -P' + ) lines = stdout.splitlines() groups = [LSBLK_PATTERN.match(line) for line in lines] tuples = [g.groups() for g in groups if g] - colnames = ('name', 'model', 'size_bytes', 'type', 'mountpoint', 'label',) + colnames = ( + 'name', + 'model', + 'size_bytes', + 'type', + 'mountpoint', + 'label', + ) blk_devices = [dict(list(zip(colnames, t))) for t in tuples] for d in blk_devices: d['model'] = d['model'].rstrip() @@ -518,18 +567,22 @@ def _IsDiskAvailable(self, blk_device): An available disk, is a disk that has not been allocated previously as a data disk, or is not being used as boot disk. """ - return (blk_device['type'] != 'part' and - blk_device['name'] != self.boot_device['name'] and - 'config' not in blk_device['label'] and - blk_device['name'] not in self.allocated_disks) + return ( + blk_device['type'] != 'part' + and blk_device['name'] != self.boot_device['name'] + and 'config' not in blk_device['label'] + and blk_device['name'] not in self.allocated_disks + ) -class Rhel7BasedRackspaceVirtualMachine(RackspaceVirtualMachine, - linux_virtual_machine.Rhel7Mixin): +class Rhel7BasedRackspaceVirtualMachine( + RackspaceVirtualMachine, linux_virtual_machine.Rhel7Mixin +): DEFAULT_IMAGE = '92f8a8b8-6019-4c27-949b-cf9910b84ffb' class VersionlessRhelBasedRackspaceVirtualMachine( linux_virtual_machine.VersionlessRhelMixin, - Rhel7BasedRackspaceVirtualMachine): + Rhel7BasedRackspaceVirtualMachine, +): pass diff --git a/perfkitbenchmarker/providers/rackspace/util.py b/perfkitbenchmarker/providers/rackspace/util.py index 1a820031ee..da68e078e7 100644 --- a/perfkitbenchmarker/providers/rackspace/util.py +++ b/perfkitbenchmarker/providers/rackspace/util.py @@ -28,12 +28,12 @@ class RackCLICommand(object): Attributes: args: list of strings. Positional args to pass to rack cli, typically - specifying an operation to perform (e.g. ['servers', 'images', 'list'] - to list available images). + specifying an operation to perform (e.g. ['servers', 'images', 'list'] to + list available images). flags: OrderedDict mapping flag name string to flag value. Flags to pass to - rack cli (e.g. {'image-id': 'some-image-id'}). + rack cli (e.g. {'image-id': 'some-image-id'}). additional_flags: list of strings. Additional flags to append unmodified to - the end of the rack cli command (e.g. ['--metadata', 'owner=user']). + the end of the rack cli command (e.g. ['--metadata', 'owner=user']). """ def __init__(self, resource, *args): @@ -42,8 +42,8 @@ def __init__(self, resource, *args): Args: resource: A Rackspace resource of type BaseResource. *args: sequence of strings. Positional args to pass to rack cli, typically - specifying an operation to perform. (e.g. ['servers', 'image', 'list'] - to list available images). + specifying an operation to perform. (e.g. ['servers', 'image', 'list'] + to list available images). """ self.resource = resource self.args = list(args) @@ -58,7 +58,7 @@ def Issue(self, **kwargs): Args: **kwargs: Keyword arguments to forward to vm_util.IssueCommand when - issuing the rack cli command. + issuing the rack cli command. Returns: A tuple of stdout, stderr, and retcode from running the rack cli command. @@ -72,7 +72,7 @@ def IssueRetryable(self, **kwargs): Args: **kwargs: Keyword arguments to forward to vm_util.IssueRetryableCommand - when issuing the rack cli command. + when issuing the rack cli command. Returns: (stdout, stderr) pair of strings from running the rack cli command. diff --git a/perfkitbenchmarker/providers/snowflake/snowflake.py b/perfkitbenchmarker/providers/snowflake/snowflake.py index 10437bfa7f..cbef2cdff8 100644 --- a/perfkitbenchmarker/providers/snowflake/snowflake.py +++ b/perfkitbenchmarker/providers/snowflake/snowflake.py @@ -30,12 +30,13 @@ # TODO(jguertin): Update these jdbc client names to reflect their function JdbcClientDict = { provider_info.AWS: 'snowflake-jdbc-client-2.5-standard.jar', - provider_info.AZURE: 'snowflake-jdbc-client-azure-external-2.0.jar' + provider_info.AZURE: 'snowflake-jdbc-client-azure-external-2.0.jar', } -def GetSnowflakeClientInterface(warehouse: str, database: str, schema: str, - cloud: str) -> edw_service.EdwClientInterface: +def GetSnowflakeClientInterface( + warehouse: str, database: str, schema: str, cloud: str +) -> edw_service.EdwClientInterface: """Builds and Returns the requested Snowflake client Interface. Args: @@ -52,8 +53,9 @@ def GetSnowflakeClientInterface(warehouse: str, database: str, schema: str, RuntimeError: if an unsupported snowflake_client_interface is requested """ if FLAGS.snowflake_client_interface == 'JDBC': - return JdbcClientInterface(warehouse, database, schema, - JdbcClientDict[cloud]) + return JdbcClientInterface( + warehouse, database, schema, JdbcClientDict[cloud] + ) raise RuntimeError('Unknown Snowflake Client Interface requested.') @@ -68,8 +70,9 @@ class JdbcClientInterface(edw_service.EdwClientInterface): Snowflake backend being tested (AWS/Azure/etc.) """ - def __init__(self, warehouse: str, database: str, schema: str, - jdbc_client: str): + def __init__( + self, warehouse: str, database: str, schema: str, jdbc_client: str + ): self.warehouse = warehouse self.database = database self.schema = schema @@ -89,8 +92,9 @@ def Prepare(self, package_name: str) -> None: self.client_vm.Install('openjdk') # Push the executable jar to the working directory on client vm - self.client_vm.InstallPreprovisionedPackageData(package_name, - [self.jdbc_client], '') + self.client_vm.InstallPreprovisionedPackageData( + package_name, [self.jdbc_client], '' + ) def ExecuteQuery(self, query_name: Text) -> Tuple[float, Dict[str, str]]: """Executes a query and returns performance details. @@ -105,19 +109,22 @@ def ExecuteQuery(self, query_name: Text) -> Tuple[float, Dict[str, str]]: successful query the value is expected to be positive. performance_details: A dictionary of query execution attributes eg. job_id """ - query_command = (f'java -cp {self.jdbc_client} ' - 'com.google.cloud.performance.edw.Single ' - f'--warehouse {self.warehouse} ' - f'--database {self.database} ' - f'--schema {self.schema} ' - f'--query_file {query_name}') + query_command = ( + f'java -cp {self.jdbc_client} ' + 'com.google.cloud.performance.edw.Single ' + f'--warehouse {self.warehouse} ' + f'--database {self.database} ' + f'--schema {self.schema} ' + f'--query_file {query_name}' + ) stdout, _ = self.client_vm.RemoteCommand(query_command) details = copy.copy(self.GetMetadata()) # Copy the base metadata details.update(json.loads(stdout)['details']) return json.loads(stdout)['query_wall_time_in_secs'], details - def ExecuteSimultaneous(self, submission_interval: int, - queries: List[str]) -> str: + def ExecuteSimultaneous( + self, submission_interval: int, queries: List[str] + ) -> str: """Executes queries simultaneously on client and return performance details. Simultaneous app expects queries as white space separated query file names. @@ -130,13 +137,15 @@ def ExecuteSimultaneous(self, submission_interval: int, Returns: A serialized dictionary of execution details. """ - query_command = (f'java -cp {self.jdbc_client} ' - 'com.google.cloud.performance.edw.Simultaneous ' - f'--warehouse {self.warehouse} ' - f'--database {self.database} ' - f'--schema {self.schema} ' - f'--submission_interval {submission_interval} ' - f'--query_files {" ".join(queries)}') + query_command = ( + f'java -cp {self.jdbc_client} ' + 'com.google.cloud.performance.edw.Simultaneous ' + f'--warehouse {self.warehouse} ' + f'--database {self.database} ' + f'--schema {self.schema} ' + f'--submission_interval {submission_interval} ' + f'--query_files {" ".join(queries)}' + ) stdout, _ = self.client_vm.RemoteCommand(query_command) return stdout @@ -169,6 +178,7 @@ def GetMetadata(self) -> Dict[str, str]: class Snowflake(edw_service.EdwService): """Object representing a Snowflake Data Warehouse Instance.""" + CLOUD = None SERVICE_TYPE = None @@ -177,9 +187,9 @@ def __init__(self, edw_service_spec): self.warehouse = FLAGS.snowflake_warehouse self.database = FLAGS.snowflake_database self.schema = FLAGS.snowflake_schema - self.client_interface = GetSnowflakeClientInterface(self.warehouse, - self.database, - self.schema, self.CLOUD) + self.client_interface = GetSnowflakeClientInterface( + self.warehouse, self.database, self.schema, self.CLOUD + ) def IsUserManaged(self, edw_service_spec): # TODO(saksena): Remove the assertion after implementing provisioning of diff --git a/perfkitbenchmarker/publisher.py b/perfkitbenchmarker/publisher.py index 7013e9e764..f98f9335f2 100755 --- a/perfkitbenchmarker/publisher.py +++ b/perfkitbenchmarker/publisher.py @@ -50,77 +50,89 @@ flags.DEFINE_string( 'product_name', 'PerfKitBenchmarker', - 'The product name to use when publishing results.') + 'The product name to use when publishing results.', +) flags.DEFINE_boolean( 'official', False, 'A boolean indicating whether results are official or not. The ' 'default is False. Official test results are treated and queried ' - 'differently from non-official test results.') + 'differently from non-official test results.', +) flags.DEFINE_boolean( 'hostname_metadata', False, 'A boolean indicating whether to publish VM hostnames as part of sample ' - 'metadata.') + 'metadata.', +) flags.DEFINE_string( 'json_path', None, 'A path to write newline-delimited JSON results ' - 'Default: write to a run-specific temporary directory') + 'Default: write to a run-specific temporary directory', +) flags.DEFINE_enum( 'json_write_mode', 'w', ['w', 'a'], - 'Open mode for file specified by --json_path. Default: overwrite file') + 'Open mode for file specified by --json_path. Default: overwrite file', +) flags.DEFINE_boolean( - 'collapse_labels', - True, - 'Collapse entries in labels in JSON output.') -flags.DEFINE_string( - 'csv_path', - None, - 'A path to write CSV-format results') + 'collapse_labels', True, 'Collapse entries in labels in JSON output.' +) +flags.DEFINE_string('csv_path', None, 'A path to write CSV-format results') flags.DEFINE_string( 'bigquery_table', None, 'The BigQuery table to publish results to. This should be of the form ' - '"[project_id:]dataset_name.table_name".') -flags.DEFINE_string( - 'bq_path', 'bq', 'Path to the "bq" executable.') + '"[project_id:]dataset_name.table_name".', +) +flags.DEFINE_string('bq_path', 'bq', 'Path to the "bq" executable.') flags.DEFINE_string( - 'bq_project', None, 'Project to use for authenticating with BigQuery.') + 'bq_project', None, 'Project to use for authenticating with BigQuery.' +) flags.DEFINE_string( - 'service_account', None, 'Service account to use to authenticate with BQ.') + 'service_account', None, 'Service account to use to authenticate with BQ.' +) flags.DEFINE_string( - 'service_account_private_key', None, - 'Service private key for authenticating with BQ.') + 'service_account_private_key', + None, + 'Service private key for authenticating with BQ.', +) flags.DEFINE_string( - 'application_default_credential_file', None, - 'Application default credentials file for authenticating with BQ.') + 'application_default_credential_file', + None, + 'Application default credentials file for authenticating with BQ.', +) -flags.DEFINE_string( - 'gsutil_path', 'gsutil', 'path to the "gsutil" executable') +flags.DEFINE_string('gsutil_path', 'gsutil', 'path to the "gsutil" executable') flags.DEFINE_string( 'cloud_storage_bucket', None, 'GCS bucket to upload records to. Bucket must exist. ' 'This flag differs from --hourly_partitioned_cloud_storage_bucket ' - 'by putting records directly in the bucket.') + 'by putting records directly in the bucket.', +) PARTITIONED_GCS_URL = flags.DEFINE_string( - 'hourly_partitioned_cloud_storage_bucket', None, + 'hourly_partitioned_cloud_storage_bucket', + None, 'GCS bucket to upload records to. Bucket must exist. This flag differs ' 'from --cloud_storage_bucket by putting records in subfolders based on ' - 'time of publish. i.e. gs://bucket/YYYY/mm/dd/HH/data.') + 'time of publish. i.e. gs://bucket/YYYY/mm/dd/HH/data.', +) flags.DEFINE_string( - 'es_uri', None, - 'The Elasticsearch address and port. e.g. http://localhost:9200') + 'es_uri', + None, + 'The Elasticsearch address and port. e.g. http://localhost:9200', +) flags.DEFINE_string( - 'es_index', 'perfkit', 'Elasticsearch index name to store documents') + 'es_index', 'perfkit', 'Elasticsearch index name to store documents' +) flags.DEFINE_string('es_type', 'result', 'Elasticsearch document type') @@ -129,20 +141,25 @@ [], 'A colon separated key-value pair that will be added to the labels field ' 'of all samples as metadata. Multiple key-value pairs may be specified ' - 'by separating each pair by commas.') + 'by separating each pair by commas.', +) flags.DEFINE_string( - 'influx_uri', None, + 'influx_uri', + None, 'The Influx DB address and port. Expects the format hostname:port' - 'If port is not passed in it assumes port 80. e.g. localhost:8086') + 'If port is not passed in it assumes port 80. e.g. localhost:8086', +) flags.DEFINE_string( - 'influx_db_name', 'perfkit', - 'Name of Influx DB database that you wish to publish to or create') + 'influx_db_name', + 'perfkit', + 'Name of Influx DB database that you wish to publish to or create', +) flags.DEFINE_boolean( - 'record_log_publisher', True, - 'Whether to use the log publisher or not.') + 'record_log_publisher', True, 'Whether to use the log publisher or not.' +) DEFAULT_JSON_OUTPUT_NAME = 'perfkitbenchmarker_results.json' DEFAULT_CREDENTIALS_JSON = 'credentials.json' @@ -162,7 +179,7 @@ _VM_METADATA_TO_LIST_PLURAL = { 'id': 'ids', 'name': 'names', - 'ip_address': 'ip_addresses' + 'ip_address': 'ip_addresses', } @@ -182,7 +199,8 @@ def PublishRunStageSamples(benchmark_spec, samples): samples: A list of samples to publish. """ events.benchmark_samples_created.send( - benchmark_spec=benchmark_spec, samples=samples) + benchmark_spec=benchmark_spec, samples=samples + ) collector = SampleCollector() collector.AddSamples(samples, benchmark_spec.name, benchmark_spec) collector.PublishSamples() @@ -231,8 +249,9 @@ def AddMetadata(self, metadata, benchmark_spec): if FLAGS.simulate_maintenance: metadata['simulate_maintenance'] = True if FLAGS.hostname_metadata: - metadata['hostnames'] = ','.join([vm.hostname - for vm in benchmark_spec.vms]) + metadata['hostnames'] = ','.join( + [vm.hostname for vm in benchmark_spec.vms] + ) if benchmark_spec.container_cluster: cluster = benchmark_spec.container_cluster for k, v in six.iteritems(cluster.GetResourceMetadata()): @@ -274,7 +293,7 @@ def AddMetadata(self, metadata, benchmark_spec): # signaled the default vm group above. vm_groups_and_all = benchmark_spec.vm_groups | {None: benchmark_spec.vms} for group_name, vms in vm_groups_and_all.items(): - name_prefix = group_name + '_' if group_name else '' + name_prefix = group_name + '_' if group_name else '' # since id and name are generic metadata prefix vm, so it is clear # what the resource is. name_prefix += 'vm_' @@ -329,9 +348,18 @@ class CSVPublisher(SamplePublisher): keys found in the data. """ - _DEFAULT_FIELDS = ('timestamp', 'test', 'metric', 'value', 'unit', - 'product_name', 'official', 'owner', 'run_uri', - 'sample_uri') + _DEFAULT_FIELDS = ( + 'timestamp', + 'test', + 'metric', + 'value', + 'unit', + 'product_name', + 'official', + 'owner', + 'run_uri', + 'sample_uri', + ) def __init__(self, path): super().__init__() @@ -341,7 +369,8 @@ def PublishSamples(self, samples): samples = list(samples) # Union of all metadata keys. meta_keys = sorted( - set(key for sample in samples for key in sample['metadata'])) + set(key for sample in samples for key in sample['metadata']) + ) logging.info('Writing CSV results to %s', self._path) with open(self._path, 'w') as fp: @@ -415,26 +444,31 @@ def _FindConstantMetadataKeys(self, samples): for k in frozenset(unique_values) - frozenset(sample['metadata']): unique_values[k].add(None) - return frozenset(k for k, v in six.iteritems(unique_values) - if len(v) == 1 and None not in v) + return frozenset( + k + for k, v in six.iteritems(unique_values) + if len(v) == 1 and None not in v + ) def _FormatMetadata(self, metadata): """Format 'metadata' as space-delimited key="value" pairs.""" - return ' '.join('{0}="{1}"'.format(k, v) - for k, v in sorted(six.iteritems(metadata))) + return ' '.join( + '{0}="{1}"'.format(k, v) for k, v in sorted(six.iteritems(metadata)) + ) def PublishSamples(self, samples): # result will store the formatted text, then be emitted to self.stream and # logged. result = six.StringIO() dashes = '-' * 25 - result.write('\n' + dashes + - 'PerfKitBenchmarker Results Summary' + - dashes + '\n') + result.write( + '\n' + dashes + 'PerfKitBenchmarker Results Summary' + dashes + '\n' + ) if not samples: - logging.debug('Pretty-printing results to %s:\n%s', self.stream, - result.getvalue()) + logging.debug( + 'Pretty-printing results to %s:\n%s', self.stream, result.getvalue() + ) self.stream.write(result.getvalue()) return @@ -445,21 +479,24 @@ def PublishSamples(self, samples): for benchmark, test_samples in itertools.groupby(samples, key): test_samples = list(test_samples) # Drop end-to-end runtime: it always has no metadata. - non_endtoend_samples = [i for i in test_samples - if i['metric'] != 'End to End Runtime'] + non_endtoend_samples = [ + i for i in test_samples if i['metric'] != 'End to End Runtime' + ] locally_constant_keys = ( - self._FindConstantMetadataKeys(non_endtoend_samples) - - globally_constant_keys) + self._FindConstantMetadataKeys(non_endtoend_samples) + - globally_constant_keys + ) all_constant_meta = globally_constant_keys.union(locally_constant_keys) - benchmark_meta = {k: v - for k, v in six.iteritems(test_samples[0]['metadata']) - if k in locally_constant_keys} + benchmark_meta = { + k: v + for k, v in six.iteritems(test_samples[0]['metadata']) + if k in locally_constant_keys + } result.write('{0}:\n'.format(benchmark.upper())) if benchmark_meta: - result.write(' {0}\n'.format( - self._FormatMetadata(benchmark_meta))) + result.write(' {0}\n'.format(self._FormatMetadata(benchmark_meta))) for sample in test_samples: meta = { @@ -476,11 +513,15 @@ def PublishSamples(self, samples): result.write(' ({0})'.format(self._FormatMetadata(meta))) result.write('\n') - global_meta = {k: v for k, v in six.iteritems(samples[0]['metadata']) - if k in globally_constant_keys} + global_meta = { + k: v + for k, v in six.iteritems(samples[0]['metadata']) + if k in globally_constant_keys + } result.write('\n' + dashes + '\n') - result.write('For all tests: {0}\n'.format( - self._FormatMetadata(global_meta))) + result.write( + 'For all tests: {0}\n'.format(self._FormatMetadata(global_meta)) + ) value = result.getvalue() logging.debug('Pretty-printing results to %s:\n%s', self.stream, value) @@ -504,8 +545,9 @@ def __init__(self, level=logging.INFO, logger=None): self._pprinter = pprint.PrettyPrinter() def __repr__(self): - return '<{0} logger={1} level={2}>'.format(type(self).__name__, self.logger, - self.level) + return '<{0} logger={1} level={2}>'.format( + type(self).__name__, self.logger, self.level + ) def PublishSamples(self, samples): header = '\n' + '-' * 25 + 'PerfKitBenchmarker Complete Results' + '-' * 25 @@ -538,11 +580,11 @@ def __init__(self, file_path, mode='wt', collapse_labels=True): def __repr__(self): return '<{0} file_path="{1}" mode="{2}">'.format( - type(self).__name__, self.file_path, self.mode) + type(self).__name__, self.file_path, self.mode + ) def PublishSamples(self, samples): - logging.info('Publishing %d samples to %s', len(samples), - self.file_path) + logging.info('Publishing %d samples to %s', len(samples), self.file_path) with open(self.file_path, self.mode) as fp: fcntl.flock(fp, fcntl.LOCK_EX) for sample in samples: @@ -568,13 +610,15 @@ class BigQueryPublisher(SamplePublisher): default credentials. Cannot be set alongside service_account. """ - def __init__(self, - bigquery_table, - project_id=None, - bq_path='bq', - service_account=None, - service_account_private_key_file=None, - application_default_credential_file=None): + def __init__( + self, + bigquery_table, + project_id=None, + bq_path='bq', + service_account=None, + service_account_private_key_file=None, + application_default_credential_file=None, + ): super().__init__() self.bigquery_table = bigquery_table self.project_id = project_id @@ -583,16 +627,24 @@ def __init__(self, self.service_account_private_key_file = service_account_private_key_file self._credentials_file = vm_util.PrependTempDir(DEFAULT_CREDENTIALS_JSON) self.application_default_credential_file = ( - application_default_credential_file) - - if ((self.service_account is None) != - (self.service_account_private_key_file is None)): - raise ValueError('service_account and service_account_private_key ' - 'must be specified together.') - if (application_default_credential_file is not None and - self.service_account is not None): - raise ValueError('application_default_credential_file cannot be used ' - 'alongside service_account.') + application_default_credential_file + ) + + if (self.service_account is None) != ( + self.service_account_private_key_file is None + ): + raise ValueError( + 'service_account and service_account_private_key ' + 'must be specified together.' + ) + if ( + application_default_credential_file is not None + and self.service_account is not None + ): + raise ValueError( + 'application_default_credential_file cannot be used ' + 'alongside service_account.' + ) def __repr__(self): return '<{0} table="{1}">'.format(type(self).__name__, self.bigquery_table) @@ -602,33 +654,40 @@ def PublishSamples(self, samples): logging.warning('No samples: not publishing to BigQuery') return - with vm_util.NamedTemporaryFile(prefix='perfkit-bq-pub', - dir=vm_util.GetTempDir(), - suffix='.json') as tf: - json_publisher = NewlineDelimitedJSONPublisher(tf.name, - collapse_labels=True) + with vm_util.NamedTemporaryFile( + prefix='perfkit-bq-pub', dir=vm_util.GetTempDir(), suffix='.json' + ) as tf: + json_publisher = NewlineDelimitedJSONPublisher( + tf.name, collapse_labels=True + ) json_publisher.PublishSamples(samples) tf.close() - logging.info('Publishing %d samples to %s', len(samples), - self.bigquery_table) + logging.info( + 'Publishing %d samples to %s', len(samples), self.bigquery_table + ) load_cmd = [self.bq_path] if self.project_id: load_cmd.append('--project_id=' + self.project_id) if self.service_account: assert self.service_account_private_key_file is not None - load_cmd.extend(['--service_account=' + self.service_account, - '--service_account_credential_file=' + - self._credentials_file, - '--service_account_private_key_file=' + - self.service_account_private_key_file]) + load_cmd.extend([ + '--service_account=' + self.service_account, + '--service_account_credential_file=' + self._credentials_file, + '--service_account_private_key_file=' + + self.service_account_private_key_file, + ]) elif self.application_default_credential_file is not None: - load_cmd.append('--application_default_credential_file=' + - self.application_default_credential_file) - load_cmd.extend(['load', - '--autodetect', - '--source_format=NEWLINE_DELIMITED_JSON', - self.bigquery_table, - tf.name]) + load_cmd.append( + '--application_default_credential_file=' + + self.application_default_credential_file + ) + load_cmd.extend([ + 'load', + '--autodetect', + '--source_format=NEWLINE_DELIMITED_JSON', + self.bigquery_table, + tf.name, + ]) vm_util.IssueRetryableCommand(load_cmd) @@ -665,9 +724,9 @@ def _GenerateObjectName(self): return object_name[:GCS_OBJECT_NAME_LENGTH] def PublishSamples(self, samples): - with vm_util.NamedTemporaryFile(prefix='perfkit-gcs-pub', - dir=vm_util.GetTempDir(), - suffix='.json') as tf: + with vm_util.NamedTemporaryFile( + prefix='perfkit-gcs-pub', dir=vm_util.GetTempDir(), suffix='.json' + ) as tf: json_publisher = NewlineDelimitedJSONPublisher(tf.name) json_publisher.PublishSamples(samples) tf.close() @@ -697,16 +756,13 @@ def __init__(self, es_uri=None, es_index=None, es_type=None): self.mapping_5_plus = { 'mappings': { 'result': { - 'numeric_detection': - True, + 'numeric_detection': True, 'properties': { 'timestamp': { 'type': 'date', - 'format': 'yyyy-MM-dd HH:mm:ss.SSSSSS' + 'format': 'yyyy-MM-dd HH:mm:ss.SSSSSS', }, - 'value': { - 'type': 'double' - } + 'value': {'type': 'double'}, }, 'dynamic_templates': [{ 'strings': { @@ -716,12 +772,12 @@ def __init__(self, es_uri=None, es_index=None, es_type=None): 'fields': { 'raw': { 'type': 'keyword', - 'ignore_above': 256 + 'ignore_above': 256, } - } - } + }, + }, } - }] + }], } } } @@ -729,16 +785,13 @@ def __init__(self, es_uri=None, es_index=None, es_type=None): self.mapping_before_5 = { 'mappings': { 'result': { - 'numeric_detection': - True, + 'numeric_detection': True, 'properties': { 'timestamp': { 'type': 'date', - 'format': 'yyyy-MM-dd HH:mm:ss.SSSSSS' + 'format': 'yyyy-MM-dd HH:mm:ss.SSSSSS', }, - 'value': { - 'type': 'double' - } + 'value': {'type': 'double'}, }, 'dynamic_templates': [{ 'strings': { @@ -748,12 +801,12 @@ def __init__(self, es_uri=None, es_index=None, es_type=None): 'fields': { 'raw': { 'type': 'string', - 'index': 'not_analyzed' + 'index': 'not_analyzed', } - } - } + }, + }, } - }] + }], } } } @@ -765,9 +818,11 @@ def PublishSamples(self, samples): from elasticsearch import Elasticsearch # pytype: disable=import-error # pylint:enable=g-import-not-at-top except ImportError: - raise ImportError('The "elasticsearch" package is required to use ' - 'the Elasticsearch publisher. Please make sure it ' - 'is installed.') + raise ImportError( + 'The "elasticsearch" package is required to use ' + 'the Elasticsearch publisher. Please make sure it ' + 'is installed.' + ) es = Elasticsearch([self.es_uri]) if not es.indices.exists(index=self.es_index): @@ -775,14 +830,18 @@ def PublishSamples(self, samples): # the version of elasticsearch that is being used if int(es.info()['version']['number'].split('.')[0]) >= 5: es.indices.create(index=self.es_index, body=self.mapping_5_plus) - logging.info('Create index %s and default mappings for' - ' elasticsearch version >= 5.0.0', - self.es_index) + logging.info( + 'Create index %s and default mappings for' + ' elasticsearch version >= 5.0.0', + self.es_index, + ) else: es.indices.create(index=self.es_index, body=self.mapping_before_5) - logging.info('Create index %s and default mappings for' - ' elasticsearch version < 5.0.0', - self.es_index) + logging.info( + 'Create index %s and default mappings for' + ' elasticsearch version < 5.0.0', + self.es_index, + ) for s in samples: sample = copy.deepcopy(s) # Make timestamp understandable by ES and human. @@ -793,11 +852,16 @@ def PublishSamples(self, samples): sample = self._deDotKeys(sample) # Add sample to the "perfkit index" of "result type" and using sample_uri # as each ES's document's unique _id - es.create(index=self.es_index, doc_type=self.es_type, - id=sample['sample_uri'], body=json.dumps(sample)) + es.create( + index=self.es_index, + doc_type=self.es_type, + id=sample['sample_uri'], + body=json.dumps(sample), + ) def _FormatTimestampForElasticsearch(self, epoch_us): """Convert the floating epoch timestamp in micro seconds epoch_us to + yyyy-MM-dd HH:mm:ss.SSSSSS in string """ ts = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(epoch_us)) @@ -822,9 +886,8 @@ class InfluxDBPublisher(SamplePublisher): Attributes: influx_uri: Takes in type string. Consists of the Influx DB address and port.Expects the format hostname:port - influx_db_name: Takes in tupe string. - Consists of the name of Influx DB database that you wish to publish to or - create. + influx_db_name: Takes in tupe string. Consists of the name of Influx DB + database that you wish to publish to or create. """ def __init__(self, influx_uri=None, influx_db_name=None): @@ -849,15 +912,23 @@ def _Publish(self, formated_samples): def _ConstructSample(self, sample): sample['product_name'] = FLAGS.product_name - timestamp = str(int((10 ** 9) * sample['timestamp'])) + timestamp = str(int((10**9) * sample['timestamp'])) measurement = 'perfkitbenchmarker' tag_set_metadata = '' if 'metadata' in sample: if sample['metadata']: tag_set_metadata = ','.join(self._FormatToKeyValue(sample['metadata'])) - tag_keys = ('test', 'official', 'owner', 'run_uri', 'sample_uri', - 'metric', 'unit', 'product_name') + tag_keys = ( + 'test', + 'official', + 'owner', + 'run_uri', + 'sample_uri', + 'metric', + 'unit', + 'product_name', + ) ordered_tags = collections.OrderedDict([(k, sample[k]) for k in tag_keys]) tag_set = ','.join(self._FormatToKeyValue(ordered_tags)) if tag_set_metadata: @@ -865,8 +936,12 @@ def _ConstructSample(self, sample): field_set = '%s=%s' % ('value', sample['value']) - sample_constructed_body = '%s,%s %s %s' % (measurement, tag_set, - field_set, timestamp) + sample_constructed_body = '%s,%s %s %s' % ( + measurement, + tag_set, + field_set, + timestamp, + ) return sample_constructed_body def _FormatToKeyValue(self, sample): @@ -887,10 +962,13 @@ def _CreateDB(self): return a 200 code without re-creating it. """ successful_http_request_codes = [200, 202, 204] - header = {'Content-type': 'application/x-www-form-urlencoded', - 'Accept': 'text/plain'} + header = { + 'Content-type': 'application/x-www-form-urlencoded', + 'Accept': 'text/plain', + } params = urllib.parse.urlencode( - {'q': 'CREATE DATABASE ' + self.influx_db_name}) + {'q': 'CREATE DATABASE ' + self.influx_db_name} + ) conn = httplib.HTTPConnection(self.influx_uri) conn.request('POST', '/query?' + params, headers=header) response = conn.getresponse() @@ -898,8 +976,11 @@ def _CreateDB(self): if response.status in successful_http_request_codes: logging.debug('Success! %s DB Created', self.influx_db_name) else: - logging.error('%d Request could not be completed due to: %s', - response.status, response.reason) + logging.error( + '%d Request could not be completed due to: %s', + response.status, + response.reason, + ) raise httplib.HTTPException def _WriteData(self, data): @@ -907,15 +988,20 @@ def _WriteData(self, data): params = data header = {'Content-type': 'application/octet-stream'} conn = httplib.HTTPConnection(self.influx_uri) - conn.request('POST', '/write?' + 'db=' + self.influx_db_name, params, - headers=header) + conn.request( + 'POST', '/write?' + 'db=' + self.influx_db_name, params, headers=header + ) response = conn.getresponse() conn.close() if response.status in successful_http_request_codes: logging.debug('Writing samples to publisher: writing samples.') else: - logging.error('%d Request could not be completed due to: %s %s', - response.status, response.reason, data) + logging.error( + '%d Request could not be completed due to: %s %s', + response.status, + response.reason, + data, + ) raise httplib.HTTPException @@ -938,8 +1024,13 @@ class SampleCollector(object): run_uri: A unique tag for the run. """ - def __init__(self, metadata_providers=None, publishers=None, - publishers_from_flags=True, add_default_publishers=True): + def __init__( + self, + metadata_providers=None, + publishers=None, + publishers_from_flags=True, + add_default_publishers=True, + ): self.samples: list[pkb_sample.SampleDict] = [] if metadata_providers is not None: @@ -968,10 +1059,13 @@ def _DefaultPublishers(cls): # Publish to the default JSON path even if we will also publish to a # different path due to flags. default_json_path = vm_util.PrependTempDir(DEFAULT_JSON_OUTPUT_NAME) - publishers.append(NewlineDelimitedJSONPublisher( - default_json_path, - mode=FLAGS.json_write_mode, - collapse_labels=FLAGS.collapse_labels)) + publishers.append( + NewlineDelimitedJSONPublisher( + default_json_path, + mode=FLAGS.json_write_mode, + collapse_labels=FLAGS.collapse_labels, + ) + ) return publishers @@ -980,10 +1074,13 @@ def _PublishersFromFlags(cls): publishers = [] if FLAGS.json_path: - publishers.append(NewlineDelimitedJSONPublisher( - FLAGS.json_path, - mode=FLAGS.json_write_mode, - collapse_labels=FLAGS.collapse_labels)) + publishers.append( + NewlineDelimitedJSONPublisher( + FLAGS.json_path, + mode=FLAGS.json_write_mode, + collapse_labels=FLAGS.collapse_labels, + ) + ) if FLAGS.bigquery_table: publishers.append( @@ -992,30 +1089,43 @@ def _PublishersFromFlags(cls): project_id=FLAGS.bq_project, bq_path=FLAGS.bq_path, service_account=FLAGS.service_account, - service_account_private_key_file=FLAGS - .service_account_private_key, - application_default_credential_file=FLAGS - .application_default_credential_file)) + service_account_private_key_file=FLAGS.service_account_private_key, + application_default_credential_file=FLAGS.application_default_credential_file, + ) + ) if FLAGS.cloud_storage_bucket: - publishers.append(CloudStoragePublisher(FLAGS.cloud_storage_bucket, - gsutil_path=FLAGS.gsutil_path)) + publishers.append( + CloudStoragePublisher( + FLAGS.cloud_storage_bucket, gsutil_path=FLAGS.gsutil_path + ) + ) if PARTITIONED_GCS_URL.value: now = datetime.datetime.now(tz=_PACIFIC_TZ) publishers.append( - CloudStoragePublisher(PARTITIONED_GCS_URL.value, - sub_folder=now.strftime('%Y/%m/%d/%H'), - gsutil_path=FLAGS.gsutil_path)) + CloudStoragePublisher( + PARTITIONED_GCS_URL.value, + sub_folder=now.strftime('%Y/%m/%d/%H'), + gsutil_path=FLAGS.gsutil_path, + ) + ) if FLAGS.csv_path: publishers.append(CSVPublisher(FLAGS.csv_path)) if FLAGS.es_uri: - publishers.append(ElasticsearchPublisher(es_uri=FLAGS.es_uri, - es_index=FLAGS.es_index, - es_type=FLAGS.es_type)) + publishers.append( + ElasticsearchPublisher( + es_uri=FLAGS.es_uri, + es_index=FLAGS.es_index, + es_type=FLAGS.es_type, + ) + ) if FLAGS.influx_uri: - publishers.append(InfluxDBPublisher(influx_uri=FLAGS.influx_uri, - influx_db_name=FLAGS.influx_db_name)) + publishers.append( + InfluxDBPublisher( + influx_uri=FLAGS.influx_uri, influx_db_name=FLAGS.influx_db_name + ) + ) return publishers @@ -1034,7 +1144,8 @@ def AddSamples(self, samples, benchmark, benchmark_spec): for meta_provider in self.metadata_providers: sample['metadata'] = meta_provider.AddMetadata( - sample['metadata'], benchmark_spec) + sample['metadata'], benchmark_spec + ) sample['product_name'] = FLAGS.product_name sample['official'] = FLAGS.official @@ -1095,8 +1206,9 @@ def RepublishJSONSamples(path): sys.exit(1) if len(argv) != 2: - logging.info('Argument number error. Usage: publisher.py ' - 'path-to-json-file') + logging.info( + 'Argument number error. Usage: publisher.py path-to-json-file' + ) sys.exit(1) json_path = argv[1] diff --git a/perfkitbenchmarker/regex_util.py b/perfkitbenchmarker/regex_util.py index f45b5e5e93..d663a772c0 100644 --- a/perfkitbenchmarker/regex_util.py +++ b/perfkitbenchmarker/regex_util.py @@ -25,11 +25,13 @@ class NoMatchError(ValueError): """Raised when no matches for a regex are found within a string.""" + pass class TooManyMatchesError(ValueError): """Raised when a regex matches a string more times than expected.""" + pass @@ -42,6 +44,7 @@ def ExtractGroup(regex, text, group=1, flags=0): group: int. Group containing a floating point value. Use '0' for the whole string. flags: int. Flags to pass to re.search(). + Returns: A string matched by 'regex' on 'text'. Raises: @@ -50,8 +53,9 @@ def ExtractGroup(regex, text, group=1, flags=0): """ match = re.search(regex, text, flags=flags) if not match: - raise NoMatchError('No match for pattern "{0}" in "{1}"'.format( - regex, text)) + raise NoMatchError( + 'No match for pattern "{0}" in "{1}"'.format(regex, text) + ) try: return match.group(group) @@ -69,28 +73,29 @@ def ExtractInt(regex, text, group=1): return int(ExtractGroup(regex, text, group=group)) -def ExtractAllFloatMetrics(text, - metric_regex=r'\w+', - value_regex=FLOAT_REGEX, - delimiter_regex='='): +def ExtractAllFloatMetrics( + text, metric_regex=r'\w+', value_regex=FLOAT_REGEX, delimiter_regex='=' +): """Extracts metrics and their values into a dict. Args: text: The text to parse to find metric and values. metric_regex: A regular expression to find metric names. The metric regex - should not contain any parenthesized groups. + should not contain any parenthesized groups. value_regex: A regular expression to find float values. By default, this - works well for floating-point numbers found via scanf. + works well for floating-point numbers found via scanf. delimiter_regex: A regular expression between the metric name and value. Returns: A dict mapping metrics to values. """ if '(' in metric_regex: - raise NotImplementedError('ExtractAllFloatMetrics does not support a ' - 'metric regex with groups.') - matches = re.findall('(%s)%s(%s)' % (metric_regex, delimiter_regex, - value_regex), text) + raise NotImplementedError( + 'ExtractAllFloatMetrics does not support a metric regex with groups.' + ) + matches = re.findall( + '(%s)%s(%s)' % (metric_regex, delimiter_regex, value_regex), text + ) return {match[0]: float(match[1]) for match in matches} @@ -99,6 +104,7 @@ def ExtractIpv4Addresses(text): Args: text: string. Text to search. + Returns: A list of ipv4 strings. Raises: @@ -123,6 +129,7 @@ def ExtractAllMatches(regex: Union[str, re.Pattern[str]], text, flags=0): regex: string. Regular expression. text: string. Text to search. flags: int. Flags to pass to re.findall(). + Returns: A list of tuples of strings that matched by 'regex' within 'text'. Raises: @@ -130,8 +137,9 @@ def ExtractAllMatches(regex: Union[str, re.Pattern[str]], text, flags=0): """ match = re.findall(regex, text, flags=flags) if not match: - raise NoMatchError('No match for pattern "{0}" in "{1}"'.format( - regex, text)) + raise NoMatchError( + 'No match for pattern "{0}" in "{1}"'.format(regex, text) + ) return match @@ -154,7 +162,8 @@ def ExtractExactlyOneMatch(regex, text): matches = ExtractAllMatches(regex, text) if len(matches) > 1: raise TooManyMatchesError( - 'Pattern "{0}" matched "{1}" non-uniquely.'.format(regex, text)) + 'Pattern "{0}" matched "{1}" non-uniquely.'.format(regex, text) + ) return matches[0] @@ -165,12 +174,14 @@ def Substitute(pattern, repl, text): pattern: string. Pattern to be replaced. repl: string. Replacement pattern. text: string. Text to search. + Returns: A string after replacing all patterns with repl. Raises: NoMatchError: when 'pattern' isn't found in string. """ if not re.search(pattern, text): - raise NoMatchError('No match for pattern "{0}" in "{1}"'.format( - pattern, text)) + raise NoMatchError( + 'No match for pattern "{0}" in "{1}"'.format(pattern, text) + ) return re.sub(pattern, repl, text) diff --git a/perfkitbenchmarker/relational_db.py b/perfkitbenchmarker/relational_db.py index 87ff7ef9d4..16420034e5 100644 --- a/perfkitbenchmarker/relational_db.py +++ b/perfkitbenchmarker/relational_db.py @@ -27,114 +27,171 @@ import six # TODO(chunla): Move IAAS flag to file -flags.DEFINE_string('db_engine', None, - 'Managed database flavor to use (mysql, postgres)') -flags.DEFINE_string('db_engine_version', None, - 'Version of the database flavor selected, e.g. 5.7') -flags.DEFINE_string('database_name', None, - 'Name of the database to create. Defaults to ' - 'pkb-db-[run-uri]') -flags.DEFINE_string('database_username', None, - 'Database username. Defaults to ' - 'pkb-db-user-[run-uri]') -flags.DEFINE_string('database_password', None, - 'Database password. Defaults to ' - 'a random 10-character alpha-numeric string') -flags.DEFINE_boolean('db_high_availability', False, - 'Specifies if the database should be high availability') -flags.DEFINE_string('db_high_availability_type', None, - 'Specifies high availableity type. (AOAG, FCIS2D, FCIMW)') -flags.DEFINE_boolean('db_backup_enabled', True, - 'Whether or not to enable automated backups') -flags.DEFINE_string('db_backup_start_time', '07:00', - 'Time in UTC that automated backups (if enabled) ' - 'will be scheduled. In the form HH:MM UTC. ' - 'Defaults to 07:00 UTC') -flags.DEFINE_list('db_zone', None, - 'zone or region to launch the database in. ' - 'Defaults to the client vm\'s zone.') -flags.DEFINE_string('client_vm_zone', None, - 'zone or region to launch the client in. ') -flags.DEFINE_string('db_machine_type', None, - 'Machine type of the database.') -flags.DEFINE_integer('db_cpus', None, - 'Number of Cpus in the database.') -flags.DEFINE_string('db_memory', None, - 'Amount of Memory in the database. Uses the same format ' - 'string as custom machine memory type.') -flags.DEFINE_integer('db_disk_size', None, - 'Size of the database disk in GB.') -flags.DEFINE_integer('db_num_striped_disks', None, - 'The number of data disks to stripe together to form one.') +flags.DEFINE_string( + 'db_engine', None, 'Managed database flavor to use (mysql, postgres)' +) +flags.DEFINE_string( + 'db_engine_version', + None, + 'Version of the database flavor selected, e.g. 5.7', +) +flags.DEFINE_string( + 'database_name', + None, + 'Name of the database to create. Defaults to pkb-db-[run-uri]', +) +flags.DEFINE_string( + 'database_username', + None, + 'Database username. Defaults to pkb-db-user-[run-uri]', +) +flags.DEFINE_string( + 'database_password', + None, + 'Database password. Defaults to a random 10-character alpha-numeric string', +) +flags.DEFINE_boolean( + 'db_high_availability', + False, + 'Specifies if the database should be high availability', +) +flags.DEFINE_string( + 'db_high_availability_type', + None, + 'Specifies high availableity type. (AOAG, FCIS2D, FCIMW)', +) +flags.DEFINE_boolean( + 'db_backup_enabled', True, 'Whether or not to enable automated backups' +) +flags.DEFINE_string( + 'db_backup_start_time', + '07:00', + 'Time in UTC that automated backups (if enabled) ' + 'will be scheduled. In the form HH:MM UTC. ' + 'Defaults to 07:00 UTC', +) +flags.DEFINE_list( + 'db_zone', + None, + 'zone or region to launch the database in. ' + "Defaults to the client vm's zone.", +) +flags.DEFINE_string( + 'client_vm_zone', None, 'zone or region to launch the client in. ' +) +flags.DEFINE_string('db_machine_type', None, 'Machine type of the database.') +flags.DEFINE_integer('db_cpus', None, 'Number of Cpus in the database.') +flags.DEFINE_string( + 'db_memory', + None, + 'Amount of Memory in the database. Uses the same format ' + 'string as custom machine memory type.', +) +flags.DEFINE_integer('db_disk_size', None, 'Size of the database disk in GB.') +flags.DEFINE_integer( + 'db_num_striped_disks', + None, + 'The number of data disks to stripe together to form one.', +) flags.DEFINE_string('db_disk_type', None, 'Disk type of the database.') -flags.DEFINE_integer('managed_db_disk_iops', None, - 'Disk iops of the database on AWS io1 disks.') - -flags.DEFINE_integer('managed_db_azure_compute_units', None, - 'Number of Dtus in the database.') -flags.DEFINE_string('managed_db_tier', None, - 'Tier in azure. (Basic, Standard, Premium).') -flags.DEFINE_string('server_vm_os_type', None, - 'OS type of the client vm.') -flags.DEFINE_string('client_vm_os_type', None, - 'OS type of the client vm.') -flags.DEFINE_string('server_gcp_min_cpu_platform', None, - 'Cpu platform of the server vm.') -flags.DEFINE_string('client_gcp_min_cpu_platform', None, - 'CPU platform of the client vm.') -flags.DEFINE_string('client_vm_machine_type', None, - 'Machine type of the client vm.') +flags.DEFINE_integer( + 'managed_db_disk_iops', None, 'Disk iops of the database on AWS io1 disks.' +) + +flags.DEFINE_integer( + 'managed_db_azure_compute_units', None, 'Number of Dtus in the database.' +) +flags.DEFINE_string( + 'managed_db_tier', None, 'Tier in azure. (Basic, Standard, Premium).' +) +flags.DEFINE_string('server_vm_os_type', None, 'OS type of the client vm.') +flags.DEFINE_string('client_vm_os_type', None, 'OS type of the client vm.') +flags.DEFINE_string( + 'server_gcp_min_cpu_platform', None, 'Cpu platform of the server vm.' +) +flags.DEFINE_string( + 'client_gcp_min_cpu_platform', None, 'CPU platform of the client vm.' +) +flags.DEFINE_string( + 'client_vm_machine_type', None, 'Machine type of the client vm.' +) flags.DEFINE_integer('client_vm_cpus', None, 'Number of Cpus in the client vm.') flags.DEFINE_string( - 'client_vm_memory', None, + 'client_vm_memory', + None, 'Amount of Memory in the vm. Uses the same format ' - 'string as custom machine memory type.') -flags.DEFINE_integer('client_vm_disk_size', None, - 'Size of the client vm disk in GB.') + 'string as custom machine memory type.', +) +flags.DEFINE_integer( + 'client_vm_disk_size', None, 'Size of the client vm disk in GB.' +) flags.DEFINE_string('client_vm_disk_type', None, 'Disk type of the client vm.') -flags.DEFINE_integer('client_vm_disk_iops', None, - 'Disk iops of the database on AWS for client vm.') +flags.DEFINE_integer( + 'client_vm_disk_iops', + None, + 'Disk iops of the database on AWS for client vm.', +) flags.DEFINE_boolean( - 'use_managed_db', True, 'If true, uses the managed MySql ' + 'use_managed_db', + True, + 'If true, uses the managed MySql ' 'service for the requested cloud provider. If false, uses ' - 'MySql installed on a VM.') + 'MySql installed on a VM.', +) flags.DEFINE_list( - 'db_flags', '', 'Flags to apply to the managed relational database ' - 'on the cloud that\'s being used. Example: ' - 'binlog_cache_size=4096,innodb_log_buffer_size=4294967295') + 'db_flags', + '', + 'Flags to apply to the managed relational database ' + "on the cloud that's being used. Example: " + 'binlog_cache_size=4096,innodb_log_buffer_size=4294967295', +) flags.DEFINE_integer( - 'innodb_buffer_pool_size', None, + 'innodb_buffer_pool_size', + None, 'Size of the innodb buffer pool size in GB. ' - 'Defaults to 25% of VM memory if unset') + 'Defaults to 25% of VM memory if unset', +) flags.DEFINE_bool( - 'mysql_bin_log', False, - 'Flag to turn binary logging on. ' - 'Defaults to False') -flags.DEFINE_integer('innodb_log_file_size', 1000, - 'Size of the log file in MB. Defaults to 1000M.') + 'mysql_bin_log', False, 'Flag to turn binary logging on. Defaults to False' +) flags.DEFINE_integer( - 'postgres_shared_buffer_size', None, + 'innodb_log_file_size', + 1000, + 'Size of the log file in MB. Defaults to 1000M.', +) +flags.DEFINE_integer( + 'postgres_shared_buffer_size', + None, 'Size of the shared buffer size in GB. ' - 'Defaults to 25% of VM memory if unset') + 'Defaults to 25% of VM memory if unset', +) OPTIMIZE_DB_SYSCTL_CONFIG = flags.DEFINE_bool( - 'optimize_db_sysctl_config', True, - 'Flag to run sysctl optimization for IAAS relational database.') + 'optimize_db_sysctl_config', + True, + 'Flag to run sysctl optimization for IAAS relational database.', +) SERVER_GCE_NUM_LOCAL_SSDS = flags.DEFINE_integer( - 'server_gce_num_local_ssds', 0, - 'The number of ssds that should be added to the Server.') + 'server_gce_num_local_ssds', + 0, + 'The number of ssds that should be added to the Server.', +) SERVER_GCE_SSD_INTERFACE = flags.DEFINE_enum( - 'server_gce_ssd_interface', 'SCSI', ['SCSI', 'NVME'], - 'The ssd interface for GCE local SSD.') + 'server_gce_ssd_interface', + 'SCSI', + ['SCSI', 'NVME'], + 'The ssd interface for GCE local SSD.', +) BACKUP_TIME_REGULAR_EXPRESSION = r'^\d\d\:\d\d$' flags.register_validator( 'db_backup_start_time', lambda value: re.search(BACKUP_TIME_REGULAR_EXPRESSION, value) is not None, - message=('--database_backup_start_time must be in the form HH:MM')) + message='--database_backup_start_time must be in the form HH:MM', +) FLAGS = flags.FLAGS @@ -176,12 +233,14 @@ def GetRelationalDbClass(cloud, is_managed_db, engine): """ try: return resource.GetResourceClass( - BaseRelationalDb, CLOUD=cloud, IS_MANAGED=is_managed_db, ENGINE=engine) + BaseRelationalDb, CLOUD=cloud, IS_MANAGED=is_managed_db, ENGINE=engine + ) except errors.Resource.SubclassNotFoundError: pass return resource.GetResourceClass( - BaseRelationalDb, CLOUD=cloud, IS_MANAGED=is_managed_db) + BaseRelationalDb, CLOUD=cloud, IS_MANAGED=is_managed_db + ) def VmsToBoot(vm_groups): @@ -189,13 +248,15 @@ def VmsToBoot(vm_groups): return { name: spec # pylint: disable=g-complex-comprehension for name, spec in six.iteritems(vm_groups) - if name == 'clients' or name == 'default' or - (not FLAGS.use_managed_db and name == 'servers') + if name == 'clients' + or name == 'default' + or (not FLAGS.use_managed_db and name == 'servers') } class BaseRelationalDb(resource.BaseResource): """Object representing a relational database Service.""" + IS_MANAGED = True RESOURCE_TYPE = 'BaseRelationalDb' REQUIRED_ATTRS = ['CLOUD', 'IS_MANAGED'] @@ -209,7 +270,8 @@ def __init__(self, relational_db_spec): super(BaseRelationalDb, self).__init__( enable_freeze_restore=relational_db_spec.enable_freeze_restore, create_on_restore_error=relational_db_spec.create_on_restore_error, - delete_on_freeze_error=relational_db_spec.delete_on_freeze_error) + delete_on_freeze_error=relational_db_spec.delete_on_freeze_error, + ) self.spec = relational_db_spec self.engine = self.spec.engine self.engine_type = sql_engine_utils.GetDbEngineType(self.engine) @@ -324,19 +386,24 @@ def GetResourceMetadata(self): 'client_vm_zone': self.spec.vm_groups['clients'].vm_spec.zone, 'use_managed_db': self.is_managed_db, 'instance_id': self.instance_id, - 'client_vm_disk_type': - self.spec.vm_groups['clients'].disk_spec.disk_type, - 'client_vm_disk_size': - self.spec.vm_groups['clients'].disk_spec.disk_size, + 'client_vm_disk_type': self.spec.vm_groups[ + 'clients' + ].disk_spec.disk_type, + 'client_vm_disk_size': self.spec.vm_groups[ + 'clients' + ].disk_spec.disk_size, } - if (hasattr(self.spec.db_spec, 'machine_type') and - self.spec.db_spec.machine_type): + if ( + hasattr(self.spec.db_spec, 'machine_type') + and self.spec.db_spec.machine_type + ): metadata.update({ 'machine_type': self.spec.db_spec.machine_type, }) - elif hasattr(self.spec.db_spec, 'cpus') and (hasattr( - self.spec.db_spec, 'memory')): + elif hasattr(self.spec.db_spec, 'cpus') and ( + hasattr(self.spec.db_spec, 'memory') + ): metadata.update({ 'cpus': self.spec.db_spec.cpus, }) @@ -354,16 +421,21 @@ def GetResourceMetadata(self): }) else: raise RelationalDbPropertyNotSetError( - 'Machine type of the database must be set.') + 'Machine type of the database must be set.' + ) - if (hasattr(self.spec.vm_groups['clients'].vm_spec, 'machine_type') and - self.spec.vm_groups['clients'].vm_spec.machine_type): + if ( + hasattr(self.spec.vm_groups['clients'].vm_spec, 'machine_type') + and self.spec.vm_groups['clients'].vm_spec.machine_type + ): metadata.update({ - 'client_vm_machine_type': - self.spec.vm_groups['clients'].vm_spec.machine_type, + 'client_vm_machine_type': self.spec.vm_groups[ + 'clients' + ].vm_spec.machine_type, }) - elif hasattr(self.spec.vm_groups['clients'].vm_spec, 'cpus') and (hasattr( - self.spec.vm_groups['clients'].vm_spec, 'memory')): + elif hasattr(self.spec.vm_groups['clients'].vm_spec, 'cpus') and ( + hasattr(self.spec.vm_groups['clients'].vm_spec, 'memory') + ): metadata.update({ 'client_vm_cpus': self.spec.vm_groups['clients'].vm_spec.cpus, }) @@ -372,7 +444,8 @@ def GetResourceMetadata(self): }) else: raise RelationalDbPropertyNotSetError( - 'Machine type of the client VM must be set.') + 'Machine type of the client VM must be set.' + ) if FLAGS.db_flags: metadata.update({ @@ -419,15 +492,17 @@ def UpdateCapacityForRun(self) -> None: def _ApplyDbFlags(self): """Apply Flags on the database.""" - raise NotImplementedError('Managed Db flags is not supported for %s' % - type(self).__name__) + raise NotImplementedError( + 'Managed Db flags is not supported for %s' % type(self).__name__ + ) def GetDefaultPort(self): """Returns default port for the db engine from the spec.""" engine = sql_engine_utils.GetDbEngineType(self.spec.engine) if engine not in DEFAULT_PORTS: - raise NotImplementedError('Default port not specified for ' - 'engine {0}'.format(engine)) + raise NotImplementedError( + 'Default port not specified for engine {0}'.format(engine) + ) return DEFAULT_PORTS[engine] def CreateDatabase(self, database_name: str) -> tuple[str, str]: diff --git a/perfkitbenchmarker/relational_db_spec.py b/perfkitbenchmarker/relational_db_spec.py index 1c262e4e32..1686a1e7d8 100644 --- a/perfkitbenchmarker/relational_db_spec.py +++ b/perfkitbenchmarker/relational_db_spec.py @@ -31,7 +31,8 @@ def GetRelationalDbSpecClass(engine): """Get the RelationalDbSpec class corresponding to 'engine'.""" if engine in [ - sql_engine_utils.SPANNER_GOOGLESQL, sql_engine_utils.SPANNER_POSTGRES + sql_engine_utils.SPANNER_GOOGLESQL, + sql_engine_utils.SPANNER_POSTGRES, ]: return spec.GetSpecClass(RelationalDbSpec, SERVICE_TYPE='spanner') return RelationalDbSpec @@ -53,6 +54,7 @@ class RelationalDbSpec(freeze_restore_spec.FreezeRestoreSpec): db_disk_spec: disk.BaseDiskSpec: Configurable disk options. db_spec: virtual_machine.BaseVmSpec: Configurable VM options. """ + SPEC_TYPE = 'RelationalDbSpec' SPEC_ATTRS = ['SERVICE_TYPE'] @@ -70,13 +72,16 @@ class RelationalDbSpec(freeze_restore_spec.FreezeRestoreSpec): def __init__(self, component_full_name, flag_values=None, **kwargs): super(RelationalDbSpec, self).__init__( - component_full_name, flag_values=flag_values, **kwargs) + component_full_name, flag_values=flag_values, **kwargs + ) # TODO(user): This is a lot of boilerplate, and is repeated # below in VmGroupSpec. See if some can be consolidated. Maybe we can # specify a VmGroupSpec instead of both vm_spec and disk_spec. ignore_package_requirements = ( getattr(flag_values, 'ignore_package_requirements', True) - if flag_values else True) + if flag_values + else True + ) providers.LoadProvider(self.cloud, ignore_package_requirements) if self.db_disk_spec: @@ -84,26 +89,30 @@ def __init__(self, component_full_name, flag_values=None, **kwargs): if disk_config is None: raise errors.Config.MissingOption( '{0}.cloud is "{1}", but {0}.db_disk_spec does not contain a ' - 'configuration for "{1}".'.format(component_full_name, self.cloud)) + 'configuration for "{1}".'.format(component_full_name, self.cloud) + ) disk_spec_class = disk.GetDiskSpecClass( self.cloud, disk_config.get('disk_type', None) ) self.db_disk_spec = disk_spec_class( '{0}.db_disk_spec.{1}'.format(component_full_name, self.cloud), flag_values=flag_values, - **disk_config) + **disk_config + ) if self.db_spec: db_vm_config = getattr(self.db_spec, self.cloud, None) if db_vm_config is None: raise errors.Config.MissingOption( '{0}.cloud is "{1}", but {0}.db_spec does not contain a ' - 'configuration for "{1}".'.format(component_full_name, self.cloud)) + 'configuration for "{1}".'.format(component_full_name, self.cloud) + ) db_vm_spec_class = virtual_machine.GetVmSpecClass(self.cloud) self.db_spec = db_vm_spec_class( '{0}.db_spec.{1}'.format(component_full_name, self.cloud), flag_values=flag_values, - **db_vm_config) + **db_vm_config + ) # Set defaults that were not able to be set in # GetOptionDecoderConstructions() @@ -148,7 +157,9 @@ def _GetOptionDecoderConstructions(cls): {'default': False}, ), 'high_availability_type': ( - option_decoders.StringDecoder, {'default': None}), + option_decoders.StringDecoder, + {'default': None}, + ), 'backup_enabled': (option_decoders.BooleanDecoder, {'default': True}), 'backup_start_time': ( option_decoders.StringDecoder, @@ -199,24 +210,29 @@ def _ApplyFlags(cls, config_values, flag_values): raise errors.Config.UnrecognizedOption( 'db_cpus/db_memory can not be specified with ' 'db_machine_type. Either specify a custom machine ' - 'with cpus and memory or specify a predefined machine type.') + 'with cpus and memory or specify a predefined machine type.' + ) - if (not has_custom_machine_type and (has_db_cpus or has_db_memory)): + if not has_custom_machine_type and (has_db_cpus or has_db_memory): raise errors.Config.MissingOption( 'To specify a custom database machine instance, both db_cpus ' - 'and db_memory must be specified.') + 'and db_memory must be specified.' + ) if has_client_custom_machine_type and has_client_machine_type: raise errors.Config.UnrecognizedOption( 'client_vm_cpus/client_vm_memory can not be specified with ' 'client_vm_machine_type. Either specify a custom machine ' - 'with cpus and memory or specify a predefined machine type.') + 'with cpus and memory or specify a predefined machine type.' + ) - if (not has_client_custom_machine_type and - (has_client_vm_cpus or has_client_vm_memory)): + if not has_client_custom_machine_type and ( + has_client_vm_cpus or has_client_vm_memory + ): raise errors.Config.MissingOption( 'To specify a custom client VM, both client_vm_cpus ' - 'and client_vm_memory must be specified.') + 'and client_vm_memory must be specified.' + ) if flag_values['use_managed_db'].present: config_values['is_managed_db'] = flag_values.use_managed_db @@ -230,48 +246,49 @@ def _ApplyFlags(cls, config_values, flag_values): if flag_values['database_name'].present: config_values['database_name'] = flag_values.database_name if flag_values['database_username'].present: - config_values['database_username'] = ( - flag_values.database_username) + config_values['database_username'] = flag_values.database_username if flag_values['database_password'].present: - config_values['database_password'] = ( - flag_values.database_password) + config_values['database_password'] = flag_values.database_password if flag_values['db_high_availability'].present: - config_values['high_availability'] = ( - flag_values.db_high_availability) + config_values['high_availability'] = flag_values.db_high_availability if flag_values['db_high_availability_type'].present: config_values['high_availability_type'] = ( - flag_values.db_high_availability_type) + flag_values.db_high_availability_type + ) if flag_values['db_backup_enabled'].present: - config_values['backup_enabled'] = (flag_values.db_backup_enabled) + config_values['backup_enabled'] = flag_values.db_backup_enabled if flag_values['db_backup_start_time'].present: - config_values['backup_start_time'] = ( - flag_values.db_backup_start_time) + config_values['backup_start_time'] = flag_values.db_backup_start_time if flag_values['db_flags'].present: config_values['db_flags'] = flag_values.db_flags cloud = config_values['cloud'] - has_unmanaged_dbs = ('vm_groups' in config_values and - 'servers' in config_values['vm_groups']) + has_unmanaged_dbs = ( + 'vm_groups' in config_values and 'servers' in config_values['vm_groups'] + ) if flag_values['db_zone'].present: config_values['db_spec'][cloud]['zone'] = flag_values.db_zone[0] config_values['zones'] = flag_values.db_zone if has_unmanaged_dbs: config_values['vm_groups']['servers']['vm_spec'][cloud]['zone'] = ( - flag_values.db_zone[0]) + flag_values.db_zone[0] + ) if flag_values['client_vm_zone'].present: - config_values['vm_groups']['clients']['vm_spec'][cloud]['zone'] = ( - flag_values.client_vm_zone) + config_values['vm_groups']['clients']['vm_spec'][cloud][ + 'zone' + ] = flag_values.client_vm_zone if has_db_machine_type: - config_values['db_spec'][cloud]['machine_type'] = ( - flag_values.db_machine_type) + config_values['db_spec'][cloud][ + 'machine_type' + ] = flag_values.db_machine_type if has_unmanaged_dbs: config_values['vm_groups']['servers']['vm_spec'][cloud][ - 'machine_type'] = ( - flag_values.db_machine_type) + 'machine_type' + ] = flag_values.db_machine_type if has_custom_machine_type: config_values['db_spec'][cloud]['machine_type'] = { 'cpus': flag_values.db_cpus, - 'memory': flag_values.db_memory + 'memory': flag_values.db_memory, } # tox and pylint have contradictory closing brace rules, so avoid having # opening and closing brackets on different lines. @@ -279,74 +296,92 @@ def _ApplyFlags(cls, config_values, flag_values): if has_unmanaged_dbs: config_values_vm_groups['servers']['vm_spec'][cloud]['machine_type'] = { 'cpus': flag_values.db_cpus, - 'memory': flag_values.db_memory + 'memory': flag_values.db_memory, } if flag_values['managed_db_azure_compute_units'].present: - config_values['db_spec'][cloud]['machine_type']['compute_units'] = ( - flag_values.managed_db_azure_compute_units) + config_values['db_spec'][cloud]['machine_type'][ + 'compute_units' + ] = flag_values.managed_db_azure_compute_units if flag_values['managed_db_tier'].present: config_values['db_tier'] = flag_values.managed_db_tier if has_client_machine_type: config_values['vm_groups']['clients']['vm_spec'][cloud][ - 'machine_type'] = ( - flag_values.client_vm_machine_type) + 'machine_type' + ] = flag_values.client_vm_machine_type if has_client_custom_machine_type: config_values_vm_groups = config_values['vm_groups'] config_values_vm_groups['clients']['vm_spec'][cloud]['machine_type'] = { 'cpus': flag_values.client_vm_cpus, - 'memory': flag_values.client_vm_memory + 'memory': flag_values.client_vm_memory, } if flag_values['db_num_striped_disks'].present and has_unmanaged_dbs: config_values['vm_groups']['servers']['disk_spec'][cloud][ - 'num_striped_disks'] = flag_values.db_num_striped_disks + 'num_striped_disks' + ] = flag_values.db_num_striped_disks if flag_values['db_disk_size'].present: - config_values['db_disk_spec'][cloud]['disk_size'] = ( - flag_values.db_disk_size) + config_values['db_disk_spec'][cloud][ + 'disk_size' + ] = flag_values.db_disk_size if has_unmanaged_dbs: config_values['vm_groups']['servers']['disk_spec'][cloud][ - 'disk_size'] = flag_values.db_disk_size + 'disk_size' + ] = flag_values.db_disk_size if flag_values['db_disk_type'].present: - config_values['db_disk_spec'][cloud]['disk_type'] = ( - flag_values.db_disk_type) + config_values['db_disk_spec'][cloud][ + 'disk_type' + ] = flag_values.db_disk_type if has_unmanaged_dbs: config_values['vm_groups']['servers']['disk_spec'][cloud][ - 'disk_type'] = flag_values.db_disk_type + 'disk_type' + ] = flag_values.db_disk_type if flag_values['managed_db_disk_iops'].present: # This value will be used in aws_relation_db.py druing db creation - config_values['db_disk_spec'][cloud]['iops'] = ( - flag_values.managed_db_disk_iops) + config_values['db_disk_spec'][cloud][ + 'iops' + ] = flag_values.managed_db_disk_iops if has_unmanaged_dbs: config_values['vm_groups']['servers']['disk_spec'][cloud][ - 'iops'] = flag_values.managed_db_disk_iops + 'iops' + ] = flag_values.managed_db_disk_iops if flag_values['client_vm_os_type'].present: config_values['vm_groups']['clients'][ - 'os_type'] = flag_values.client_vm_os_type + 'os_type' + ] = flag_values.client_vm_os_type if flag_values['server_vm_os_type'].present: config_values['vm_groups']['servers'][ - 'os_type'] = flag_values.server_vm_os_type + 'os_type' + ] = flag_values.server_vm_os_type if flag_values['client_gcp_min_cpu_platform'].present: config_values['vm_groups']['clients']['vm_spec'][cloud][ - 'min_cpu_platform'] = flag_values.client_gcp_min_cpu_platform + 'min_cpu_platform' + ] = flag_values.client_gcp_min_cpu_platform if flag_values['server_gcp_min_cpu_platform'].present: config_values['vm_groups']['servers']['vm_spec'][cloud][ - 'min_cpu_platform'] = flag_values.server_gcp_min_cpu_platform + 'min_cpu_platform' + ] = flag_values.server_gcp_min_cpu_platform if flag_values['server_gce_num_local_ssds'].present and has_unmanaged_dbs: config_values['vm_groups']['servers']['vm_spec'][cloud][ - 'num_local_ssds'] = flag_values.server_gce_num_local_ssds + 'num_local_ssds' + ] = flag_values.server_gce_num_local_ssds if flag_values['server_gce_ssd_interface'].present and has_unmanaged_dbs: config_values['vm_groups']['servers']['vm_spec'][cloud][ - 'ssd_interface'] = flag_values.server_gce_ssd_interface + 'ssd_interface' + ] = flag_values.server_gce_ssd_interface config_values['vm_groups']['servers']['disk_spec'][cloud][ - 'interface'] = flag_values.server_gce_ssd_interface + 'interface' + ] = flag_values.server_gce_ssd_interface if flag_values['client_vm_disk_size'].present: - config_values['vm_groups']['clients']['disk_spec'][cloud]['disk_size'] = ( - flag_values.client_vm_disk_size) + config_values['vm_groups']['clients']['disk_spec'][cloud][ + 'disk_size' + ] = flag_values.client_vm_disk_size if flag_values['client_vm_disk_type'].present: - config_values['vm_groups']['clients']['disk_spec'][cloud]['disk_type'] = ( - flag_values.client_vm_disk_type) + config_values['vm_groups']['clients']['disk_spec'][cloud][ + 'disk_type' + ] = flag_values.client_vm_disk_type if flag_values['client_vm_disk_iops'].present: - config_values['vm_groups']['clients']['disk_spec'][cloud]['disk_iops'] = ( - flag_values.client_vm_disk_iops) + config_values['vm_groups']['clients']['disk_spec'][cloud][ + 'disk_iops' + ] = flag_values.client_vm_disk_iops logging.warning('Relational db config values: %s', config_values) diff --git a/perfkitbenchmarker/requirements.py b/perfkitbenchmarker/requirements.py index 2d84fe41eb..643ec249bd 100644 --- a/perfkitbenchmarker/requirements.py +++ b/perfkitbenchmarker/requirements.py @@ -15,9 +15,8 @@ from collections import deque import os -import pkg_resources - from perfkitbenchmarker import errors +import pkg_resources # Path of the root of the current git branch. @@ -44,8 +43,10 @@ def _CheckRequirements(requirements_file_path): lines = deque(fp.readlines()) elif line: pkg_resources.require(line) - except (pkg_resources.DistributionNotFound, - pkg_resources.VersionConflict) as e: + except ( + pkg_resources.DistributionNotFound, + pkg_resources.VersionConflict, + ) as e: # In newer versions of setuptools, these exception classes have a report # method that provides a readable description of the error. report = getattr(e, 'report', None) @@ -56,7 +57,9 @@ def _CheckRequirements(requirements_file_path): 'command:{linesep}pip install -r "{path}"{linesep}To bypass package ' 'requirement checks, run PerfKit Benchmarker with the ' '--ignore_package_requirements flag.'.format( - linesep=os.linesep, msg=err_msg, path=requirements_file_path)) + linesep=os.linesep, msg=err_msg, path=requirements_file_path + ) + ) def CheckBasicRequirements(): @@ -68,8 +71,7 @@ def CheckBasicRequirements(): directory. If such a file does not exist, then the requirements check is skipped. """ - requirements_file_path = os.path.join(_BRANCH_ROOT_DIR, - 'requirements.txt') + requirements_file_path = os.path.join(_BRANCH_ROOT_DIR, 'requirements.txt') if os.path.isfile(requirements_file_path): _CheckRequirements(requirements_file_path) @@ -87,7 +89,11 @@ def CheckProviderRequirements(provider): provider: string. Lowercase name of the cloud provider (e.g. 'gcp'). """ requirements_file_path = os.path.join( - _BRANCH_ROOT_DIR, 'perfkitbenchmarker', 'providers', provider, - 'requirements.txt') + _BRANCH_ROOT_DIR, + 'perfkitbenchmarker', + 'providers', + provider, + 'requirements.txt', + ) if os.path.isfile(requirements_file_path): _CheckRequirements(requirements_file_path) diff --git a/perfkitbenchmarker/sample.py b/perfkitbenchmarker/sample.py index 98c716def2..89638dede7 100644 --- a/perfkitbenchmarker/sample.py +++ b/perfkitbenchmarker/sample.py @@ -43,7 +43,11 @@ TIMESTAMPS = 'timestamps' INTERVAL = 'interval' TIME_SERIES_METADATA = [ - RAMP_UP_ENDS, RAMP_DOWN_STARTS, VALUES, TIMESTAMPS, INTERVAL + RAMP_UP_ENDS, + RAMP_DOWN_STARTS, + VALUES, + TIMESTAMPS, + INTERVAL, ] @@ -61,7 +65,6 @@ def PercentileCalculator(numbers, percentiles=PERCENTILES_LIST): Raises: ValueError, if numbers is empty or if a percentile is outside of [0, 100]. - """ # 'if not numbers' will fail if numbers is an np.Array or pd.Series. @@ -85,8 +88,8 @@ def PercentileCalculator(numbers, percentiles=PERCENTILES_LIST): average = total / float(count) result['average'] = average if count > 1: - total_of_squares = sum([(i - average)**2 for i in numbers]) - result['stddev'] = (total_of_squares / (count - 1))**0.5 + total_of_squares = sum([(i - average) ** 2 for i in numbers]) + result['stddev'] = (total_of_squares / (count - 1)) ** 0.5 else: result['stddev'] = 0 @@ -108,7 +111,7 @@ def GeoMean(iterable): arr = np.fromiter(iterable, dtype='float') if not arr.size: raise ValueError("Can't compute geomean of empty list.") - return arr.prod()**(1 / len(arr)) + return arr.prod() ** (1 / len(arr)) # The Sample is converted via collections.namedtuple._asdict for publishing @@ -126,13 +129,9 @@ class Sample(collections.namedtuple('Sample', _SAMPLE_FIELDS)): timestamp: float. Unix timestamp. """ - def __new__(cls, - metric, - value, - unit, - metadata=None, - timestamp=None, - **kwargs): + def __new__( + cls, metric, value, unit, metadata=None, timestamp=None, **kwargs + ): if timestamp is None: timestamp = time.time() @@ -143,7 +142,8 @@ def __new__(cls, unit, metadata=metadata or {}, timestamp=timestamp, - **kwargs) + **kwargs, + ) def __eq__(self, other) -> bool: if not isinstance(other, Sample): @@ -160,7 +160,7 @@ def __eq__(self, other) -> bool: return False return True - def asdict(self)-> Dict[str, Any]: # pylint:disable=invalid-name + def asdict(self) -> Dict[str, Any]: # pylint:disable=invalid-name """Converts the Sample to a dictionary.""" return self._asdict() @@ -168,9 +168,9 @@ def asdict(self)-> Dict[str, Any]: # pylint:disable=invalid-name _Histogram = collections.OrderedDict -def MakeHistogram(values: List[float], - round_bottom: float = 0.0, - round_to_sig_fig: int = 3) -> _Histogram[float, int]: +def MakeHistogram( + values: List[float], round_bottom: float = 0.0, round_to_sig_fig: int = 3 +) -> _Histogram[float, int]: """Take a list of float values and returns a ordered dict of values and frequency. Args: @@ -195,7 +195,8 @@ def MakeHistogram(values: List[float], if value > 0: rounded_value = round( value, - round_to_sig_fig - int(math.floor(math.log10(abs(value)))) - 1) + round_to_sig_fig - int(math.floor(math.log10(abs(value)))) - 1, + ) else: rounded_value = 0.0 histogram[rounded_value] = histogram.get(rounded_value, 0) + 1 @@ -206,17 +207,20 @@ def MakeHistogram(values: List[float], def _ConvertHistogramToString(histogram: _Histogram[float, int]) -> str: histogram_label_values = ','.join( - f'"{key}": {value}' for (key, value) in histogram.items()) + f'"{key}": {value}' for (key, value) in histogram.items() + ) histogram_labels = '{%s}' % histogram_label_values return histogram_labels -def CreateHistogramSample(histogram: _Histogram[float, int], - name: str, - subname: str, - units: str, - additional_metadata=None, - metric='') -> Sample: +def CreateHistogramSample( + histogram: _Histogram[float, int], + name: str, + subname: str, + units: str, + additional_metadata=None, + metric='', +) -> Sample: """Given a histogram of values, create a sample. Args: @@ -229,26 +233,27 @@ def CreateHistogramSample(histogram: _Histogram[float, int], Returns: sample: One sample object that reports the histogram passed in. - """ metadata = { 'histogram': _ConvertHistogramToString(histogram), 'Name': name, - 'Subname': subname + 'Subname': subname, } if additional_metadata: metadata.update(additional_metadata) return Sample(metric, 0, units, metadata) -def CreateTimeSeriesSample(values: List[Any], - timestamps: List[float], - metric: str, - units: str, - interval: float, - ramp_up_ends=None, - ramp_down_starts=None, - additional_metadata=None) -> Sample: +def CreateTimeSeriesSample( + values: List[Any], + timestamps: List[float], + metric: str, + units: str, + interval: float, + ramp_up_ends=None, + ramp_down_starts=None, + additional_metadata=None, +) -> Sample: """Create time series samples. Given a list of values and the timestamp the values @@ -269,7 +274,6 @@ def CreateTimeSeriesSample(values: List[Any], Returns: sample: One sample object that reports the time series passed in. - """ if len(values) != len(timestamps): raise errors.Error('Length of values is different to length of timestamps') diff --git a/perfkitbenchmarker/scripts/database_scripts/plot_sysbench_results.py b/perfkitbenchmarker/scripts/database_scripts/plot_sysbench_results.py index 4e7a4419fb..a39e1d348e 100755 --- a/perfkitbenchmarker/scripts/database_scripts/plot_sysbench_results.py +++ b/perfkitbenchmarker/scripts/database_scripts/plot_sysbench_results.py @@ -101,12 +101,10 @@ def add_file(self, filename): try: f = open(filename, 'r') except: - raise STDERRFileDoesNotExistError( - ( - 'Unable to open file (%s). Assume this is because run failed.' - ' Will raise exception to kill run now.' % filename - ) - ) + raise STDERRFileDoesNotExistError(( + 'Unable to open file (%s). Assume this is because run failed.' + ' Will raise exception to kill run now.' % filename + )) data = self._parse_file(f) f.close() self._add_data(data) diff --git a/perfkitbenchmarker/scripts/tensorflow_serving_client_workload.py b/perfkitbenchmarker/scripts/tensorflow_serving_client_workload.py index 750c4a672a..b010e0311a 100644 --- a/perfkitbenchmarker/scripts/tensorflow_serving_client_workload.py +++ b/perfkitbenchmarker/scripts/tensorflow_serving_client_workload.py @@ -69,13 +69,11 @@ def get_files_in_directory_sorted(directory): """Returns a list of files in directory, sorted alphabetically.""" - return sorted( - [ - os.path.join(directory, name) - for name in os.listdir(directory) - if os.path.isfile(os.path.join(directory, name)) - ] - ) + return sorted([ + os.path.join(directory, name) + for name in os.listdir(directory) + if os.path.isfile(os.path.join(directory, name)) + ]) class TfServingClientWorkload(object): diff --git a/perfkitbenchmarker/smb_service.py b/perfkitbenchmarker/smb_service.py index 1749ac6f1b..92d3379265 100644 --- a/perfkitbenchmarker/smb_service.py +++ b/perfkitbenchmarker/smb_service.py @@ -86,16 +86,25 @@ def __init__(self, disk_spec, zone): # SMB service does not have to have a list of smb_tiers nor does it have # to be implemented by a provider raise errors.Config.InvalidValue( - ('smb_tier "%s" not in acceptable list "%s" ' - 'for cloud %s') % (self.smb_tier, self.SMB_TIERS, self.CLOUD)) - logging.debug('%s SMB service with smb_tier %s zone %s default version %s', - self.CLOUD, self.smb_tier, self.zone, - self.DEFAULT_SMB_VERSION) + 'smb_tier "%s" not in acceptable list "%s" for cloud %s' + % (self.smb_tier, self.SMB_TIERS, self.CLOUD) + ) + logging.debug( + '%s SMB service with smb_tier %s zone %s default version %s', + self.CLOUD, + self.smb_tier, + self.zone, + self.DEFAULT_SMB_VERSION, + ) def CreateSmbDisk(self): - return disk.SmbDisk(self.disk_spec, self.GetRemoteAddress(), - self.GetStorageAccountAndKey(), - self.DEFAULT_SMB_VERSION, self.smb_tier) + return disk.SmbDisk( + self.disk_spec, + self.GetRemoteAddress(), + self.GetStorageAccountAndKey(), + self.DEFAULT_SMB_VERSION, + self.smb_tier, + ) @abc.abstractmethod def _IsReady(self): diff --git a/perfkitbenchmarker/spark_service.py b/perfkitbenchmarker/spark_service.py index 364bf8d68d..9fab31bfe3 100644 --- a/perfkitbenchmarker/spark_service.py +++ b/perfkitbenchmarker/spark_service.py @@ -36,12 +36,17 @@ from perfkitbenchmarker import resource from perfkitbenchmarker.linux_packages import hadoop -flags.DEFINE_string('spark_static_cluster_id', None, - 'If set, the name of the Spark cluster, assumed to be ' - 'ready.') -flags.DEFINE_enum('spark_service_log_level', 'INFO', ['DEBUG', 'INFO', 'FATAL'], - 'Supported log levels when submitting jobs to spark service' - ' clusters.') +flags.DEFINE_string( + 'spark_static_cluster_id', + None, + 'If set, the name of the Spark cluster, assumed to be ready.', +) +flags.DEFINE_enum( + 'spark_service_log_level', + 'INFO', + ['DEBUG', 'INFO', 'FATAL'], + 'Supported log levels when submitting jobs to spark service clusters.', +) # Cloud to use for pkb-created Spark service. @@ -79,11 +84,13 @@ class BaseSparkService(resource.BaseResource): CLOUD = None SERVICE_NAME = None - SPARK_SAMPLE_LOCATION = ('file:///usr/lib/spark/examples/jars/' - 'spark-examples.jar') + SPARK_SAMPLE_LOCATION = ( + 'file:///usr/lib/spark/examples/jars/spark-examples.jar' + ) - HADOOP_SAMPLE_LOCATION = ('file:///usr/lib/hadoop-mapreduce/' - 'hadoop-mapreduce-examples.jar') + HADOOP_SAMPLE_LOCATION = ( + 'file:///usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar' + ) def __init__(self, spark_service_spec): """Initialize the Apache Spark Service object. @@ -100,16 +107,23 @@ def __init__(self, spark_service_spec): self.spec.master_group = copy.copy(self.spec.worker_group) self.spec.master_group.vm_count = 1 self.cluster_id = spark_service_spec.static_cluster_id - assert (spark_service_spec.master_group.vm_spec.zone == - spark_service_spec.worker_group.vm_spec.zone) + assert ( + spark_service_spec.master_group.vm_spec.zone + == spark_service_spec.worker_group.vm_spec.zone + ) self.zone = spark_service_spec.master_group.vm_spec.zone @abc.abstractmethod - def SubmitJob(self, job_jar, class_name, - job_script=None, - job_poll_interval=None, - job_stdout_file=None, job_arguments=None, - job_type=SPARK_JOB_TYPE): + def SubmitJob( + self, + job_jar, + class_name, + job_script=None, + job_poll_interval=None, + job_stdout_file=None, + job_arguments=None, + job_type=SPARK_JOB_TYPE, + ): """Submit a job to the spark service. Submits a job and waits for it to complete. @@ -118,14 +132,12 @@ def SubmitJob(self, job_jar, class_name, job_jar: Jar file to execute. class_name: Name of the main class. job_script: PySpark script to run. job_jar and class_name must be None. - job_poll_interval: integer saying how often to poll for job - completion. Not used by providers for which submit job is a - synchronous operation. - job_stdout_file: String giving the location of the file in - which to put the standard out of the job. - job_arguments: Arguments to pass to class_name. These are - not the arguments passed to the wrapper that submits the - job. + job_poll_interval: integer saying how often to poll for job completion. + Not used by providers for which submit job is a synchronous operation. + job_stdout_file: String giving the location of the file in which to put + the standard out of the job. + job_arguments: Arguments to pass to class_name. These are not the + arguments passed to the wrapper that submits the job. job_type: The type of the job. Returns: @@ -162,13 +174,16 @@ def GetMetadata(self): 'spark_service': self.SERVICE_NAME, 'spark_svc_cloud': self.CLOUD, 'spark_cluster_id': self.cluster_id, - 'spark_cluster_zone': getattr(self, 'zone', None) or 'unknown' + 'spark_cluster_zone': getattr(self, 'zone', None) or 'unknown', } # TODO grab this information for user_managed clusters. if not self.user_managed: - basic_data.update({'num_workers': str(self.spec.worker_group.vm_count), - 'worker_machine_type': - str(self.spec.worker_group.vm_spec.machine_type)}) + basic_data.update({ + 'num_workers': str(self.spec.worker_group.vm_count), + 'worker_machine_type': str( + self.spec.worker_group.vm_spec.machine_type + ), + }) return basic_data @classmethod @@ -209,26 +224,31 @@ def InstallHadoop(vm): if 'worker_group' not in self.vms: raise errors.Resource.CreationError( - 'PkbSparkService requires worker_group VMs.') + 'PkbSparkService requires worker_group VMs.' + ) background_tasks.RunThreaded( InstallHadoop, self.vms['worker_group'] + self.vms['master_group'] ) self.leader = self.vms['master_group'][0] - hadoop.ConfigureAndStart(self.leader, - self.vms['worker_group']) + hadoop.ConfigureAndStart(self.leader, self.vms['worker_group']) def _Delete(self): pass - def SubmitJob(self, jar_file, class_name, job_poll_interval=None, # pytype: disable=signature-mismatch # overriding-default-value-checks - job_stdout_file=None, job_arguments=None, - job_type=SPARK_JOB_TYPE): + def SubmitJob( + self, + jar_file, + class_name, + job_poll_interval=None, # pytype: disable=signature-mismatch # overriding-default-value-checks + job_stdout_file=None, + job_arguments=None, + job_type=SPARK_JOB_TYPE, + ): """Submit the jar file.""" if job_type == SPARK_JOB_TYPE: raise NotImplementedError() - cmd_list = [posixpath.join(hadoop.HADOOP_BIN, 'hadoop'), - 'jar', jar_file] + cmd_list = [posixpath.join(hadoop.HADOOP_BIN, 'hadoop'), 'jar', jar_file] if class_name: cmd_list.append(class_name) if job_arguments: @@ -240,15 +260,18 @@ def SubmitJob(self, jar_file, class_name, job_poll_interval=None, # pytype: dis if job_stdout_file: with open(job_stdout_file, 'w') as f: f.write(stdout) - return {SUCCESS: True, - RUNTIME: (end_time - start_time).total_seconds()} + return {SUCCESS: True, RUNTIME: (end_time - start_time).total_seconds()} @classmethod def GetExampleJar(cls, job_type): if job_type == HADOOP_JOB_TYPE: return posixpath.join( - hadoop.HADOOP_DIR, 'share', 'hadoop', 'mapreduce', - 'hadoop-mapreduce-examples-{0}.jar'.format(FLAGS.hadoop_version)) + hadoop.HADOOP_DIR, + 'share', + 'hadoop', + 'mapreduce', + 'hadoop-mapreduce-examples-{0}.jar'.format(FLAGS.hadoop_version), + ) else: raise NotImplementedError() diff --git a/perfkitbenchmarker/stages.py b/perfkitbenchmarker/stages.py index efc00fc293..e2215eb4ce 100644 --- a/perfkitbenchmarker/stages.py +++ b/perfkitbenchmarker/stages.py @@ -26,18 +26,22 @@ STAGES = [PROVISION, PREPARE, RUN, CLEANUP, TEARDOWN] -_NEXT_STAGE = {PROVISION: [PREPARE, TEARDOWN], - PREPARE: [RUN, CLEANUP], RUN: [CLEANUP], - CLEANUP: [TEARDOWN]} +_NEXT_STAGE = { + PROVISION: [PREPARE, TEARDOWN], + PREPARE: [RUN, CLEANUP], + RUN: [CLEANUP], + CLEANUP: [TEARDOWN], +} _ALL = 'all' _VALID_FLAG_VALUES = PROVISION, PREPARE, RUN, CLEANUP, TEARDOWN, _ALL _SYNTACTIC_HELP = ( - "A complete benchmark execution consists of {0} stages: {1}. Possible flag " - "values include an individual stage, a comma-separated list of stages, or " + 'A complete benchmark execution consists of {0} stages: {1}. Possible flag ' + 'values include an individual stage, a comma-separated list of stages, or ' "'all'. If a list of stages is provided, they must be in order without " - "skipping any stage.".format(len(STAGES), ', '.join(STAGES))) + 'skipping any stage.'.format(len(STAGES), ', '.join(STAGES)) +) class RunStageParser(flags.ListParser): @@ -66,39 +70,54 @@ def parse(self, argument): stage_list = super(RunStageParser, self).parse(argument) if not stage_list: - raise ValueError('Unable to parse {0}. Stage list cannot be ' - 'empty.'.format(repr(argument))) + raise ValueError( + 'Unable to parse {0}. Stage list cannot be empty.'.format( + repr(argument) + ) + ) invalid_items = set(stage_list).difference(_VALID_FLAG_VALUES) if invalid_items: raise ValueError( 'Unable to parse {0}. Unrecognized stages were found: {1}'.format( - repr(argument), ', '.join(sorted(invalid_items)))) + repr(argument), ', '.join(sorted(invalid_items)) + ) + ) if _ALL in stage_list: if len(stage_list) > 1: raise ValueError( "Unable to parse {0}. If 'all' stages are specified, individual " - "stages cannot also be specified.".format(repr(argument))) + 'stages cannot also be specified.'.format(repr(argument)) + ) return list(STAGES) previous_stage = stage_list[0] for stage in itertools.islice(stage_list, 1, None): expected_stages = _NEXT_STAGE.get(previous_stage) if not expected_stages: - raise ValueError("Unable to parse {0}. '{1}' should be the last " - "stage.".format(repr(argument), previous_stage)) + raise ValueError( + "Unable to parse {0}. '{1}' should be the last stage.".format( + repr(argument), previous_stage + ) + ) if stage not in expected_stages: raise ValueError( "Unable to parse {0}. The stage after '{1}' should be one of '{2}'," " not '{3}'.".format( - repr(argument), previous_stage, expected_stages, stage)) + repr(argument), previous_stage, expected_stages, stage + ) + ) previous_stage = stage return stage_list flags.DEFINE( - RunStageParser(), 'run_stage', STAGES, + RunStageParser(), + 'run_stage', + STAGES, 'The stage or stages of perfkitbenchmarker to run.', - flags.FLAGS, flags.ListSerializer(',')) + flags.FLAGS, + flags.ListSerializer(','), +) diff --git a/perfkitbenchmarker/static_virtual_machine.py b/perfkitbenchmarker/static_virtual_machine.py index dd0af0cf7c..4a408ee6f4 100644 --- a/perfkitbenchmarker/static_virtual_machine.py +++ b/perfkitbenchmarker/static_virtual_machine.py @@ -40,14 +40,20 @@ FLAGS = flags.FLAGS -flags.DEFINE_list('static_vm_tags', None, - 'The tags of static VMs for PKB to run with. Even if other ' - 'VMs are specified in a config, if they aren\'t in this list ' - 'they will be skipped during VM creation.') - -flags.DEFINE_bool('copy_ssh_private_keys_into_static_vms', False, - 'A flag to allow the VM to copy ssh private key to ' - 'authenticate static VMs.') +flags.DEFINE_list( + 'static_vm_tags', + None, + 'The tags of static VMs for PKB to run with. Even if other ' + "VMs are specified in a config, if they aren't in this list " + 'they will be skipped during VM creation.', +) + +flags.DEFINE_bool( + 'copy_ssh_private_keys_into_static_vms', + False, + 'A flag to allow the VM to copy ssh private key to ' + 'authenticate static VMs.', +) class StaticVmSpec(virtual_machine.BaseVmSpec): @@ -55,28 +61,39 @@ class StaticVmSpec(virtual_machine.BaseVmSpec): CLOUD = 'Static' - def __init__(self, component_full_name, ip_address=None, user_name=None, - ssh_private_key=None, internal_ip=None, ssh_port=22, - password=None, disk_specs=None, os_type=None, tag=None, - zone=None, **kwargs): + def __init__( + self, + component_full_name, + ip_address=None, + user_name=None, + ssh_private_key=None, + internal_ip=None, + ssh_port=22, + password=None, + disk_specs=None, + os_type=None, + tag=None, + zone=None, + **kwargs + ): """Initialize the StaticVmSpec object. Args: component_full_name: string. Fully qualified name of the configurable - component containing the config options. + component containing the config options. ip_address: The public ip address of the VM. user_name: The username of the VM that the keyfile corresponds to. - ssh_private_key: The absolute path to the private keyfile to use to ssh - to the VM. + ssh_private_key: The absolute path to the private keyfile to use to ssh to + the VM. internal_ip: The internal ip address of the VM. ssh_port: The port number to use for SSH and SCP commands. password: The password used to log into the VM (Windows Only). disk_specs: None or a list of dictionaries containing kwargs used to - create disk.BaseDiskSpecs. + create disk.BaseDiskSpecs. os_type: The OS type of the VM. See the flag of the same name for more - information. - tag: A string that allows the VM to be included or excluded from a run - by using the 'static_vm_tags' flag. + information. + tag: A string that allows the VM to be included or excluded from a run by + using the 'static_vm_tags' flag. zone: The VM's zone. **kwargs: Other args for the superclass. """ @@ -93,8 +110,11 @@ def __init__(self, component_full_name, ip_address=None, user_name=None, self.disk_specs = [ disk.BaseDiskSpec( '{0}.disk_specs[{1}]'.format(component_full_name, i), - flag_values=kwargs.get('flag_values'), **disk_spec) - for i, disk_spec in enumerate(disk_specs or ())] + flag_values=kwargs.get('flag_values'), + **disk_spec + ) + for i, disk_spec in enumerate(disk_specs or ()) + ] class StaticDisk(disk.BaseDisk): @@ -136,8 +156,9 @@ def __init__(self, vm_spec): self.user_name = vm_spec.user_name self.ssh_private_key = vm_spec.ssh_private_key self.internal_ip = vm_spec.internal_ip - self.zone = self.zone or ('Static - %s@%s' % (self.user_name, - self.ip_address)) + self.zone = self.zone or ( + 'Static - %s@%s' % (self.user_name, self.ip_address) + ) self.ssh_port = vm_spec.ssh_port self.password = vm_spec.password self.disk_specs = vm_spec.disk_specs @@ -213,8 +234,9 @@ def ReadStaticVirtualMachineFile(cls, file_obj): vm_arr = json.load(file_obj) if not isinstance(vm_arr, list): - raise ValueError('Invalid static VM file. Expected array, got: %s.' % - type(vm_arr)) + raise ValueError( + 'Invalid static VM file. Expected array, got: %s.' % type(vm_arr) + ) required_keys = frozenset(['ip_address', 'user_name']) @@ -231,9 +253,15 @@ def ReadStaticVirtualMachineFile(cls, file_obj): # assume linux_required_keys for unknown os_type required_keys = required_keys_by_os.get(FLAGS.os_type, linux_required_keys) - optional_keys = frozenset(['internal_ip', 'zone', 'local_disks', - 'scratch_disk_mountpoints', 'os_type', - 'ssh_port', 'install_packages']) + optional_keys = frozenset([ + 'internal_ip', + 'zone', + 'local_disks', + 'scratch_disk_mountpoints', + 'os_type', + 'ssh_port', + 'install_packages', + ]) allowed_keys = required_keys | optional_keys def VerifyItemFormat(item): @@ -244,8 +272,9 @@ def VerifyItemFormat(item): if extra_keys: raise ValueError('Unexpected keys: {0}'.format(', '.join(extra_keys))) elif missing_keys: - raise ValueError('Missing required keys: {0}'.format( - ', '.join(missing_keys))) + raise ValueError( + 'Missing required keys: {0}'.format(', '.join(missing_keys)) + ) for item in vm_arr: VerifyItemFormat(item) @@ -259,21 +288,27 @@ def VerifyItemFormat(item): password = item.get('password') if not isinstance(local_disks, list): - raise ValueError('Expected a list of local disks, got: {0}'.format( - local_disks)) + raise ValueError( + 'Expected a list of local disks, got: {0}'.format(local_disks) + ) scratch_disk_mountpoints = item.get('scratch_disk_mountpoints', []) if not isinstance(scratch_disk_mountpoints, list): raise ValueError( 'Expected a list of disk mount points, got: {0}'.format( - scratch_disk_mountpoints)) + scratch_disk_mountpoints + ) + ) ssh_port = item.get('ssh_port', 22) os_type = item.get('os_type') install_packages = item.get('install_packages', True) - if ((os_type == os_types.WINDOWS and FLAGS.os_type != os_types.WINDOWS) or - (os_type != os_types.WINDOWS and FLAGS.os_type == os_types.WINDOWS)): - raise ValueError('Please only use Windows VMs when using ' - '--os_type=windows and vice versa.') + if ( + os_type == os_types.WINDOWS and FLAGS.os_type != os_types.WINDOWS + ) or (os_type != os_types.WINDOWS and FLAGS.os_type == os_types.WINDOWS): + raise ValueError( + 'Please only use Windows VMs when using ' + '--os_type=windows and vice versa.' + ) disk_kwargs_list = [] for path in scratch_disk_mountpoints: @@ -282,11 +317,18 @@ def VerifyItemFormat(item): disk_kwargs_list.append({'device_path': local_disk}) vm_spec = StaticVmSpec( - 'static_vm_file', ip_address=ip_address, user_name=user_name, - ssh_port=ssh_port, install_packages=install_packages, - ssh_private_key=keyfile_path, internal_ip=internal_ip, zone=zone, - disk_specs=disk_kwargs_list, password=password, - flag_values=flags.FLAGS) + 'static_vm_file', + ip_address=ip_address, + user_name=user_name, + ssh_port=ssh_port, + install_packages=install_packages, + ssh_private_key=keyfile_path, + internal_ip=internal_ip, + zone=zone, + disk_specs=disk_kwargs_list, + password=password, + flag_values=flags.FLAGS, + ) vm_class = GetStaticVmClass(os_type) vm = vm_class(vm_spec) # pytype: disable=not-instantiable @@ -316,28 +358,34 @@ def GetStaticVmClass(os_type) -> type[virtual_machine.BaseVirtualMachine]: if not os_type: os_type = os_types.DEFAULT logging.warning('Could not find os type for VM. Defaulting to %s.', os_type) - return resource.GetResourceClass(virtual_machine.BaseVirtualMachine, - CLOUD=StaticVirtualMachine.CLOUD, - OS_TYPE=os_type) + return resource.GetResourceClass( + virtual_machine.BaseVirtualMachine, + CLOUD=StaticVirtualMachine.CLOUD, + OS_TYPE=os_type, + ) class Ubuntu1604BasedStaticVirtualMachine( - StaticVirtualMachine, linux_virtual_machine.Ubuntu1604Mixin): + StaticVirtualMachine, linux_virtual_machine.Ubuntu1604Mixin +): pass class Ubuntu1804BasedStaticVirtualMachine( - StaticVirtualMachine, linux_virtual_machine.Ubuntu1804Mixin): + StaticVirtualMachine, linux_virtual_machine.Ubuntu1804Mixin +): pass class Ubuntu2004BasedStaticVirtualMachine( - StaticVirtualMachine, linux_virtual_machine.Ubuntu2004Mixin): + StaticVirtualMachine, linux_virtual_machine.Ubuntu2004Mixin +): pass class Ubuntu2204BasedStaticVirtualMachine( - StaticVirtualMachine, linux_virtual_machine.Ubuntu2204Mixin): + StaticVirtualMachine, linux_virtual_machine.Ubuntu2204Mixin +): pass @@ -347,63 +395,75 @@ class Ubuntu2304BasedStaticVirtualMachine( pass -class ClearBasedStaticVirtualMachine(StaticVirtualMachine, - linux_virtual_machine.ClearMixin): +class ClearBasedStaticVirtualMachine( + StaticVirtualMachine, linux_virtual_machine.ClearMixin +): pass -class Rhel7BasedStaticVirtualMachine(StaticVirtualMachine, - linux_virtual_machine.Rhel7Mixin): +class Rhel7BasedStaticVirtualMachine( + StaticVirtualMachine, linux_virtual_machine.Rhel7Mixin +): pass -class Rhel8BasedStaticVirtualMachine(StaticVirtualMachine, - linux_virtual_machine.Rhel8Mixin): +class Rhel8BasedStaticVirtualMachine( + StaticVirtualMachine, linux_virtual_machine.Rhel8Mixin +): pass -class Rhel9BasedStaticVirtualMachine(StaticVirtualMachine, - linux_virtual_machine.Rhel9Mixin): +class Rhel9BasedStaticVirtualMachine( + StaticVirtualMachine, linux_virtual_machine.Rhel9Mixin +): pass -class Fedora36BasedStaticVirtualMachine(StaticVirtualMachine, - linux_virtual_machine.Fedora36Mixin): +class Fedora36BasedStaticVirtualMachine( + StaticVirtualMachine, linux_virtual_machine.Fedora36Mixin +): pass -class Fedora37BasedStaticVirtualMachine(StaticVirtualMachine, - linux_virtual_machine.Fedora37Mixin): +class Fedora37BasedStaticVirtualMachine( + StaticVirtualMachine, linux_virtual_machine.Fedora37Mixin +): pass -class CentOs7BasedStaticVirtualMachine(StaticVirtualMachine, - linux_virtual_machine.CentOs7Mixin): +class CentOs7BasedStaticVirtualMachine( + StaticVirtualMachine, linux_virtual_machine.CentOs7Mixin +): pass -class CentOs8BasedStaticVirtualMachine(StaticVirtualMachine, - linux_virtual_machine.CentOs8Mixin): +class CentOs8BasedStaticVirtualMachine( + StaticVirtualMachine, linux_virtual_machine.CentOs8Mixin +): pass -class Debian9BasedStaticVirtualMachine(StaticVirtualMachine, - linux_virtual_machine.Debian9Mixin): +class Debian9BasedStaticVirtualMachine( + StaticVirtualMachine, linux_virtual_machine.Debian9Mixin +): pass -class Debian10BasedStaticVirtualMachine(StaticVirtualMachine, - linux_virtual_machine.Debian10Mixin): +class Debian10BasedStaticVirtualMachine( + StaticVirtualMachine, linux_virtual_machine.Debian10Mixin +): pass -class Debian11BasedStaticVirtualMachine(StaticVirtualMachine, - linux_virtual_machine.Debian11Mixin): +class Debian11BasedStaticVirtualMachine( + StaticVirtualMachine, linux_virtual_machine.Debian11Mixin +): pass -class Debian12BasedStaticVirtualMachine(StaticVirtualMachine, - linux_virtual_machine.Debian12Mixin): +class Debian12BasedStaticVirtualMachine( + StaticVirtualMachine, linux_virtual_machine.Debian12Mixin +): pass diff --git a/perfkitbenchmarker/temp_dir.py b/perfkitbenchmarker/temp_dir.py index 5916ad1a78..8a9560d448 100644 --- a/perfkitbenchmarker/temp_dir.py +++ b/perfkitbenchmarker/temp_dir.py @@ -57,8 +57,7 @@ def GetAllRunsDirPath(): @functools.lru_cache() def GetRunDirPath(): """Gets path to the directory containing files specific to a PKB run.""" - return os.path.join( - FLAGS.temp_dir, _RUNS, str(flags.FLAGS.run_uri)) + return os.path.join(FLAGS.temp_dir, _RUNS, str(flags.FLAGS.run_uri)) def GetSshConnectionsDir(): diff --git a/perfkitbenchmarker/test_util.py b/perfkitbenchmarker/test_util.py index 67ec5b1efe..7571788001 100644 --- a/perfkitbenchmarker/test_util.py +++ b/perfkitbenchmarker/test_util.py @@ -31,6 +31,7 @@ class SamplesTestMixin(object): """A mixin for unittest.TestCase that adds a type-specific equality + predicate for samples. """ @@ -42,20 +43,31 @@ def __init__(self, *args, **kwargs): def assertSamplesEqualUpToTimestamp(self, a, b, msg=None): """Assert that two samples are equal, ignoring timestamp differences.""" - self.assertEqual(a.metric, b.metric, msg or - 'Samples %s and %s have different metrics' % (a, b)) + self.assertEqual( + a.metric, + b.metric, + msg or 'Samples %s and %s have different metrics' % (a, b), + ) if isinstance(a.value, float) and isinstance(b.value, float): self.assertAlmostEqual( - a.value, b.value, msg=msg or - 'Samples %s and %s have different values' % (a, b)) + a.value, + b.value, + msg=msg or 'Samples %s and %s have different values' % (a, b), + ) else: self.assertEqual( - a.value, b.value, msg or - 'Samples %s and %s have different values' % (a, b)) - self.assertEqual(a.unit, b.unit, msg or - 'Samples %s and %s have different units' % (a, b)) - self.assertDictEqual(a.metadata, b.metadata, msg or - 'Samples %s and %s have different metadata' % (a, b)) + a.value, + b.value, + msg or 'Samples %s and %s have different values' % (a, b), + ) + self.assertEqual( + a.unit, b.unit, msg or 'Samples %s and %s have different units' % (a, b) + ) + self.assertDictEqual( + a.metadata, + b.metadata, + msg or 'Samples %s and %s have different metadata' % (a, b), + ) # Deliberately don't compare the timestamp fields of the samples. def assertSampleListsEqualUpToTimestamp(self, a, b, msg=None): @@ -67,21 +79,28 @@ def assertSampleListsEqualUpToTimestamp(self, a, b, msg=None): custom test for that. """ - self.assertEqual(len(a), len(b), - msg or 'Lists %s and %s are not the same length' % (a, b)) + self.assertEqual( + len(a), + len(b), + msg or 'Lists %s and %s are not the same length' % (a, b), + ) for i in range(len(a)): - self.assertIsInstance(a[i], sample.Sample, - msg or ('%s (item %s in list) is ' - 'not a sample.Sample object' % - (a[i], i))) - self.assertIsInstance(b[i], sample.Sample, - msg or ('%s (item %s in list) is ' - 'not a sample.Sample object' % - (b[i], i))) + self.assertIsInstance( + a[i], + sample.Sample, + msg + or ('%s (item %s in list) is not a sample.Sample object' % (a[i], i)), + ) + self.assertIsInstance( + b[i], + sample.Sample, + msg + or ('%s (item %s in list) is not a sample.Sample object' % (b[i], i)), + ) try: self.assertSamplesEqualUpToTimestamp(a[i], b[i], msg=msg) except self.failureException as ex: - ex.message = str(ex) + (' (was item %s in list)' % i) + ex.message = str(ex) + ' (was item %s in list)' % i ex.args = (ex.message,) raise ex @@ -108,9 +127,8 @@ def assertDiskMounts(benchmark_config, mount_point): otherwise raises an exception. Args: - benchmark_config: a dict in the format of - benchmark_spec.BenchmarkSpec. The config must specify exactly - one virtual machine. + benchmark_config: a dict in the format of benchmark_spec.BenchmarkSpec. The + config must specify exactly one virtual machine. mount_point: a path, represented as a string. Raises: @@ -127,9 +145,9 @@ def assertDiskMounts(benchmark_config, mount_point): m = mock.MagicMock() m.BENCHMARK_NAME = _BENCHMARK_NAME config_spec = benchmark_config_spec.BenchmarkConfigSpec( - _BENCHMARK_NAME, flag_values=flags.FLAGS, **benchmark_config) - spec = benchmark_spec.BenchmarkSpec( - m, config_spec, _BENCHMARK_UID) + _BENCHMARK_NAME, flag_values=flags.FLAGS, **benchmark_config + ) + spec = benchmark_spec.BenchmarkSpec(m, config_spec, _BENCHMARK_UID) with spec.RedirectGlobalFlags(): try: spec.ConstructVirtualMachines() diff --git a/perfkitbenchmarker/time_triggers/__init__.py b/perfkitbenchmarker/time_triggers/__init__.py index 3f52f33041..3b34b8672d 100644 --- a/perfkitbenchmarker/time_triggers/__init__.py +++ b/perfkitbenchmarker/time_triggers/__init__.py @@ -31,5 +31,3 @@ def RegisterAll(unused_sender, parsed_flags): module.Register(parsed_flags) events.RegisterTracingEvents() - - diff --git a/perfkitbenchmarker/time_triggers/base_time_trigger.py b/perfkitbenchmarker/time_triggers/base_time_trigger.py index efd4d83ff5..6858edc946 100644 --- a/perfkitbenchmarker/time_triggers/base_time_trigger.py +++ b/perfkitbenchmarker/time_triggers/base_time_trigger.py @@ -28,9 +28,12 @@ FLAGS = flags.FLAGS _TRIGGER_VM_GROUPS = flags.DEFINE_list( - 'trigger_vm_groups', None, 'Run trigger on all vms in the vm groups.' + 'trigger_vm_groups', + None, + 'Run trigger on all vms in the vm groups.' 'By default trigger runs on all VMs, for client and server achitecture,' - 'specify trace vm groups to servers to only collect metrics on server vms.') + 'specify trace vm groups to servers to only collect metrics on server vms.', +) class BaseTimeTrigger(metaclass=abc.ABCMeta): @@ -48,7 +51,7 @@ def __init__(self, delay=0): self.trigger_time = None self.metadata = { self.trigger_name: True, - self.trigger_name + '_delay': delay + self.trigger_name + '_delay': delay, } def CreateAndStartTriggerThread(self, vm) -> None: @@ -60,6 +63,7 @@ def CreateAndStartTriggerThread(self, vm) -> None: Args: vm: A virtual machine. """ + def TriggerEvent(): time.sleep(self.delay) logging.info('Triggering %s on %s', self.trigger_name, vm.name) @@ -87,9 +91,9 @@ def RunTrigger(self, unused_sender): """Run the trigger event.""" for vm in self.vms: self.CreateAndStartTriggerThread(vm) - self.trigger_time = ( - datetime.datetime.now() + - datetime.timedelta(seconds=self.delay)) + self.trigger_time = datetime.datetime.now() + datetime.timedelta( + seconds=self.delay + ) self.metadata[self.trigger_name + '_time'] = str(self.trigger_time) # pylint: disable=unused-argument @@ -99,13 +103,10 @@ def UpdateMetadata(self, unused_sender, benchmark_spec, samples): s.metadata.update(self.metadata) def Register(self): - events.before_phase.connect( - self.SetUpTrigger, stages.RUN, weak=False) - events.trigger_phase.connect( - self.RunTrigger, weak=False) + events.before_phase.connect(self.SetUpTrigger, stages.RUN, weak=False) + events.trigger_phase.connect(self.RunTrigger, weak=False) events.benchmark_samples_created.connect(self.AppendSamples, weak=False) - events.all_samples_created.connect( - self.UpdateMetadata, weak=False) + events.all_samples_created.connect(self.UpdateMetadata, weak=False) @property def trigger_name(self) -> str: @@ -126,4 +127,3 @@ def AppendSamples(self, unused_sender, benchmark_spec, samples): def Register(unused_parsed_flags): pass - diff --git a/perfkitbenchmarker/time_triggers/maintenance_simulation_trigger.py b/perfkitbenchmarker/time_triggers/maintenance_simulation_trigger.py index 96b755aed2..1871a7323b 100644 --- a/perfkitbenchmarker/time_triggers/maintenance_simulation_trigger.py +++ b/perfkitbenchmarker/time_triggers/maintenance_simulation_trigger.py @@ -17,7 +17,7 @@ import copy import logging import statistics -from typing import Any, List, Dict +from typing import Any, Dict, List from absl import flags from perfkitbenchmarker import benchmark_spec as bm_spec @@ -182,7 +182,7 @@ def _AggregateThroughputSample(self, s: sample.Sample) -> List[sample.Sample]: # If more than 1 sequential value is missing from the time series. # Distrubute the ops throughout the time series if i > 0: - time_gap_in_seconds = ((time - time_series[i - 1]) / 1000) + time_gap_in_seconds = (time - time_series[i - 1]) / 1000 missing_entry_count = int((time_gap_in_seconds / interval) - 1) if missing_entry_count > 1: interval_values.extend( diff --git a/perfkitbenchmarker/timing_util.py b/perfkitbenchmarker/timing_util.py index fcc6f37a5b..eeb9f4a44f 100644 --- a/perfkitbenchmarker/timing_util.py +++ b/perfkitbenchmarker/timing_util.py @@ -28,23 +28,39 @@ MEASUREMENTS_RUNTIMES = 'runtimes' MEASUREMENTS_TIMESTAMPS = 'timestamps' MEASUREMENTS_ALL = OrderedDict([ - (MEASUREMENTS_NONE, ( - 'No measurements included (same as providing an empty list, and cannot ' - 'be combined with other options).')), - (MEASUREMENTS_END_TO_END_RUNTIME, ( - 'Includes an end-to-end runtime measurement.')), - (MEASUREMENTS_RUNTIMES, ( - 'Includes runtimes of all measured intervals, including the end-to-end ' - 'runtime, the time taken by the benchmark module Prepare, Run, and ' - 'Cleanup functions, and other important intervals.')), - (MEASUREMENTS_TIMESTAMPS, ( - 'Includes start and stop timestamps of all measured intervals.'))]) + ( + MEASUREMENTS_NONE, + ( + 'No measurements included (same as providing an empty list, and' + ' cannot be combined with other options).' + ), + ), + ( + MEASUREMENTS_END_TO_END_RUNTIME, + 'Includes an end-to-end runtime measurement.', + ), + ( + MEASUREMENTS_RUNTIMES, + ( + 'Includes runtimes of all measured intervals, including the' + ' end-to-end runtime, the time taken by the benchmark module' + ' Prepare, Run, and Cleanup functions, and other important' + ' intervals.' + ), + ), + ( + MEASUREMENTS_TIMESTAMPS, + 'Includes start and stop timestamps of all measured intervals.', + ), +]) def EndToEndRuntimeMeasurementEnabled(): """Returns whether end-to-end runtime measurement is globally enabled.""" - return (MEASUREMENTS_END_TO_END_RUNTIME in flags.FLAGS.timing_measurements or - RuntimeMeasurementsEnabled()) + return ( + MEASUREMENTS_END_TO_END_RUNTIME in flags.FLAGS.timing_measurements + or RuntimeMeasurementsEnabled() + ) def RuntimeMeasurementsEnabled(): @@ -64,8 +80,7 @@ def ValidateMeasurementsFlag(options_list): must be valid. The NONE option cannot be combined with other options. Args: - options_list: A list of strings parsed from the provided value for the - flag. + options_list: A list of strings parsed from the provided value for the flag. Returns: True if the list of options provided as the value for the flag meets all @@ -78,24 +93,31 @@ def ValidateMeasurementsFlag(options_list): for option in options_list: if option not in MEASUREMENTS_ALL: raise flags.ValidationError( - '%s: Invalid value for --%s' % (option, MEASUREMENTS_FLAG_NAME)) + '%s: Invalid value for --%s' % (option, MEASUREMENTS_FLAG_NAME) + ) if option == MEASUREMENTS_NONE and len(options_list) != 1: raise flags.ValidationError( - '%s: Cannot combine with other --%s options' % ( - option, MEASUREMENTS_FLAG_NAME)) + '%s: Cannot combine with other --%s options' + % (option, MEASUREMENTS_FLAG_NAME) + ) return True flags.DEFINE_list( - MEASUREMENTS_FLAG_NAME, MEASUREMENTS_END_TO_END_RUNTIME, + MEASUREMENTS_FLAG_NAME, + MEASUREMENTS_END_TO_END_RUNTIME, 'Comma-separated list of values from <%s> that selects which timing ' 'measurements to enable. Measurements will be included as samples in the ' - 'benchmark results. %s' % ('|'.join(MEASUREMENTS_ALL), ' '.join([ - '%s: %s' % (option, description) - for option, description in MEASUREMENTS_ALL.items() - ]))) -flags.register_validator( - MEASUREMENTS_FLAG_NAME, ValidateMeasurementsFlag) + 'benchmark results. %s' + % ( + '|'.join(MEASUREMENTS_ALL), + ' '.join([ + '%s: %s' % (option, description) + for option, description in MEASUREMENTS_ALL.items() + ]), + ), +) +flags.register_validator(MEASUREMENTS_FLAG_NAME, ValidateMeasurementsFlag) def _GenerateIntervalSamples(interval, include_timestamps): @@ -118,10 +140,12 @@ def _GenerateIntervalSamples(interval, include_timestamps): elapsed_time = stop_time - start_time samples.append(sample.Sample(name + ' Runtime', elapsed_time, 'seconds')) if include_timestamps: - samples.append(sample.Sample( - name + ' Start Timestamp', start_time, 'seconds')) - samples.append(sample.Sample( - name + ' Stop Timestamp', stop_time, 'seconds')) + samples.append( + sample.Sample(name + ' Start Timestamp', start_time, 'seconds') + ) + samples.append( + sample.Sample(name + ' Stop Timestamp', stop_time, 'seconds') + ) return samples @@ -160,5 +184,7 @@ def GenerateSamples(self): """ include_timestamps = TimestampMeasurementsEnabled() return [ - sample for interval in self.intervals for sample in - _GenerateIntervalSamples(interval, include_timestamps)] + sample + for interval in self.intervals + for sample in _GenerateIntervalSamples(interval, include_timestamps) + ] diff --git a/perfkitbenchmarker/traces/asadm.py b/perfkitbenchmarker/traces/asadm.py index c31d8916df..03059fbc4d 100644 --- a/perfkitbenchmarker/traces/asadm.py +++ b/perfkitbenchmarker/traces/asadm.py @@ -57,8 +57,9 @@ 'TB': 1000.0, 'GB': 1.0, 'MB': 1 / 1000.0, - 'KB': 1 / (1000.0 ** 2), - 'B': 1 / (1000.0 ** 3)} + 'KB': 1 / (1000.0**2), + 'B': 1 / (1000.0**3), +} class _AsadmSummaryCollector(base_collector.BaseCollector): @@ -384,7 +385,7 @@ def _AnalyzeAsadmLatencyResults(metadata, output, samples): len(write_iops), len(write_lat_1_ms), len(write_lat_8_ms), - len(write_lat_64_ms) + len(write_lat_64_ms), ) effective_metric_length = min(effective_read_length, effective_write_length) samples.extend([ diff --git a/perfkitbenchmarker/traces/collectd.py b/perfkitbenchmarker/traces/collectd.py index 6d7c822078..42ca9074f4 100644 --- a/perfkitbenchmarker/traces/collectd.py +++ b/perfkitbenchmarker/traces/collectd.py @@ -26,8 +26,9 @@ from perfkitbenchmarker import vm_util from perfkitbenchmarker.linux_packages import collectd -flags.DEFINE_boolean('collectd', False, - 'Install and run collectd on the guest.') +flags.DEFINE_boolean( + 'collectd', False, 'Install and run collectd on the guest.' +) flags.DEFINE_string('collectd_output', None, 'Path to store collectd results.') @@ -51,7 +52,7 @@ def Before(self, unused_sender, benchmark_spec): Args: benchmark_spec: benchmark_spec.BenchmarkSpec. The benchmark currently - running. + running. """ logging.info('Installing collectd') vms = benchmark_spec.vms @@ -62,7 +63,7 @@ def After(self, unused_sender, benchmark_spec): Args: benchmark_spec: benchmark_spec.BenchmarkSpec. The benchmark that stopped - running. + running. """ logging.info('Stopping collectd') vms = benchmark_spec.vms @@ -78,8 +79,9 @@ def Register(parsed_flags): output_directory = parsed_flags.collectd_output or vm_util.GetTempDir() if not os.path.isdir(output_directory): - raise IOError('collectd output directory does not exist: {0}'.format( - output_directory)) + raise IOError( + 'collectd output directory does not exist: {0}'.format(output_directory) + ) collector = _CollectdCollector(output_directory) events.before_phase.connect(collector.Before, stages.RUN, weak=False) events.after_phase.connect(collector.After, stages.RUN, weak=False) diff --git a/perfkitbenchmarker/traces/dstat.py b/perfkitbenchmarker/traces/dstat.py index 1b96866885..6064cd8540 100644 --- a/perfkitbenchmarker/traces/dstat.py +++ b/perfkitbenchmarker/traces/dstat.py @@ -23,7 +23,6 @@ import re from absl import flags import numpy as np - from perfkitbenchmarker import background_tasks from perfkitbenchmarker import events from perfkitbenchmarker import sample @@ -33,26 +32,40 @@ from perfkitbenchmarker.traces import base_collector import six -flags.DEFINE_boolean('dstat', False, - 'Run dstat (http://dag.wiee.rs/home-made/dstat/) ' - 'on each VM to collect system performance metrics during ' - 'each benchmark run.') -flags.DEFINE_integer('dstat_interval', None, - 'dstat sample collection frequency, in seconds. Only ' - 'applicable when --dstat is specified.') -flags.DEFINE_string('dstat_output', None, - 'Output directory for dstat output. ' - 'Only applicable when --dstat is specified. ' - 'Default: run temporary directory.') -flags.DEFINE_boolean('dstat_publish', False, - 'Whether to publish average dstat statistics.') -flags.DEFINE_string('dstat_publish_regex', None, 'Requires setting ' - 'dstat_publish to true. If specified, any dstat statistic ' - 'matching this regular expression will be published such ' - 'that each individual statistic will be in a sample with ' - 'the time since the epoch in the metadata. Examples. Use ' - '".*" to record all samples. Use "net" to record ' - 'networking statistics.') +flags.DEFINE_boolean( + 'dstat', + False, + 'Run dstat (http://dag.wiee.rs/home-made/dstat/) ' + 'on each VM to collect system performance metrics during ' + 'each benchmark run.', +) +flags.DEFINE_integer( + 'dstat_interval', + None, + 'dstat sample collection frequency, in seconds. Only ' + 'applicable when --dstat is specified.', +) +flags.DEFINE_string( + 'dstat_output', + None, + 'Output directory for dstat output. ' + 'Only applicable when --dstat is specified. ' + 'Default: run temporary directory.', +) +flags.DEFINE_boolean( + 'dstat_publish', False, 'Whether to publish average dstat statistics.' +) +flags.DEFINE_string( + 'dstat_publish_regex', + None, + 'Requires setting ' + 'dstat_publish to true. If specified, any dstat statistic ' + 'matching this regular expression will be published such ' + 'that each individual statistic will be in a sample with ' + 'the time since the epoch in the metadata. Examples. Use ' + '".*" to record all samples. Use "net" to record ' + 'networking statistics.', +) FLAGS = flags.FLAGS @@ -69,7 +82,6 @@ def _InstallCollector(self, vm): vm.Install('dstat') def _CollectorRunCommand(self, vm, collector_file): - # List block devices so that I/O to each block device can be recorded. block_devices, _ = vm.RemoteCommand( 'lsblk --nodeps --output NAME --noheadings' @@ -92,7 +104,8 @@ def Analyze(self, unused_sender, benchmark_spec, samples): def _AnalyzeEvent(role, labels, out, event): # Find out index of rows belong to event according to timestamp. cond = (out[:, 0] > event.start_timestamp) & ( - out[:, 0] < event.end_timestamp) + out[:, 0] < event.end_timestamp + ) # Skip analyzing event if none of rows falling into time range. if not cond.any(): return @@ -104,7 +117,8 @@ def _AnalyzeEvent(role, labels, out, event): samples.extend([ sample.Sample(label, avg[idx], '', metadata) - for idx, label in enumerate(labels[1:])]) + for idx, label in enumerate(labels[1:]) + ]) dstat_publish_regex = FLAGS.dstat_publish_regex if dstat_publish_regex: @@ -116,11 +130,13 @@ def _AnalyzeEvent(role, labels, out, event): individual_sample_metadata = copy.deepcopy(metadata) individual_sample_metadata['dstat_epoch'] = out[sample_idx, 0] samples.append( - sample.Sample(label, value, '', individual_sample_metadata)) + sample.Sample(label, value, '', individual_sample_metadata) + ) def _Analyze(role, file): - with open(os.path.join(self.output_directory, - os.path.basename(file)), 'r') as f: + with open( + os.path.join(self.output_directory, os.path.basename(file)), 'r' + ) as f: fp = iter(f) labels, out = dstat.ParseCsvFile(fp) background_tasks.RunThreaded( @@ -138,17 +154,23 @@ def Register(parsed_flags): if not parsed_flags.dstat: return - output_directory = (parsed_flags.dstat_output - if parsed_flags['dstat_output'].present - else vm_util.GetTempDir()) + output_directory = ( + parsed_flags.dstat_output + if parsed_flags['dstat_output'].present + else vm_util.GetTempDir() + ) - logging.debug('Registering dstat collector with interval %s, output to %s.', - parsed_flags.dstat_interval, output_directory) + logging.debug( + 'Registering dstat collector with interval %s, output to %s.', + parsed_flags.dstat_interval, + output_directory, + ) if not os.path.isdir(output_directory): os.makedirs(output_directory) - collector = _DStatCollector(interval=parsed_flags.dstat_interval, - output_directory=output_directory) + collector = _DStatCollector( + interval=parsed_flags.dstat_interval, output_directory=output_directory + ) events.before_phase.connect(collector.Start, stages.RUN, weak=False) events.after_phase.connect(collector.Stop, stages.RUN, weak=False) if parsed_flags.dstat_publish: diff --git a/perfkitbenchmarker/traces/mpstat.py b/perfkitbenchmarker/traces/mpstat.py index 6e2eb1f269..3c89f6179e 100644 --- a/perfkitbenchmarker/traces/mpstat.py +++ b/perfkitbenchmarker/traces/mpstat.py @@ -41,7 +41,6 @@ outstanding disk I/O request. For more details, see https://linux.die.net/man/1/mpstat. - """ @@ -60,30 +59,43 @@ import six _MPSTAT = flags.DEFINE_boolean( - 'mpstat', False, 'Run mpstat (https://linux.die.net/man/1/mpstat) ' - 'to collect system performance metrics during benchmark run.') + 'mpstat', + False, + 'Run mpstat (https://linux.die.net/man/1/mpstat) ' + 'to collect system performance metrics during benchmark run.', +) _MPSTAT_BREAKDOWN = flags.DEFINE_enum( - 'mpstat_breakdown', 'SUM', ['SUM', 'CPU', 'ALL'], + 'mpstat_breakdown', + 'SUM', + ['SUM', 'CPU', 'ALL'], 'Level of aggregation for statistics. Accepted ' 'values are "SUM", "CPU", "ALL". Defaults to SUM. See ' - 'https://linux.die.net/man/1/mpstat for details.') + 'https://linux.die.net/man/1/mpstat for details.', +) _MPSTAT_CPUS = flags.DEFINE_string( - 'mpstat_cpus', 'ALL', 'Comma delimited string of CPU ids or ALL. ' - 'Defaults to ALL.') + 'mpstat_cpus', + 'ALL', + 'Comma delimited string of CPU ids or ALL. Defaults to ALL.', +) _MPSTAT_INTERVAL = flags.DEFINE_integer( - 'mpstat_interval', 1, - 'The amount of time in seconds between each mpstat report.' - 'Defaults to 1.') + 'mpstat_interval', + 1, + 'The amount of time in seconds between each mpstat report.Defaults to 1.', +) _MPSTAT_COUNT = flags.DEFINE_integer( - 'mpstat_count', 1, 'The number of reports generated at interval apart.' - 'Defaults to 1.') + 'mpstat_count', + 1, + 'The number of reports generated at interval apart.Defaults to 1.', +) _MPSTAT_PUBLISH = flags.DEFINE_boolean( - 'mpstat_publish', False, - 'Whether to publish mpstat statistics.') + 'mpstat_publish', False, 'Whether to publish mpstat statistics.' +) _MPSTAT_PUBLISH_PER_INTERVAL_SAMPLES = flags.DEFINE_boolean( - 'mpstat_publish_per_interval_samples', False, + 'mpstat_publish_per_interval_samples', + False, 'Whether to publish a separate mpstat statistics sample ' - 'for each interval. If True, --mpstat_publish must be True.') + 'for each interval. If True, --mpstat_publish must be True.', +) FLAGS = flags.FLAGS @@ -92,15 +104,21 @@ flags.register_validator( _MPSTAT_INTERVAL.name, lambda value: value < _TWENTY_THREE_HOURS_IN_SECONDS, - message=('If --mpstat_interval must be less than 23 hours (if it\'s set ' - 'near or above 24 hours, it becomes hard to infer sample ' - 'timestamp from mpstat output.')) + message=( + "If --mpstat_interval must be less than 23 hours (if it's set " + 'near or above 24 hours, it becomes hard to infer sample ' + 'timestamp from mpstat output.' + ), +) flags.register_validator( _MPSTAT_PUBLISH_PER_INTERVAL_SAMPLES.name, lambda value: FLAGS.mpstat_publish or not value, - message=('If --mpstat_publish_per_interval is True, --mpstat_publish must ' - 'be True.')) + message=( + 'If --mpstat_publish_per_interval is True, --mpstat_publish must ' + 'be True.' + ), +) def _ParseStartTime(output: str) -> float: @@ -115,7 +133,6 @@ def _ParseStartTime(output: str) -> float: Example input: third_party/py/perfkitbenchmarker/tests/data/mpstat_output.json - """ hosts = output['sysstat']['hosts'] date = hosts[0]['date'] @@ -124,8 +141,8 @@ def _ParseStartTime(output: str) -> float: start_datetime_string = ' '.join([date, time]) # As a sysstat utility, this is printed in UTC by default start_datetime = datetime.datetime.strptime( - start_datetime_string, - '%Y-%m-%d %H:%M:%S').replace(tzinfo=datetime.timezone.utc) + start_datetime_string, '%Y-%m-%d %H:%M:%S' + ).replace(tzinfo=datetime.timezone.utc) return start_datetime.timestamp() @@ -151,7 +168,8 @@ def _GetCPUAverageMetrics( host_stats: List[Dict[str, Any]], number_of_cpus: int, metadata: Dict[str, Any], - timestamp: Optional[float] = None): + timestamp: Optional[float] = None, +): """Get average metrics for all CPUs. Args: @@ -208,12 +226,15 @@ def _GetCPUAverageMetrics( metric_name = 'mpstat_avg_' + cpu_metric meta = metadata.copy() meta['mpstat_cpu_id'] = cpu_id - samples.append(sample.Sample( - metric=metric_name, - value=average, - unit='%', - metadata=meta, - timestamp=timestamp)) + samples.append( + sample.Sample( + metric=metric_name, + value=average, + unit='%', + metadata=meta, + timestamp=timestamp, + ) + ) return samples @@ -221,7 +242,8 @@ def _GetCPUAverageInterruptions( host_stats: List[Dict[str, Any]], number_of_cpus: int, metadata: Dict[str, Any], - timestamp: Optional[float] = None): + timestamp: Optional[float] = None, +): """Get average interruption for all CPUs. Args: @@ -255,21 +277,24 @@ def _GetCPUAverageInterruptions( ] """ samples = [] - for cpu_id in range(number_of_cpus+1): + for cpu_id in range(number_of_cpus + 1): measurements = [] for report in host_stats: value = report['sum-interrupts'][cpu_id]['intr'] measurements.append(value) - average = sum(measurements)/len(measurements) + average = sum(measurements) / len(measurements) metric_name = 'mpstat_avg_intr' meta = metadata.copy() - meta['mpstat_cpu_id'] = cpu_id-1 - samples.append(sample.Sample( - metric=metric_name, - value=average, - unit='interrupts/sec', - metadata=meta, - timestamp=timestamp)) + meta['mpstat_cpu_id'] = cpu_id - 1 + samples.append( + sample.Sample( + metric=metric_name, + value=average, + unit='interrupts/sec', + metadata=meta, + timestamp=timestamp, + ) + ) return samples @@ -277,7 +302,8 @@ def _GetPerIntervalSamples( host_stats: List[Dict[str, Any]], metadata: Dict[str, Any], start_timestamp: int, - interval: int) -> List[sample.Sample]: + interval: int, +) -> List[sample.Sample]: """Generate samples for all CPU related metrics in every run of mpstat. Args: @@ -306,12 +332,15 @@ def _GetPerIntervalSamples( meta = metadata.copy() meta['mpstat_cpu_id'] = cpu_id meta['ordinal'] = ordinal - samples.append(sample.Sample( - metric=metric_name, - value=metric_value, - unit='%', - metadata=meta, - timestamp=sample_timestamp)) + samples.append( + sample.Sample( + metric=metric_name, + value=metric_value, + unit='%', + metadata=meta, + timestamp=sample_timestamp, + ) + ) return samples @@ -320,7 +349,7 @@ def _MpstatResults( output: Dict[str, Any], interval: int, per_interval_samples: bool = False, - ): +): """Parses and appends mpstat results to the samples list. Args: @@ -343,23 +372,17 @@ def _MpstatResults( metadata['nodename'] = host['nodename'] samples += _GetCPUAverageMetrics( - host_stats, - number_of_cpus, - metadata, - start_timestamp) + host_stats, number_of_cpus, metadata, start_timestamp + ) samples += _GetCPUAverageInterruptions( - host_stats, - number_of_cpus, - metadata, - start_timestamp) + host_stats, number_of_cpus, metadata, start_timestamp + ) if per_interval_samples: samples += _GetPerIntervalSamples( - host_stats, - metadata, - start_timestamp, - interval) + host_stats, metadata, start_timestamp, interval + ) return samples @@ -371,10 +394,8 @@ class MpstatCollector(base_collector.BaseCollector): """ def __init__( - self, - interval=None, - output_directory=None, - per_interval_samples=False): + self, interval=None, output_directory=None, per_interval_samples=False + ): super().__init__(interval, output_directory=output_directory) self.per_interval_samples = per_interval_samples @@ -387,14 +408,17 @@ def _InstallCollector(self, vm): def _CollectorRunCommand(self, vm, collector_file): # We set the environment variable S_TIME_FORMAT=ISO to ensure consistent # time formatting from mpstat - return ('export S_TIME_FORMAT=ISO; mpstat -I {breakdown} -u -P ' - '{processor_number} {interval} {count} -o JSON > {output} 2>&1 &' - .format( - breakdown=FLAGS.mpstat_breakdown, - processor_number=FLAGS.mpstat_cpus, - interval=self.interval, - count=FLAGS.mpstat_count, - output=collector_file)) + return ( + 'export S_TIME_FORMAT=ISO; mpstat -I {breakdown} -u -P ' + '{processor_number} {interval} {count} -o JSON > {output} 2>&1 &' + .format( + breakdown=FLAGS.mpstat_breakdown, + processor_number=FLAGS.mpstat_cpus, + interval=self.interval, + count=FLAGS.mpstat_count, + output=collector_file, + ) + ) def Analyze(self, unused_sender, benchmark_spec, samples): """Analyze mpstat file and record samples. @@ -407,8 +431,8 @@ def Analyze(self, unused_sender, benchmark_spec, samples): def _Analyze(role, output): """Parse file and record samples.""" with open( - os.path.join(self.output_directory, os.path.basename(output)), - 'r') as fp: + os.path.join(self.output_directory, os.path.basename(output)), 'r' + ) as fp: output = json.loads(fp.read()) metadata = { 'event': 'mpstat', @@ -420,7 +444,8 @@ def _Analyze(role, output): output, self.interval, per_interval_samples=self.per_interval_samples, - )) + ) + ) background_tasks.RunThreaded( _Analyze, [((k, w), {}) for k, w in six.iteritems(self._role_mapping)] @@ -436,7 +461,8 @@ def Register(parsed_flags): collector = MpstatCollector( interval=parsed_flags.mpstat_interval, - per_interval_samples=parsed_flags.mpstat_publish_per_interval_samples) + per_interval_samples=parsed_flags.mpstat_publish_per_interval_samples, + ) events.before_phase.connect(collector.Start, stages.RUN, weak=False) events.after_phase.connect(collector.Stop, stages.RUN, weak=False) if parsed_flags.mpstat_publish: diff --git a/perfkitbenchmarker/traces/nvidia_power.py b/perfkitbenchmarker/traces/nvidia_power.py index a0396068a0..5ae4ca796b 100644 --- a/perfkitbenchmarker/traces/nvidia_power.py +++ b/perfkitbenchmarker/traces/nvidia_power.py @@ -31,17 +31,24 @@ import six _NVIDIA_POWER = flags.DEFINE_boolean( - 'nvidia_power', False, 'Run nvidia power on each VM to collect power in ' - 'each benchmark run.') -_INTERVAL = flags.DEFINE_integer('nvidia_power_interval', 1, - 'Nvidia power sample collection frequency.') -_PUBLISH = flags.DEFINE_boolean('nvidia_power_publish', True, - 'Whether to publish Nvidia Power.') -_SELECTIONS = flags.DEFINE_string('nvidia_power_selections', 'power', - 'Choice(s) of data to collect; all or any ' - 'combinations of power, pstate, utilization, ' - 'temperature, clocks, and ' - 'clocks_throttle_reasons.') + 'nvidia_power', + False, + 'Run nvidia power on each VM to collect power in each benchmark run.', +) +_INTERVAL = flags.DEFINE_integer( + 'nvidia_power_interval', 1, 'Nvidia power sample collection frequency.' +) +_PUBLISH = flags.DEFINE_boolean( + 'nvidia_power_publish', True, 'Whether to publish Nvidia Power.' +) +_SELECTIONS = flags.DEFINE_string( + 'nvidia_power_selections', + 'power', + 'Choice(s) of data to collect; all or any ' + 'combinations of power, pstate, utilization, ' + 'temperature, clocks, and ' + 'clocks_throttle_reasons.', +) FLAGS = flags.FLAGS # queries to give to 'nvidia-smi --query-gpu='; refer to the output of @@ -142,8 +149,10 @@ def _NvidiaPowerResults( for line in output: gpu_metadata = line.copy() timestamp = datetime.datetime.timestamp( - datetime.datetime.strptime(gpu_metadata[' timestamp'], - ' %Y/%m/%d %H:%M:%S.%f')) + datetime.datetime.strptime( + gpu_metadata[' timestamp'], ' %Y/%m/%d %H:%M:%S.%f' + ) + ) gpu_metadata.update(metadata) for query_item in query_items: if query_item in ['index', 'timestamp']: @@ -162,7 +171,9 @@ def _NvidiaPowerResults( value=value, unit=table_row['unit'], metadata=new_metadata, - timestamp=timestamp)) + timestamp=timestamp, + ) + ) class _NvidiaPowerCollector(base_collector.BaseCollector): @@ -171,9 +182,12 @@ class _NvidiaPowerCollector(base_collector.BaseCollector): Runs Nvidia on the VMs. """ - def __init__(self, selections: str = 'power', - interval: float = 1.0, - output_directory: str = '') -> None: + def __init__( + self, + selections: str = 'power', + interval: float = 1.0, + output_directory: str = '', + ) -> None: super(_NvidiaPowerCollector, self).__init__(interval, output_directory) self.selections = selections self.query_items = ['index', 'timestamp'] @@ -193,8 +207,9 @@ def _CollectorName(self) -> str: """See base class.""" return 'nvidia_power' - def _CollectorRunCommand(self, vm: virtual_machine.BaseVirtualMachine, - collector_file: str) -> str: + def _CollectorRunCommand( + self, vm: virtual_machine.BaseVirtualMachine, collector_file: str + ) -> str: """See base class.""" interval_ms = int(self.interval * 1000 + 1e-6) return ( @@ -213,14 +228,16 @@ def Analyze( def _Analyze(role: str, collector_file: str) -> None: with open( os.path.join(self.output_directory, os.path.basename(collector_file)), - 'r') as fp: + 'r', + ) as fp: metadata = { 'event': 'nvidia_power', 'nvidia_interval': self.interval, - 'role': role + 'role': role, } - _NvidiaPowerResults(metadata, csv.DictReader(fp), samples, - self.query_items) + _NvidiaPowerResults( + metadata, csv.DictReader(fp), samples, self.query_items + ) background_tasks.RunThreaded( _Analyze, [((k, w), {}) for k, w in six.iteritems(self._role_mapping)] @@ -233,8 +250,10 @@ def Register(parsed_flags: flags) -> None: return collector = _NvidiaPowerCollector( - selections=_SELECTIONS.value, interval=_INTERVAL.value, - output_directory=vm_util.GetTempDir()) + selections=_SELECTIONS.value, + interval=_INTERVAL.value, + output_directory=vm_util.GetTempDir(), + ) events.before_phase.connect(collector.Start, stages.RUN, weak=False) events.after_phase.connect(collector.Stop, stages.RUN, weak=False) if _PUBLISH.value: diff --git a/perfkitbenchmarker/traces/ops_agent.py b/perfkitbenchmarker/traces/ops_agent.py index 00878d10d8..86c59fda96 100644 --- a/perfkitbenchmarker/traces/ops_agent.py +++ b/perfkitbenchmarker/traces/ops_agent.py @@ -28,20 +28,24 @@ flags.DEFINE_string( 'ops_agent_debian_rapture_repo', 'google-cloud-ops-agent-${CODENAME}-${REPO_SUFFIX:-all}', - 'The Cloud Rapture repo name used by ops agent installation bash script Debian distribution.' + 'The Cloud Rapture repo name used by ops agent installation bash script' + ' Debian distribution.', ) flags.DEFINE_string( 'ops_agent_rpm_rapture_repo', r'google-cloud-ops-agent-${CODENAME}-\$basearch-${REPO_SUFFIX:-all}', - 'The Cloud Rapture repo name used by ops agent installation bash script for RPM distribution .' + 'The Cloud Rapture repo name used by ops agent installation bash script for' + ' RPM distribution .', ) flags.DEFINE_string( 'ops_agent_suse_rapture_repo', r'google-cloud-ops-agent-${CODENAME}-\$basearch-${REPO_SUFFIX:-all}', - 'The Cloud Rapture repo name used by ops agent installation bash script for SUSE distribution.' + 'The Cloud Rapture repo name used by ops agent installation bash script for' + ' SUSE distribution.', +) +flags.DEFINE_string( + 'ops_agent_config_file', '', 'Path of the configuration file for Ops Agent.' ) -flags.DEFINE_string('ops_agent_config_file', '', - 'Path of the configuration file for Ops Agent.') OPS_AGENT_BASH_SCRIPT_PATH = './ops_agent/add-google-cloud-ops-agent-repo.sh' FLAGS = flags.FLAGS @@ -69,19 +73,25 @@ def _InstallCollector(self, vm): 'DEBIAN_REPO_NAME': str(FLAGS.ops_agent_debian_rapture_repo), 'RPM_REPO_NAME': str(FLAGS.ops_agent_rpm_rapture_repo), 'SUSE_REPO_NAME': str(FLAGS.ops_agent_suse_rapture_repo), - }) + }, + ) vm.RobustRemoteCommand( - f'sudo bash {OPS_AGENT_BASH_SCRIPT_PATH} --also-install') + f'sudo bash {OPS_AGENT_BASH_SCRIPT_PATH} --also-install' + ) if FLAGS.ops_agent_config_file: vm.PushFile(FLAGS.ops_agent_config_file, '/tmp/config.yaml') vm.RemoteCommand( - 'sudo cp /tmp/config.yaml /etc/google-cloud-ops-agent/config.yaml') + 'sudo cp /tmp/config.yaml /etc/google-cloud-ops-agent/config.yaml' + ) def _CollectorRunCommand(self, vm, collector_file): """See base class.""" - return f'sudo service google-cloud-ops-agent restart > {collector_file} 2>&1 & echo $!' + return ( + f'sudo service google-cloud-ops-agent restart > {collector_file} 2>&1 &' + ' echo $!' + ) def Register(parsed_flags): diff --git a/perfkitbenchmarker/traces/perfspect.py b/perfkitbenchmarker/traces/perfspect.py index 980009c6df..4c21ddadc9 100644 --- a/perfkitbenchmarker/traces/perfspect.py +++ b/perfkitbenchmarker/traces/perfspect.py @@ -39,22 +39,27 @@ FLAGS = flags.FLAGS -flags.DEFINE_boolean('perfspect', False, - 'Install and run perfspect on the target system.') -flags.DEFINE_string('perfspect_tarball', None, - 'Local path to perfspect tarball.') -flags.DEFINE_string('perfspect_url', None, - 'URL for downloading perfspect tarball.') +flags.DEFINE_boolean( + 'perfspect', False, 'Install and run perfspect on the target system.' +) +flags.DEFINE_string( + 'perfspect_tarball', None, 'Local path to perfspect tarball.' +) +flags.DEFINE_string( + 'perfspect_url', None, 'URL for downloading perfspect tarball.' +) PERFSPECT_ARCHIVE_URL = 'https://github.com/intel/PerfSpect/releases/download/v1.1.0/perfspect_1.1.0.tgz' PREREQ_UBUNTU = [ - 'linux-tools-common', 'linux-tools-generic', 'linux-tools-`uname -r`' + 'linux-tools-common', + 'linux-tools-generic', + 'linux-tools-`uname -r`', ] PREREQ_CENTOS = ['perf'] PREREQ_PKGS = ['python3-pip'] -class PerfspectCollector(): +class PerfspectCollector: """Manages running telemetry during a test, and fetching the results folder.""" telemetry_dir = '/opt/perf_telemetry' @@ -89,12 +94,12 @@ def _StartTelemetry(self, vm): except errors.VirtualMachine.RemoteCommandError as ex: logging.exception('Failed executing perf. Is it installed?') raise ex - perf_collect_file = posixpath.join(self.telemetry_dir, 'perfspect', - 'perf-collect.sh') + perf_collect_file = posixpath.join( + self.telemetry_dir, 'perfspect', 'perf-collect.sh' + ) vm.RemoteCommand(f'sudo chmod +x {perf_collect_file}') perf_dir = posixpath.join(self.telemetry_dir, 'perfspect') - stdout, _ = vm.RemoteCommand( - f'cd {perf_dir} && sudo ./perf-collect.sh') + stdout, _ = vm.RemoteCommand(f'cd {perf_dir} && sudo ./perf-collect.sh') self.pid = stdout.strip() logging.debug('fpid of PerfSpect collector process: %s', self.pid) @@ -106,7 +111,8 @@ def _StopTelemetry(self, vm): logging.info('Post processing PerfSpect raw metrics') perf_dir = posixpath.join(self.telemetry_dir, 'perfspect') vm.RemoteCommand( - f'cd {perf_dir} && sudo ./perf-postprocess -r results/perfstat.csv') + f'cd {perf_dir} && sudo ./perf-postprocess -r results/perfstat.csv' + ) def _FetchResults(self, vm): """Fetches PerfSpect telemetry results.""" @@ -136,8 +142,9 @@ def _GetLocalArchive(self): logging.info('downloading PerfSpect from: %s', url) filename = os.path.basename(urlparse(url).path) local_archive_path = posixpath.join(vm_util.GetTempDir(), filename) - vm_util.IssueCommand(['curl', '-k', '-L', '-o', local_archive_path, url], - timeout=None) + vm_util.IssueCommand( + ['curl', '-k', '-L', '-o', local_archive_path, url], timeout=None + ) return local_archive_path def Before(self, unused_sender, benchmark_spec): @@ -154,12 +161,12 @@ def Before(self, unused_sender, benchmark_spec): self.perf_dir = posixpath.join(vm_util.GetTempDir(), 'perfspect') self.perfspect_archive = self._GetLocalArchive() vm_util.IssueCommand( - ['tar', '-C', - vm_util.GetTempDir(), '-xf', self.perfspect_archive]) + ['tar', '-C', vm_util.GetTempDir(), '-xf', self.perfspect_archive] + ) vm_util.IssueCommand([ 'cp', data.ResourcePath(posixpath.join('perfspect', 'perf-collect.sh')), - self.perf_dir + '/' + self.perf_dir + '/', ]) background_tasks.RunThreaded(self._InstallTelemetry, vms) @@ -187,9 +194,11 @@ def Register(parsed_flags): logging.info('Registering PerfSpect telemetry collector') telemetry_collector = PerfspectCollector() events.before_phase.connect( - telemetry_collector.Before, events.RUN_PHASE, weak=False) + telemetry_collector.Before, events.RUN_PHASE, weak=False + ) events.after_phase.connect( - telemetry_collector.After, events.RUN_PHASE, weak=False) + telemetry_collector.After, events.RUN_PHASE, weak=False + ) def IsEnabled(): diff --git a/perfkitbenchmarker/traces/sar.py b/perfkitbenchmarker/traces/sar.py index cd57af8914..dceaa2c067 100644 --- a/perfkitbenchmarker/traces/sar.py +++ b/perfkitbenchmarker/traces/sar.py @@ -26,23 +26,35 @@ import six flags.DEFINE_boolean( - 'sar', False, 'Run sar (https://linux.die.net/man/1/sar) ' + 'sar', + False, + 'Run sar (https://linux.die.net/man/1/sar) ' 'on each VM to collect system performance metrics during ' - 'each benchmark run.') + 'each benchmark run.', +) flags.DEFINE_integer( - 'sar_interval', 5, 'sar sample collection frequency, in seconds. Only ' - 'applicable when --sar is specified.') + 'sar_interval', + 5, + 'sar sample collection frequency, in seconds. Only ' + 'applicable when --sar is specified.', +) flags.DEFINE_integer( - 'sar_samples', None, + 'sar_samples', + None, 'Number of sar samples to collect. When undefined sar is ' 'ran indefinitely. This must be set to record average ' - 'statistics. Only applicable when --sar is specified.') + 'statistics. Only applicable when --sar is specified.', +) flags.DEFINE_string( - 'sar_output', None, 'Output directory for sar output. ' + 'sar_output', + None, + 'Output directory for sar output. ' 'Only applicable when --sar is specified. ' - 'Default: run temporary directory.') -flags.DEFINE_boolean('sar_publish', True, - 'Whether to publish average sar statistics.') + 'Default: run temporary directory.', +) +flags.DEFINE_boolean( + 'sar_publish', True, 'Whether to publish average sar statistics.' +) FLAGS = flags.FLAGS @@ -85,7 +97,9 @@ def _AddStealResults(metadata, output, samples): samples.append( sample.Sample( - metric=metric, value=value, unit='%', metadata=my_metadata)) + metric=metric, value=value, unit='%', metadata=my_metadata + ) + ) class _SarCollector(base_collector.BaseCollector): @@ -101,11 +115,13 @@ def _InstallCollector(self, vm): vm.InstallPackages('sysstat') def _CollectorRunCommand(self, vm, collector_file): - cmd = ('sar -u {sar_interval} {sar_samples} > {output} 2>&1 & ' - 'echo $!').format( - output=collector_file, - sar_interval=FLAGS.sar_interval, - sar_samples=FLAGS.sar_samples if FLAGS.sar_samples else '') + cmd = ( + 'sar -u {sar_interval} {sar_samples} > {output} 2>&1 & echo $!' + ).format( + output=collector_file, + sar_interval=FLAGS.sar_interval, + sar_samples=FLAGS.sar_samples if FLAGS.sar_samples else '', + ) return cmd def Analyze(self, unused_sender, benchmark_spec, samples): @@ -113,8 +129,9 @@ def Analyze(self, unused_sender, benchmark_spec, samples): def _Analyze(role, f): """Parse file and record samples.""" - with open(os.path.join(self.output_directory, os.path.basename(f)), - 'r') as fp: + with open( + os.path.join(self.output_directory, os.path.basename(f)), 'r' + ) as fp: output = fp.read() metadata = { 'event': 'sar', @@ -135,15 +152,21 @@ def Register(parsed_flags): output_directory = ( parsed_flags.sar_output - if parsed_flags['sar_output'].present else vm_util.GetTempDir()) + if parsed_flags['sar_output'].present + else vm_util.GetTempDir() + ) - logging.debug('Registering sar collector with interval %s, output to %s.', - parsed_flags.sar_interval, output_directory) + logging.debug( + 'Registering sar collector with interval %s, output to %s.', + parsed_flags.sar_interval, + output_directory, + ) if not os.path.isdir(output_directory): os.makedirs(output_directory) collector = _SarCollector( - interval=parsed_flags.sar_interval, output_directory=output_directory) + interval=parsed_flags.sar_interval, output_directory=output_directory + ) events.before_phase.connect(collector.Start, stages.RUN, weak=False) events.after_phase.connect(collector.Stop, stages.RUN, weak=False) if parsed_flags.sar_publish: diff --git a/perfkitbenchmarker/traces/tcpdump.py b/perfkitbenchmarker/traces/tcpdump.py index 90906ce78a..3ff2f8d642 100644 --- a/perfkitbenchmarker/traces/tcpdump.py +++ b/perfkitbenchmarker/traces/tcpdump.py @@ -25,18 +25,28 @@ from perfkitbenchmarker.traces import base_collector flags.DEFINE_boolean( - 'tcpdump', False, 'Run tcpdump on each VM to collect network packets in ' - 'each benchmark run.') -flags.DEFINE_list('tcpdump_ignore_ports', [22], - 'Ports to ignore when running tcpdump') + 'tcpdump', + False, + 'Run tcpdump on each VM to collect network packets in each benchmark run.', +) flags.DEFINE_list( - 'tcpdump_include_ports', [], 'Ports to include when running tcpdump. By ' - 'default collects all ports except those in --tcpdump_ignore_ports') -flags.DEFINE_integer('tcpdump_snaplen', 96, - 'Tcpdump snaplen, see man tcpdump "-s"') + 'tcpdump_ignore_ports', [22], 'Ports to ignore when running tcpdump' +) +flags.DEFINE_list( + 'tcpdump_include_ports', + [], + 'Ports to include when running tcpdump. By ' + 'default collects all ports except those in --tcpdump_ignore_ports', +) +flags.DEFINE_integer( + 'tcpdump_snaplen', 96, 'Tcpdump snaplen, see man tcpdump "-s"' +) flags.DEFINE_integer( - 'tcpdump_packet_count', None, 'Number of packets to collect. Default ' - 'is to collect all packets in the run phase') + 'tcpdump_packet_count', + None, + 'Number of packets to collect. Default ' + 'is to collect all packets in the run phase', +) FLAGS = flags.FLAGS @@ -61,11 +71,13 @@ class _TcpdumpCollector(base_collector.BaseCollector): Installs tcpdump and runs it on the VMs. """ - def __init__(self, - ignore_ports=None, - include_ports=None, - snaplen=None, - packet_count=None): + def __init__( + self, + ignore_ports=None, + include_ports=None, + snaplen=None, + packet_count=None, + ): super(_TcpdumpCollector, self).__init__(None, None) self.snaplen = snaplen self.packet_count = packet_count @@ -120,7 +132,8 @@ def _CreateCollector(parsed_flags): ignore_ports=parsed_flags.tcpdump_ignore_ports, include_ports=parsed_flags.tcpdump_include_ports, snaplen=parsed_flags.tcpdump_snaplen, - packet_count=parsed_flags.tcpdump_packet_count) + packet_count=parsed_flags.tcpdump_packet_count, + ) def Register(parsed_flags): diff --git a/perfkitbenchmarker/virtual_machine.py b/perfkitbenchmarker/virtual_machine.py index 9b8237d96c..52a4a626ba 100644 --- a/perfkitbenchmarker/virtual_machine.py +++ b/perfkitbenchmarker/virtual_machine.py @@ -60,8 +60,7 @@ def ValidateVmMetadataFlag(options_list): All provided options must be in the form of key:value. Args: - options_list: A list of strings parsed from the provided value for the - flag. + options_list: A list of strings parsed from the provided value for the flag. Returns: True if the list of options provided as the value for the flag meets @@ -74,50 +73,66 @@ def ValidateVmMetadataFlag(options_list): for option in options_list: if ':' not in option[1:-1]: raise flags.ValidationError( - '"%s" not in expected key:value format' % option) + '"%s" not in expected key:value format' % option + ) return True + # vm_metadata flag name VM_METADATA = 'vm_metadata' flags.DEFINE_boolean( - 'dedicated_hosts', False, + 'dedicated_hosts', + False, 'If True, use hosts that only have VMs from the same ' - 'benchmark running on them.') + 'benchmark running on them.', +) flags.DEFINE_integer( - 'num_vms_per_host', None, + 'num_vms_per_host', + None, 'The number of VMs per dedicated host. If None, VMs will be packed on a ' 'single host until no more can be packed at which point a new host will ' - 'be created.') + 'be created.', +) flags.DEFINE_integer( - 'num_cpus_override', None, + 'num_cpus_override', + None, 'Rather than detecting the number of CPUs present on the machine, use this ' 'value if set. Some benchmarks will use this number to automatically ' 'scale their configurations; this can be used as a method to control ' 'benchmark scaling. It will also change the num_cpus metadata ' - 'published along with the benchmark data.') -flags.DEFINE_list(VM_METADATA, [], 'Metadata to add to the vm. It expects' - 'key:value pairs.') + 'published along with the benchmark data.', +) +flags.DEFINE_list( + VM_METADATA, [], 'Metadata to add to the vm. It expectskey:value pairs.' +) flags.register_validator(VM_METADATA, ValidateVmMetadataFlag) flags.DEFINE_bool( - 'skip_firewall_rules', False, + 'skip_firewall_rules', + False, 'If set, this run will not create firewall rules. This is useful if the ' 'user project already has all of the firewall rules in place and/or ' - 'creating new ones is expensive') + 'creating new ones is expensive', +) flags.DEFINE_bool( - 'preprovision_ignore_checksum', False, + 'preprovision_ignore_checksum', + False, 'Ignore checksum verification for preprovisioned data. ' - 'Not recommended, please use with caution') + 'Not recommended, please use with caution', +) flags.DEFINE_boolean( - 'connect_via_internal_ip', False, + 'connect_via_internal_ip', + False, 'Whether to use internal IP addresses for running commands on and pushing ' 'data to VMs. By default, PKB interacts with VMs using external IP ' - 'addresses.') + 'addresses.', +) _ASSIGN_EXTERNAL_IP = flags.DEFINE_boolean( 'assign_external_ip', True, 'If True, an external (public) IP will be created for VMs. ' - 'If False, --connect_via_internal_ip may also be needed.') + 'If False, --connect_via_internal_ip may also be needed.', +) flags.DEFINE_string( 'boot_startup_script', None, @@ -148,12 +163,17 @@ class BootCompletionIpSubset(enum.Enum): ) # Deprecated. Use connect_via_internal_ip. flags.DEFINE_boolean( - 'ssh_via_internal_ip', False, + 'ssh_via_internal_ip', + False, 'Whether to use internal IP addresses for running commands on and pushing ' 'data to VMs. By default, PKB interacts with VMs using external IP ' - 'addresses.') -flags.DEFINE_boolean('retry_on_rate_limited', True, - 'Whether to retry commands when rate limited.') + 'addresses.', +) +flags.DEFINE_boolean( + 'retry_on_rate_limited', + True, + 'Whether to retry commands when rate limited.', +) GPU_K80 = 'k80' GPU_P100 = 'p100' @@ -180,13 +200,16 @@ class BootCompletionIpSubset(enum.Enum): CPUARCH_AARCH64 = 'aarch64' flags.DEFINE_integer( - 'gpu_count', None, - 'Number of gpus to attach to the VM. Requires gpu_type to be ' - 'specified.') + 'gpu_count', + None, + 'Number of gpus to attach to the VM. Requires gpu_type to be specified.', +) flags.DEFINE_enum( - 'gpu_type', None, VALID_GPU_TYPES, - 'Type of gpus to attach to the VM. Requires gpu_count to be ' - 'specified.') + 'gpu_type', + None, + VALID_GPU_TYPES, + 'Type of gpus to attach to the VM. Requires gpu_count to be specified.', +) def GetVmSpecClass(cloud): @@ -196,8 +219,9 @@ def GetVmSpecClass(cloud): def GetVmClass(cloud, os_type): """Returns the VM class corresponding to 'cloud' and 'os_type'.""" - return resource.GetResourceClass(BaseVirtualMachine, - CLOUD=cloud, OS_TYPE=os_type) + return resource.GetResourceClass( + BaseVirtualMachine, CLOUD=cloud, OS_TYPE=os_type + ) class BaseVmSpec(spec.BaseSpec): @@ -211,14 +235,14 @@ class BaseVmSpec(spec.BaseSpec): gpu_type: None or string. Type of gpus to attach to the VM. image: The disk image to boot from. assign_external_ip: Bool. If true, create an external (public) IP. - install_packages: If false, no packages will be installed. This is - useful if benchmark dependencies have already been installed. - background_cpu_threads: The number of threads of background CPU usage - while running the benchmark. + install_packages: If false, no packages will be installed. This is useful if + benchmark dependencies have already been installed. + background_cpu_threads: The number of threads of background CPU usage while + running the benchmark. background_network_mbits_per_sec: The number of megabits per second of - background network traffic during the benchmark. - background_network_ip_type: The IP address type (INTERNAL or - EXTERNAL) to use for generating background network workload. + background network traffic during the benchmark. + background_network_ip_type: The IP address type (INTERNAL or EXTERNAL) to + use for generating background network workload. disable_interrupt_moderation: If true, disables interrupt moderation. disable_rss: = If true, disables rss. boot_startup_script: Startup script to run during boot. @@ -255,9 +279,9 @@ def _ApplyFlags(cls, config_values, flag_values): Args: config_values: dict mapping config option names to provided values. Is - modified by this function. + modified by this function. flag_values: flags.FlagValues. Runtime flags that may override the - provided config values. + provided config values. Returns: dict mapping config option names to values derived from the config @@ -272,13 +296,16 @@ def _ApplyFlags(cls, config_values, flag_values): config_values['machine_type'] = flag_values.machine_type if flag_values['background_cpu_threads'].present: config_values['background_cpu_threads'] = ( - flag_values.background_cpu_threads) + flag_values.background_cpu_threads + ) if flag_values['background_network_mbits_per_sec'].present: config_values['background_network_mbits_per_sec'] = ( - flag_values.background_network_mbits_per_sec) + flag_values.background_network_mbits_per_sec + ) if flag_values['background_network_ip_type'].present: config_values['background_network_ip_type'] = ( - flag_values.background_network_ip_type) + flag_values.background_network_ip_type + ) if flag_values['dedicated_hosts'].present: config_values['use_dedicated_host'] = flag_values.dedicated_hosts if flag_values['num_vms_per_host'].present: @@ -291,7 +318,8 @@ def _ApplyFlags(cls, config_values, flag_values): config_values['assign_external_ip'] = flag_values.assign_external_ip if flag_values['disable_interrupt_moderation'].present: config_values['disable_interrupt_moderation'] = ( - flag_values.disable_interrupt_moderation) + flag_values.disable_interrupt_moderation + ) if flag_values['disable_rss'].present: config_values['disable_rss'] = flag_values.disable_rss if flag_values['vm_metadata'].present: @@ -301,10 +329,12 @@ def _ApplyFlags(cls, config_values, flag_values): if 'gpu_count' in config_values and 'gpu_type' not in config_values: raise errors.Config.MissingOption( - 'gpu_type must be specified if gpu_count is set') + 'gpu_type must be specified if gpu_count is set' + ) if 'gpu_type' in config_values and 'gpu_count' not in config_values: raise errors.Config.MissingOption( - 'gpu_count must be specified if gpu_type is set') + 'gpu_count must be specified if gpu_type is set' + ) @classmethod def _GetOptionDecoderConstructions(cls): @@ -320,43 +350,78 @@ def _GetOptionDecoderConstructions(cls): """ result = super(BaseVmSpec, cls)._GetOptionDecoderConstructions() result.update({ - 'disable_interrupt_moderation': (option_decoders.BooleanDecoder, { - 'default': False}), + 'disable_interrupt_moderation': ( + option_decoders.BooleanDecoder, + {'default': False}, + ), 'disable_rss': (option_decoders.BooleanDecoder, {'default': False}), - 'image': (option_decoders.StringDecoder, {'none_ok': True, - 'default': None}), - 'install_packages': (option_decoders.BooleanDecoder, {'default': True}), - 'machine_type': (option_decoders.StringDecoder, {'none_ok': True, - 'default': None}), + 'image': ( + option_decoders.StringDecoder, + {'none_ok': True, 'default': None}, + ), + 'install_packages': ( + option_decoders.BooleanDecoder, + {'default': True}, + ), + 'machine_type': ( + option_decoders.StringDecoder, + {'none_ok': True, 'default': None}, + ), 'assign_external_ip': ( option_decoders.BooleanDecoder, {'default': True}, ), - 'gpu_type': (option_decoders.EnumDecoder, { - 'valid_values': VALID_GPU_TYPES, - 'default': None}), - 'gpu_count': (option_decoders.IntDecoder, {'min': 1, 'default': None}), - 'zone': (option_decoders.StringDecoder, {'none_ok': True, - 'default': None}), - 'cidr': (option_decoders.StringDecoder, {'none_ok': True, - 'default': None}), - 'use_dedicated_host': (option_decoders.BooleanDecoder, - {'default': False}), - 'num_vms_per_host': (option_decoders.IntDecoder, - {'default': None}), - 'background_network_mbits_per_sec': (option_decoders.IntDecoder, { - 'none_ok': True, 'default': None}), - 'background_network_ip_type': (option_decoders.EnumDecoder, { - 'default': vm_util.IpAddressSubset.EXTERNAL, - 'valid_values': [vm_util.IpAddressSubset.EXTERNAL, - vm_util.IpAddressSubset.INTERNAL]}), - 'background_cpu_threads': (option_decoders.IntDecoder, { - 'none_ok': True, 'default': None}), - 'boot_startup_script': (option_decoders.StringDecoder, { - 'none_ok': True, 'default': None}), - 'vm_metadata': (option_decoders.ListDecoder, { - 'item_decoder': option_decoders.StringDecoder(), - 'default': []})}) + 'gpu_type': ( + option_decoders.EnumDecoder, + {'valid_values': VALID_GPU_TYPES, 'default': None}, + ), + 'gpu_count': ( + option_decoders.IntDecoder, + {'min': 1, 'default': None}, + ), + 'zone': ( + option_decoders.StringDecoder, + {'none_ok': True, 'default': None}, + ), + 'cidr': ( + option_decoders.StringDecoder, + {'none_ok': True, 'default': None}, + ), + 'use_dedicated_host': ( + option_decoders.BooleanDecoder, + {'default': False}, + ), + 'num_vms_per_host': (option_decoders.IntDecoder, {'default': None}), + 'background_network_mbits_per_sec': ( + option_decoders.IntDecoder, + {'none_ok': True, 'default': None}, + ), + 'background_network_ip_type': ( + option_decoders.EnumDecoder, + { + 'default': vm_util.IpAddressSubset.EXTERNAL, + 'valid_values': [ + vm_util.IpAddressSubset.EXTERNAL, + vm_util.IpAddressSubset.INTERNAL, + ], + }, + ), + 'background_cpu_threads': ( + option_decoders.IntDecoder, + {'none_ok': True, 'default': None}, + ), + 'boot_startup_script': ( + option_decoders.StringDecoder, + {'none_ok': True, 'default': None}, + ), + 'vm_metadata': ( + option_decoders.ListDecoder, + { + 'item_decoder': option_decoders.StringDecoder(), + 'default': [], + }, + ), + }) return result @@ -370,9 +435,10 @@ class BaseOsMixin(six.with_metaclass(abc.ABCMeta, object)): Attributes: bootable_time: The time when the VM finished booting. hostname: The VM's hostname. - remote_access_ports: A list of ports which must be opened on the firewall - in order to access the VM. + remote_access_ports: A list of ports which must be opened on the firewall in + order to access the VM. """ + # Represents whether the VM type can be (cleanly) rebooted. Should be false # for a class if rebooting causes issues, e.g. for KubernetesVirtualMachine # needing to reboot often indicates a design problem since restarting a @@ -420,9 +486,9 @@ def __init__(self): # Update to Json type if ever available: # https://github.com/python/typing/issues/182 self.os_metadata: Dict[str, Union[str, int, list[str]]] = {} - assert type( - self).BASE_OS_TYPE in os_types.BASE_OS_TYPES, '%s is not in %s' % ( - type(self).BASE_OS_TYPE, os_types.BASE_OS_TYPES) + assert ( + type(self).BASE_OS_TYPE in os_types.BASE_OS_TYPES + ), '%s is not in %s' % (type(self).BASE_OS_TYPE, os_types.BASE_OS_TYPES) @property @classmethod @@ -449,7 +515,7 @@ def RemoteCommand( command: str, ignore_failure: bool = False, timeout: Optional[float] = None, - **kwargs + **kwargs, ) -> Tuple[str, str]: """Runs a command on the VM. @@ -459,8 +525,8 @@ def RemoteCommand( Args: command: A valid bash command. ignore_failure: Ignore any failure if set to true. - timeout: The time to wait in seconds for the command before exiting. - None means no timeout. + timeout: The time to wait in seconds for the command before exiting. None + means no timeout. **kwargs: Additional command arguments. Returns: @@ -520,7 +586,8 @@ def Reboot(self): """ if not self.IS_REBOOTABLE: raise errors.VirtualMachine.VirtualMachineError( - "Trying to reboot a VM that isn't rebootable.") + "Trying to reboot a VM that isn't rebootable." + ) vm_bootable_time = None @@ -681,8 +748,7 @@ def _WaitForSSH(self, ip_address: Union[str, None] = None): raise NotImplementedError() def VMLastBootTime(self): - """Returns the time the VM was last rebooted as reported by the VM. - """ + """Returns the time the VM was last rebooted as reported by the VM.""" raise NotImplementedError() def OnStartup(self): @@ -752,8 +818,8 @@ def PushFile(self, source_path, remote_path=''): Args: source_path: The location of the file or directory on the LOCAL machine. - remote_path: The destination of the file on the REMOTE machine, default - is the home directory. + remote_path: The destination of the file on the REMOTE machine, default is + the home directory. """ self.RemoteCopy(source_path, remote_path) @@ -762,9 +828,9 @@ def PullFile(self, local_path, remote_path): Args: local_path: string. The destination path of the file or directory on the - local machine. + local machine. remote_path: string. The source path of the file or directory on the - remote machine. + remote machine. """ self.RemoteCopy(local_path, remote_path, copy_to=False) @@ -776,6 +842,7 @@ def PushDataFile(self, data_file, remote_path='', should_double_copy=None): remote_path: The destination for 'data_file' on the VM. If not specified, the file will be placed in the user's home directory. should_double_copy: Indicates whether to first copy to the home directory + Raises: perfkitbenchmarker.data.ResourceNotFound: if 'data_file' does not exist. """ @@ -783,7 +850,7 @@ def PushDataFile(self, data_file, remote_path='', should_double_copy=None): if should_double_copy: home_file_path = '~/' + data_file self.PushFile(file_path, home_file_path) - copy_cmd = (' '.join(['cp', home_file_path, remote_path])) + copy_cmd = ' '.join(['cp', home_file_path, remote_path]) self.RemoteCommand(copy_cmd) else: self.PushFile(file_path, remote_path) @@ -811,8 +878,9 @@ def RenderTemplate(self, template_path, remote_path, context): template = environment.from_string(template_contents) prefix = 'pkb-' + os.path.basename(template_path) - with vm_util.NamedTemporaryFile(prefix=prefix, dir=vm_util.GetTempDir(), - delete=False, mode='w') as tf: + with vm_util.NamedTemporaryFile( + prefix=prefix, dir=vm_util.GetTempDir(), delete=False, mode='w' + ) as tf: tf.write(template.render(vm=self, **context)) tf.close() self.RemoteCopy(tf.name, remote_path) @@ -872,8 +940,8 @@ def NumCpusForBenchmark(self, report_only_physical_cpus=False): disabled. Args: - report_only_physical_cpus: Whether to report only the physical - (non-SMT) CPUs. Default is to report all vCPUs. + report_only_physical_cpus: Whether to report only the physical (non-SMT) + CPUs. Default is to report all vCPUs. Returns: The number of CPUs for benchmark configuration purposes. @@ -991,8 +1059,9 @@ def GetSha256sum(self, path, filename): """ raise NotImplementedError() - def CheckPreprovisionedData(self, install_path, module_name, filename, - expected_sha256): + def CheckPreprovisionedData( + self, install_path, module_name, filename, expected_sha256 + ): """Checks preprovisioned data for a checksum. Checks the expected 256sum against the actual sha256sum. Called after the @@ -1002,7 +1071,7 @@ def CheckPreprovisionedData(self, install_path, module_name, filename, Args: install_path: The install path on this VM. The benchmark is installed at - this path in a subdirectory of the benchmark name. + this path in a subdirectory of the benchmark name. module_name: Name of the benchmark associated with this data file. filename: The name of the file that was downloaded. expected_sha256: The expected sha256 checksum value. @@ -1012,26 +1081,28 @@ def CheckPreprovisionedData(self, install_path, module_name, filename, raise errors.Setup.BadPreprovisionedDataError( 'Invalid sha256sum for %s/%s: %s (actual) != %s (expected). Might ' 'want to run using --preprovision_ignore_checksum ' - '(not recommended).' % ( - module_name, filename, actual_sha256, expected_sha256)) + '(not recommended).' + % (module_name, filename, actual_sha256, expected_sha256) + ) def TestConnectRemoteAccessPort(self, port=None, socket_timeout=0.5): """Tries to connect to remote access port and throw if it fails. Args: - port: Integer of the port to connect to. Defaults to - the default remote connection port of the VM. + port: Integer of the port to connect to. Defaults to the default remote + connection port of the VM. socket_timeout: The number of seconds to wait on the socket before failing - and retrying. If this is too low, the connection may never succeed. If - this is too high it will add latency (because the current connection - may fail after a time that a new connection would succeed). - Defaults to 500ms. + and retrying. If this is too low, the connection may never succeed. If + this is too high it will add latency (because the current connection may + fail after a time that a new connection would succeed). Defaults to + 500ms. """ if not port: port = self.primary_remote_access_port # TODO(user): refactor to reuse sockets? - with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) \ - as sock: + with contextlib.closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as sock: # Before the IP is reachable the socket times out (and throws). After that # it throws immediately. sock.settimeout(socket_timeout) # seconds @@ -1069,7 +1140,9 @@ def __init__(self): assert self.OS_TYPE assert self.END_OF_LIFE warning = "os_type '%s' is deprecated and will be removed after %s." % ( - self.OS_TYPE, self.END_OF_LIFE) + self.OS_TYPE, + self.END_OF_LIFE, + ) if self.ALTERNATIVE_OS: warning += " Use '%s' instead." % self.ALTERNATIVE_OS logging.warning(warning) @@ -1101,8 +1174,8 @@ class BaseVirtualMachine(BaseOsMixin, resource.BaseResource): scratch_disks: list of BaseDisk objects. Scratch disks attached to the VM. max_local_disks: The number of local disks on the VM that can be used as scratch disks or that can be striped together. - background_cpu_threads: The number of threads of background CPU usage - while running the benchmark. + background_cpu_threads: The number of threads of background CPU usage while + running the benchmark. background_network_mbits_per_sec: Number of mbits/sec of background network usage while running the benchmark. background_network_ip_type: Type of IP address to use for generating @@ -1142,8 +1215,9 @@ def __init__(self, vm_spec: BaseVmSpec): self.gpu_type = vm_spec.gpu_type self.image = vm_spec.image self.install_packages = vm_spec.install_packages - self.can_connect_via_internal_ip = (FLAGS.ssh_via_internal_ip - or FLAGS.connect_via_internal_ip) + self.can_connect_via_internal_ip = ( + FLAGS.ssh_via_internal_ip or FLAGS.connect_via_internal_ip + ) self.boot_completion_ip_subset = _BOOT_COMPLETION_IP_SUBSET.value self.assign_external_ip = vm_spec.assign_external_ip self.ip_address = None @@ -1159,7 +1233,8 @@ def __init__(self, vm_spec: BaseVmSpec): self.remote_disk_counter = 0 self.background_cpu_threads = vm_spec.background_cpu_threads self.background_network_mbits_per_sec = ( - vm_spec.background_network_mbits_per_sec) + vm_spec.background_network_mbits_per_sec + ) self.background_network_ip_type = vm_spec.background_network_ip_type self.use_dedicated_host = None self.num_vms_per_host = None @@ -1180,7 +1255,8 @@ def __init__(self, vm_spec: BaseVmSpec): self.boot_startup_script = vm_spec.boot_startup_script if self.OS_TYPE == os_types.CORE_OS and self.boot_startup_script: raise errors.Setup.InvalidConfigurationError( - 'Startup script are not supported on CoreOS.') + 'Startup script are not supported on CoreOS.' + ) @property @classmethod @@ -1190,7 +1266,8 @@ def CLOUD(cls): def __repr__(self): return ''.format( - self.ip_address, self.internal_ip) + self.ip_address, self.internal_ip + ) def __str__(self): if self.ip_address: @@ -1202,7 +1279,8 @@ def GetConnectionIp(self): if not self.created: raise errors.VirtualMachine.VirtualMachineError( 'VM was not properly created, but PKB is attempting to connect to ' - 'it anyways. Caller should guard against VM not being created.') + 'it anyways. Caller should guard against VM not being created.' + ) if self.can_connect_via_internal_ip: return self.internal_ip if not self.ip_address: @@ -1280,14 +1358,15 @@ def GetScratchDir(self, disk_num=0): Args: disk_num: The number of the disk to mount. + Returns: The mounted disk directory. - """ if disk_num >= len(self.scratch_disks): raise errors.Error( - 'GetScratchDir(disk_num=%s) is invalid, max disk_num is %s' % ( - disk_num, len(self.scratch_disks))) + 'GetScratchDir(disk_num=%s) is invalid, max disk_num is %s' + % (disk_num, len(self.scratch_disks)) + ) return self.scratch_disks[disk_num].mount_point def AllowIcmp(self): @@ -1398,15 +1477,21 @@ def CollectLMNotificationsTime(self): """Extract LM notifications from log file.""" raise NotImplementedError() - def _InstallData(self, preprovisioned_data, module_name, filenames, - install_path, fallback_url): + def _InstallData( + self, + preprovisioned_data, + module_name, + filenames, + install_path, + fallback_url, + ): """Installs preprovisioned_data on this VM. Args: preprovisioned_data: The dict mapping filenames to sha256sum hashes. module_name: The name of the module defining the preprovisioned data. filenames: An iterable of preprovisioned data filenames for a particular - module. + module. install_path: The path to download the data file. fallback_url: The dict mapping filenames to fallback url for downloading. @@ -1424,11 +1509,14 @@ def _InstallData(self, preprovisioned_data, module_name, filenames, sha256sum = preprovisioned_data.get(filename) try: preprovisioned = self.ShouldDownloadPreprovisionedData( - module_name, filename) + module_name, filename + ) except NotImplementedError: - logging.info('The provider does not implement ' - 'ShouldDownloadPreprovisionedData. Attempting to ' - 'download the data via URL') + logging.info( + 'The provider does not implement ' + 'ShouldDownloadPreprovisionedData. Attempting to ' + 'download the data via URL' + ) preprovisioned = False if not FLAGS.preprovision_ignore_checksum and not sha256sum: raise errors.Setup.BadPreprovisionedDataError( @@ -1436,7 +1524,8 @@ def _InstallData(self, preprovisioned_data, module_name, filenames, 'to run using --preprovision_ignore_checksum (not recommended). ' 'See README.md for information about preprovisioned data. ' 'Cannot find file in /data directory either, fail to upload from ' - 'local directory.' % (filename, module_name)) + 'local directory.' % (filename, module_name) + ) if preprovisioned: self.DownloadPreprovisionedData(install_path, module_name, filename) @@ -1444,8 +1533,8 @@ def _InstallData(self, preprovisioned_data, module_name, filenames, self.Install('wget') file_name = os.path.basename(url) self.RemoteCommand( - 'wget -O {0} {1}'.format( - os.path.join(install_path, file_name), url)) + 'wget -O {0} {1}'.format(os.path.join(install_path, file_name), url) + ) else: raise errors.Setup.BadPreprovisionedDataError( 'Cannot find preprovisioned file %s inside preprovisioned bucket ' @@ -1453,13 +1542,16 @@ def _InstallData(self, preprovisioned_data, module_name, filenames, 'preprovisioned data. ' 'Cannot find fallback url of the file to download from web. ' 'Cannot find file in /data directory either, fail to upload from ' - 'local directory.' % (filename, module_name)) + 'local directory.' % (filename, module_name) + ) if not FLAGS.preprovision_ignore_checksum: self.CheckPreprovisionedData( - install_path, module_name, filename, sha256sum) + install_path, module_name, filename, sha256sum + ) - def InstallPreprovisionedBenchmarkData(self, benchmark_name, filenames, - install_path): + def InstallPreprovisionedBenchmarkData( + self, benchmark_name, filenames, install_path + ): """Installs preprovisioned benchmark data on this VM. Some benchmarks require importing many bytes of data into the virtual @@ -1479,10 +1571,10 @@ def InstallPreprovisionedBenchmarkData(self, benchmark_name, filenames, Args: benchmark_name: The name of the benchmark defining the preprovisioned - data. The benchmark's module must define the dict BENCHMARK_DATA - mapping filenames to sha256sum hashes. + data. The benchmark's module must define the dict BENCHMARK_DATA mapping + filenames to sha256sum hashes. filenames: An iterable of preprovisioned data filenames for a particular - benchmark. + benchmark. install_path: The path to download the data file. Raises: @@ -1493,21 +1585,29 @@ def InstallPreprovisionedBenchmarkData(self, benchmark_name, filenames, benchmark_module = benchmark_lookup.BenchmarkModule(benchmark_name) if not benchmark_module: raise errors.Setup.BadPreprovisionedDataError( - 'Cannot install preprovisioned data for undefined benchmark %s.' % - benchmark_name) + 'Cannot install preprovisioned data for undefined benchmark %s.' + % benchmark_name + ) try: # TODO(user): Change BENCHMARK_DATA to PREPROVISIONED_DATA. preprovisioned_data = benchmark_module.BENCHMARK_DATA except AttributeError: raise errors.Setup.BadPreprovisionedDataError( 'Benchmark %s does not define a BENCHMARK_DATA dict with ' - 'preprovisioned data.' % benchmark_name) + 'preprovisioned data.' % benchmark_name + ) fallback_url = getattr(benchmark_module, 'BENCHMARK_DATA_URL', {}) - self._InstallData(preprovisioned_data, benchmark_name, filenames, - install_path, fallback_url) - - def InstallPreprovisionedPackageData(self, package_name, filenames, - install_path): + self._InstallData( + preprovisioned_data, + benchmark_name, + filenames, + install_path, + fallback_url, + ) + + def InstallPreprovisionedPackageData( + self, package_name, filenames, install_path + ): """Installs preprovisioned Package data on this VM. Some benchmarks require importing many bytes of data into the virtual @@ -1527,11 +1627,11 @@ def InstallPreprovisionedPackageData(self, package_name, filenames, Args: package_name: The name of the package file defining the preprovisoned - data. The default vaule is None. If the package_name is provided, the - package file must define the dict PREPROVISIONED_DATA mapping filenames to - sha256sum hashes. + data. The default vaule is None. If the package_name is provided, the + package file must define the dict PREPROVISIONED_DATA mapping filenames + to sha256sum hashes. filenames: An iterable of preprovisioned data filenames for a particular - package. + package. install_path: The path to download the data file. Raises: @@ -1542,17 +1642,20 @@ def InstallPreprovisionedPackageData(self, package_name, filenames, package_module = package_lookup.PackageModule(package_name) if not package_module: raise errors.Setup.BadPreprovisionedDataError( - 'Cannot install preprovisioned data for undefined package %s.' % - package_name) + 'Cannot install preprovisioned data for undefined package %s.' + % package_name + ) try: preprovisioned_data = package_module.PREPROVISIONED_DATA except AttributeError: raise errors.Setup.BadPreprovisionedDataError( 'Package %s does not define a PREPROVISIONED_DATA dict with ' - 'preprovisioned data.' % package_name) + 'preprovisioned data.' % package_name + ) fallback_url = getattr(package_module, 'PACKAGE_DATA_URL', {}) - self._InstallData(preprovisioned_data, package_name, filenames, - install_path, fallback_url) + self._InstallData( + preprovisioned_data, package_name, filenames, install_path, fallback_url + ) def ShouldDownloadPreprovisionedData(self, module_name, filename): """Returns whether or not preprovisioned data is available. @@ -1619,9 +1722,9 @@ def UpdateInterruptibleVmStatus(self, use_api=False): Args: use_api: boolean, If use_api is false, method will attempt to query - metadata service to check vm preemption. If use_api is true, method will - attempt to use API to detect vm preemption query if metadata service - detecting fails. + metadata service to check vm preemption. If use_api is true, method will + attempt to use API to detect vm preemption query if metadata service + detecting fails. """ if not self.IsInterruptible(): return diff --git a/perfkitbenchmarker/vpn_service.py b/perfkitbenchmarker/vpn_service.py index 8d80797dff..339440c4ae 100644 --- a/perfkitbenchmarker/vpn_service.py +++ b/perfkitbenchmarker/vpn_service.py @@ -20,31 +20,38 @@ module includes the BaseVpnGateway class to manage VPN gateway endpoints. """ +from enum import Enum import itertools import json import logging import re import threading -import uuid -from enum import Enum - from typing import List, Tuple +import uuid +from absl import flags from perfkitbenchmarker import background_tasks from perfkitbenchmarker import context from perfkitbenchmarker import errors from perfkitbenchmarker import network -from absl import flags from perfkitbenchmarker import resource from perfkitbenchmarker import vm_util -flags.DEFINE_integer('vpn_service_tunnel_count', None, - 'Number of tunnels to create for each VPN Gateway pair.') -flags.DEFINE_integer('vpn_service_gateway_count', None, - 'Number of VPN Gateways to create for each vm_group.') -flags.DEFINE_string('vpn_service_name', None, - 'If set, use this name for VPN Service.') -flags.DEFINE_string('vpn_service_shared_key', None, - 'If set, use this PSK for VPNs.') +flags.DEFINE_integer( + 'vpn_service_tunnel_count', + None, + 'Number of tunnels to create for each VPN Gateway pair.', +) +flags.DEFINE_integer( + 'vpn_service_gateway_count', + None, + 'Number of VPN Gateways to create for each vm_group.', +) +flags.DEFINE_string( + 'vpn_service_name', None, 'If set, use this name for VPN Service.' +) +flags.DEFINE_string( + 'vpn_service_shared_key', None, 'If set, use this PSK for VPNs.' +) flags.DEFINE_integer('vpn_service_ike_version', None, 'IKE version') @@ -52,10 +59,13 @@ class VPN_ROUTING_TYPE(Enum): STATIC = 'static' DYNAMIC = 'dynamic' + flags.DEFINE_enum( - 'vpn_service_routing_type', None, + 'vpn_service_routing_type', + None, [VPN_ROUTING_TYPE.STATIC.value, VPN_ROUTING_TYPE.DYNAMIC.value], - 'static or dynamic(BGP)') + 'static or dynamic(BGP)', +) FLAGS = flags.FLAGS @@ -79,8 +89,9 @@ class VPN(object): def __init__(self, *args, **kwargs): # set by Create - self.gateway_pair: Tuple[network.BaseVpnGateway, - network.BaseVpnGateway] = None + self.gateway_pair: Tuple[network.BaseVpnGateway, network.BaseVpnGateway] = ( + None + ) self.tunnel_config: TunnelConfig = None return object.__init__(self, *args, **kwargs) @@ -93,10 +104,13 @@ def getKeyFromGatewayPair(self, gateway_pair, suffix=''): Returns: string. The VPN key. - """ - key = 'vpn' + ''.join( - gateway for gateway in gateway_pair) + suffix + FLAGS.run_uri + key = ( + 'vpn' + + ''.join(gateway for gateway in gateway_pair) + + suffix + + FLAGS.run_uri + ) return key def Create(self, gateway_pair, suffix=''): @@ -111,13 +125,13 @@ def GetVPN(self, gateway_pair, suffix=''): """Gets a VPN object for the gateway_pair or creates one if none exists. Args: + gateway_pair: a tuple of two VpnGateways """ benchmark_spec = context.GetThreadBenchmarkSpec() if benchmark_spec is None: - raise errors.Error('GetVPN called in a thread without a ' - 'BenchmarkSpec.') + raise errors.Error('GetVPN called in a thread without a BenchmarkSpec.') with benchmark_spec.vpns_lock: key = self.getKeyFromGatewayPair(gateway_pair, suffix) if key not in benchmark_spec.vpns: @@ -132,8 +146,10 @@ def ConfigureTunnel(self): vpn_gateway_0 = benchmark_spec.vpn_gateways[self.gateway_pair[0]] vpn_gateway_1 = benchmark_spec.vpn_gateways[self.gateway_pair[1]] - both_gateways_are_passive = (vpn_gateway_0.require_target_to_init - and vpn_gateway_1.require_target_to_init) + both_gateways_are_passive = ( + vpn_gateway_0.require_target_to_init + and vpn_gateway_1.require_target_to_init + ) assert not both_gateways_are_passive, 'Cant connect 2 passive VPN Gateways' tunnel_config_hash = None @@ -161,9 +177,12 @@ def isTunnelConfigured(self): """ is_tunnel_configured = False if len(self.tunnel_config.endpoints) == 2: - if (self.tunnel_config.endpoints[self.gateway_pair[0]]['is_configured'] - and - self.tunnel_config.endpoints[self.gateway_pair[1]]['is_configured']): + if ( + self.tunnel_config.endpoints[self.gateway_pair[0]]['is_configured'] + and self.tunnel_config.endpoints[self.gateway_pair[1]][ + 'is_configured' + ] + ): logging.debug('Tunnel is configured.') is_tunnel_configured = True return is_tunnel_configured @@ -177,11 +196,11 @@ def isTunnelReady(self): """ benchmark_spec = context.GetThreadBenchmarkSpec() logging.debug('Tunnel endpoints configured. Waiting for tunnel...') - ready = ( - benchmark_spec.vpn_gateways[self.gateway_pair[0]].IsTunnelReady( - self.tunnel_config.endpoints[self.gateway_pair[0]]['tunnel_id']) and - benchmark_spec.vpn_gateways[self.gateway_pair[1]].IsTunnelReady( - self.tunnel_config.endpoints[self.gateway_pair[1]]['tunnel_id'])) + ready = benchmark_spec.vpn_gateways[self.gateway_pair[0]].IsTunnelReady( + self.tunnel_config.endpoints[self.gateway_pair[0]]['tunnel_id'] + ) and benchmark_spec.vpn_gateways[self.gateway_pair[1]].IsTunnelReady( + self.tunnel_config.endpoints[self.gateway_pair[1]]['tunnel_id'] + ) if not ready: raise errors.Resource.RetryableCreationError() @@ -207,7 +226,6 @@ class TunnelConfig(object): } } - """ _tunnelconfig_lock = threading.Lock() @@ -234,7 +252,6 @@ def hash(self): Returns: int: An integer that changes if any properties have changes. - """ return hash(json.dumps(self.__dict__, sort_keys=True, default=str)) @@ -292,11 +309,13 @@ def _Create(self): benchmark_spec = context.GetThreadBenchmarkSpec() if benchmark_spec is None: - raise errors.Error('CreateVPN Service. called in a thread without a ' - 'BenchmarkSpec.') + raise errors.Error( + 'CreateVPN Service. called in a thread without a BenchmarkSpec.' + ) self.vpn_gateway_pairs = self.GetVpnGatewayPairs( - benchmark_spec.vpn_gateways) + benchmark_spec.vpn_gateways + ) for gateway_pair in self.vpn_gateway_pairs: # creates the vpn if it doesn't exist and registers in bm_spec.vpns @@ -349,14 +368,15 @@ def GetVpnGatewayPairs( Returns: list. The list of tuples of gateway pairs to create VPNs for. - """ vpn_gateway_pairs = itertools.combinations(vpn_gateways, 2) r = re.compile( - r'(?P.*-.*-.*)?-(?P[0-9])-(?P.*)') + r'(?P.*-.*-.*)?-(?P[0-9])-(?P.*)' + ) def filterGateways(gateway_pair): return r.search(gateway_pair[0]).group('gateway_prefix') != r.search( - gateway_pair[1]).group('gateway_prefix') + gateway_pair[1] + ).group('gateway_prefix') return list(filter(filterGateways, vpn_gateway_pairs)) diff --git a/perfkitbenchmarker/windows_benchmarks/__init__.py b/perfkitbenchmarker/windows_benchmarks/__init__.py index c7dc000aa4..c1e30300b0 100644 --- a/perfkitbenchmarker/windows_benchmarks/__init__.py +++ b/perfkitbenchmarker/windows_benchmarks/__init__.py @@ -29,6 +29,8 @@ def _LoadBenchmarks(): VALID_BENCHMARKS = {} for module in BENCHMARKS: if module.BENCHMARK_NAME in VALID_BENCHMARKS: - raise ValueError('There are multiple benchmarks with BENCHMARK_NAME "%s"' % - (module.BENCHMARK_NAME)) + raise ValueError( + 'There are multiple benchmarks with BENCHMARK_NAME "%s"' + % (module.BENCHMARK_NAME) + ) VALID_BENCHMARKS[module.BENCHMARK_NAME] = module diff --git a/perfkitbenchmarker/windows_benchmarks/cluster_boot_benchmark.py b/perfkitbenchmarker/windows_benchmarks/cluster_boot_benchmark.py index f0b46b074c..adeb5014bf 100644 --- a/perfkitbenchmarker/windows_benchmarks/cluster_boot_benchmark.py +++ b/perfkitbenchmarker/windows_benchmarks/cluster_boot_benchmark.py @@ -23,8 +23,10 @@ from perfkitbenchmarker.linux_benchmarks import cluster_boot_benchmark flags.DEFINE_boolean( - 'cluster_boot_test_rdp_port_listening', False, - 'Test the time it takes to successfully connect to the RDP port.') + 'cluster_boot_test_rdp_port_listening', + False, + 'Test the time it takes to successfully connect to the RDP port.', +) FLAGS = flags.FLAGS BENCHMARK_NAME = cluster_boot_benchmark.BENCHMARK_NAME diff --git a/perfkitbenchmarker/windows_benchmarks/diskspd_benchmark.py b/perfkitbenchmarker/windows_benchmarks/diskspd_benchmark.py index c6e67cdfac..91ae3f478b 100644 --- a/perfkitbenchmarker/windows_benchmarks/diskspd_benchmark.py +++ b/perfkitbenchmarker/windows_benchmarks/diskspd_benchmark.py @@ -46,7 +46,7 @@ def Run(benchmark_spec): Args: benchmark_spec: The benchmark specification. Contains all data that is - required to run the benchmark. + required to run the benchmark. Returns: A list of sample.Sample objects with the benchmark results. diff --git a/perfkitbenchmarker/windows_benchmarks/fio_benchmark.py b/perfkitbenchmarker/windows_benchmarks/fio_benchmark.py index 76b57adf91..52f63c12e7 100644 --- a/perfkitbenchmarker/windows_benchmarks/fio_benchmark.py +++ b/perfkitbenchmarker/windows_benchmarks/fio_benchmark.py @@ -30,8 +30,9 @@ FLAGS = flags.FLAGS -flags.DEFINE_integer('fio_file_size', 10, - '"size" field of the fio config in GB.') +flags.DEFINE_integer( + 'fio_file_size', 10, '"size" field of the fio config in GB.' +) BENCHMARK_NAME = 'fio' BENCHMARK_CONFIG = """ @@ -140,12 +141,12 @@ def Run(benchmark_spec): # Ignored when --fio_generate_scenarios is used. job_file_contents = DEFAULT_JOB.format( - filesize=FLAGS.fio_file_size, - runtime=FLAGS.fio_runtime + filesize=FLAGS.fio_file_size, runtime=FLAGS.fio_runtime ) - return linux_fio.RunWithExec(vm, fio_exec, remote_job_file_path, - job_file_contents) + return linux_fio.RunWithExec( + vm, fio_exec, remote_job_file_path, job_file_contents + ) def Cleanup(unused_benchmark_spec): diff --git a/perfkitbenchmarker/windows_benchmarks/hammerdbcli_benchmark.py b/perfkitbenchmarker/windows_benchmarks/hammerdbcli_benchmark.py index 880c176055..87a5f6d665 100644 --- a/perfkitbenchmarker/windows_benchmarks/hammerdbcli_benchmark.py +++ b/perfkitbenchmarker/windows_benchmarks/hammerdbcli_benchmark.py @@ -22,7 +22,6 @@ from perfkitbenchmarker import configs from perfkitbenchmarker import errors from perfkitbenchmarker import sql_engine_utils - from perfkitbenchmarker.windows_packages import hammerdb diff --git a/perfkitbenchmarker/windows_benchmarks/ntttcp_benchmark.py b/perfkitbenchmarker/windows_benchmarks/ntttcp_benchmark.py index 7fd87ea906..63c5ec061a 100644 --- a/perfkitbenchmarker/windows_benchmarks/ntttcp_benchmark.py +++ b/perfkitbenchmarker/windows_benchmarks/ntttcp_benchmark.py @@ -21,7 +21,6 @@ from perfkitbenchmarker import configs from perfkitbenchmarker import publisher from perfkitbenchmarker import vm_util - from perfkitbenchmarker.windows_packages import ntttcp @@ -60,18 +59,28 @@ def Prepare(benchmark_spec): vm.AllowPort(ntttcp.BASE_DATA_PORT, ntttcp.BASE_DATA_PORT + num_ports) -def _RunTest(benchmark_spec, sender, receiver, dest_ip, ip_type, conf, - cooldown_s): +def _RunTest( + benchmark_spec, sender, receiver, dest_ip, ip_type, conf, cooldown_s +): """Run a single NTTTCP test, and publish the results.""" try: - results = ntttcp.RunNtttcp(sender, receiver, dest_ip, ip_type, conf.udp, - conf.threads, conf.time_s, conf.packet_size, - cooldown_s) + results = ntttcp.RunNtttcp( + sender, + receiver, + dest_ip, + ip_type, + conf.udp, + conf.threads, + conf.time_s, + conf.packet_size, + cooldown_s, + ) publisher.PublishRunStageSamples(benchmark_spec, results) return True except IOError: - logging.info('Failed to publish %s IP results for config %s', ip_type, - str(conf)) + logging.info( + 'Failed to publish %s IP results for config %s', ip_type, str(conf) + ) return False @@ -80,14 +89,16 @@ def Run(benchmark_spec): Args: benchmark_spec: The benchmark specification. Contains all data that is - required to run the benchmark. + required to run the benchmark. Returns: A list of sample.Sample objects with the benchmark results. """ - vm_sets = [(benchmark_spec.vms[0], benchmark_spec.vms[1]), - (benchmark_spec.vms[1], benchmark_spec.vms[0])] + vm_sets = [ + (benchmark_spec.vms[0], benchmark_spec.vms[1]), + (benchmark_spec.vms[1], benchmark_spec.vms[0]), + ] parsed_configs = ntttcp.ParseConfigList() @@ -95,23 +106,37 @@ def Run(benchmark_spec): failed_confs = [] # Send traffic in both directions - for ((sender, receiver), conf) in itertools.product(vm_sets, parsed_configs): + for (sender, receiver), conf in itertools.product(vm_sets, parsed_configs): # Send using external IP addresses if vm_util.ShouldRunOnExternalIpAddress(conf.ip_type): - if not _RunTest(benchmark_spec, sender, receiver, receiver.ip_address, - 'external', conf, True): + if not _RunTest( + benchmark_spec, + sender, + receiver, + receiver.ip_address, + 'external', + conf, + True, + ): failed_confs.append(('external', conf)) # Send using internal IP addresses if vm_util.ShouldRunOnInternalIpAddress(sender, receiver, conf.ip_type): - if not _RunTest(benchmark_spec, sender, receiver, receiver.internal_ip, - 'internal', conf, - len(parsed_configs) > 1): + if not _RunTest( + benchmark_spec, + sender, + receiver, + receiver.internal_ip, + 'internal', + conf, + len(parsed_configs) > 1, + ): failed_confs.append(('internal', conf)) if failed_confs: - logging.info('Failed to run test and/or gather results for %s', - str(failed_confs)) + logging.info( + 'Failed to run test and/or gather results for %s', str(failed_confs) + ) return [] diff --git a/perfkitbenchmarker/windows_benchmarks/nuttcp_benchmark.py b/perfkitbenchmarker/windows_benchmarks/nuttcp_benchmark.py index 2c936c1ad1..196f62a6ec 100644 --- a/perfkitbenchmarker/windows_benchmarks/nuttcp_benchmark.py +++ b/perfkitbenchmarker/windows_benchmarks/nuttcp_benchmark.py @@ -17,7 +17,6 @@ from absl import flags from perfkitbenchmarker import configs from perfkitbenchmarker import vm_util - from perfkitbenchmarker.windows_packages import nuttcp from six.moves import range @@ -62,12 +61,26 @@ def RunNuttcp(vms, exec_path): def _RunNuttcpTest(sending_vm, receiving_vm, iteration): if vm_util.ShouldRunOnExternalIpAddress(): results.extend( - nuttcp.RunNuttcp(sending_vm, receiving_vm, exec_path, - receiving_vm.ip_address, 'external', iteration)) + nuttcp.RunNuttcp( + sending_vm, + receiving_vm, + exec_path, + receiving_vm.ip_address, + 'external', + iteration, + ) + ) if vm_util.ShouldRunOnInternalIpAddress(sending_vm, receiving_vm): results.extend( - nuttcp.RunNuttcp(sending_vm, receiving_vm, exec_path, - receiving_vm.internal_ip, 'internal', iteration)) + nuttcp.RunNuttcp( + sending_vm, + receiving_vm, + exec_path, + receiving_vm.internal_ip, + 'internal', + iteration, + ) + ) # run in both directions just for completeness for iteration in range(FLAGS.nuttcp_udp_iterations): diff --git a/perfkitbenchmarker/windows_benchmarks/psping_benchmark.py b/perfkitbenchmarker/windows_benchmarks/psping_benchmark.py index 530f59aa3d..1b4d5e35a9 100644 --- a/perfkitbenchmarker/windows_benchmarks/psping_benchmark.py +++ b/perfkitbenchmarker/windows_benchmarks/psping_benchmark.py @@ -17,7 +17,6 @@ from absl import flags from perfkitbenchmarker import configs from perfkitbenchmarker import vm_util - from perfkitbenchmarker.windows_packages import psping FLAGS = flags.FLAGS @@ -60,14 +59,14 @@ def Run(benchmark_spec): def _RunTest(sending_vm, receiving_vm): if vm_util.ShouldRunOnExternalIpAddress(): - results.extend(psping.RunLatencyTest(sending_vm, - receiving_vm, - use_internal_ip=False)) + results.extend( + psping.RunLatencyTest(sending_vm, receiving_vm, use_internal_ip=False) + ) if vm_util.ShouldRunOnInternalIpAddress(sending_vm, receiving_vm): - results.extend(psping.RunLatencyTest(sending_vm, - receiving_vm, - use_internal_ip=True)) + results.extend( + psping.RunLatencyTest(sending_vm, receiving_vm, use_internal_ip=True) + ) _RunTest(vms[0], vms[1]) _RunTest(vms[1], vms[0]) diff --git a/perfkitbenchmarker/windows_packages/__init__.py b/perfkitbenchmarker/windows_packages/__init__.py index 70c060653a..4f80ec4d67 100644 --- a/perfkitbenchmarker/windows_packages/__init__.py +++ b/perfkitbenchmarker/windows_packages/__init__.py @@ -30,8 +30,10 @@ def _LoadPackages(): This imports all package modules in this directory and then creates a mapping from module names to the modules themselves and returns it. """ - return {module.__name__.split('.')[-1]: module for module in - import_util.LoadModulesForPath(__path__, __name__)} + return { + module.__name__.split('.')[-1]: module + for module in import_util.LoadModulesForPath(__path__, __name__) + } PACKAGES = _LoadPackages() diff --git a/perfkitbenchmarker/windows_packages/diskspd.py b/perfkitbenchmarker/windows_packages/diskspd.py index 881a47878c..2e917ad41e 100644 --- a/perfkitbenchmarker/windows_packages/diskspd.py +++ b/perfkitbenchmarker/windows_packages/diskspd.py @@ -33,98 +33,144 @@ FLAGS = flags.FLAGS -flags.DEFINE_integer('diskspd_duration', 20, - 'The number of seconds to run diskspd test.' - 'Defaults to 30s. Unit: seconds.') - -flags.DEFINE_integer('diskspd_warmup', 5, - 'The warm up time for diskspd, the time needed to enter' - 'steady state of I/O operation. ' - 'Defaults to 5s. Unit: seconds.') - -flags.DEFINE_integer('diskspd_cooldown', 5, - 'The cool down time for diskspd, the time to ensure that' - 'each instance of diskspd is active during each' - 'measurement period of each instance. ' - 'Defaults: 5s. Unit: seconds') - -flags.DEFINE_integer('diskspd_thread_number_per_file', 1, - 'The thread number created per file to' - 'perform read and write. ' - 'Defaults: 1.') - -flags.DEFINE_enum('diskspd_access_pattern', 's', ['s', 'r'], - 'the access patten of the read and write' - 'the performance will be downgrade a little bit if use' - 'different hints' - 'available option: r|s, ' - 'r: random access' - 's: sequential access. ' - 'Defaults: s.') - -flags.DEFINE_integer('diskspd_write_read_ratio', 0, - 'The ratio of write workload to read workload.' - 'Example: 50 means 50%, and write and read each takes' - '50% of the total I/O data.' - 'To test read speed, set this value to 0. ' - 'To test write speed, set this value to 100. ' - 'Defaults: 0. Unit: percent.') - -flags.DEFINE_integer('diskspd_block_size', 64, - 'The block size used when reading and writing data. ' - 'Defaults: 64K. Unit: KB, ' - 'can be set via --diskspd_block_unit') - -flags.DEFINE_enum('diskspd_block_unit', 'K', ['K', 'M', 'G'], - 'The unit of the block size, available option: K|M|G. ' - 'Will be used as the unit for --diskspd_block_size ' - 'Defaults: K.') - -flags.DEFINE_integer('diskspd_stride_or_alignment', 64, - 'If the access pattern is sequential, then this value' - 'means the stride for the access' - 'If the access pattern is random, then this value means' - 'the specified number of bytes that random I/O aligns to.' - 'Defaults: 64K. Unit: KB, can be set') - -flags.DEFINE_enum('diskspd_stride_or_alignment_unit', 'K', ['K', 'M', 'G', 'b'], - 'The unit of the stride_or_alignment,' - 'available option: K|M|G|b' - 'Defaults: K.') - -flags.DEFINE_bool('diskspd_large_page', False, - 'Whether use large page for IO buffers. ' - 'Defaults: False') - -flags.DEFINE_bool('diskspd_latency_stats', False, - 'Whether measure the latency statistics' - 'Defaults: False') - -flags.DEFINE_bool('diskspd_disable_affinity', False, - 'Whether to diable the group affinity,' - 'group affinity is to round robin tasks. ' - 'across processor group. ' - 'Defaults: False') - -flags.DEFINE_bool('diskspd_write_through', True, - 'Whether to enable write through IO. ' - 'Defaults: True') - -flags.DEFINE_bool('diskspd_software_cache', True, - 'Whether to disable software caching' - 'Defaults: True') - -flags.DEFINE_integer('diskspd_outstanding_io', '2', - 'The number of outstanding I/O per thread per target.' - 'Defaults: 2.') - -flags.DEFINE_integer('diskspd_throughput_per_ms', None, - 'The throughput per thread per target. ' - 'Defaults: None. Unit: bytes per ms.') - -flags.DEFINE_integer('diskspd_file_size', 819200, - 'The file size DiskSpd will create when testing. ' - 'Defaults: 819200. Unit: KB.') +flags.DEFINE_integer( + 'diskspd_duration', + 20, + 'The number of seconds to run diskspd test.Defaults to 30s. Unit: seconds.', +) + +flags.DEFINE_integer( + 'diskspd_warmup', + 5, + 'The warm up time for diskspd, the time needed to enter' + 'steady state of I/O operation. ' + 'Defaults to 5s. Unit: seconds.', +) + +flags.DEFINE_integer( + 'diskspd_cooldown', + 5, + 'The cool down time for diskspd, the time to ensure that' + 'each instance of diskspd is active during each' + 'measurement period of each instance. ' + 'Defaults: 5s. Unit: seconds', +) + +flags.DEFINE_integer( + 'diskspd_thread_number_per_file', + 1, + 'The thread number created per file toperform read and write. Defaults: 1.', +) + +flags.DEFINE_enum( + 'diskspd_access_pattern', + 's', + ['s', 'r'], + 'the access patten of the read and write' + 'the performance will be downgrade a little bit if use' + 'different hints' + 'available option: r|s, ' + 'r: random access' + 's: sequential access. ' + 'Defaults: s.', +) + +flags.DEFINE_integer( + 'diskspd_write_read_ratio', + 0, + 'The ratio of write workload to read workload.' + 'Example: 50 means 50%, and write and read each takes' + '50% of the total I/O data.' + 'To test read speed, set this value to 0. ' + 'To test write speed, set this value to 100. ' + 'Defaults: 0. Unit: percent.', +) + +flags.DEFINE_integer( + 'diskspd_block_size', + 64, + 'The block size used when reading and writing data. ' + 'Defaults: 64K. Unit: KB, ' + 'can be set via --diskspd_block_unit', +) + +flags.DEFINE_enum( + 'diskspd_block_unit', + 'K', + ['K', 'M', 'G'], + 'The unit of the block size, available option: K|M|G. ' + 'Will be used as the unit for --diskspd_block_size ' + 'Defaults: K.', +) + +flags.DEFINE_integer( + 'diskspd_stride_or_alignment', + 64, + 'If the access pattern is sequential, then this value' + 'means the stride for the access' + 'If the access pattern is random, then this value means' + 'the specified number of bytes that random I/O aligns to.' + 'Defaults: 64K. Unit: KB, can be set', +) + +flags.DEFINE_enum( + 'diskspd_stride_or_alignment_unit', + 'K', + ['K', 'M', 'G', 'b'], + 'The unit of the stride_or_alignment,available option: K|M|G|bDefaults: K.', +) + +flags.DEFINE_bool( + 'diskspd_large_page', + False, + 'Whether use large page for IO buffers. Defaults: False', +) + +flags.DEFINE_bool( + 'diskspd_latency_stats', + False, + 'Whether measure the latency statisticsDefaults: False', +) + +flags.DEFINE_bool( + 'diskspd_disable_affinity', + False, + 'Whether to diable the group affinity,' + 'group affinity is to round robin tasks. ' + 'across processor group. ' + 'Defaults: False', +) + +flags.DEFINE_bool( + 'diskspd_write_through', + True, + 'Whether to enable write through IO. Defaults: True', +) + +flags.DEFINE_bool( + 'diskspd_software_cache', + True, + 'Whether to disable software cachingDefaults: True', +) + +flags.DEFINE_integer( + 'diskspd_outstanding_io', + '2', + 'The number of outstanding I/O per thread per target.Defaults: 2.', +) + +flags.DEFINE_integer( + 'diskspd_throughput_per_ms', + None, + 'The throughput per thread per target. Defaults: None. Unit: bytes per ms.', +) + +flags.DEFINE_integer( + 'diskspd_file_size', + 819200, + 'The file size DiskSpd will create when testing. ' + 'Defaults: 819200. Unit: KB.', +) flags.DEFINE_list( 'diskspd_config_list', @@ -134,13 +180,17 @@ 'for example FALSE:TRUE:64. ' 'Default Behavior: diskspd benchmark test will try to combine' '--diskspd_access_pattern, --diskspd_write_read_ratio, ' - '--diskspd_block_size together and form a set a config to run.') + '--diskspd_block_size together and form a set a config to run.', +) DISKSPD_RETRIES = 10 DISKSPD_DIR = 'DiskSpd-2.0.21a' DISKSPD_ZIP = DISKSPD_DIR + '.zip' -DISKSPD_URL = ('https://gallery.technet.microsoft.com/DiskSpd-A-Robust-Storage' - '-6ef84e62/file/199535/2/' + DISKSPD_ZIP) +DISKSPD_URL = ( + 'https://gallery.technet.microsoft.com/DiskSpd-A-Robust-Storage' + '-6ef84e62/file/199535/2/' + + DISKSPD_ZIP +) DISKSPD_TMPFILE = 'testfile.dat' DISKSPD_XMLFILE = 'result.xml' DISKSPD_TIMEOUT_MULTIPLIER = 3 @@ -152,9 +202,9 @@ _NUM_PARAMS_IN_CONFIG = 3 # named tuple used in passing configs around -DiskspdConf = collections.namedtuple('DiskspdConf', - ['access_pattern', 'write_ratio', - 'block_size']) +DiskspdConf = collections.namedtuple( + 'DiskspdConf', ['access_pattern', 'write_ratio', 'block_size'] +) def DiskspdConfigListValidator(value): @@ -183,8 +233,9 @@ def DiskspdConfigListValidator(value): return True -flags.register_validator('diskspd_config_list', DiskspdConfigListValidator, - 'malformed config list') +flags.register_validator( + 'diskspd_config_list', DiskspdConfigListValidator, 'malformed config list' +) def ParseConfigList(): @@ -196,7 +247,8 @@ def ParseConfigList(): DiskspdConf( access_pattern=FLAGS.diskspd_access_pattern, write_ratio=FLAGS.diskspd_write_read_ratio, - block_size=FLAGS.diskspd_block_size) + block_size=FLAGS.diskspd_block_size, + ) ] for config in FLAGS.diskspd_config_list: @@ -206,7 +258,9 @@ def ParseConfigList(): DiskspdConf( access_pattern='r' if (confs[0] in TRUE_VALS) else 's', write_ratio=0 if (confs[1] in TRUE_VALS) else 100, - block_size=int(confs[2]))) + block_size=int(confs[2]), + ) + ) return conf_list @@ -218,27 +272,31 @@ def Install(vm): def _RunDiskSpdWithOptions(vm, options): - total_runtime = FLAGS.diskspd_warmup + FLAGS.diskspd_cooldown + \ - FLAGS.diskspd_duration + total_runtime = ( + FLAGS.diskspd_warmup + FLAGS.diskspd_cooldown + FLAGS.diskspd_duration + ) timeout_duration = total_runtime * DISKSPD_TIMEOUT_MULTIPLIER diskspd_exe_dir = ntpath.join(vm.temp_dir, 'x86') command = 'cd {diskspd_exe_dir}; .\\diskspd.exe {diskspd_options}'.format( - diskspd_exe_dir=diskspd_exe_dir, diskspd_options=options) + diskspd_exe_dir=diskspd_exe_dir, diskspd_options=options + ) vm.RobustRemoteCommand(command, timeout=timeout_duration) def _RemoveXml(vm): diskspd_exe_dir = ntpath.join(vm.temp_dir, 'x86') rm_command = 'cd {diskspd_exe_dir}; rm xml.txt'.format( - diskspd_exe_dir=diskspd_exe_dir) + diskspd_exe_dir=diskspd_exe_dir + ) vm.RemoteCommand(rm_command, ignore_failure=True) def _CatXml(vm): diskspd_exe_dir = ntpath.join(vm.temp_dir, 'x86') cat_command = 'cd {diskspd_exe_dir}; cat {result_xml}'.format( - diskspd_exe_dir=diskspd_exe_dir, result_xml=DISKSPD_XMLFILE) + diskspd_exe_dir=diskspd_exe_dir, result_xml=DISKSPD_XMLFILE + ) diskspd_xml, _ = vm.RemoteCommand(cat_command) return diskspd_xml @@ -246,16 +304,18 @@ def _CatXml(vm): def _RemoveTempFile(vm): diskspd_exe_dir = ntpath.join(vm.temp_dir, 'x86') rm_command = 'cd {diskspd_exe_dir}; rm .\\{tmp_file_name}'.format( - diskspd_exe_dir=diskspd_exe_dir, tmp_file_name=DISKSPD_TMPFILE) + diskspd_exe_dir=diskspd_exe_dir, tmp_file_name=DISKSPD_TMPFILE + ) vm.RemoteCommand(rm_command, ignore_failure=True) -def _RunDiskSpd(running_vm, access_pattern, - diskspd_write_read_ratio, block_size, metadata): +def _RunDiskSpd( + running_vm, access_pattern, diskspd_write_read_ratio, block_size, metadata +): """Run single iteration of Diskspd test.""" - sending_options = _GenerateOption(access_pattern, - diskspd_write_read_ratio, - block_size) + sending_options = _GenerateOption( + access_pattern, diskspd_write_read_ratio, block_size + ) process_args = [(_RunDiskSpdWithOptions, (running_vm, sending_options), {})] background_tasks.RunParallelProcesses(process_args, 200) result_xml = _CatXml(running_vm) @@ -284,38 +344,43 @@ def _GenerateOption(access_pattern, diskspd_write_read_ratio, block_size): software_cache_string = '-Su' if FLAGS.diskspd_software_cache else '' write_through_string = '-Sw' if FLAGS.diskspd_write_through else '' block_size_string = str(block_size) + str(FLAGS.diskspd_block_unit) - access_pattern_string = str(access_pattern) + \ - str(FLAGS.diskspd_stride_or_alignment) + \ - str(FLAGS.diskspd_stride_or_alignment_unit) + access_pattern_string = ( + str(access_pattern) + + str(FLAGS.diskspd_stride_or_alignment) + + str(FLAGS.diskspd_stride_or_alignment_unit) + ) throughput_per_ms_string = '' if FLAGS.diskspd_throughput_per_ms: throughput_per_ms_string = '-g' + str(FLAGS.diskspd_throughput_per_ms) - sending_options = ('-c{filesize}K -d{duration} -t{threadcount} ' - '-W{warmup} -C{cooldown} -Rxml -w{ratio} ' - '{large_page} {latency_stats} {disable_affinity} ' - '{software_cache} {write_through} {throughput}' - '-b{block_size} -f{hint_string} -{access_pattern} ' - '-o{outstanding_io} -L ' - 'C:\\scratch\\{tempfile} > {xmlfile}').format( - filesize=FLAGS.diskspd_file_size, - duration=FLAGS.diskspd_duration, - threadcount=FLAGS.diskspd_thread_number_per_file, - warmup=FLAGS.diskspd_warmup, - cooldown=FLAGS.diskspd_cooldown, - ratio=diskspd_write_read_ratio, - tempfile=DISKSPD_TMPFILE, - xmlfile=DISKSPD_XMLFILE, - large_page=large_page_string, - latency_stats=latency_stats_string, - disable_affinity=disable_affinity_string, - software_cache=software_cache_string, - write_through=write_through_string, - access_pattern=access_pattern_string, - block_size=block_size_string, - hint_string=access_pattern, - throughput=throughput_per_ms_string, - outstanding_io=FLAGS.diskspd_outstanding_io) + sending_options = ( + '-c{filesize}K -d{duration} -t{threadcount} ' + '-W{warmup} -C{cooldown} -Rxml -w{ratio} ' + '{large_page} {latency_stats} {disable_affinity} ' + '{software_cache} {write_through} {throughput}' + '-b{block_size} -f{hint_string} -{access_pattern} ' + '-o{outstanding_io} -L ' + 'C:\\scratch\\{tempfile} > {xmlfile}' + ).format( + filesize=FLAGS.diskspd_file_size, + duration=FLAGS.diskspd_duration, + threadcount=FLAGS.diskspd_thread_number_per_file, + warmup=FLAGS.diskspd_warmup, + cooldown=FLAGS.diskspd_cooldown, + ratio=diskspd_write_read_ratio, + tempfile=DISKSPD_TMPFILE, + xmlfile=DISKSPD_XMLFILE, + large_page=large_page_string, + latency_stats=latency_stats_string, + disable_affinity=disable_affinity_string, + software_cache=software_cache_string, + write_through=write_through_string, + access_pattern=access_pattern_string, + block_size=block_size_string, + hint_string=access_pattern, + throughput=throughput_per_ms_string, + outstanding_io=FLAGS.diskspd_outstanding_io, + ) return sending_options @@ -331,7 +396,9 @@ def RunDiskSpd(running_vm): # some of the flags information has been included in the xml file metadata['diskspd_block_size_unit'] = FLAGS.diskspd_block_unit metadata['diskspd_stride_or_alignment'] = FLAGS.diskspd_stride_or_alignment - metadata['diskspd_stride_or_alignment_unit'] = FLAGS.diskspd_stride_or_alignment_unit + metadata['diskspd_stride_or_alignment_unit'] = ( + FLAGS.diskspd_stride_or_alignment_unit + ) metadata['diskspd_large_page'] = FLAGS.diskspd_large_page metadata['diskspd_latency_stats'] = FLAGS.diskspd_latency_stats metadata['diskspd_disable_affinity'] = FLAGS.diskspd_disable_affinity @@ -345,9 +412,15 @@ def RunDiskSpd(running_vm): # run diskspd in four different scenario, will generate a metadata list for conf in conf_list: - sample_list.append(_RunDiskSpd(running_vm, conf.access_pattern, - conf.write_ratio, conf.block_size, - metadata)) + sample_list.append( + _RunDiskSpd( + running_vm, + conf.access_pattern, + conf.write_ratio, + conf.block_size, + metadata, + ) + ) return sample_list @@ -436,5 +509,4 @@ def ParseDiskSpdResults(result_xml, metadata, main_metric): metadata['TotalSpeed'] = total_speed metadata['TotalIops'] = total_iops - return sample.Sample(main_metric, metadata[main_metric], 'MB/s', - metadata) + return sample.Sample(main_metric, metadata[main_metric], 'MB/s', metadata) diff --git a/perfkitbenchmarker/windows_packages/fio.py b/perfkitbenchmarker/windows_packages/fio.py index 65ee7bf24d..e502db811b 100644 --- a/perfkitbenchmarker/windows_packages/fio.py +++ b/perfkitbenchmarker/windows_packages/fio.py @@ -22,9 +22,9 @@ def GetFioExec(vm): - return ntpath.join(vm.temp_dir, - '{fio_dir}\\fio.exe --thread'.format( - fio_dir=WINDOWS_FIO_DIR)) + return ntpath.join( + vm.temp_dir, '{fio_dir}\\fio.exe --thread'.format(fio_dir=WINDOWS_FIO_DIR) + ) def GetRemoteJobFilePath(vm): diff --git a/perfkitbenchmarker/windows_packages/hammerdb.py b/perfkitbenchmarker/windows_packages/hammerdb.py index 4b30e4e8f0..325634ccec 100644 --- a/perfkitbenchmarker/windows_packages/hammerdb.py +++ b/perfkitbenchmarker/windows_packages/hammerdb.py @@ -33,7 +33,10 @@ HAMMERDB = 'HammerDB-{0}' HAMMERDB_DIR = HAMMERDB + '-Win' HAMMERDB_ZIP = HAMMERDB_DIR + '.zip' -HAMMERDB_URL = 'https://github.com/TPC-Council/HammerDB/releases/download/v{0}/' + HAMMERDB_ZIP +HAMMERDB_URL = ( + 'https://github.com/TPC-Council/HammerDB/releases/download/v{0}/' + + HAMMERDB_ZIP +) # import linux flags HAMMERDB_SCRIPT = linux_hammerdb.HAMMERDB_SCRIPT @@ -54,13 +57,15 @@ def Install(self, vm, tcl_script_parameters: Any): PushTestFile(vm, self.tcl_script_name, self.path) for parameter in self.needed_parameters: - tcl_script_parameters.SearchAndReplaceInScript(vm, self.tcl_script_name, - parameter) + tcl_script_parameters.SearchAndReplaceInScript( + vm, self.tcl_script_name, parameter + ) - def Run(self, vm, timeout: Optional[int] = 60*60*6) -> str: + def Run(self, vm, timeout: Optional[int] = 60 * 60 * 6) -> str: """Run hammerdbcli script.""" hammerdb_exe_dir = ntpath.join( - vm.temp_dir, HAMMERDB.format(linux_hammerdb.HAMMERDB_VERSION.value)) + vm.temp_dir, HAMMERDB.format(linux_hammerdb.HAMMERDB_VERSION.value) + ) stdout, _ = vm.RobustRemoteCommand( f'cd {hammerdb_exe_dir} ; ' f'.\\hammerdbcli.bat auto {self.tcl_script_name}', @@ -75,9 +80,9 @@ class WindowsTclScriptParameters(linux_hammerdb.TclScriptParameters): """Handle of the parameters that may be needed by a TCL script.""" def SearchAndReplaceInScript(self, vm, script_name: str, parameter: str): - SearchAndReplaceTclScript(vm, parameter, - self.map_search_to_replace[parameter], - script_name) + SearchAndReplaceTclScript( + vm, parameter, self.map_search_to_replace[parameter], script_name + ) def _GetFileContent(vm, file_path: str) -> str: @@ -104,12 +109,14 @@ def ParseTpcCResults(stdout: str, vm) -> List[sample.Sample]: tpcc_metrics = linux_hammerdb.ParseBasicTpcCResults(stdout) if linux_hammerdb.HAMMERDB_TPCC_TIME_PROFILE.value: tpcc_results = _GetFileContent( - vm, ntpath.join(vm.temp_dir, '..', 'hdbxtprofile.log')) + vm, ntpath.join(vm.temp_dir, '..', 'hdbxtprofile.log') + ) tpcc_metrics += ParseTpcCTimeProfileResultsFromFile(tpcc_results) if linux_hammerdb.TPCC_LOG_TRANSACTIONS.value: tpcc_results = _GetFileContent( - vm, ntpath.join(vm.temp_dir, '..', 'hdbtcount.log')) + vm, ntpath.join(vm.temp_dir, '..', 'hdbtcount.log') + ) tpcc_metrics += ParseTpcCTPMResultsFromFile(tpcc_results) return tpcc_metrics @@ -121,11 +128,14 @@ def ParseTpcHResults(stdout: str) -> List[sample.Sample]: def SearchAndReplaceTclScript(vm, search: str, replace: str, script_name: str): hammerdb_exe_dir = ntpath.join( - vm.temp_dir, HAMMERDB.format(linux_hammerdb.HAMMERDB_VERSION.value)) - vm.RemoteCommand(f'cd {hammerdb_exe_dir} ; ' - f'(Get-Content {script_name}) ' - f'-replace "{search}", "{replace}" | ' - f'Set-Content {script_name} -encoding ASCII ; ') + vm.temp_dir, HAMMERDB.format(linux_hammerdb.HAMMERDB_VERSION.value) + ) + vm.RemoteCommand( + f'cd {hammerdb_exe_dir} ; ' + f'(Get-Content {script_name}) ' + f'-replace "{search}", "{replace}" | ' + f'Set-Content {script_name} -encoding ASCII ; ' + ) def Install(vm): @@ -133,37 +143,59 @@ def Install(vm): version = linux_hammerdb.HAMMERDB_VERSION.value if version not in [linux_hammerdb.HAMMERDB_4_5, linux_hammerdb.HAMMERDB_4_3]: raise errors.Setup.InvalidFlagConfigurationError( - f'Hammerdb version {version} is not supported on Windows. ') + f'Hammerdb version {version} is not supported on Windows. ' + ) vm.Install('mssql_tools') zip_path = ntpath.join(vm.temp_dir, HAMMERDB_ZIP.format(version)) vm.DownloadFile(HAMMERDB_URL.format(version), zip_path) vm.UnzipFile(zip_path, vm.temp_dir) -def SetupConfig(vm, db_engine: str, hammerdb_script: str, ip: str, port: int, - password: str, user: str, is_managed_azure: bool): +def SetupConfig( + vm, + db_engine: str, + hammerdb_script: str, + ip: str, + port: int, + password: str, + user: str, + is_managed_azure: bool, +): """Sets up the necessary scripts on the VM with the necessary parameters.""" db_engine = sql_engine_utils.GetDbEngineType(db_engine) if db_engine not in linux_hammerdb.SCRIPT_MAPPING: - raise ValueError('{0} is currently not supported for running ' - 'hammerdb benchmarks.'.format(db_engine)) + raise ValueError( + '{0} is currently not supported for running ' + 'hammerdb benchmarks.'.format(db_engine) + ) if hammerdb_script not in linux_hammerdb.SCRIPT_MAPPING[db_engine]: - raise ValueError('{0} is not a known hammerdb script.'.format( - hammerdb_script)) + raise ValueError( + '{0} is not a known hammerdb script.'.format(hammerdb_script) + ) linux_scripts = linux_hammerdb.SCRIPT_MAPPING[db_engine][hammerdb_script] windows_scripts = [ - WindowsHammerDbTclScript(script.tcl_script_name, script.needed_parameters, - script.path, script.script_type) + WindowsHammerDbTclScript( + script.tcl_script_name, + script.needed_parameters, + script.path, + script.script_type, + ) for script in linux_scripts ] for script in windows_scripts: script_parameters = WindowsTclScriptParameters( - ip, port, password, user, is_managed_azure, - hammerdb_script, script.script_type) + ip, + port, + password, + user, + is_managed_azure, + hammerdb_script, + script.script_type, + ) script.Install(vm, script_parameters) # Run all the build script or scripts before actual run phase @@ -186,12 +218,12 @@ def Run( TPCH gathers the latency of the 22 TPCH queries. Args: - vm: The virtual machine to run on that has - Install and SetupConfig already invoked on it. + vm: The virtual machine to run on that has Install and SetupConfig already + invoked on it. db_engine: The type of database that the script is running on hammerdb_script: An enumeration from HAMMERDB_SCRIPT indicating which - script to run. Must have been prior setup with - SetupConfig method on the vm to work. + script to run. Must have been prior setup with SetupConfig method on the + vm to work. timeout: Timeout when running hammerdbcli Returns: @@ -201,19 +233,25 @@ def Run( linux_scripts = linux_hammerdb.SCRIPT_MAPPING[db_engine][hammerdb_script] windows_scripts = [ - WindowsHammerDbTclScript(script.tcl_script_name, script.needed_parameters, - script.path, script.script_type) + WindowsHammerDbTclScript( + script.tcl_script_name, + script.needed_parameters, + script.path, + script.script_type, + ) for script in linux_scripts ] # Run the run phase script. script = [ - script for script in windows_scripts + script + for script in windows_scripts if script.script_type == linux_hammerdb.RUN_SCRIPT_TYPE ] if len(script) != 1: raise errors.Benchmarks.RunError( - f'1 run script expected but {len(script)} found. Exiting.') + f'1 run script expected but {len(script)} found. Exiting.' + ) stdout = script[0].Run(vm, timeout=timeout) if hammerdb_script == linux_hammerdb.HAMMERDB_SCRIPT_TPC_H: @@ -225,9 +263,12 @@ def Run( def PushTestFile(vm, data_file: str, path: str): vm.PushFile( data.ResourcePath(posixpath.join(path, data_file)), - ntpath.join(vm.temp_dir, - HAMMERDB.format(linux_hammerdb.HAMMERDB_VERSION.value), - data_file)) + ntpath.join( + vm.temp_dir, + HAMMERDB.format(linux_hammerdb.HAMMERDB_VERSION.value), + data_file, + ), + ) def GetMetadata(db_engine: str): diff --git a/perfkitbenchmarker/windows_packages/ntttcp.py b/perfkitbenchmarker/windows_packages/ntttcp.py index 098636ae07..cf1ef2259d 100644 --- a/perfkitbenchmarker/windows_packages/ntttcp.py +++ b/perfkitbenchmarker/windows_packages/ntttcp.py @@ -34,42 +34,61 @@ FLAGS = flags.FLAGS -flags.DEFINE_integer('ntttcp_threads', 1, - 'The number of client and server threads for NTttcp ' - 'to run with.') +flags.DEFINE_integer( + 'ntttcp_threads', + 1, + 'The number of client and server threads for NTttcp to run with.', +) -flags.DEFINE_integer('ntttcp_time', 60, - 'The number of seconds for NTttcp to run.') +flags.DEFINE_integer( + 'ntttcp_time', 60, 'The number of seconds for NTttcp to run.' +) flags.DEFINE_bool('ntttcp_udp', False, 'Whether to run a UDP test.') -flags.DEFINE_integer('ntttcp_cooldown_time', 60, - 'Time to wait between the test runs.') - -flags.DEFINE_integer('ntttcp_packet_size', None, - 'The size of the packet being used in the test.') - -flags.DEFINE_integer('ntttcp_sender_sb', -1, - 'The size of the send buffer, in Kilo Bytes, on the ' - 'sending VM. The default is the OS default.') - -flags.DEFINE_integer('ntttcp_sender_rb', -1, - 'The size of the receive buffer, in Kilo Bytes, on the ' - 'sending VM. The default is the OS default.') - -flags.DEFINE_integer('ntttcp_receiver_sb', -1, - 'The size of the send buffer, in Kilo Bytes, on the ' - 'receiving VM. The default is the OS default.') - -flags.DEFINE_integer('ntttcp_receiver_rb', -1, - 'The size of the receive buffer, in Kilo Bytes, on the ' - 'receiving VM. The default is the OS default.') +flags.DEFINE_integer( + 'ntttcp_cooldown_time', 60, 'Time to wait between the test runs.' +) + +flags.DEFINE_integer( + 'ntttcp_packet_size', None, 'The size of the packet being used in the test.' +) + +flags.DEFINE_integer( + 'ntttcp_sender_sb', + -1, + 'The size of the send buffer, in Kilo Bytes, on the ' + 'sending VM. The default is the OS default.', +) + +flags.DEFINE_integer( + 'ntttcp_sender_rb', + -1, + 'The size of the receive buffer, in Kilo Bytes, on the ' + 'sending VM. The default is the OS default.', +) + +flags.DEFINE_integer( + 'ntttcp_receiver_sb', + -1, + 'The size of the send buffer, in Kilo Bytes, on the ' + 'receiving VM. The default is the OS default.', +) + +flags.DEFINE_integer( + 'ntttcp_receiver_rb', + -1, + 'The size of the receive buffer, in Kilo Bytes, on the ' + 'receiving VM. The default is the OS default.', +) flags.DEFINE_list( - 'ntttcp_config_list', '', + 'ntttcp_config_list', + '', 'comma separated list of configs to run with ntttcp. The ' 'format for a single config is UDP:THREADS:RUNTIME_S:IP_TYPE:PACKET_SIZE, ' - 'for example True:4:60:INTERNAL:0,False:8:60:EXTERNAL:150') + 'for example True:4:60:INTERNAL:0,False:8:60:EXTERNAL:150', +) # When adding new configs to ntttcp_config_list, increase this value _NUM_PARAMS_IN_CONFIG = 5 @@ -79,14 +98,19 @@ NTTTCP_RETRIES = 10 NTTTCP_VERSION = 'v5.36' NTTTCP_EXE = 'NTttcp.exe' -NTTTCP_URL = ('https://github.com/microsoft/ntttcp/releases/download/' + - NTTTCP_VERSION + '/' + NTTTCP_EXE) +NTTTCP_URL = ( + 'https://github.com/microsoft/ntttcp/releases/download/' + + NTTTCP_VERSION + + '/' + + NTTTCP_EXE +) TRUE_VALS = ['True', 'true', 't'] FALSE_VALS = ['False', 'false', 'f'] # named tuple used in passing configs around -NtttcpConf = collections.namedtuple('NtttcpConf', - 'udp threads time_s ip_type packet_size') +NtttcpConf = collections.namedtuple( + 'NtttcpConf', 'udp threads time_s ip_type packet_size' +) def NtttcpConfigListValidator(value): @@ -122,15 +146,17 @@ def NtttcpConfigListValidator(value): # verify the ip type if ip_type not in [ - vm_util.IpAddressSubset.EXTERNAL, vm_util.IpAddressSubset.INTERNAL + vm_util.IpAddressSubset.EXTERNAL, + vm_util.IpAddressSubset.INTERNAL, ]: return False return True -flags.register_validator('ntttcp_config_list', NtttcpConfigListValidator, - 'malformed config list') +flags.register_validator( + 'ntttcp_config_list', NtttcpConfigListValidator, 'malformed config list' +) def ParseConfigList(): @@ -143,7 +169,8 @@ def ParseConfigList(): threads=FLAGS.ntttcp_threads, time_s=FLAGS.ntttcp_time, ip_type=FLAGS.ip_addresses, - packet_size=FLAGS.ntttcp_packet_size) + packet_size=FLAGS.ntttcp_packet_size, + ) ] conf_list = [] @@ -156,7 +183,9 @@ def ParseConfigList(): threads=int(confs[1]), time_s=int(confs[2]), ip_type=confs[3], - packet_size=int(confs[4]))) + packet_size=int(confs[4]), + ) + ) return conf_list @@ -179,14 +208,16 @@ def _RunNtttcp(vm, options): command = 'cd {ntttcp_exe_dir}; .\\{ntttcp_exe} {ntttcp_options}'.format( ntttcp_exe=NTTTCP_EXE, ntttcp_exe_dir=ntttcp_exe_dir, - ntttcp_options=options) + ntttcp_options=options, + ) vm.RobustRemoteCommand(command, timeout=timeout_duration) def _RemoveXml(vm): ntttcp_exe_dir = vm.temp_dir rm_command = 'cd {ntttcp_exe_dir}; rm xml.txt'.format( - ntttcp_exe_dir=ntttcp_exe_dir) + ntttcp_exe_dir=ntttcp_exe_dir + ) vm.RemoteCommand(rm_command, ignore_failure=True) @@ -194,7 +225,8 @@ def _RemoveXml(vm): def _CatXml(vm): ntttcp_exe_dir = vm.temp_dir cat_command = 'cd {ntttcp_exe_dir}; cat xml.txt'.format( - ntttcp_exe_dir=ntttcp_exe_dir) + ntttcp_exe_dir=ntttcp_exe_dir + ) ntttcp_xml, _ = vm.RemoteCommand(cat_command, timeout=10) return ntttcp_xml @@ -204,8 +236,17 @@ def _GetSockBufferSize(sock_buff_size): @vm_util.Retry(max_retries=NTTTCP_RETRIES) -def RunNtttcp(sending_vm, receiving_vm, receiving_ip_address, ip_type, udp, - threads, time_s, packet_size, cooldown): +def RunNtttcp( + sending_vm, + receiving_vm, + receiving_ip_address, + ip_type, + udp, + threads, + time_s, + packet_size, + cooldown, +): """Run NTttcp and return the samples collected from the run.""" if cooldown: @@ -220,22 +261,27 @@ def RunNtttcp(sending_vm, receiving_vm, receiving_ip_address, ip_type, udp, packet_size_string = ' -l %d ' % packet_size shared_options = '-xml -t {time} -p {port} {packet_size}'.format( - time=time_s, port=BASE_DATA_PORT, packet_size=packet_size_string) + time=time_s, port=BASE_DATA_PORT, packet_size=packet_size_string + ) udp_string = '-u' if udp else '' sending_options = shared_options + ( - '-s {udp} -m \'{threads},*,{ip}\' -rb {rb} -sb {sb}').format( - udp=udp_string, - threads=threads, - ip=receiving_ip_address, - rb=_GetSockBufferSize(FLAGS.ntttcp_sender_rb), - sb=_GetSockBufferSize(FLAGS.ntttcp_sender_sb)) + "-s {udp} -m '{threads},*,{ip}' -rb {rb} -sb {sb}" + ).format( + udp=udp_string, + threads=threads, + ip=receiving_ip_address, + rb=_GetSockBufferSize(FLAGS.ntttcp_sender_rb), + sb=_GetSockBufferSize(FLAGS.ntttcp_sender_sb), + ) receiving_options = shared_options + ( - '-r {udp} -m \'{threads},*,0.0.0.0\' -rb {rb} -sb {sb}').format( - udp=udp_string, - threads=threads, - rb=_GetSockBufferSize(FLAGS.ntttcp_receiver_rb), - sb=_GetSockBufferSize(FLAGS.ntttcp_receiver_sb)) + "-r {udp} -m '{threads},*,0.0.0.0' -rb {rb} -sb {sb}" + ).format( + udp=udp_string, + threads=threads, + rb=_GetSockBufferSize(FLAGS.ntttcp_receiver_rb), + sb=_GetSockBufferSize(FLAGS.ntttcp_receiver_sb), + ) # NTttcp will append to the xml file when it runs, which causes parsing # to fail if there was a preexisting xml file. To be safe, try deleting @@ -311,8 +357,10 @@ def ParseNtttcpResults(sender_xml_results, receiver_xml_results, metadata): throughput_element = sender_xml_root.find('./throughput[@metric="mbps"]') samples.append( - sample.Sample('Total Throughput', float(throughput_element.text), 'Mbps', - metadata)) + sample.Sample( + 'Total Throughput', float(throughput_element.text), 'Mbps', metadata + ) + ) thread_elements = sender_xml_root.findall('./thread') if len(thread_elements) > 1: @@ -320,8 +368,12 @@ def ParseNtttcpResults(sender_xml_results, receiver_xml_results, metadata): throughput_element = element.find('./throughput[@metric="mbps"]') metadata = metadata.copy() metadata['thread_index'] = element.attrib['index'] - samples.append(sample.Sample('Thread Throughput', - float(throughput_element.text), - 'Mbps', - metadata)) + samples.append( + sample.Sample( + 'Thread Throughput', + float(throughput_element.text), + 'Mbps', + metadata, + ) + ) return samples diff --git a/perfkitbenchmarker/windows_packages/nuttcp.py b/perfkitbenchmarker/windows_packages/nuttcp.py index 5344b3216e..a5c3b15789 100644 --- a/perfkitbenchmarker/windows_packages/nuttcp.py +++ b/perfkitbenchmarker/windows_packages/nuttcp.py @@ -31,35 +31,55 @@ NUTTCP_OUT_FILE = 'nuttcp_results' CPU_OUT_FILE = 'cpu_results' -flags.DEFINE_integer('nuttcp_max_bandwidth_mb', 10000, - 'The maximum bandwidth, in megabytes, to test in a ' - 'UDP stream.') - -flags.DEFINE_integer('nuttcp_min_bandwidth_mb', 100, - 'The minimum bandwidth, in megabytes, to test in a ' - 'UDP stream.') - -flags.DEFINE_integer('nuttcp_bandwidth_step_mb', 1000, - 'The amount of megabytes to increase bandwidth in each ' - 'UDP stream test.') - -flags.DEFINE_integer('nuttcp_udp_stream_seconds', 60, - 'The amount of time to run the UDP stream test.') - -flags.DEFINE_integer('nuttcp_udp_packet_size', 1420, - 'The size of each UDP packet sent in the UDP stream.') - -flags.DEFINE_bool('nuttcp_udp_run_both_directions', False, - 'Run the test twice, using each VM as a source.') - -flags.DEFINE_integer('nuttcp_udp_iterations', 1, - 'The number of consecutive tests to run.') - -flags.DEFINE_bool('nuttcp_udp_unlimited_bandwidth', False, - 'Run an "unlimited bandwidth" test') - -flags.DEFINE_integer('nuttcp_cpu_sample_time', 3, - 'Time, in seconds, to take the CPU usage sample.') +flags.DEFINE_integer( + 'nuttcp_max_bandwidth_mb', + 10000, + 'The maximum bandwidth, in megabytes, to test in a UDP stream.', +) + +flags.DEFINE_integer( + 'nuttcp_min_bandwidth_mb', + 100, + 'The minimum bandwidth, in megabytes, to test in a UDP stream.', +) + +flags.DEFINE_integer( + 'nuttcp_bandwidth_step_mb', + 1000, + 'The amount of megabytes to increase bandwidth in each UDP stream test.', +) + +flags.DEFINE_integer( + 'nuttcp_udp_stream_seconds', + 60, + 'The amount of time to run the UDP stream test.', +) + +flags.DEFINE_integer( + 'nuttcp_udp_packet_size', + 1420, + 'The size of each UDP packet sent in the UDP stream.', +) + +flags.DEFINE_bool( + 'nuttcp_udp_run_both_directions', + False, + 'Run the test twice, using each VM as a source.', +) + +flags.DEFINE_integer( + 'nuttcp_udp_iterations', 1, 'The number of consecutive tests to run.' +) + +flags.DEFINE_bool( + 'nuttcp_udp_unlimited_bandwidth', False, 'Run an "unlimited bandwidth" test' +) + +flags.DEFINE_integer( + 'nuttcp_cpu_sample_time', + 3, + 'Time, in seconds, to take the CPU usage sample.', +) NUTTCP_DIR = 'nuttcp-8.1.4.win64' NUTTCP_ZIP = NUTTCP_DIR + '.zip' @@ -79,19 +99,22 @@ def Install(vm): @vm_util.Retry() def DownloadWithRetry(): vm.DownloadFile(NUTTCP_URL, zip_path) + try: DownloadWithRetry() except errors.VirtualMachine.RemoteCommandError as e: # The mirror to download nuttcp from is temporarily unavailable. raise errors.Benchmarks.KnownIntermittentError( - 'Failed to download nuttcp package: %s' % e) + 'Failed to download nuttcp package: %s' % e + ) vm.UnzipFile(zip_path, vm.temp_dir) def CheckPrerequisites(): if FLAGS.nuttcp_udp_stream_seconds <= FLAGS.nuttcp_cpu_sample_time: raise errors.Config.InvalidValue( - 'nuttcp_udp_stream_seconds must be greater than nuttcp_cpu_sample_time') + 'nuttcp_udp_stream_seconds must be greater than nuttcp_cpu_sample_time' + ) def GetExecPath(): @@ -107,9 +130,8 @@ def _RunNuttcp(vm, options, exec_path): exec_path: string path to the nuttcp executable """ command = 'cd {exec_dir}; .\\{exec_path} {options}'.format( - exec_dir=vm.temp_dir, - exec_path=exec_path, - options=options) + exec_dir=vm.temp_dir, exec_path=exec_path, options=options + ) # Timeout after expected duration, 5sec server wait plus 120sec buffer timeout_duration = FLAGS.nuttcp_udp_stream_seconds + _COMMAND_TIMEOUT_BUFFER vm.RobustRemoteCommand(command, timeout=timeout_duration) @@ -125,16 +147,19 @@ def _GetCpuUsage(vm): NuttcpNotRunningError: raised if nuttcp is not running when the CPU usage data gathering has finished. """ - command = ('cd {exec_path}; ' - "Get-Counter -Counter '\\Processor(*)\\% Processor Time' " - '-SampleInterval {sample_time} | ' - 'select -ExpandProperty CounterSamples | ' - 'select InstanceName,CookedValue > {out_file};' - 'if ((ps | select-string nuttcp | measure-object).Count -eq 0) ' - '{{echo "FAIL"}}').format( - exec_path=vm.temp_dir, - sample_time=FLAGS.nuttcp_cpu_sample_time, - out_file=CPU_OUT_FILE) + command = ( + 'cd {exec_path}; ' + "Get-Counter -Counter '\\Processor(*)\\% Processor Time' " + '-SampleInterval {sample_time} | ' + 'select -ExpandProperty CounterSamples | ' + 'select InstanceName,CookedValue > {out_file};' + 'if ((ps | select-string nuttcp | measure-object).Count -eq 0) ' + '{{echo "FAIL"}}' + ).format( + exec_path=vm.temp_dir, + sample_time=FLAGS.nuttcp_cpu_sample_time, + out_file=CPU_OUT_FILE, + ) # returning from the command should never take longer than 120 seconds over # the actual sample time. If it does, it is hung. timeout_duration = FLAGS.nuttcp_cpu_sample_time + _COMMAND_TIMEOUT_BUFFER @@ -159,25 +184,29 @@ def RunSingleBandwidth(bandwidth, sending_vm, receiving_vm, dest_ip, exec_path): Returns: output from the client nuttcp process. """ - sender_args = ('-u -p{data_port} -P{control_port} -R{bandwidth} ' - '-T{time} -l{packet_size} {dest_ip} > {out_file}').format( - data_port=UDP_PORT, - control_port=CONTROL_PORT, - bandwidth=bandwidth, - time=FLAGS.nuttcp_udp_stream_seconds, - packet_size=FLAGS.nuttcp_udp_packet_size, - dest_ip=dest_ip, - out_file=NUTTCP_OUT_FILE) + sender_args = ( + '-u -p{data_port} -P{control_port} -R{bandwidth} ' + '-T{time} -l{packet_size} {dest_ip} > {out_file}' + ).format( + data_port=UDP_PORT, + control_port=CONTROL_PORT, + bandwidth=bandwidth, + time=FLAGS.nuttcp_udp_stream_seconds, + packet_size=FLAGS.nuttcp_udp_packet_size, + dest_ip=dest_ip, + out_file=NUTTCP_OUT_FILE, + ) receiver_args = '-p{data_port} -P{control_port} -1'.format( - data_port=UDP_PORT, - control_port=CONTROL_PORT) + data_port=UDP_PORT, control_port=CONTROL_PORT + ) # Process to run the nuttcp server server_process = multiprocessing.Process( name='server', target=_RunNuttcp, - args=(receiving_vm, receiver_args, exec_path)) + args=(receiving_vm, receiver_args, exec_path), + ) server_process.start() receiving_vm.WaitForProcessRunning('nuttcp', 30) @@ -186,14 +215,16 @@ def RunSingleBandwidth(bandwidth, sending_vm, receiving_vm, dest_ip, exec_path): client_process = multiprocessing.Process( name='client', target=_RunNuttcp, - args=(sending_vm, sender_args, exec_path)) + args=(sending_vm, sender_args, exec_path), + ) client_process.start() sending_vm.WaitForProcessRunning('nuttcp', 30) process_args = [ (_GetCpuUsage, (receiving_vm,), {}), - (_GetCpuUsage, (sending_vm,), {})] + (_GetCpuUsage, (sending_vm,), {}), + ] background_tasks.RunParallelProcesses(process_args, 200) @@ -213,14 +244,16 @@ def GatherResults(vm, out_file): The contents of 'out_file' as a string. """ cat_command = 'cd {results_dir}; cat {out_file}' - results_command = cat_command.format(results_dir=vm.temp_dir, - out_file=out_file) + results_command = cat_command.format( + results_dir=vm.temp_dir, out_file=out_file + ) results, _ = vm.RemoteCommand(results_command) return results -def RunNuttcp(sending_vm, receiving_vm, exec_path, dest_ip, network_type, - iteration): +def RunNuttcp( + sending_vm, receiving_vm, exec_path, dest_ip, network_type, iteration +): """Run nuttcp tests. Args: @@ -242,14 +275,14 @@ def RunNuttcp(sending_vm, receiving_vm, exec_path, dest_ip, network_type, for b in range( FLAGS.nuttcp_min_bandwidth_mb, FLAGS.nuttcp_max_bandwidth_mb, - FLAGS.nuttcp_bandwidth_step_mb) + FLAGS.nuttcp_bandwidth_step_mb, + ) ] if FLAGS.nuttcp_udp_unlimited_bandwidth: bandwidths.append('u') for bandwidth in bandwidths: - RunSingleBandwidth(bandwidth, sending_vm, receiving_vm, dest_ip, exec_path) # retrieve the results and parse them @@ -262,9 +295,17 @@ def RunNuttcp(sending_vm, receiving_vm, exec_path, dest_ip, network_type, receiving_cpu_results = GatherResults(receiving_vm, CPU_OUT_FILE) samples.append( - GetUDPStreamSample(udp_results, sender_cpu_results, - receiving_cpu_results, sending_vm, receiving_vm, - bandwidth, network_type, iteration)) + GetUDPStreamSample( + udp_results, + sender_cpu_results, + receiving_cpu_results, + sending_vm, + receiving_vm, + bandwidth, + network_type, + iteration, + ) + ) return samples @@ -286,6 +327,7 @@ def _GetCpuResults(cpu_results): Args: cpu_results: string of the output of the cpu usage command. + Returns: Array of (cpu_num, percentage) """ @@ -300,9 +342,16 @@ def _GetCpuResults(cpu_results): # drop/pkt 6.72 %loss -def GetUDPStreamSample(command_out, sender_cpu_results, receiving_cpu_results, - sending_vm, receiving_vm, request_bandwidth, - network_type, iteration): +def GetUDPStreamSample( + command_out, + sender_cpu_results, + receiving_cpu_results, + sending_vm, + receiving_vm, + request_bandwidth, + network_type, + iteration, +): """Get a sample from the nuttcp string results. Args: @@ -328,7 +377,8 @@ def GetUDPStreamSample(command_out, sender_cpu_results, receiving_cpu_results, except IndexError as e: # nuttcp connection timed out during the run. raise errors.Benchmarks.KnownIntermittentError( - f'Not enough data to parse nuttcp results, command timed out: {e}') + f'Not enough data to parse nuttcp results, command timed out: {e}' + ) packet_loss = data_line[16] diff --git a/perfkitbenchmarker/windows_packages/psping.py b/perfkitbenchmarker/windows_packages/psping.py index dd8289a3ce..7a821a01c8 100644 --- a/perfkitbenchmarker/windows_packages/psping.py +++ b/perfkitbenchmarker/windows_packages/psping.py @@ -16,7 +16,6 @@ """Module containing psping installation and cleanup functions. psping is a tool made for benchmarking Windows networking. - """ import json @@ -36,17 +35,17 @@ PSPING_ZIP = PSPING_DIR + '.zip' PSPING_URL = 'https://download.sysinternals.com/files/' + PSPING_ZIP -flags.DEFINE_integer('psping_packet_size', 1, - 'The size of the packet to test the ping with.') +flags.DEFINE_integer( + 'psping_packet_size', 1, 'The size of the packet to test the ping with.' +) -flags.DEFINE_integer('psping_bucket_count', 100, - 'For the results histogram, number of columns') +flags.DEFINE_integer( + 'psping_bucket_count', 100, 'For the results histogram, number of columns' +) -flags.DEFINE_integer('psping_rr_count', 1000, - 'The number of pings to attempt') +flags.DEFINE_integer('psping_rr_count', 1000, 'The number of pings to attempt') -flags.DEFINE_integer('psping_timeout', 10, - 'The time to allow psping to run') +flags.DEFINE_integer('psping_timeout', 10, 'The time to allow psping to run') def Install(vm): @@ -62,9 +61,8 @@ def StartPspingServer(vm): server_command = ( 'Start-Job -ScriptBlock {{' '{psping_exec_dir}\\psping.exe /accepteula -s 0.0.0.0:{port};' - '}}').format( - psping_exec_dir=vm.temp_dir, - port=TEST_PORT) + '}}' + ).format(psping_exec_dir=vm.temp_dir, port=TEST_PORT) vm.RemoteCommand(server_command) @@ -91,42 +89,47 @@ def RunLatencyTest(sending_vm, receiving_vm, use_internal_ip=True): Returns: list of samples representing latency between the two VMs. """ - server_ip = (receiving_vm.internal_ip if use_internal_ip - else receiving_vm.ip_address) + server_ip = ( + receiving_vm.internal_ip if use_internal_ip else receiving_vm.ip_address + ) client_command = ( 'cd {psping_exec_dir}; ' - 'sleep 2;' # sleep to make sure the server starts first. + 'sleep 2;' '.\\psping.exe /accepteula -l {packet_size} -i 0 -q ' '-n {rr_count} -h {bucket_count} {ip}:{port}' - ' > {out_file}').format( - psping_exec_dir=sending_vm.temp_dir, - packet_size=FLAGS.psping_packet_size, - rr_count=FLAGS.psping_rr_count, - bucket_count=FLAGS.psping_bucket_count, - ip=server_ip, - port=TEST_PORT, - out_file=PSPING_OUTPUT_FILE) + ' > {out_file}' # sleep to make sure the server starts first. + ).format( + psping_exec_dir=sending_vm.temp_dir, + packet_size=FLAGS.psping_packet_size, + rr_count=FLAGS.psping_rr_count, + bucket_count=FLAGS.psping_bucket_count, + ip=server_ip, + port=TEST_PORT, + out_file=PSPING_OUTPUT_FILE, + ) # PSPing does not have a configurable timeout. To get around this, start the # server as a background job, then kill it after 10 seconds server_command = ( - '{psping_exec_dir}\\psping.exe /accepteula -s 0.0.0.0:{port};').format( - psping_exec_dir=receiving_vm.temp_dir, - port=TEST_PORT) + '{psping_exec_dir}\\psping.exe /accepteula -s 0.0.0.0:{port};' + ).format(psping_exec_dir=receiving_vm.temp_dir, port=TEST_PORT) - process_args = [(_RunPsping, (receiving_vm, server_command), {}), - (_RunPsping, (sending_vm, client_command), {})] + process_args = [ + (_RunPsping, (receiving_vm, server_command), {}), + (_RunPsping, (sending_vm, client_command), {}), + ] background_tasks.RunParallelProcesses(process_args, 200, 1) cat_command = 'cd {psping_exec_dir}; cat {out_file}'.format( - psping_exec_dir=sending_vm.temp_dir, - out_file=PSPING_OUTPUT_FILE) + psping_exec_dir=sending_vm.temp_dir, out_file=PSPING_OUTPUT_FILE + ) output, _ = sending_vm.RemoteCommand(cat_command) return ParsePspingResults(output, sending_vm, receiving_vm, use_internal_ip) + # example output # PsPing v2.10 - PsPing - ping, latency, bandwidth measurement utility # Copyright (C) 2012-2016 Mark Russinovich @@ -215,9 +218,9 @@ def ParsePspingResults(results, client_vm, server_vm, internal_ip_used): latency = float(entry_data[0]) count = int(entry_data[1]) - histogram.append({'latency': latency, - 'count': count, - 'bucket_number': index}) + histogram.append( + {'latency': latency, 'count': count, 'bucket_number': index} + ) index += 1 @@ -225,6 +228,7 @@ def ParsePspingResults(results, client_vm, server_vm, internal_ip_used): histogram_metadata.update({'histogram': json.dumps(histogram)}) samples.append( - sample.Sample('latency:histogram', 0, 'ms', histogram_metadata)) + sample.Sample('latency:histogram', 0, 'ms', histogram_metadata) + ) return samples diff --git a/perfkitbenchmarker/windows_virtual_machine.py b/perfkitbenchmarker/windows_virtual_machine.py index 87bf0e0ded..4b2f196f5c 100644 --- a/perfkitbenchmarker/windows_virtual_machine.py +++ b/perfkitbenchmarker/windows_virtual_machine.py @@ -36,14 +36,18 @@ FLAGS = flags.FLAGS flags.DEFINE_bool( - 'log_windows_password', False, + 'log_windows_password', + False, 'Whether to log passwords for Windows machines. This can be useful in ' - 'the event of needing to manually RDP to the instance.') + 'the event of needing to manually RDP to the instance.', +) flags.DEFINE_bool( - 'set_cpu_priority_high', False, + 'set_cpu_priority_high', + False, 'Allows executables to be set to High (up from Normal) CPU priority ' - 'through the SetProcessPriorityToHigh function.') + 'through the SetProcessPriorityToHigh function.', +) # Windows disk letter starts from C, use a larger disk letter for attached disk # to avoid conflict. On Azure, D is reserved for DvD drive. @@ -73,7 +77,9 @@ """.format(winrm_port=WINRM_PORT) STARTUP_SCRIPT = 'powershell -EncodedCommand {encoded_command}'.format( encoded_command=six.ensure_str( - base64.b64encode(_STARTUP_SCRIPT.encode('utf-16-le')))) + base64.b64encode(_STARTUP_SCRIPT.encode('utf-16-le')) + ) +) class WaitTimeoutError(Exception): @@ -148,9 +154,8 @@ def RobustRemoteCommand( start_command_time = time.time() try: self.RemoteCommand( - logged_command, - ignore_failure=ignore_failure, - timeout=timeout) + logged_command, ignore_failure=ignore_failure, timeout=timeout + ) except errors.VirtualMachine.RemoteCommandError: logging.exception( 'Exception while running %s on %s, waiting for command to finish', @@ -200,8 +205,8 @@ def RemoteCommand( command: A valid powershell command. ignore_failure: Ignore any failure if set to true. timeout: Float. A timeout in seconds for the command. If None is passed, - no timeout is applied. Timeout kills the winrm session which then - kills the process being executed. + no timeout is applied. Timeout kills the winrm session which then kills + the process being executed. Returns: A tuple of stdout and stderr from running the command. @@ -214,9 +219,11 @@ def RemoteCommand( s = winrm.Session( 'https://%s:%s' % (self.GetConnectionIp(), self.winrm_port), auth=(self.user_name, self.password), - server_cert_validation='ignore') + server_cert_validation='ignore', + ) encoded_command = six.ensure_str( - base64.b64encode(command.encode('utf_16_le'))) + base64.b64encode(command.encode('utf_16_le')) + ) @timeout_decorator.timeout( timeout, @@ -227,17 +234,26 @@ def run_command(): return s.run_cmd('powershell -encodedcommand %s' % encoded_command) r = run_command() - retcode, stdout, stderr = r.status_code, six.ensure_str( - r.std_out), six.ensure_str(r.std_err) + retcode, stdout, stderr = ( + r.status_code, + six.ensure_str(r.std_out), + six.ensure_str(r.std_err), + ) - debug_text = ('Ran %s on %s. Return code (%s).\nSTDOUT: %s\nSTDERR: %s' % - (command, self, retcode, stdout, stderr)) + debug_text = 'Ran %s on %s. Return code (%s).\nSTDOUT: %s\nSTDERR: %s' % ( + command, + self, + retcode, + stdout, + stderr, + ) logging.info(debug_text, stacklevel=2) if retcode and not ignore_failure: - error_text = ('Got non-zero return code (%s) executing %s\n' - 'STDOUT: %sSTDERR: %s' % - (retcode, command, stdout, stderr)) + error_text = ( + 'Got non-zero return code (%s) executing %s\nSTDOUT: %sSTDERR: %s' + % (retcode, command, stdout, stderr) + ) raise errors.VirtualMachine.RemoteCommandError(error_text) return stdout, stderr @@ -281,8 +297,9 @@ def RemoteCopy(self, local_path, remote_path='', copy_to=True): else: self._SmbclientRemoteCopy(local_path, remote_path, copy_to, network_drive) - def _SmbclientRemoteCopy(self, local_path, remote_path, - copy_to, network_drive): + def _SmbclientRemoteCopy( + self, local_path, remote_path, copy_to, network_drive + ): """Copies a file to or from the VM using smbclient. Args: @@ -290,7 +307,7 @@ def _SmbclientRemoteCopy(self, local_path, remote_path, remote_path: Optional path of where to copy file on remote host. copy_to: True to copy to vm, False to copy from vm. network_drive: The smb specification for the remote drive - (//{ip_address}/{share_name}). + (//{ip_address}/{share_name}). Raises: RemoteCommandError: If there was a problem copying the file. @@ -304,22 +321,28 @@ def _SmbclientRemoteCopy(self, local_path, remote_path, else: smb_command += 'get %s %s' % (remote_file, local_file) smb_copy = [ - 'smbclient', network_drive, - '--max-protocol', 'SMB3', - '--user', '%s%%%s' % (self.user_name, self.password), - '--port', str(self.smb_port), - '--command', smb_command + 'smbclient', + network_drive, + '--max-protocol', + 'SMB3', + '--user', + '%s%%%s' % (self.user_name, self.password), + '--port', + str(self.smb_port), + '--command', + smb_command, ] - stdout, stderr, retcode = vm_util.IssueCommand(smb_copy, - raise_on_failure=False) + stdout, stderr, retcode = vm_util.IssueCommand( + smb_copy, raise_on_failure=False + ) if retcode: - error_text = ('Got non-zero return code (%s) executing %s\n' - 'STDOUT: %sSTDERR: %s' % - (retcode, smb_copy, stdout, stderr)) + error_text = ( + 'Got non-zero return code (%s) executing %s\nSTDOUT: %sSTDERR: %s' + % (retcode, smb_copy, stdout, stderr) + ) raise errors.VirtualMachine.RemoteCommandError(error_text) - def _PsDriveRemoteCopy(self, local_path, remote_path, - copy_to, network_drive): + def _PsDriveRemoteCopy(self, local_path, remote_path, copy_to, network_drive): """Copies a file to or from the VM using New-PSDrive and Copy-Item. Args: @@ -327,7 +350,7 @@ def _PsDriveRemoteCopy(self, local_path, remote_path, remote_path: Optional path of where to copy file on remote host. copy_to: True to copy to vm, False to copy from vm. network_drive: The smb specification for the remote drive - (//{ip_address}/{share_name}). + (//{ip_address}/{share_name}). Raises: RemoteCommandError: If there was a problem copying the file. @@ -336,14 +359,16 @@ def _PsDriveRemoteCopy(self, local_path, remote_path, password = self.password.replace("'", "''") create_cred = ( - '$pw = convertto-securestring -AsPlainText -Force \'%s\';' + "$pw = convertto-securestring -AsPlainText -Force '%s';" '$cred = new-object -typename System.Management.Automation' - '.PSCredential -argumentlist %s,$pw' % (password, self.user_name)) + '.PSCredential -argumentlist %s,$pw' % (password, self.user_name) + ) psdrive_name = self.name create_psdrive = ( - 'New-PSDrive -Name %s -PSProvider filesystem -Root ' - '%s -Credential $cred' % (psdrive_name, network_drive)) + 'New-PSDrive -Name %s -PSProvider filesystem -Root %s -Credential $cred' + % (psdrive_name, network_drive) + ) remote_path = '%s:%s' % (psdrive_name, remote_path) if copy_to: @@ -355,16 +380,23 @@ def _PsDriveRemoteCopy(self, local_path, remote_path, delete_connection = 'net use %s /delete' % network_drive - cmd = ';'.join([set_error_pref, create_cred, create_psdrive, - copy_item, delete_connection]) + cmd = ';'.join([ + set_error_pref, + create_cred, + create_psdrive, + copy_item, + delete_connection, + ]) stdout, stderr, retcode = vm_util.IssueCommand( - ['powershell', '-Command', cmd], timeout=None, raise_on_failure=False) + ['powershell', '-Command', cmd], timeout=None, raise_on_failure=False + ) if retcode: - error_text = ('Got non-zero return code (%s) executing %s\n' - 'STDOUT: %sSTDERR: %s' % - (retcode, cmd, stdout, stderr)) + error_text = ( + 'Got non-zero return code (%s) executing %s\nSTDOUT: %sSTDERR: %s' + % (retcode, cmd, stdout, stderr) + ) raise errors.VirtualMachine.RemoteCommandError(error_text) def WaitForBootCompletion(self): @@ -373,7 +405,8 @@ def WaitForBootCompletion(self): if FLAGS.cluster_boot_test_rdp_port_listening: to_wait_for.append(self._WaitForRdpPort) background_tasks.RunParallelThreads( - [(method, [], {}) for method in to_wait_for], 2) + [(method, [], {}) for method in to_wait_for], 2 + ) @vm_util.Retry(log_errors=False, poll_interval=1, timeout=2400) def _WaitForRdpPort(self): @@ -386,8 +419,10 @@ def _WaitForWinRmCommand(self): """Waits for WinRM command and optionally for the WinRM port to listen.""" # Test for listening on the port first, because this will happen strictly # first. - if (FLAGS.cluster_boot_test_port_listening and - self.port_listening_time is None): + if ( + FLAGS.cluster_boot_test_port_listening + and self.port_listening_time is None + ): self.TestConnectRemoteAccessPort() self.port_listening_time = time.time() @@ -494,11 +529,12 @@ def WaitForProcessRunning(self, process, timeout): WaitTimeoutError: raised if the process does not run within "timeout" seconds. """ - command = ('$count={timeout};' - 'while( (ps | select-string {process} | measure-object).Count ' - '-eq 0 -and $count -gt 0) {{sleep 1; $count=$count-1}}; ' - 'if ($count -eq 0) {{echo "FAIL"}}').format( - timeout=timeout, process=process) + command = ( + '$count={timeout};' + 'while( (ps | select-string {process} | measure-object).Count ' + '-eq 0 -and $count -gt 0) {{sleep 1; $count=$count-1}}; ' + 'if ($count -eq 0) {{echo "FAIL"}}' + ).format(timeout=timeout, process=process) stdout, _ = self.RemoteCommand(command) if 'FAIL' in stdout: raise WaitTimeoutError() @@ -525,7 +561,8 @@ def _GetNumCpus(self): """ stdout, _ = self.RemoteCommand( 'Get-WmiObject -class Win32_processor | ' - 'Select-Object -ExpandProperty NumberOfLogicalProcessors') + 'Select-Object -ExpandProperty NumberOfLogicalProcessors' + ) # In the case that there are multiple Win32_processor instances, the result # of this command can be a string like '4 4 '. return sum(int(i) for i in stdout.split()) @@ -540,8 +577,8 @@ def _GetTotalMemoryKb(self): This method does not cache results (unlike "total_memory_kb"). """ stdout, _ = self.RemoteCommand( - 'Get-WmiObject -class Win32_PhysicalMemory | ' - 'select -exp Capacity') + 'Get-WmiObject -class Win32_PhysicalMemory | select -exp Capacity' + ) result = sum(int(capacity) for capacity in stdout.split('\n') if capacity) return result / 1024 @@ -561,18 +598,21 @@ def DownloadFile(self, url, dest): # Allow more security protocols to make it easier to download from # sites where we don't know the security protocol beforehand - command = ('[Net.ServicePointManager]::SecurityProtocol = ' - '[System.Net.SecurityProtocolType] ' - '"tls, tls11, tls12";' - 'Invoke-WebRequest {url} -OutFile {dest}').format( - url=url, dest=dest) + command = ( + '[Net.ServicePointManager]::SecurityProtocol = ' + '[System.Net.SecurityProtocolType] ' + '"tls, tls11, tls12";' + 'Invoke-WebRequest {url} -OutFile {dest}' + ).format(url=url, dest=dest) self.RemoteCommand(command) def UnzipFile(self, zip_file, dest): """Unzips the file with the given path.""" - command = ('Add-Type -A System.IO.Compression.FileSystem; ' - '[IO.Compression.ZipFile]::ExtractToDirectory(\'{zip_file}\', ' - '\'{dest}\')').format(zip_file=zip_file, dest=dest) + command = ( + 'Add-Type -A System.IO.Compression.FileSystem; ' + "[IO.Compression.ZipFile]::ExtractToDirectory('{zip_file}', " + "'{dest}')" + ).format(zip_file=zip_file, dest=dest) self.RemoteCommand(command) def DisableGuestFirewall(self): @@ -593,8 +633,9 @@ def RunDiskpartScript(self, script): tf.close() script_path = ntpath.join(self.temp_dir, os.path.basename(tf.name)) self.RemoteCopy(tf.name, script_path) - self.RemoteCommand('diskpart /s {script_path}'.format( - script_path=script_path)) + self.RemoteCommand( + 'diskpart /s {script_path}'.format(script_path=script_path) + ) def DiskDriveIsLocal(self, device, model): """Helper method to determine if a disk drive is a local ssd to stripe.""" @@ -642,59 +683,71 @@ def _IsSmtEnabled(self): class Windows2016CoreMixin(BaseWindowsMixin): """Class holding Windows Server 2016 Server Core VM specifics.""" + OS_TYPE = os_types.WINDOWS2016_CORE class Windows2019CoreMixin(BaseWindowsMixin): """Class holding Windows Server 2019 Server Core VM specifics.""" + OS_TYPE = os_types.WINDOWS2019_CORE class Windows2022CoreMixin(BaseWindowsMixin): """Class holding Windows Server 2022 Server Core VM specifics.""" + OS_TYPE = os_types.WINDOWS2022_CORE class Windows2016DesktopMixin(BaseWindowsMixin): """Class holding Windows Server 2016 with Desktop Experience VM specifics.""" + OS_TYPE = os_types.WINDOWS2016_DESKTOP class Windows2019DesktopMixin(BaseWindowsMixin): """Class holding Windows Server 2019 with Desktop Experience VM specifics.""" + OS_TYPE = os_types.WINDOWS2019_DESKTOP class Windows2022DesktopMixin(BaseWindowsMixin): """Class holding Windows Server 2019 with Desktop Experience VM specifics.""" + OS_TYPE = os_types.WINDOWS2022_DESKTOP class Windows2019SQLServer2017Standard(BaseWindowsMixin): """Class holding Windows Server 2019 with Desktop Experience VM specifics.""" + OS_TYPE = os_types.WINDOWS2019_SQLSERVER_2017_STANDARD class Windows2019SQLServer2017Enterprise(BaseWindowsMixin): """Class holding Windows Server 2019 with Desktop Experience VM specifics.""" + OS_TYPE = os_types.WINDOWS2019_SQLSERVER_2017_ENTERPRISE class Windows2019SQLServer2019Standard(BaseWindowsMixin): """Class holding Windows Server 2019 with Desktop Experience VM specifics.""" + OS_TYPE = os_types.WINDOWS2019_SQLSERVER_2019_STANDARD class Windows2019SQLServer2019Enterprise(BaseWindowsMixin): """Class holding Windows Server 2019 with Desktop Experience VM specifics.""" + OS_TYPE = os_types.WINDOWS2019_SQLSERVER_2019_ENTERPRISE class Windows2022SQLServer2019Standard(BaseWindowsMixin): """Class holding Windows Server 2022 with Desktop Experience VM specifics.""" + OS_TYPE = os_types.WINDOWS2022_SQLSERVER_2019_STANDARD class Windows2022SQLServer2019Enterprise(BaseWindowsMixin): """Class holding Windows Server 2022 with Desktop Experience VM specifics.""" + OS_TYPE = os_types.WINDOWS2022_SQLSERVER_2019_ENTERPRISE diff --git a/setup.py b/setup.py index daf19e65dd..41a522a6c7 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import setup, find_packages +from setuptools import find_packages, setup setup( name='perfkitbenchmarker', @@ -20,6 +20,5 @@ license='Apache 2.0', packages=find_packages(exclude=['tests']), scripts=['pkb.py'], - install_requires=['absl-py', - 'jinja2>=2.7', - 'setuptools']) + install_requires=['absl-py', 'jinja2>=2.7', 'setuptools'], +) diff --git a/tests/aws_test.py b/tests/aws_test.py index 8050fc931f..500db30c26 100644 --- a/tests/aws_test.py +++ b/tests/aws_test.py @@ -21,7 +21,6 @@ from absl.testing import flagsaver from absl.testing import parameterized import mock - from perfkitbenchmarker import benchmark_spec from perfkitbenchmarker import context from perfkitbenchmarker import errors @@ -43,7 +42,12 @@ def _AwsCommand(region, topic, *args): return [ - 'aws', '--output', 'json', 'ec2', topic, '--region={}'.format(region) + 'aws', + '--output', + 'json', + 'ec2', + topic, + '--region={}'.format(region), ] + list(args) @@ -56,22 +60,28 @@ def setUp(self): self.addCleanup(p.stop) self.disk = aws_disk.AwsDisk( aws_disk.AwsDiskSpec(_COMPONENT, disk_type='standard'), - 'us-east-1', 'm4.2xlarge') + 'us-east-1', + 'm4.2xlarge', + ) self.disk.id = 'vol-foo' def testVolumePresent(self): - response = ('{"Volumes":[{"Size":500,"Attachments":[],"SnapshotId":null,' - '"CreateTime": "2015-05-04T23:47:31.726Z","VolumeId":' - '"vol-5859691f","AvailabilityZone":"us-east-1a","VolumeType":' - '"standard","State":"creating"}]}') + response = ( + '{"Volumes":[{"Size":500,"Attachments":[],"SnapshotId":null,' + '"CreateTime": "2015-05-04T23:47:31.726Z","VolumeId":' + '"vol-5859691f","AvailabilityZone":"us-east-1a","VolumeType":' + '"standard","State":"creating"}]}' + ) util.IssueRetryableCommand.side_effect = [(response, None)] self.assertTrue(self.disk._Exists()) def testVolumeDeleted(self): - response = ('{"Volumes":[{"VolumeId":"vol-e45b6ba3","Size": 500,' - '"AvailabilityZone": "us-east-1a","CreateTime":' - '"2015-05-04T23:53:42.952Z","Attachments":[],"State":' - '"deleting","SnapshotId":null,"VolumeType": "standard"}]}') + response = ( + '{"Volumes":[{"VolumeId":"vol-e45b6ba3","Size": 500,' + '"AvailabilityZone": "us-east-1a","CreateTime":' + '"2015-05-04T23:53:42.952Z","Attachments":[],"State":' + '"deleting","SnapshotId":null,"VolumeType": "standard"}]}' + ) util.IssueRetryableCommand.side_effect = [(response, None)] self.assertFalse(self.disk._Exists()) @@ -80,8 +90,9 @@ class AwsVpcTestCase(pkb_common_test_case.PkbCommonTestCase): def setUp(self): super(AwsVpcTestCase, self).setUp() - p = mock.patch('perfkitbenchmarker.providers.aws.' - 'util.IssueRetryableCommand') + p = mock.patch( + 'perfkitbenchmarker.providers.aws.util.IssueRetryableCommand' + ) p.start() self.addCleanup(p.stop) self.vpc = aws_network.AwsVpc('region') @@ -93,8 +104,10 @@ def testCreateNormalVpc(self, mock_cmd): res1 = {'Vpc': {'VpcId': vpc_id}} res2 = {'Vpcs': [1]} res3 = {'SecurityGroups': [{'GroupId': security_group_id}]} - mock_cmd.side_effect = [(json.dumps(res1), '', 0), - (json.dumps(res3), '', 0)] + mock_cmd.side_effect = [ + (json.dumps(res1), '', 0), + (json.dumps(res3), '', 0), + ] util.IssueRetryableCommand.return_value = json.dumps(res2), '' self.assertFalse(self.vpc.user_managed) self.vpc.Create() @@ -102,15 +115,17 @@ def testCreateNormalVpc(self, mock_cmd): self.assertEqual(security_group_id, self.vpc.default_security_group_id) def testInstanceVpcQuotaExceededDuringCreate(self): - stderr = ('An error occurred (VpcLimitExceeded) when calling the CreateVpc ' - 'operation: The maximum number of VPCs has been reached.') + stderr = ( + 'An error occurred (VpcLimitExceeded) when calling the CreateVpc ' + 'operation: The maximum number of VPCs has been reached.' + ) self.MockIssueCommand(None, stderr, 255) with self.assertRaises(errors.Benchmarks.QuotaFailure) as e: self.vpc.Create() self.assertEqual(str(e.exception), stderr) def testVpcCreateError(self): - stderr = ('An unknown error occurred when calling the CreateVpc opeartion.') + stderr = 'An unknown error occurred when calling the CreateVpc opeartion.' retcode = 1 error_msg = 'Failed to create Vpc: %s return code: %s' % (retcode, stderr) self.MockIssueCommand(None, stderr, retcode) @@ -124,10 +139,12 @@ def testVpcDeleted(self): self.assertFalse(self.vpc._Exists()) def testVpcPresent(self): - response = ('{"Vpcs":[{"InstanceTenancy":"default","CidrBlock":' - '"10.0.0.0/16","IsDefault":false,"DhcpOptionsId":' - '"dopt-59b12f38","State":"available","VpcId":"vpc-2289a647"' - '}]}') + response = ( + '{"Vpcs":[{"InstanceTenancy":"default","CidrBlock":' + '"10.0.0.0/16","IsDefault":false,"DhcpOptionsId":' + '"dopt-59b12f38","State":"available","VpcId":"vpc-2289a647"' + '}]}' + ) util.IssueRetryableCommand.side_effect = [(response, None)] self.assertTrue(self.vpc._Exists()) @@ -149,10 +166,15 @@ def testInternetGatewayLifecycle(self, mock_cmd): delete_response = {} create_cmd = _AwsCommand(region, 'create-internet-gateway') describe_cmd = _AwsCommand( - region, 'describe-internet-gateways', - '--filter=Name=internet-gateway-id,Values={}'.format(gateway_id)) - delete_cmd = _AwsCommand(region, 'delete-internet-gateway', - '--internet-gateway-id={}'.format(gateway_id)) + region, + 'describe-internet-gateways', + '--filter=Name=internet-gateway-id,Values={}'.format(gateway_id), + ) + delete_cmd = _AwsCommand( + region, + 'delete-internet-gateway', + '--internet-gateway-id={}'.format(gateway_id), + ) # Test the Create() command mock_cmd.side_effect = [(json.dumps(create_response), '', 0)] @@ -170,7 +192,7 @@ def testInternetGatewayLifecycle(self, mock_cmd): mock_cmd.side_effect = [(json.dumps(delete_response), '', 0)] * 2 util.IssueRetryableCommand.side_effect = [ (json.dumps(exists_response), ''), - (json.dumps(exists_response_no_resources), '') + (json.dumps(exists_response_no_resources), ''), ] gateway.Delete() mock_cmd.assert_called_with(delete_cmd, raise_on_failure=False) @@ -186,16 +208,22 @@ def testSubnetLifecycle(self, mock_cmd): exists_response = {'Subnets': [{'SubnetId': subnet_id, 'VpcId': vpc_id}]} exists_response_no_resources = {'Subnets': []} delete_response = {} - create_cmd = _AwsCommand(region, 'create-subnet', - '--vpc-id={}'.format(vpc_id), - '--cidr-block=10.0.0.0/24', - '--availability-zone={}'.format(zone)) + create_cmd = _AwsCommand( + region, + 'create-subnet', + '--vpc-id={}'.format(vpc_id), + '--cidr-block=10.0.0.0/24', + '--availability-zone={}'.format(zone), + ) describe_cmd = _AwsCommand( - region, 'describe-subnets', + region, + 'describe-subnets', '--filter=Name=vpc-id,Values={}'.format(vpc_id), - '--filter=Name=subnet-id,Values={}'.format(subnet_id)) - delete_cmd = _AwsCommand(region, 'delete-subnet', - '--subnet-id={}'.format(subnet_id)) + '--filter=Name=subnet-id,Values={}'.format(subnet_id), + ) + delete_cmd = _AwsCommand( + region, 'delete-subnet', '--subnet-id={}'.format(subnet_id) + ) # Test the Create() command mock_cmd.side_effect = [(json.dumps(create_response), '', 0)] @@ -209,9 +237,10 @@ def testSubnetLifecycle(self, mock_cmd): mock_cmd.reset_mock() util.IssueRetryableCommand.reset_mock() mock_cmd.side_effect = [(json.dumps(delete_response), '', 0)] - util.IssueRetryableCommand.side_effect = [ - (json.dumps(exists_response_no_resources), '') - ] + util.IssueRetryableCommand.side_effect = [( + json.dumps(exists_response_no_resources), + '', + )] subnet.Delete() mock_cmd.assert_called_with(delete_cmd, raise_on_failure=False) util.IssueRetryableCommand.assert_called_with(describe_cmd) @@ -224,9 +253,10 @@ def testStaticVpc(self, mock_cmd): does_exist_res = {'Vpcs': [1]} does_not_exist_res = {'Vpcs': []} mock_cmd.return_value = json.dumps(security_group_res), '', 0 - util.IssueRetryableCommand.side_effect = [(json.dumps(does_exist_res), ''), - (json.dumps(does_not_exist_res), - '')] + util.IssueRetryableCommand.side_effect = [ + (json.dumps(does_exist_res), ''), + (json.dumps(does_not_exist_res), ''), + ] vpc = aws_network.AwsVpc('region', vpc_id) self.assertTrue(vpc.user_managed) # show Create() works @@ -242,11 +272,7 @@ def testNetworkSpecWithVpcId(self): subnet_id = 'subnet-1234' cidr = '10.0.0.0/24' res = { - 'Subnets': [{ - 'VpcId': vpc_id, - 'SubnetId': subnet_id, - 'CidrBlock': cidr - }] + 'Subnets': [{'VpcId': vpc_id, 'SubnetId': subnet_id, 'CidrBlock': cidr}] } util.IssueRetryableCommand.return_value = json.dumps(res), None spec = aws_network.AwsNetworkSpec(zone, vpc_id) @@ -256,8 +282,9 @@ def testNetworkSpecWithVpcId(self): self.assertEqual(cidr, spec.cidr_block) -class TestAwsVirtualMachine(pkb_common_test_case.TestOsMixin, - aws_virtual_machine.AwsVirtualMachine): +class TestAwsVirtualMachine( + pkb_common_test_case.TestOsMixin, aws_virtual_machine.AwsVirtualMachine +): pass @@ -266,12 +293,12 @@ def TestVmSpec(): 'test_vm_spec.AWS', zone='us-east-1a', machine_type='c3.large', - spot_price=123.45) + spot_price=123.45, + ) def OpenJSONData(filename): - path = os.path.join(os.path.dirname(__file__), - 'data', filename) + path = os.path.join(os.path.dirname(__file__), 'data', filename) with open(path) as f: return f.read() @@ -287,20 +314,22 @@ def setUp(self): FLAGS.cloud = provider_info.AWS FLAGS.run_uri = 'aaaaaa' FLAGS.temp_dir = 'tmp' - p = mock.patch('perfkitbenchmarker.providers.aws.' - 'util.IssueRetryableCommand') + p = mock.patch( + 'perfkitbenchmarker.providers.aws.util.IssueRetryableCommand' + ) p.start() self.addCleanup(p.stop) - p2 = mock.patch('perfkitbenchmarker.' - 'vm_util.IssueCommand') + p2 = mock.patch('perfkitbenchmarker.vm_util.IssueCommand') p2.start() self.addCleanup(p2.stop) # VM Creation depends on there being a BenchmarkSpec. config_spec = benchmark_config_spec.BenchmarkConfigSpec( - _BENCHMARK_NAME, flag_values=FLAGS, vm_groups={}) - self.spec = benchmark_spec.BenchmarkSpec(mock.MagicMock(), config_spec, - _BENCHMARK_UID) + _BENCHMARK_NAME, flag_values=FLAGS, vm_groups={} + ) + self.spec = benchmark_spec.BenchmarkSpec( + mock.MagicMock(), config_spec, _BENCHMARK_UID + ) self.addCleanup(context.SetThreadBenchmarkSpec, None) self.vm = CreateTestAwsVm() @@ -317,8 +346,7 @@ def setUp(self): self.vm.placement_group = placement_group self.response = OpenJSONData('aws-describe-instance.json') - self.sir_response = OpenJSONData( - 'aws-describe-spot-instance-requests.json') + self.sir_response = OpenJSONData('aws-describe-spot-instance-requests.json') self.vm.network.is_static = False self.vm.network.regional_network.vpc.default_security_group_id = 'sg-1234' @@ -387,7 +415,7 @@ def testInstancePresent(self): 'does not exist' ), 'expected_error': errors.Benchmarks.KnownIntermittentError, - } + }, ) def testVMCreationError(self, stderr, expected_error): vm_util.IssueCommand.side_effect = [('', '', 0)] @@ -401,14 +429,17 @@ def testVMCreationError(self, stderr, expected_error): { 'testcase_name': 'retry_rate_limited', 'retry_rate_limited': True, - 'expected_error': errors.Resource.RetryableCreationError - }, { + 'expected_error': errors.Resource.RetryableCreationError, + }, + { 'testcase_name': 'do_not_retry_rate_limited', 'retry_rate_limited': False, - 'expected_error': errors.Benchmarks.QuotaFailure - }) - def testInstanceRequestLimitExceededCreate(self, retry_rate_limited, - expected_error): + 'expected_error': errors.Benchmarks.QuotaFailure, + }, + ) + def testInstanceRequestLimitExceededCreate( + self, retry_rate_limited, expected_error + ): FLAGS.retry_on_rate_limited = retry_rate_limited stderr = ( 'An error occurred (RequestLimitExceeded) when calling the RunInstances' @@ -429,10 +460,14 @@ def testInstanceStockedOutAfterCreate(self): response = OpenJSONData('aws-describe-instance-stockout.json') util.IssueRetryableCommand.side_effect = [(response, None)] with self.assertRaises( - errors.Benchmarks.InsufficientCapacityCloudFailure) as e: + errors.Benchmarks.InsufficientCapacityCloudFailure + ) as e: self.vm._Exists() - self.assertEqual(str(e.exception), 'Server.InsufficientInstanceCapacity:' - ' Insufficient capacity to satisfy instance request') + self.assertEqual( + str(e.exception), + 'Server.InsufficientInstanceCapacity:' + ' Insufficient capacity to satisfy instance request', + ) def testInternalServerErrorAfterCreate(self): """This tests when run-instances succeeds and returns a pending instance. @@ -500,44 +535,55 @@ def testCreateSpot(self, mock_cmd): vm_util.IssueCommand.side_effect = [(None, '', None)] self.vm._Create() - vm_util.IssueCommand.assert_called_with([ - 'aws', - '--output', - 'json', - 'ec2', - 'run-instances', - '--region=us-east-1', - '--client-token=00000000-1111-2222-3333-444444444444', - '--image-id=ami-12345', - '--instance-type=c3.large', - '--key-name=perfkit-key-aaaaaa', - '--tag-specifications=foobar', - '--associate-public-ip-address', - '--subnet-id=subnet-id', - ('--placement=AvailabilityZone=us-east-1a,' - 'GroupName=placement_group_name'), - ('--instance-market-options={"MarketType": "spot", ' - '"SpotOptions": {"SpotInstanceType": "one-time", ' - '"InstanceInterruptionBehavior": "terminate", "MaxPrice": "123.45"}}') - ], - raise_on_failure=False) + vm_util.IssueCommand.assert_called_with( + [ + 'aws', + '--output', + 'json', + 'ec2', + 'run-instances', + '--region=us-east-1', + '--client-token=00000000-1111-2222-3333-444444444444', + '--image-id=ami-12345', + '--instance-type=c3.large', + '--key-name=perfkit-key-aaaaaa', + '--tag-specifications=foobar', + '--associate-public-ip-address', + '--subnet-id=subnet-id', + ( + '--placement=AvailabilityZone=us-east-1a,' + 'GroupName=placement_group_name' + ), + ( + '--instance-market-options={"MarketType": "spot",' + ' "SpotOptions": {"SpotInstanceType": "one-time",' + ' "InstanceInterruptionBehavior": "terminate", "MaxPrice":' + ' "123.45"}}' + ), + ], + raise_on_failure=False, + ) self.vm.use_spot_instance = False def testCreateSpotFailure(self): - stderr = ('An error occurred (InsufficientInstanceCapacity) when calling ' - 'the RunInstances operation (reached max retries: 4): ' - 'Insufficient capacity.') + stderr = ( + 'An error occurred (InsufficientInstanceCapacity) when calling ' + 'the RunInstances operation (reached max retries: 4): ' + 'Insufficient capacity.' + ) vm_util.IssueCommand.side_effect = [('', '', 0)] self.vm._CreateDependencies() vm_util.IssueCommand.side_effect = [(None, stderr, None)] with self.assertRaises( - errors.Benchmarks.InsufficientCapacityCloudFailure) as e: + errors.Benchmarks.InsufficientCapacityCloudFailure + ) as e: self.vm.use_spot_instance = True self.vm._Create() self.assertEqual(str(e.exception), stderr) - self.assertEqual(self.vm.spot_status_code, - 'InsufficientSpotInstanceCapacity') + self.assertEqual( + self.vm.spot_status_code, 'InsufficientSpotInstanceCapacity' + ) self.assertTrue(self.vm.spot_early_termination) self.vm.use_spot_instance = False @@ -547,14 +593,17 @@ def testDeleteCancelsSpotInstanceRequest(self): self.vm._Delete() vm_util.IssueCommand.assert_called_with( - ['aws', - '--output', - 'json', - '--region=us-east-1', - 'ec2', - 'cancel-spot-instance-requests', - '--spot-instance-request-ids=sir-abc'], - raise_on_failure=False) + [ + 'aws', + '--output', + 'json', + '--region=us-east-1', + 'ec2', + 'cancel-spot-instance-requests', + '--spot-instance-request-ids=sir-abc', + ], + raise_on_failure=False, + ) self.vm.use_spot_instance = False def testFirewallAllowPortNonStatic(self): @@ -562,9 +611,14 @@ def testFirewallAllowPortNonStatic(self): util.IssueRetryableCommand.return_value = json.dumps(check_firewall), '' self.vm.firewall.AllowPort(self.vm, 22) # only checking that the 2nd of the tcp/udp calls are done - cmd = _AwsCommand('us-east-1', 'authorize-security-group-ingress', - '--group-id=sg-1234', '--port=22-22', '--cidr=0.0.0.0/0', - '--protocol=udp') + cmd = _AwsCommand( + 'us-east-1', + 'authorize-security-group-ingress', + '--group-id=sg-1234', + '--port=22-22', + '--cidr=0.0.0.0/0', + '--protocol=udp', + ) util.IssueRetryableCommand.assert_called_with(cmd) def testFirewallAllowPortStaticVm(self): @@ -607,15 +661,19 @@ def testInstallEfaCentos(self): vm = InitCentosVm() vm._ConfigureElasticIp = mock.Mock() vm._PostCreate() - aws_cmd = '; '.join([ - cmd[0][0] for cmd in vm.RemoteHostCommandWithReturnCode.call_args_list - ]) + aws_cmd = '; '.join( + [cmd[0][0] for cmd in vm.RemoteHostCommandWithReturnCode.call_args_list] + ) self.assertRegex( - aws_cmd, '.*'.join([ - 'curl -O https://s3-us-west-2', 'yum upgrade -y kernel', - 'yum install -y kernel-devel', 'efa_installer.sh -y', - './efa_test.sh' - ])) + aws_cmd, + '.*'.join([ + 'curl -O https://s3-us-west-2', + 'yum upgrade -y kernel', + 'yum install -y kernel-devel', + 'efa_installer.sh -y', + './efa_test.sh', + ]), + ) vm.Reboot.assert_called_once() @@ -640,9 +698,7 @@ def InitCentosVm(): 'Instances': [{ 'PublicIpAddress': '10.0.0.1', 'PrivateIpAddress': '10.0.0.2', - 'SecurityGroups': [{ - 'GroupId': None - }] + 'SecurityGroups': [{'GroupId': None}], }] }] } @@ -692,13 +748,16 @@ def setUp(self): p.start() self.addCleanup(p.stop) config_spec = benchmark_config_spec.BenchmarkConfigSpec( - _BENCHMARK_NAME, flag_values=FLAGS, vm_groups={}) - self.spec = benchmark_spec.BenchmarkSpec(mock.MagicMock(), config_spec, - _BENCHMARK_UID) + _BENCHMARK_NAME, flag_values=FLAGS, vm_groups={} + ) + self.spec = benchmark_spec.BenchmarkSpec( + mock.MagicMock(), config_spec, _BENCHMARK_UID + ) self.aws_vm = CreateTestAwsVm() - path = os.path.join(os.path.dirname(__file__), - 'data', 'describe_image_output.txt') + path = os.path.join( + os.path.dirname(__file__), 'data', 'describe_image_output.txt' + ) with open(path) as fp: self.describe_image_output = fp.read() @@ -712,23 +771,30 @@ def testValidMachineTypeWithNoRootVolumeSize(self): self.assertIsNone(actual) def testValidMachineTypeWithSpecifiedRootVolumeSize(self): - util.IssueRetryableCommand.side_effect = [(self.describe_image_output, - None)] + util.IssueRetryableCommand.side_effect = [( + self.describe_image_output, + None, + )] self.aws_vm.boot_disk_size = 35 self.aws_vm.machine_type = 'c1.medium' self.aws_vm.image = 'ami-a9d276c9' self.aws_vm.region = 'us-west-2' - expected = [{'DeviceName': '/dev/sda1', - 'Ebs': {'SnapshotId': 'snap-826344d5', - 'DeleteOnTermination': True, - 'VolumeType': 'gp2', - 'VolumeSize': 35}}] + expected = [{ + 'DeviceName': '/dev/sda1', + 'Ebs': { + 'SnapshotId': 'snap-826344d5', + 'DeleteOnTermination': True, + 'VolumeType': 'gp2', + 'VolumeSize': 35, + }, + }] actual = json.loads(aws_virtual_machine.GetBlockDeviceMap(self.aws_vm)) self.assertEqual(actual, expected) class AwsGetRootBlockDeviceSpecForImageTestCase( - pkb_common_test_case.PkbCommonTestCase): + pkb_common_test_case.PkbCommonTestCase +): def setUp(self): super(AwsGetRootBlockDeviceSpecForImageTestCase, self).setUp() @@ -736,14 +802,17 @@ def setUp(self): p.start() self.addCleanup(p.stop) - path = os.path.join(os.path.dirname(__file__), - 'data', 'describe_image_output.txt') + path = os.path.join( + os.path.dirname(__file__), 'data', 'describe_image_output.txt' + ) with open(path) as fp: self.describe_image_output = fp.read() def testOk(self): - util.IssueRetryableCommand.side_effect = [(self.describe_image_output, - None)] + util.IssueRetryableCommand.side_effect = [( + self.describe_image_output, + None, + )] image_id = 'ami-a9d276c9' region = 'us-west-2' expected = { @@ -753,11 +822,12 @@ def testOk(self): 'DeleteOnTermination': True, 'VolumeType': 'gp2', 'VolumeSize': 8, - 'Encrypted': False - } + 'Encrypted': False, + }, } - actual = aws_virtual_machine.GetRootBlockDeviceSpecForImage(image_id, - region) + actual = aws_virtual_machine.GetRootBlockDeviceSpecForImage( + image_id, region + ) self.assertEqual(actual, expected) @@ -767,10 +837,14 @@ class AwsKeyFileManagerTestCase(pkb_common_test_case.PkbCommonTestCase): @mock.patch.object(vm_util, 'IssueCommand') def testKeyPairLimitExceeded(self, import_cmd, cat_cmd): cat_cmd.side_effect = [('key_content', None)] - import_cmd.side_effect = [('', ( - 'An error occurred (KeyPairLimitExceeded) when calling the ' - 'ImportKeyPair operation: Maximum of 5000 keypairs reached.' - ), 255)] + import_cmd.side_effect = [( + '', + ( + 'An error occurred (KeyPairLimitExceeded) when calling the ' + 'ImportKeyPair operation: Maximum of 5000 keypairs reached.' + ), + 255, + )] with self.assertRaises(errors.Benchmarks.QuotaFailure): aws_virtual_machine.AwsKeyFileManager.ImportKeyfile('region') diff --git a/tests/background_cpu_test.py b/tests/background_cpu_test.py index 5378bb97fb..b66ca20c40 100644 --- a/tests/background_cpu_test.py +++ b/tests/background_cpu_test.py @@ -20,7 +20,6 @@ from absl import flags import contextlib2 import mock - from perfkitbenchmarker import benchmark_spec from perfkitbenchmarker import configs from perfkitbenchmarker import context @@ -70,32 +69,51 @@ def setUp(self): def _CreateBenchmarkSpec(self, benchmark_config_yaml): config = configs.LoadConfig(benchmark_config_yaml, {}, NAME) config_spec = benchmark_config_spec.BenchmarkConfigSpec( - NAME, flag_values=FLAGS, **config) + NAME, flag_values=FLAGS, **config + ) return benchmark_spec.BenchmarkSpec(ping_benchmark, config_spec, UID) - def _CheckVmCallCounts(self, spec, working_groups, working_expected_counts, - non_working_groups, non_working_expected_counts): + def _CheckVmCallCounts( + self, + spec, + working_groups, + working_expected_counts, + non_working_groups, + non_working_expected_counts, + ): # TODO(user): This is also used in TestBackgroundNetworkWorkload. # Consider moving to a shared function or base class. - expected_call_counts = {group: working_expected_counts - for group in working_groups} - expected_call_counts.update({group: non_working_expected_counts - for group in non_working_groups}) + expected_call_counts = { + group: working_expected_counts for group in working_groups + } + expected_call_counts.update( + {group: non_working_expected_counts for group in non_working_groups} + ) for group_name, vm_expected_call_counts in six.iteritems( - expected_call_counts): + expected_call_counts + ): group_vms = spec.vm_groups[group_name] - self.assertEqual(len(group_vms), 1, - msg='VM group "{0}" had {1} VMs'.format(group_name, - len(group_vms))) + self.assertEqual( + len(group_vms), + 1, + msg='VM group "{0}" had {1} VMs'.format(group_name, len(group_vms)), + ) vm = group_vms[0] - iter_mocked_functions = zip_longest(_MOCKED_VM_FUNCTIONS, - vm_expected_call_counts) + iter_mocked_functions = zip_longest( + _MOCKED_VM_FUNCTIONS, vm_expected_call_counts + ) for function_name, expected_call_count in iter_mocked_functions: call_count = getattr(vm, function_name).call_count - self.assertEqual(call_count, expected_call_count, msg=( - 'Expected {0} from VM group "{1}" to be called {2} times, but it ' - 'was called {3} times.'.format(function_name, group_name, - expected_call_count, call_count))) + self.assertEqual( + call_count, + expected_call_count, + msg=( + 'Expected {0} from VM group "{1}" to be called {2} times, but' + ' it was called {3} times.'.format( + function_name, group_name, expected_call_count, call_count + ) + ), + ) def _CheckVMFromSpec(self, spec, working_groups=(), non_working_groups=()): with contextlib2.ExitStack() as stack: @@ -171,8 +189,9 @@ def testBackgroundWorkloadConfig(self): self.assertIsNone(vm.background_cpu_threads) for vm in spec.vm_groups[_GROUP_2]: self.assertEqual(vm.background_cpu_threads, 3) - self._CheckVMFromSpec(spec, working_groups=[_GROUP_2], - non_working_groups=[_GROUP_1]) + self._CheckVMFromSpec( + spec, working_groups=[_GROUP_2], non_working_groups=[_GROUP_1] + ) if __name__ == '__main__': diff --git a/tests/background_network_workload_test.py b/tests/background_network_workload_test.py index aea375761f..28c453d2e9 100644 --- a/tests/background_network_workload_test.py +++ b/tests/background_network_workload_test.py @@ -20,7 +20,6 @@ from absl import flags import contextlib2 import mock - from perfkitbenchmarker import benchmark_spec from perfkitbenchmarker import configs from perfkitbenchmarker import context @@ -99,29 +98,47 @@ def setUp(self): FLAGS.temp_dir = 'tmp' self.addCleanup(context.SetThreadBenchmarkSpec, None) - def _CheckVmCallCounts(self, spec, working_groups, working_expected_counts, - non_working_groups, non_working_expected_counts): + def _CheckVmCallCounts( + self, + spec, + working_groups, + working_expected_counts, + non_working_groups, + non_working_expected_counts, + ): # TODO(user): This is also used in TestBackgroundNetworkWorkload. # Consider moving to a shared function or base class. - expected_call_counts = {group: working_expected_counts - for group in working_groups} - expected_call_counts.update({group: non_working_expected_counts - for group in non_working_groups}) + expected_call_counts = { + group: working_expected_counts for group in working_groups + } + expected_call_counts.update( + {group: non_working_expected_counts for group in non_working_groups} + ) for group_name, vm_expected_call_counts in six.iteritems( - expected_call_counts): + expected_call_counts + ): group_vms = spec.vm_groups[group_name] - self.assertEqual(len(group_vms), 1, - msg='VM group "{0}" had {1} VMs'.format(group_name, - len(group_vms))) + self.assertEqual( + len(group_vms), + 1, + msg='VM group "{0}" had {1} VMs'.format(group_name, len(group_vms)), + ) vm = group_vms[0] - iter_mocked_functions = zip_longest(_MOCKED_VM_FUNCTIONS, - vm_expected_call_counts) + iter_mocked_functions = zip_longest( + _MOCKED_VM_FUNCTIONS, vm_expected_call_counts + ) for function_name, expected_call_count in iter_mocked_functions: call_count = getattr(vm, function_name).call_count - self.assertEqual(call_count, expected_call_count, msg=( - 'Expected {0} from VM group "{1}" to be called {2} times, but it ' - 'was called {3} times.'.format(function_name, group_name, - expected_call_count, call_count))) + self.assertEqual( + call_count, + expected_call_count, + msg=( + 'Expected {0} from VM group "{1}" to be called {2} times, but' + ' it was called {3} times.'.format( + function_name, group_name, expected_call_count, call_count + ) + ), + ) def _CheckVMFromSpec(self, spec, working_groups=(), non_working_groups=()): with contextlib2.ExitStack() as stack: @@ -145,7 +162,8 @@ def _CheckVMFromSpec(self, spec, working_groups=(), non_working_groups=()): def makeSpec(self, yaml_benchmark_config=ping_benchmark.BENCHMARK_CONFIG): config = configs.LoadConfig(yaml_benchmark_config, {}, NAME) config_spec = benchmark_config_spec.BenchmarkConfigSpec( - NAME, flag_values=FLAGS, **config) + NAME, flag_values=FLAGS, **config + ) spec = benchmark_spec.BenchmarkSpec(ping_benchmark, config_spec, UID) spec.ConstructVirtualMachines() return spec @@ -212,35 +230,39 @@ def testBackgroundWorkloadVanillaConfigBadIpTypeFlag(self): def testBackgroundWorkloadConfig(self): """Check that the config can be used to set the background iperf.""" - spec = self.makeSpec( - yaml_benchmark_config=CONFIG_WITH_BACKGROUND_NETWORK) + spec = self.makeSpec(yaml_benchmark_config=CONFIG_WITH_BACKGROUND_NETWORK) for vm in spec.vm_groups[_GROUP_1]: self.assertIsNone(vm.background_network_mbits_per_sec) for vm in spec.vm_groups[_GROUP_2]: self.assertEqual(vm.background_network_mbits_per_sec, 300) self.assertEqual(vm.background_network_ip_type, 'EXTERNAL') - self._CheckVMFromSpec(spec, working_groups=[_GROUP_2], - non_working_groups=[_GROUP_1]) + self._CheckVMFromSpec( + spec, working_groups=[_GROUP_2], non_working_groups=[_GROUP_1] + ) def testBackgroundWorkloadConfigBadIp(self): """Check that the config with invalid ip type gets an error.""" - config = configs.LoadConfig(CONFIG_WITH_BACKGROUND_NETWORK_BAD_IPFLAG, - {}, NAME) + config = configs.LoadConfig( + CONFIG_WITH_BACKGROUND_NETWORK_BAD_IPFLAG, {}, NAME + ) with self.assertRaises(errors.Config.InvalidValue): benchmark_config_spec.BenchmarkConfigSpec( - NAME, flag_values=FLAGS, **config) + NAME, flag_values=FLAGS, **config + ) def testBackgroundWorkloadConfigGoodIp(self): """Check that the config can be used with an internal ip address.""" spec = self.makeSpec( - yaml_benchmark_config=CONFIG_WITH_BACKGROUND_NETWORK_IPFLAG) + yaml_benchmark_config=CONFIG_WITH_BACKGROUND_NETWORK_IPFLAG + ) for vm in spec.vm_groups[_GROUP_1]: self.assertIsNone(vm.background_network_mbits_per_sec) for vm in spec.vm_groups[_GROUP_2]: self.assertEqual(vm.background_network_mbits_per_sec, 300) self.assertEqual(vm.background_network_ip_type, 'INTERNAL') - self._CheckVMFromSpec(spec, working_groups=[_GROUP_2], - non_working_groups=[_GROUP_1]) + self._CheckVMFromSpec( + spec, working_groups=[_GROUP_2], non_working_groups=[_GROUP_1] + ) if __name__ == '__main__': diff --git a/tests/background_tasks_test.py b/tests/background_tasks_test.py index 26c1c5721e..75be9517c9 100644 --- a/tests/background_tasks_test.py +++ b/tests/background_tasks_test.py @@ -49,7 +49,7 @@ def _WaitAndAppendInt(int_list, int_to_append, event=None, timeout=None): int_list.append(int_to_append) -class Counter(): +class Counter: def __init__(self): self.value = 0 @@ -70,16 +70,18 @@ def testKwargs(self): self.assertEqual(result, '_ReturnArgs(x=8)') def testArgsAndKwargs(self): - result = background_tasks._GetCallString((_ReturnArgs, ('blue', 5), - {'x': 8})) + result = background_tasks._GetCallString( + (_ReturnArgs, ('blue', 5), {'x': 8}) + ) self.assertEqual(result, '_ReturnArgs(blue, 5, x=8)') def testSinglePartial(self): _ReturnArgs2 = functools.partial(_ReturnArgs, 1, x=2) result = background_tasks._GetCallString((_ReturnArgs2, (), {})) self.assertEqual(result, '_ReturnArgs(1, x=2)') - result = background_tasks._GetCallString((_ReturnArgs2, ('blue', 5), - {'x': 8})) + result = background_tasks._GetCallString( + (_ReturnArgs2, ('blue', 5), {'x': 8}) + ) self.assertEqual(result, '_ReturnArgs(1, blue, 5, x=8)') def testDoublePartial(self): @@ -87,8 +89,9 @@ def testDoublePartial(self): _ReturnArgs3 = functools.partial(_ReturnArgs2, 3, x=4) result = background_tasks._GetCallString((_ReturnArgs3, (), {})) self.assertEqual(result, '_ReturnArgs(1, 3, x=4)') - result = background_tasks._GetCallString((_ReturnArgs3, ('blue', 5), - {'x': 8})) + result = background_tasks._GetCallString( + (_ReturnArgs3, ('blue', 5), {'x': 8}) + ) self.assertEqual(result, '_ReturnArgs(1, 3, blue, 5, x=8)') @@ -106,8 +109,11 @@ def testMoreThreadsThanConcurrencyLimit(self): def testException(self): int_list = [] - calls = [(_AppendLength, (int_list,), {}), (_RaiseValueError, (), {}), - (_AppendLength, (int_list,), {})] + calls = [ + (_AppendLength, (int_list,), {}), + (_RaiseValueError, (), {}), + (_AppendLength, (int_list,), {}), + ] with self.assertRaises(errors.VmUtil.ThreadException): background_tasks.RunParallelThreads(calls, max_concurrency=1) self.assertEqual(int_list, [0, 1]) @@ -123,10 +129,12 @@ def testInterrupt(self): # thread 1 nor 3 is able to append to int_list. int_list = [] event = threading.Event() - calls = [(_WaitAndAppendInt, (int_list, 0, event, 5), {}), - (_WaitAndAppendInt, (int_list, 1), {}), - (os.kill, (os.getpid(), signal.SIGINT), {}), - (_WaitAndAppendInt, (int_list, 3, event, 5), {})] + calls = [ + (_WaitAndAppendInt, (int_list, 0, event, 5), {}), + (_WaitAndAppendInt, (int_list, 1), {}), + (os.kill, (os.getpid(), signal.SIGINT), {}), + (_WaitAndAppendInt, (int_list, 3, event, 5), {}), + ] with self.assertRaises(KeyboardInterrupt): background_tasks.RunParallelThreads(calls, max_concurrency=2) self.assertEqual(int_list, [1]) @@ -152,7 +160,8 @@ def testSimpleListParams(self): def testListOfTupleParams(self): result = background_tasks.RunThreaded( - _ReturnArgs, [(('red',), {}), (('green',), {'b': 'blue'})]) + _ReturnArgs, [(('red',), {}), (('green',), {'b': 'blue'})] + ) self.assertEqual(result, [(None, 'red'), ('blue', 'green')]) @@ -170,9 +179,11 @@ def testMoreThreadsThanConcurrencyLimit(self): def testException(self): counter = Counter() - calls = [(_IncrementCounter, (counter,), {}), - (_RaiseValueError, (), {}), - (_IncrementCounter, (counter,), {})] + calls = [ + (_IncrementCounter, (counter,), {}), + (_RaiseValueError, (), {}), + (_IncrementCounter, (counter,), {}), + ] with self.assertRaises(errors.VmUtil.CalledProcessException): background_tasks.RunParallelProcesses(calls, max_concurrency=1) diff --git a/tests/background_workload_framework_test.py b/tests/background_workload_framework_test.py index 9fbeb931cb..ab3db3ead9 100644 --- a/tests/background_workload_framework_test.py +++ b/tests/background_workload_framework_test.py @@ -18,7 +18,6 @@ import unittest from absl import flags import mock - from perfkitbenchmarker import benchmark_spec from perfkitbenchmarker import configs from perfkitbenchmarker import context @@ -53,7 +52,8 @@ def testBackgroundWorkloadSpec(self): config = configs.LoadConfig(ping_benchmark.BENCHMARK_CONFIG, {}, NAME) config_spec = benchmark_config_spec.BenchmarkConfigSpec( - NAME, flag_values=FLAGS, **config) + NAME, flag_values=FLAGS, **config + ) spec = benchmark_spec.BenchmarkSpec(ping_benchmark, config_spec, UID) vm0 = mock.MagicMock() vm1 = mock.MagicMock() @@ -68,7 +68,8 @@ def testBackgroundWorkloadSpec(self): with mock.patch(ping_benchmark.__name__ + '.Run'): vm0.StopBackgroundWorkload.side_effect = functools.partial( - self._CheckAndIncrement, expected_last_call=0) + self._CheckAndIncrement, expected_last_call=0 + ) pkb.DoCleanupPhase(spec, timer) for vm in spec.vms: self.assertEqual(vm.StartBackgroundWorkload.call_count, 1) diff --git a/tests/beam_benchmark_helper_test.py b/tests/beam_benchmark_helper_test.py index 432fdae7c5..f9e4c54d07 100644 --- a/tests/beam_benchmark_helper_test.py +++ b/tests/beam_benchmark_helper_test.py @@ -18,7 +18,6 @@ import unittest from absl import flags import mock - from perfkitbenchmarker import beam_benchmark_helper from perfkitbenchmarker import dpb_constants from perfkitbenchmarker import vm_util @@ -37,30 +36,34 @@ def setUp(self): def test_runner_option_override_use_override(self): test_option_val = 'TestVal' actual_options = [] - beam_benchmark_helper.AddRunnerPipelineOption(actual_options, None, - test_option_val) + beam_benchmark_helper.AddRunnerPipelineOption( + actual_options, None, test_option_val + ) self.assertListEqual(['--runner=' + test_option_val], actual_options) def test_runner_option_override_empty_override(self): test_option_val = '' actual_options = [] - beam_benchmark_helper.AddRunnerPipelineOption(actual_options, None, - test_option_val) + beam_benchmark_helper.AddRunnerPipelineOption( + actual_options, None, test_option_val + ) self.assertListEqual([], actual_options) def test_dataflow_runner_name_added(self): test_option_val = 'dataflow' actual_command = [] beam_benchmark_helper.AddRunnerArgument(actual_command, test_option_val) - self.assertListEqual(['-DintegrationTestRunner=' + test_option_val], - actual_command) + self.assertListEqual( + ['-DintegrationTestRunner=' + test_option_val], actual_command + ) def test_direct_runner_name_added(self): test_option_val = 'direct' actual_command = [] beam_benchmark_helper.AddRunnerArgument(actual_command, test_option_val) - self.assertListEqual(['-DintegrationTestRunner=' + test_option_val], - actual_command) + self.assertListEqual( + ['-DintegrationTestRunner=' + test_option_val], actual_command + ) def test_runner_name_empty(self): test_option_val = '' @@ -77,30 +80,34 @@ def test_extra_property_empty_property(self): def test_extra_property_single_property(self): test_option_val = '[key=value]' actual_mvn_command = [] - beam_benchmark_helper.AddExtraProperties(actual_mvn_command, - test_option_val) + beam_benchmark_helper.AddExtraProperties( + actual_mvn_command, test_option_val + ) self.assertListEqual(['-Dkey=value'], actual_mvn_command) def test_extra_property_single_property_quoted(self): test_option_val = '["key=value"]' actual_mvn_command = [] - beam_benchmark_helper.AddExtraProperties(actual_mvn_command, - test_option_val) + beam_benchmark_helper.AddExtraProperties( + actual_mvn_command, test_option_val + ) self.assertListEqual(['-Dkey=value'], actual_mvn_command) def test_extra_property_multiple_properties(self): test_option_val = '["key=value", "key2=value2"]' actual_mvn_command = [] - beam_benchmark_helper.AddExtraProperties(actual_mvn_command, - test_option_val) + beam_benchmark_helper.AddExtraProperties( + actual_mvn_command, test_option_val + ) self.assertListEqual(['-Dkey=value', '-Dkey2=value2'], actual_mvn_command) def test_integrationPipelineOptions_rejection(self): test_option_val = '["integrationTestPipelineOptions=..."]' actual_mvn_command = [] with self.assertRaises(ValueError): - beam_benchmark_helper.AddExtraProperties(actual_mvn_command, - test_option_val) + beam_benchmark_helper.AddExtraProperties( + actual_mvn_command, test_option_val + ) def test_hdfs_filesystem_addition(self): test_option_val = 'hdfs' @@ -118,23 +125,25 @@ def test_add_task(self): test_module_val = ':sdks:java:io' test_task_val = 'tests' actual_command = [] - beam_benchmark_helper.AddTaskArgument(actual_command, test_task_val, - test_module_val) + beam_benchmark_helper.AddTaskArgument( + actual_command, test_task_val, test_module_val + ) self.assertListEqual([':sdks:java:io:tests'], actual_command) def test_add_empty_task(self): test_option_val = '' actual_command = [] with self.assertRaises(ValueError): - beam_benchmark_helper.AddTaskArgument(actual_command, test_option_val, - test_option_val) + beam_benchmark_helper.AddTaskArgument( + actual_command, test_option_val, test_option_val + ) def test_initialize_beam_repo_beam_exists(self): FLAGS.beam_location = tempfile.mkdtemp() - with mock.patch.object(beam_benchmark_helper, '_PrebuildBeam') as mock_prebuild, \ - mock.patch.object(vm_util, 'GenTempDir'): - + with mock.patch.object( + beam_benchmark_helper, '_PrebuildBeam' + ) as mock_prebuild, mock.patch.object(vm_util, 'GenTempDir'): mock_spec = mock.MagicMock() mock_spec.dpb_service.SERVICE_TYPE = dpb_constants.DATAFLOW @@ -144,19 +153,22 @@ def test_initialize_beam_repo_beam_exists(self): def test_initialize_beam_repo_beam_not_exists(self): FLAGS.beam_location = None - with mock.patch.object(beam_benchmark_helper, '_PrebuildBeam') as mock_prebuild, \ - mock.patch.object(vm_util, 'GenTempDir'), \ - mock.patch.object(vm_util, 'GetTempDir'), \ - mock.patch.object(vm_util, 'IssueCommand') as mock_run: - + with mock.patch.object( + beam_benchmark_helper, '_PrebuildBeam' + ) as mock_prebuild, mock.patch.object( + vm_util, 'GenTempDir' + ), mock.patch.object( + vm_util, 'GetTempDir' + ), mock.patch.object( + vm_util, 'IssueCommand' + ) as mock_run: mock_spec = mock.MagicMock() mock_spec.dpb_service.SERVICE_TYPE = dpb_constants.DATAFLOW beam_benchmark_helper.InitializeBeamRepo(mock_spec) expected_cmd = ['git', 'clone', 'https://github.com/apache/beam.git'] - mock_run.assert_called_once_with(expected_cmd, - cwd=vm_util.GetTempDir()) + mock_run.assert_called_once_with(expected_cmd, cwd=vm_util.GetTempDir()) mock_prebuild.assert_called_once() def test_beam_prebuild(self): @@ -166,10 +178,13 @@ def test_beam_prebuild(self): FLAGS.beam_filesystem = 'hdfs' FLAGS.beam_extra_properties = '[extra_key=extra_value]' - with mock.patch.object(beam_benchmark_helper, '_GetGradleCommand') as mock_gradle, \ - mock.patch.object(beam_benchmark_helper, '_GetBeamDir'), \ - mock.patch.object(vm_util, 'IssueCommand') as mock_run: - + with mock.patch.object( + beam_benchmark_helper, '_GetGradleCommand' + ) as mock_gradle, mock.patch.object( + beam_benchmark_helper, '_GetBeamDir' + ), mock.patch.object( + vm_util, 'IssueCommand' + ) as mock_run: mock_gradle.return_value = 'gradlew' beam_benchmark_helper._PrebuildBeam() @@ -181,11 +196,11 @@ def test_beam_prebuild(self): ':sdks:java:assemble', '-DintegrationTestRunner=dataflow', '-Dfilesystem=hdfs', - '-Dextra_key=extra_value' + '-Dextra_key=extra_value', ] - mock_run.assert_called_once_with(expected_cmd, - cwd=beam_benchmark_helper._GetBeamDir(), - timeout=1500) + mock_run.assert_called_once_with( + expected_cmd, cwd=beam_benchmark_helper._GetBeamDir(), timeout=1500 + ) def test_build_python_gradle_command(self): FLAGS.beam_python_attr = 'IT' @@ -194,24 +209,29 @@ def test_build_python_gradle_command(self): FLAGS.beam_python_sdk_location = 'py/location.tar' FLAGS.beam_sdk = beam_benchmark_helper.BEAM_PYTHON_SDK - with mock.patch.object(beam_benchmark_helper, '_GetGradleCommand') as mock_gradle, \ - mock.patch.object(beam_benchmark_helper, '_GetBeamDir'), \ - mock.patch.object(vm_util, 'ExecutableOnPath', return_value=True) as exec_check: - + with mock.patch.object( + beam_benchmark_helper, '_GetGradleCommand' + ) as mock_gradle, mock.patch.object( + beam_benchmark_helper, '_GetBeamDir' + ), mock.patch.object( + vm_util, 'ExecutableOnPath', return_value=True + ) as exec_check: mock_gradle.return_value = 'gradlew' mock_spec = mock.MagicMock() mock_spec.service_type = dpb_constants.DATAFLOW - actual_cmd, _ = beam_benchmark_helper.BuildBeamCommand(mock_spec, - 'apache_beam.py', - ['--args']) + actual_cmd, _ = beam_benchmark_helper.BuildBeamCommand( + mock_spec, 'apache_beam.py', ['--args'] + ) expected_cmd = [ 'gradlew', ':sdks:python:integrationTest', '-Dtests=apache_beam.py', '-Dattr=IT', - '-DpipelineOptions=--args "--runner=TestRunner" ' - '"--sdk_location=py/location.tar"', + ( + '-DpipelineOptions=--args "--runner=TestRunner" ' + '"--sdk_location=py/location.tar"' + ), '--info', '--scan', ] @@ -225,16 +245,20 @@ def test_build_java_gradle_command(self): FLAGS.beam_extra_properties = '["extra_key=extra_value"]' FLAGS.beam_sdk = beam_benchmark_helper.BEAM_JAVA_SDK - with mock.patch.object(beam_benchmark_helper, '_GetGradleCommand') as mock_gradle, \ - mock.patch.object(beam_benchmark_helper, '_GetBeamDir'), \ - mock.patch.object(vm_util, 'ExecutableOnPath', return_value=True) as exec_check: - + with mock.patch.object( + beam_benchmark_helper, '_GetGradleCommand' + ) as mock_gradle, mock.patch.object( + beam_benchmark_helper, '_GetBeamDir' + ), mock.patch.object( + vm_util, 'ExecutableOnPath', return_value=True + ) as exec_check: mock_gradle.return_value = 'gradlew' mock_spec = mock.MagicMock() mock_spec.service_type = dpb_constants.DATAFLOW actual_cmd, _ = beam_benchmark_helper.BuildBeamCommand( - mock_spec, 'org.apache.beam.sdk.java', ['--args']) + mock_spec, 'org.apache.beam.sdk.java', ['--args'] + ) expected_cmd = [ 'gradlew', ':sdks:java:integrationTest', diff --git a/tests/benchmark_sets_test.py b/tests/benchmark_sets_test.py index d800d1c4ee..ed26448593 100644 --- a/tests/benchmark_sets_test.py +++ b/tests/benchmark_sets_test.py @@ -17,13 +17,11 @@ import unittest from absl import flags import mock - from perfkitbenchmarker import benchmark_sets from perfkitbenchmarker import configs from perfkitbenchmarker import linux_benchmarks # This import to ensure required FLAGS are defined. from perfkitbenchmarker import pkb # NOQA - import six import yaml @@ -48,7 +46,7 @@ {'machine_type': 'n1-standard-1', 'zones': 'us-central1-a'}, {'machine_type': 'n1-standard-1', 'zones': 'us-central1-b'}, {'machine_type': 'n1-standard-4', 'zones': 'us-central1-a'}, - {'machine_type': 'n1-standard-4', 'zones': 'us-central1-b'} + {'machine_type': 'n1-standard-4', 'zones': 'us-central1-b'}, ] ZIP_CONFIG_DIFFERENT_AXES_LENGTH = """ netperf: @@ -72,7 +70,7 @@ """ EXPECTED_ZIP_FLAGS = [ {'machine_type': 'n1-standard-4', 'gpu_count': 1, 'gpu_type': 'k80'}, - {'machine_type': 'n1-standard-8', 'gpu_count': 2, 'gpu_type': 'k80'} + {'machine_type': 'n1-standard-8', 'gpu_count': 2, 'gpu_type': 'k80'}, ] SINGLE_ZIP_CONFIG = """ netperf: @@ -85,7 +83,7 @@ """ EXPECTED_SINGLE_ZIP_FLAGS = [ {'machine_type': 'n1-standard-4', 'gpu_type': 'k80'}, - {'machine_type': 'n1-standard-8', 'gpu_type': 'k80'} + {'machine_type': 'n1-standard-8', 'gpu_type': 'k80'}, ] ZIP_AND_MATRIX_CONFIG = """ netperf: @@ -102,14 +100,30 @@ zones: [us-central1-a, us-central1-b] """ EXPECTED_ZIP_AND_MATRIX_FLAGS = [ - {'zones': 'us-central1-a', 'gpu_type': 'k80', - 'machine_type': 'n1-standard-4', 'gpu_count': 1}, - {'zones': 'us-central1-b', 'gpu_type': 'k80', - 'machine_type': 'n1-standard-4', 'gpu_count': 1}, - {'zones': 'us-central1-b', 'gpu_type': 'k80', - 'machine_type': 'n1-standard-8', 'gpu_count': 2}, - {'zones': 'us-central1-a', 'gpu_type': 'k80', - 'machine_type': 'n1-standard-8', 'gpu_count': 2} + { + 'zones': 'us-central1-a', + 'gpu_type': 'k80', + 'machine_type': 'n1-standard-4', + 'gpu_count': 1, + }, + { + 'zones': 'us-central1-b', + 'gpu_type': 'k80', + 'machine_type': 'n1-standard-4', + 'gpu_count': 1, + }, + { + 'zones': 'us-central1-b', + 'gpu_type': 'k80', + 'machine_type': 'n1-standard-8', + 'gpu_count': 2, + }, + { + 'zones': 'us-central1-a', + 'gpu_type': 'k80', + 'machine_type': 'n1-standard-8', + 'gpu_count': 2, + }, ] FILTER_CONFIG = """ netperf: @@ -168,15 +182,17 @@ def setUp(self): def testStandardSet(self): self.assertIn(benchmark_sets.STANDARD_SET, benchmark_sets.BENCHMARK_SETS) - standard_set = (benchmark_sets.BENCHMARK_SETS[ - benchmark_sets.STANDARD_SET])[benchmark_sets.BENCHMARK_LIST] + standard_set = (benchmark_sets.BENCHMARK_SETS[benchmark_sets.STANDARD_SET])[ + benchmark_sets.BENCHMARK_LIST + ] self.assertIn('iperf', standard_set) self.assertIn('fio', standard_set) def testBenchmarkSetsHaveValidNames(self): # check all the benchmark sets to make sure they contain valid names - valid_benchmark_and_set_names = (self.valid_benchmark_names | - self.valid_benchmark_set_names) + valid_benchmark_and_set_names = ( + self.valid_benchmark_names | self.valid_benchmark_set_names + ) benchmark_set_items = list(benchmark_sets.BENCHMARK_SETS.items()) for _, key_value in benchmark_set_items: benchmark_def_list = key_value[benchmark_sets.BENCHMARK_LIST] @@ -187,19 +203,22 @@ def testBenchmarkDerivedSets(self): # make sure that sets which are derived from the standard_set # expands into a valid set of benchmarks with mock.patch.dict( - benchmark_sets.BENCHMARK_SETS, { + benchmark_sets.BENCHMARK_SETS, + { 'test_derived_set': { benchmark_sets.MESSAGE: 'test derived benchmark set.', - benchmark_sets.BENCHMARK_LIST: [benchmark_sets.STANDARD_SET] + benchmark_sets.BENCHMARK_LIST: [benchmark_sets.STANDARD_SET], } - }): + }, + ): self.mock_flags.benchmarks = ['test_derived_set'] benchmark_tuple_list = benchmark_sets.GetBenchmarksFromFlags() self.assertIsNotNone(benchmark_tuple_list) self.assertGreater(len(benchmark_tuple_list), 0) for benchmark_tuple in benchmark_tuple_list: - self.assertIn(benchmark_tuple[0].BENCHMARK_NAME, - self.valid_benchmark_names) + self.assertIn( + benchmark_tuple[0].BENCHMARK_NAME, self.valid_benchmark_names + ) def testBenchmarkNestedDerivedSets(self): # make sure that sets which are derived from the standard_set @@ -207,24 +226,27 @@ def testBenchmarkNestedDerivedSets(self): self.mock_flags.benchmarks = [benchmark_sets.STANDARD_SET] standard_module_list = benchmark_sets.GetBenchmarksFromFlags() with mock.patch.dict( - benchmark_sets.BENCHMARK_SETS, { + benchmark_sets.BENCHMARK_SETS, + { 'test_derived_set': { benchmark_sets.MESSAGE: 'test derived benchmark set.', - benchmark_sets.BENCHMARK_LIST: [benchmark_sets.STANDARD_SET] + benchmark_sets.BENCHMARK_LIST: [benchmark_sets.STANDARD_SET], }, 'test_nested_derived_set': { benchmark_sets.MESSAGE: 'test nested derived benchmark set.', - benchmark_sets.BENCHMARK_LIST: ['test_derived_set'] - } - }): + benchmark_sets.BENCHMARK_LIST: ['test_derived_set'], + }, + }, + ): # TODO(voellm): better check would be to make sure both lists are the same benchmark_tuple_list = benchmark_sets.GetBenchmarksFromFlags() self.assertIsNotNone(benchmark_tuple_list) self.assertIsNotNone(standard_module_list) self.assertEqual(len(benchmark_tuple_list), len(standard_module_list)) for benchmark_tuple in benchmark_tuple_list: - self.assertIn(benchmark_tuple[0].BENCHMARK_NAME, - self.valid_benchmark_names) + self.assertIn( + benchmark_tuple[0].BENCHMARK_NAME, self.valid_benchmark_names + ) def testBenchmarkValidCommandLine1(self): # make sure the standard_set expands to a valid set of benchmarks @@ -233,8 +255,9 @@ def testBenchmarkValidCommandLine1(self): self.assertIsNotNone(benchmark_tuple_list) self.assertGreater(len(benchmark_tuple_list), 0) for benchmark_tuple in benchmark_tuple_list: - self.assertIn(benchmark_tuple[0].BENCHMARK_NAME, - self.valid_benchmark_names) + self.assertIn( + benchmark_tuple[0].BENCHMARK_NAME, self.valid_benchmark_names + ) @staticmethod def _ContainsModule(module_name, module_list): @@ -251,11 +274,13 @@ def testBenchmarkValidCommandLine2(self): self.assertIsNotNone(benchmark_tuple_list) self.assertGreater(len(benchmark_tuple_list), 0) for benchmark_tuple in benchmark_tuple_list: - self.assertIn(benchmark_tuple[0].BENCHMARK_NAME, - self.valid_benchmark_names) + self.assertIn( + benchmark_tuple[0].BENCHMARK_NAME, self.valid_benchmark_names + ) # make sure bonnieplusplus is a listed benchmark - self.assertTrue(self._ContainsModule('bonnieplusplus', - benchmark_tuple_list)) + self.assertTrue( + self._ContainsModule('bonnieplusplus', benchmark_tuple_list) + ) def testBenchmarkValidCommandLine3(self): # make sure the command with two benchmarks is processed correctly @@ -264,8 +289,9 @@ def testBenchmarkValidCommandLine3(self): self.assertIsNotNone(benchmark_tuple_list) self.assertEqual(len(benchmark_tuple_list), 2) for benchmark_tuple in benchmark_tuple_list: - self.assertIn(benchmark_tuple[0].BENCHMARK_NAME, - self.valid_benchmark_names) + self.assertIn( + benchmark_tuple[0].BENCHMARK_NAME, self.valid_benchmark_names + ) # make sure listed benchmarks are present self.assertTrue(self._ContainsModule('iperf', benchmark_tuple_list)) self.assertTrue(self._ContainsModule('fio', benchmark_tuple_list)) @@ -284,7 +310,8 @@ def testConfigNames(self): self.mock_flags.benchmarks = ['internal_iprf', 'netperf'] with mock.patch( 'perfkitbenchmarker.configs.GetUserConfig', - return_value=yaml.safe_load(USER_CONFIG)): + return_value=yaml.safe_load(USER_CONFIG), + ): benchmark_tuple_list = benchmark_sets.GetBenchmarksFromFlags() self.assertTrue(self._ContainsModule('iperf', benchmark_tuple_list)) self.assertTrue(self._ContainsModule('netperf', benchmark_tuple_list)) @@ -293,82 +320,99 @@ def testMatrices(self): self.mock_flags.benchmarks = ['netperf'] with mock.patch( 'perfkitbenchmarker.configs.GetUserConfig', - return_value=yaml.safe_load(MATRIX_CONFIG)): + return_value=yaml.safe_load(MATRIX_CONFIG), + ): benchmark_tuple_list = benchmark_sets.GetBenchmarksFromFlags() self.assertEqual(len(benchmark_tuple_list), 4) - flag_list = [benchmark_tuple[1]['flags'] - for benchmark_tuple in benchmark_tuple_list] + flag_list = [ + benchmark_tuple[1]['flags'] for benchmark_tuple in benchmark_tuple_list + ] six.assertCountEqual(self, flag_list, EXPECTED_MATRIX_FLAGS) def testZipWithDifferentAxesLengths(self): self.mock_flags.benchmarks = ['netperf'] with mock.patch( 'perfkitbenchmarker.configs.GetUserConfig', - return_value=yaml.safe_load(ZIP_CONFIG_DIFFERENT_AXES_LENGTH)): + return_value=yaml.safe_load(ZIP_CONFIG_DIFFERENT_AXES_LENGTH), + ): self.assertRaises(ValueError, benchmark_sets.GetBenchmarksFromFlags) def testZip(self): self.mock_flags.benchmarks = ['netperf'] with mock.patch( 'perfkitbenchmarker.configs.GetUserConfig', - return_value=yaml.safe_load(ZIP_CONFIG)): + return_value=yaml.safe_load(ZIP_CONFIG), + ): benchmark_tuple_list = benchmark_sets.GetBenchmarksFromFlags() self.assertEqual(len(benchmark_tuple_list), 2) - flag_list = [benchmark_tuple[1]['flags'] - for benchmark_tuple in benchmark_tuple_list] + flag_list = [ + benchmark_tuple[1]['flags'] for benchmark_tuple in benchmark_tuple_list + ] six.assertCountEqual(self, flag_list, EXPECTED_ZIP_FLAGS) def testZipSingleAxis(self): self.mock_flags.benchmarks = ['netperf'] with mock.patch( 'perfkitbenchmarker.configs.GetUserConfig', - return_value=yaml.safe_load(SINGLE_ZIP_CONFIG)): + return_value=yaml.safe_load(SINGLE_ZIP_CONFIG), + ): benchmark_tuple_list = benchmark_sets.GetBenchmarksFromFlags() self.assertEqual(len(benchmark_tuple_list), 2) - flag_list = [benchmark_tuple[1]['flags'] - for benchmark_tuple in benchmark_tuple_list] + flag_list = [ + benchmark_tuple[1]['flags'] for benchmark_tuple in benchmark_tuple_list + ] six.assertCountEqual(self, flag_list, EXPECTED_SINGLE_ZIP_FLAGS) def testZipAndMatrix(self): self.mock_flags.benchmarks = ['netperf'] with mock.patch( 'perfkitbenchmarker.configs.GetUserConfig', - return_value=yaml.safe_load(ZIP_AND_MATRIX_CONFIG)): + return_value=yaml.safe_load(ZIP_AND_MATRIX_CONFIG), + ): benchmark_tuple_list = benchmark_sets.GetBenchmarksFromFlags() self.assertEqual(len(benchmark_tuple_list), 4) - flag_list = [benchmark_tuple[1]['flags'] - for benchmark_tuple in benchmark_tuple_list] + flag_list = [ + benchmark_tuple[1]['flags'] for benchmark_tuple in benchmark_tuple_list + ] six.assertCountEqual(self, flag_list, EXPECTED_ZIP_AND_MATRIX_FLAGS) def testFilters(self): self.mock_flags.benchmarks = ['netperf'] with mock.patch( 'perfkitbenchmarker.configs.GetUserConfig', - return_value=yaml.safe_load(FILTER_CONFIG)): + return_value=yaml.safe_load(FILTER_CONFIG), + ): benchmark_tuple_list = benchmark_sets.GetBenchmarksFromFlags() self.assertEqual(len(benchmark_tuple_list), 1) - self.assertEqual(benchmark_tuple_list[0][1]['flags'], - {'zones': 'us-central1-a', - 'machine_type': 'n1-standard-1'}) + self.assertEqual( + benchmark_tuple_list[0][1]['flags'], + {'zones': 'us-central1-a', 'machine_type': 'n1-standard-1'}, + ) def testFlagPrecedence(self): self.mock_flags.benchmarks = ['netperf'] with mock.patch( 'perfkitbenchmarker.configs.GetUserConfig', - return_value=yaml.safe_load(FLAG_PRECEDENCE_CONFIG)): + return_value=yaml.safe_load(FLAG_PRECEDENCE_CONFIG), + ): benchmark_tuple_list = benchmark_sets.GetBenchmarksFromFlags() self.assertEqual(len(benchmark_tuple_list), 1) - self.assertEqual(benchmark_tuple_list[0][1]['flags'], - {'netperf_benchmarks': 'TCP_STREAM', - 'netperf_test_length': 40, - 'netperf_max_iter': 3}) + self.assertEqual( + benchmark_tuple_list[0][1]['flags'], + { + 'netperf_benchmarks': 'TCP_STREAM', + 'netperf_test_length': 40, + 'netperf_max_iter': 3, + }, + ) def testFlagMatrixNotFound(self): self.mock_flags.benchmarks = ['netperf'] self.mock_flags.flag_matrix = 'bad_flag_matrix_name' with mock.patch( 'perfkitbenchmarker.configs.GetUserConfig', - return_value=yaml.safe_load(USER_CONFIG)): + return_value=yaml.safe_load(USER_CONFIG), + ): with self.assertRaises(benchmark_sets.FlagMatrixNotFoundException): benchmark_sets.GetBenchmarksFromFlags() @@ -377,7 +421,8 @@ def testFlagZipNotFound(self): self.mock_flags.flag_zip = 'bad_flag_zip_name' with mock.patch( 'perfkitbenchmarker.configs.GetUserConfig', - return_value=yaml.safe_load(USER_CONFIG)): + return_value=yaml.safe_load(USER_CONFIG), + ): with self.assertRaises(benchmark_sets.FlagZipNotFoundException): benchmark_sets.GetBenchmarksFromFlags() diff --git a/tests/benchmark_spec_test.py b/tests/benchmark_spec_test.py index 881359ec6e..b3e79bc060 100644 --- a/tests/benchmark_spec_test.py +++ b/tests/benchmark_spec_test.py @@ -131,11 +131,13 @@ class ConstructEdwServiceTestCase(_BenchmarkSpecTestCase): def testSimpleConfig(self): spec = pkb_common_test_case.CreateBenchmarkSpecFromYaml( - yaml_string=_SIMPLE_EDW_CONFIG, benchmark_name='edw_benchmark') + yaml_string=_SIMPLE_EDW_CONFIG, benchmark_name='edw_benchmark' + ) spec.ConstructEdwService() self.assertEqual('snowflake_aws', spec.edw_service.SERVICE_TYPE) - self.assertIsInstance(spec.edw_service, - providers.aws.snowflake_aws.Snowflake) + self.assertIsInstance( + spec.edw_service, providers.aws.snowflake_aws.Snowflake + ) class ConstructSpannerTestCase(_BenchmarkSpecTestCase): @@ -155,14 +157,16 @@ def setUp(self): create_on_restore_error: True """) self.test_bm_spec = pkb_common_test_case.CreateBenchmarkSpecFromYaml( - yaml_string=test_spec, benchmark_name='cloud_spanner_ycsb') + yaml_string=test_spec, benchmark_name='cloud_spanner_ycsb' + ) @flagsaver.flagsaver(run_uri='test_uri') def testInitialization(self): self.test_bm_spec.ConstructRelationalDb() spanner_instance = self.test_bm_spec.relational_db - self.assertIsInstance(spanner_instance, - gcp_spanner.GoogleSqlGcpSpannerInstance) + self.assertIsInstance( + spanner_instance, gcp_spanner.GoogleSqlGcpSpannerInstance + ) self.assertTrue(spanner_instance.enable_freeze_restore) self.assertTrue(spanner_instance.delete_on_freeze_error) self.assertTrue(spanner_instance.create_on_restore_error) @@ -180,18 +184,22 @@ def testInitializationDefaults(self): engine: spanner-googlesql """) self.test_bm_spec = pkb_common_test_case.CreateBenchmarkSpecFromYaml( - yaml_string=test_spec, benchmark_name='cloud_spanner_ycsb') + yaml_string=test_spec, benchmark_name='cloud_spanner_ycsb' + ) self.test_bm_spec.ConstructRelationalDb() spanner_instance = self.test_bm_spec.relational_db - self.assertIsInstance(spanner_instance, - gcp_spanner.GoogleSqlGcpSpannerInstance) + self.assertIsInstance( + spanner_instance, gcp_spanner.GoogleSqlGcpSpannerInstance + ) self.assertEqual(spanner_instance.instance_id, 'pkb-instance-test_uri') self.assertEqual(spanner_instance.database, 'pkb-database-test_uri') - self.assertEqual(spanner_instance._description, - gcp_spanner._DEFAULT_DESCRIPTION) - self.assertEqual(spanner_instance._config, - f'regional-{gcp_spanner._DEFAULT_REGION}') + self.assertEqual( + spanner_instance._description, gcp_spanner._DEFAULT_DESCRIPTION + ) + self.assertEqual( + spanner_instance._config, f'regional-{gcp_spanner._DEFAULT_REGION}' + ) self.assertEqual(spanner_instance.nodes, gcp_spanner._DEFAULT_NODES) self.assertFalse(spanner_instance.user_managed) @@ -204,8 +212,12 @@ def testRestoreInstanceCopiedFromPreviousSpec(self): engine: spanner-googlesql """) # Set up the restore spec Spanner instance - self.test_bm_spec.restore_spec = pkb_common_test_case.CreateBenchmarkSpecFromYaml( - yaml_string=restore_spanner_spec, benchmark_name='cloud_spanner_ycsb') + self.test_bm_spec.restore_spec = ( + pkb_common_test_case.CreateBenchmarkSpecFromYaml( + yaml_string=restore_spanner_spec, + benchmark_name='cloud_spanner_ycsb', + ) + ) self.test_bm_spec.restore_spec.ConstructRelationalDb() self.test_bm_spec.ConstructRelationalDb() @@ -253,14 +265,16 @@ def testStaticVms(self): def testValidConfigWithDiskSpec(self): spec = pkb_common_test_case.CreateBenchmarkSpecFromYaml( - VALID_CONFIG_WITH_DISK_SPEC) + VALID_CONFIG_WITH_DISK_SPEC + ) spec.ConstructVirtualMachines() vms = spec.vm_groups['default'] self.assertEqual(len(vms), 2) for vm in vms: self.assertEqual(len(vm.disk_specs), 3) - self.assertTrue(all(disk_spec.disk_size == 75 - for disk_spec in vm.disk_specs)) + self.assertTrue( + all(disk_spec.disk_size == 75 for disk_spec in vm.disk_specs) + ) @flagsaver.flagsaver def testZoneFlag(self): @@ -287,7 +301,8 @@ class BenchmarkSupportTestCase(_BenchmarkSpecTestCase): def createBenchmarkSpec(self, config, benchmark): spec = pkb_common_test_case.CreateBenchmarkSpecFromConfigDict( - config, benchmark) + config, benchmark + ) spec.ConstructVirtualMachines() return True @@ -300,8 +315,9 @@ def testBenchmarkSupportFlag(self): and returns None if the benchmark isn't in either list. """ FLAGS.cloud = 'Kubernetes' - config = configs.LoadConfig(iperf_benchmark.BENCHMARK_CONFIG, {}, - ALWAYS_SUPPORTED) + config = configs.LoadConfig( + iperf_benchmark.BENCHMARK_CONFIG, {}, ALWAYS_SUPPORTED + ) self.assertTrue(self.createBenchmarkSpec(config, ALWAYS_SUPPORTED)) with self.assertRaises(ValueError): self.createBenchmarkSpec(config, NEVER_SUPPORTED) @@ -309,7 +325,8 @@ def testBenchmarkSupportFlag(self): FLAGS.benchmark_compatibility_checking = 'permissive' self.assertTrue( self.createBenchmarkSpec(config, ALWAYS_SUPPORTED), - 'benchmark is supported, mode is permissive') + 'benchmark is supported, mode is permissive', + ) with self.assertRaises(ValueError): self.createBenchmarkSpec(config, NEVER_SUPPORTED) @@ -322,7 +339,8 @@ class RedirectGlobalFlagsTestCase(pkb_common_test_case.PkbCommonTestCase): def testNoFlagOverride(self): config_spec = benchmark_config_spec.BenchmarkConfigSpec( - NAME, flag_values=FLAGS, vm_groups={}) + NAME, flag_values=FLAGS, vm_groups={} + ) spec = benchmark_spec.BenchmarkSpec(mock.MagicMock(), config_spec, UID) self.assertEqual(FLAGS.benchmark_spec_test_flag, 0) with spec.RedirectGlobalFlags(): @@ -331,8 +349,11 @@ def testNoFlagOverride(self): def testFlagOverride(self): config_spec = benchmark_config_spec.BenchmarkConfigSpec( - NAME, flag_values=FLAGS, flags={'benchmark_spec_test_flag': 1}, - vm_groups={}) + NAME, + flag_values=FLAGS, + flags={'benchmark_spec_test_flag': 1}, + vm_groups={}, + ) spec = benchmark_spec.BenchmarkSpec(mock.MagicMock(), config_spec, UID) self.assertEqual(FLAGS.benchmark_spec_test_flag, 0) with spec.RedirectGlobalFlags(): diff --git a/tests/benchmark_status_test.py b/tests/benchmark_status_test.py index 9b2d38eb13..5e6c7ab8a2 100644 --- a/tests/benchmark_status_test.py +++ b/tests/benchmark_status_test.py @@ -17,7 +17,6 @@ import unittest from absl.testing import parameterized - import mock from perfkitbenchmarker import benchmark_status from perfkitbenchmarker import errors @@ -42,9 +41,13 @@ def __init__(self, name, uid, status, failed_substatus=None): _BENCHMARK_SPECS = [ MockSpec('iperf', 'iperf0', benchmark_status.SUCCEEDED), MockSpec('iperf', 'iperf1', benchmark_status.FAILED), - MockSpec('iperf', 'iperf2', benchmark_status.FAILED, - benchmark_status.FailedSubstatus.QUOTA), - MockSpec('cluster_boot', 'cluster_boot0', benchmark_status.SKIPPED) + MockSpec( + 'iperf', + 'iperf2', + benchmark_status.FAILED, + benchmark_status.FailedSubstatus.QUOTA, + ), + MockSpec('cluster_boot', 'cluster_boot0', benchmark_status.SKIPPED), ] _STATUS_TABLE = os.linesep.join(( '--------------------------------------------------------', @@ -54,7 +57,8 @@ def __init__(self, name, uid, status, failed_substatus=None): 'iperf iperf1 FAILED ', 'iperf iperf2 FAILED QUOTA_EXCEEDED ', 'cluster_boot cluster_boot0 SKIPPED ', - '--------------------------------------------------------')) + '--------------------------------------------------------', +)) _STATUS_SUMMARY = os.linesep.join(( 'Benchmark run statuses:', '--------------------------------------------------------', @@ -65,7 +69,8 @@ def __init__(self, name, uid, status, failed_substatus=None): 'iperf iperf2 FAILED QUOTA_EXCEEDED ', 'cluster_boot cluster_boot0 SKIPPED ', '--------------------------------------------------------', - 'Success rate: 25.00% (1/4)')) + 'Success rate: 25.00% (1/4)', +)) class CreateSummaryTableTestCase(unittest.TestCase): @@ -91,20 +96,18 @@ class FailedSubstatusTestCase(pkb_common_test_case.PkbCommonTestCase): 'expected_substatus': benchmark_status.FailedSubstatus.QUOTA, }, { - 'testcase_name': - 'Capacity', - 'exception_class': - errors.Benchmarks.InsufficientCapacityCloudFailure, - 'expected_substatus': - benchmark_status.FailedSubstatus.INSUFFICIENT_CAPACITY, + 'testcase_name': 'Capacity', + 'exception_class': errors.Benchmarks.InsufficientCapacityCloudFailure, + 'expected_substatus': ( + benchmark_status.FailedSubstatus.INSUFFICIENT_CAPACITY + ), }, { - 'testcase_name': - 'KnownIntermittent', - 'exception_class': - errors.Benchmarks.KnownIntermittentError, - 'expected_substatus': - benchmark_status.FailedSubstatus.KNOWN_INTERMITTENT, + 'testcase_name': 'KnownIntermittent', + 'exception_class': errors.Benchmarks.KnownIntermittentError, + 'expected_substatus': ( + benchmark_status.FailedSubstatus.KNOWN_INTERMITTENT + ), }, { 'testcase_name': 'RestoreError', @@ -122,11 +125,14 @@ class FailedSubstatusTestCase(pkb_common_test_case.PkbCommonTestCase): 'expected_substatus': benchmark_status.FailedSubstatus.UNCATEGORIZED, }, ) - def testRunBenchmarkExceptionHasCorrectFailureStatus(self, exception_class, - expected_substatus): + def testRunBenchmarkExceptionHasCorrectFailureStatus( + self, exception_class, expected_substatus + ): self.enter_context( mock.patch.object( - pkb, 'DoProvisionPhase', side_effect=[exception_class()])) + pkb, 'DoProvisionPhase', side_effect=[exception_class()] + ) + ) test_spec = pkb_common_test_case.CreateBenchmarkSpecFromYaml() # Skip pickling the spec. self.enter_context(mock.patch.object(test_spec, 'Pickle')) diff --git a/tests/cloud_harmony_util_test.py b/tests/cloud_harmony_util_test.py index fec9a60fa5..66c295fae0 100644 --- a/tests/cloud_harmony_util_test.py +++ b/tests/cloud_harmony_util_test.py @@ -1,4 +1,5 @@ """Tests for perfkitbenchmarker.cloud_harmony_util.""" + import os import unittest @@ -23,8 +24,9 @@ def setUp(self): def test_ParseCsvResultsIntoMetadata(self): prefix = 'foobar' - path = os.path.join(os.path.dirname(__file__), - 'data/cloud_harmony_sample.csv') + path = os.path.join( + os.path.dirname(__file__), 'data/cloud_harmony_sample.csv' + ) with open(path, mode='r') as fp: sample_csv = fp.read() metadata = cloud_harmony_util.ParseCsvResultsFromString(sample_csv, prefix) @@ -47,7 +49,7 @@ def test_GetCommonMetadata(self): '--meta_region us-east3 ' '--meta_zone us-east3 ' '--meta_test_id None' - ) + ) self.assertEqual(no_override, expected_no_override) expected_override = ( @@ -59,10 +61,12 @@ def test_GetCommonMetadata(self): '--meta_region us-east3 ' '--meta_zone us-east3 ' '--meta_test_id None' - ) + ) override = cloud_harmony_util.GetCommonMetadata( - {'meta_instance_id': 'n1-standard-8'}) + {'meta_instance_id': 'n1-standard-8'} + ) self.assertEqual(override, expected_override) + if __name__ == '__main__': unittest.main() diff --git a/tests/cloud_tpu_test.py b/tests/cloud_tpu_test.py index 00150a9b09..410ab76f61 100644 --- a/tests/cloud_tpu_test.py +++ b/tests/cloud_tpu_test.py @@ -15,7 +15,6 @@ import unittest from absl import flags - from perfkitbenchmarker import cloud_tpu from perfkitbenchmarker.configs import benchmark_config_spec from tests import pkb_common_test_case @@ -69,9 +68,7 @@ def setUp(self): 'cloud': 'GCP', } - cloud_tpu._CLOUD_TPU_REGISTRY = { - 'GCP': FakeTpu(None) - } + cloud_tpu._CLOUD_TPU_REGISTRY = {'GCP': FakeTpu(None)} def tearDown(self): super(TpuSpecTestCase, self).tearDown() @@ -79,84 +76,99 @@ def tearDown(self): def testMinimalConfig(self): result = benchmark_config_spec._TpuGroupSpec( - _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.minimal_spec) + _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.minimal_spec + ) self.assertEqual(result.cloud, 'GCP') def testDefaultTpuName(self): result = benchmark_config_spec._TpuGroupSpec( - _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.minimal_spec) + _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.minimal_spec + ) self.assertEqual(result.tpu_name, 'pkb-tpu-train-123') def testCustomTpuName(self): spec = MergeDicts(self.minimal_spec, {'tpu_name': 'pkb-tpu'}) result = benchmark_config_spec._TpuGroupSpec( - _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **spec) + _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **spec + ) self.assertEqual(result.tpu_name, 'pkb-tpu') def testDefaultTpuCidrRange(self): result = benchmark_config_spec._TpuGroupSpec( - _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.minimal_spec) + _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.minimal_spec + ) self.assertEqual(result.tpu_cidr_range, None) def testCustomTpuCidrRange(self): spec = MergeDicts(self.minimal_spec, {'tpu_cidr_range': '192.168.0.0/29'}) result = benchmark_config_spec._TpuGroupSpec( - _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **spec) + _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **spec + ) self.assertEqual(result.tpu_cidr_range, '192.168.0.0/29') def testDefaultTpuAcceleratorType(self): result = benchmark_config_spec._TpuGroupSpec( - _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.minimal_spec) + _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.minimal_spec + ) self.assertEqual(result.tpu_accelerator_type, None) def testCustomTpuAcceleratorType(self): spec = MergeDicts(self.minimal_spec, {'tpu_accelerator_type': 'tpu-v2'}) result = benchmark_config_spec._TpuGroupSpec( - _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **spec) + _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **spec + ) self.assertEqual(result.tpu_accelerator_type, 'tpu-v2') def testDefaultTpuDescription(self): result = benchmark_config_spec._TpuGroupSpec( - _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.minimal_spec) + _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.minimal_spec + ) self.assertEqual(result.tpu_description, None) def testCustomTpuDescription(self): spec = MergeDicts(self.minimal_spec, {'tpu_description': 'My TF Node'}) result = benchmark_config_spec._TpuGroupSpec( - _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **spec) + _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **spec + ) self.assertEqual(result.tpu_description, 'My TF Node') def testDefaultTpuNetwork(self): result = benchmark_config_spec._TpuGroupSpec( - _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.minimal_spec) + _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.minimal_spec + ) self.assertEqual(result.tpu_network, None) def testCustomTpuNetwork(self): spec = MergeDicts(self.minimal_spec, {'tpu_network': 'default'}) result = benchmark_config_spec._TpuGroupSpec( - _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **spec) + _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **spec + ) self.assertEqual(result.tpu_network, 'default') def testDefaultTpuZone(self): result = benchmark_config_spec._TpuGroupSpec( - _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.minimal_spec) + _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.minimal_spec + ) self.assertEqual(result.tpu_zone, None) def testCustomTpuZone(self): spec = MergeDicts(self.minimal_spec, {'tpu_zone': 'us-central1-a'}) result = benchmark_config_spec._TpuGroupSpec( - _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **spec) + _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **spec + ) self.assertEqual(result.tpu_zone, 'us-central1-a') def testDefaultTpuVersion(self): result = benchmark_config_spec._TpuGroupSpec( - _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.minimal_spec) + _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.minimal_spec + ) self.assertEqual(result.tpu_tf_version, None) def testCustomTpuVersion(self): spec = MergeDicts(self.minimal_spec, {'tpu_tf_version': 'nightly'}) result = benchmark_config_spec._TpuGroupSpec( - _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **spec) + _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **spec + ) self.assertEqual(result.tpu_tf_version, 'nightly') @@ -174,12 +186,10 @@ def setUp(self): 'tpu_description': 'My TF Node', 'tpu_network': 'default', 'tpu_tf_version': 'nightly', - 'tpu_zone': 'us-central1-a' + 'tpu_zone': 'us-central1-a', } - cloud_tpu._CLOUD_TPU_REGISTRY = { - 'GCP': FakeTpu(None) - } + cloud_tpu._CLOUD_TPU_REGISTRY = {'GCP': FakeTpu(None)} def tearDown(self): super(TpuSpecFlagsTestCase, self).tearDown() @@ -191,43 +201,50 @@ def testCloudFlag(self): def testTpuNameFlag(self): FLAGS['tpu_name'].parse('pkb-tpu') result = benchmark_config_spec._TpuGroupSpec( - _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.full_spec) + _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.full_spec + ) self.assertEqual(result.tpu_name, 'pkb-tpu') def testTpuCidrRangeFlag(self): FLAGS['tpu_cidr_range'].parse('10.240.0.0/29') result = benchmark_config_spec._TpuGroupSpec( - _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.full_spec) + _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.full_spec + ) self.assertEqual(result.tpu_cidr_range, '10.240.0.0/29') def testTpuAcceleratorTypeFlag(self): FLAGS['tpu_accelerator_type'].parse('tpu-v1') result = benchmark_config_spec._TpuGroupSpec( - _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.full_spec) + _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.full_spec + ) self.assertEqual(result.tpu_accelerator_type, 'tpu-v1') def testTpuDescriptionFlag(self): FLAGS['tpu_description'].parse('MyTfNode') result = benchmark_config_spec._TpuGroupSpec( - _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.full_spec) + _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.full_spec + ) self.assertEqual(result.tpu_description, 'MyTfNode') def testTpuNetworkFlag(self): FLAGS['tpu_network'].parse('my-tf-network') result = benchmark_config_spec._TpuGroupSpec( - _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.full_spec) + _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.full_spec + ) self.assertEqual(result.tpu_network, 'my-tf-network') def testTpuTfVersion(self): FLAGS['tpu_tf_version'].parse('1.2') result = benchmark_config_spec._TpuGroupSpec( - _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.full_spec) + _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.full_spec + ) self.assertEqual(result.tpu_tf_version, '1.2') def testTpuZone(self): FLAGS['tpu_zone'].parse('us-central1-c') result = benchmark_config_spec._TpuGroupSpec( - _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.full_spec) + _COMPONENT, _GROUP_NAME, flag_values=FLAGS, **self.full_spec + ) self.assertEqual(result.tpu_zone, 'us-central1-c') diff --git a/tests/config_test.py b/tests/config_test.py index 8cbe7e9b89..92567a1fe5 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -71,14 +71,14 @@ class ConfigsTestCase(unittest.TestCase): def testLoadAllDefaultConfigs(self): - all_benchmarks = (linux_benchmarks.BENCHMARKS + - windows_benchmarks.BENCHMARKS) + all_benchmarks = linux_benchmarks.BENCHMARKS + windows_benchmarks.BENCHMARKS for benchmark_module in all_benchmarks: self.assertIsInstance(benchmark_module.GetConfig({}), dict) def testLoadValidConfig(self): self.assertIsInstance( - configs.LoadMinimalConfig(VALID_CONFIG, CONFIG_NAME), dict) + configs.LoadMinimalConfig(VALID_CONFIG, CONFIG_NAME), dict + ) def testWrongName(self): with self.assertRaises(KeyError): @@ -113,7 +113,8 @@ def testMergeConfigWithNoOverrides(self): def testLoadConfigWithExternalReference(self): self.assertIsInstance( - configs.LoadMinimalConfig(REF_CONFIG, CONFIG_NAME), dict) + configs.LoadMinimalConfig(REF_CONFIG, CONFIG_NAME), dict + ) def testLoadConfigWithBadReference(self): with self.assertRaises(errors.Config.ParseError): @@ -123,11 +124,10 @@ def testConfigOverrideFlag(self): p = mock.patch(configs.__name__ + '.FLAGS') self.addCleanup(p.stop) mock_flags = p.start() - config_override = [ - 'a.vm_groups.default.vm_count=5', - 'a.flags.flag=value'] - mock_flags.configure_mock(config_override=config_override, - benchmark_config_file=None) + config_override = ['a.vm_groups.default.vm_count=5', 'a.flags.flag=value'] + mock_flags.configure_mock( + config_override=config_override, benchmark_config_file=None + ) config = configs.GetUserConfig() self.assertEqual(config['a']['vm_groups']['default']['vm_count'], 5) self.assertEqual(config['a']['flags']['flag'], 'value') diff --git a/tests/configs/benchmark_config_spec_test.py b/tests/configs/benchmark_config_spec_test.py index 9e37bbdeaf..7e577905ce 100644 --- a/tests/configs/benchmark_config_spec_test.py +++ b/tests/configs/benchmark_config_spec_test.py @@ -18,7 +18,6 @@ import unittest from absl import flags import mock - from perfkitbenchmarker import errors from perfkitbenchmarker import os_types from perfkitbenchmarker import provider_info @@ -39,8 +38,10 @@ _COMPONENT = 'test_component' _OPTION = 'test_option' _GCP_ONLY_VM_CONFIG = {'GCP': {'machine_type': 'n1-standard-1'}} -_GCP_AWS_VM_CONFIG = {'GCP': {'machine_type': 'n1-standard-1'}, - 'AWS': {'machine_type': 'm4.large'}} +_GCP_AWS_VM_CONFIG = { + 'GCP': {'machine_type': 'n1-standard-1'}, + 'AWS': {'machine_type': 'm4.large'}, +} _GCP_AWS_DISK_CONFIG = {'GCP': {}, 'AWS': {}} @@ -70,15 +71,21 @@ def testDict(self): def testNonDict(self): with self.assertRaises(errors.Config.InvalidValue) as cm: self._spec_class(_COMPONENT, GCP=[]) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.GCP value: "[]" (of type "list"). Value must ' - 'be one of the following types: dict.')) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.GCP value: "[]" (of type "list"). Value' + ' must be one of the following types: dict.' + ), + ) def testUnrecognizedCloud(self): with self.assertRaises(errors.Config.UnrecognizedOption) as cm: self._spec_class(_COMPONENT, fake_provider={}) - self.assertEqual(str(cm.exception), ( - 'Unrecognized options were found in test_component: fake_provider.')) + self.assertEqual( + str(cm.exception), + 'Unrecognized options were found in test_component: fake_provider.', + ) class PerCloudConfigDecoderTestCase(pkb_common_test_case.PkbCommonTestCase): @@ -98,8 +105,9 @@ def testAcceptNone(self): def testEmptyDict(self): result = self._decoder.Decode({}, _COMPONENT, {}) self.assertIsInstance(result, spec.PerCloudConfigSpec) - self.assertEqual(result.__dict__, { - cloud: None for cloud in provider_info.VALID_CLOUDS}) + self.assertEqual( + result.__dict__, {cloud: None for cloud in provider_info.VALID_CLOUDS} + ) def testNonEmptyDict(self): result = self._decoder.Decode(_GCP_ONLY_VM_CONFIG, _COMPONENT, {}) @@ -133,11 +141,9 @@ def testVmSpecFlag(self): def testDiskSpecFlag(self): FLAGS.scratch_dir = '/path/from/flag' FLAGS['scratch_dir'].present = True - result = self._decoder.Decode({ - 'disk_specs': [{ - 'mount_point': '/path/from/spec' - }] - }, _COMPONENT, FLAGS) + result = self._decoder.Decode( + {'disk_specs': [{'mount_point': '/path/from/spec'}]}, _COMPONENT, FLAGS + ) self.assertEqual(result.disk_specs[0].mount_point, '/path/from/flag') @@ -161,8 +167,10 @@ def testInvalidList(self): input_list = [{'ssh_port': 0}, {'ssh_port': 1}, {'ssh_pory': 2}] with self.assertRaises(errors.Config.UnrecognizedOption) as cm: self._decoder.Decode(input_list, _COMPONENT, {}) - self.assertEqual(str(cm.exception), ( - 'Unrecognized options were found in test_component[2]: ssh_pory.')) + self.assertEqual( + str(cm.exception), + 'Unrecognized options were found in test_component[2]: ssh_pory.', + ) class VmGroupSpecTestCase(pkb_common_test_case.PkbCommonTestCase): @@ -170,15 +178,22 @@ class VmGroupSpecTestCase(pkb_common_test_case.PkbCommonTestCase): def setUp(self): super(VmGroupSpecTestCase, self).setUp() self._spec_class = vm_group_decoders.VmGroupSpec - self._kwargs = {'cloud': provider_info.GCP, 'os_type': os_types.DEFAULT, - 'vm_spec': _GCP_AWS_VM_CONFIG} + self._kwargs = { + 'cloud': provider_info.GCP, + 'os_type': os_types.DEFAULT, + 'vm_spec': _GCP_AWS_VM_CONFIG, + } def testMissingValues(self): with self.assertRaises(errors.Config.MissingOption) as cm: self._spec_class(_COMPONENT) - self.assertEqual(str(cm.exception), ( - 'Required options were missing from test_component: cloud, os_type, ' - 'vm_spec.')) + self.assertEqual( + str(cm.exception), + ( + 'Required options were missing from test_component: cloud, os_type,' + ' vm_spec.' + ), + ) def testDefaults(self): result = self._spec_class(_COMPONENT, **self._kwargs) @@ -207,61 +222,95 @@ def testInvalidDiskCount(self): self._kwargs['disk_count'] = -1 with self.assertRaises(errors.Config.InvalidValue) as cm: self._spec_class(_COMPONENT, **self._kwargs) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.disk_count value: "-1". ' - 'Value must be at least 0.')) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.disk_count value: "-1". ' + 'Value must be at least 0.' + ), + ) def testInvalidDiskSpec(self): self._kwargs['disk_spec'] = {'GCP': None} with self.assertRaises(errors.Config.InvalidValue) as cm: self._spec_class(_COMPONENT, **self._kwargs) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.disk_spec.GCP value: "None" (of type ' - '"NoneType"). Value must be one of the following types: dict.')) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.disk_spec.GCP value: "None" (of type ' + '"NoneType"). Value must be one of the following types: dict.' + ), + ) def testInvalidOsType(self): self._kwargs['os_type'] = 'fake_os_type' with self.assertRaises(errors.Config.InvalidValue) as cm: self._spec_class(_COMPONENT, **self._kwargs) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.os_type value: "fake_os_type". Value must be ' - 'one of the following: {0}.'.format(', '.join(os_types.ALL)))) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.os_type value: "fake_os_type". Value must' + ' be one of the following: {0}.'.format(', '.join(os_types.ALL)) + ), + ) def testInvalidStaticVms(self): self._kwargs['static_vms'] = [{'fake_option': None}] with self.assertRaises(errors.Config.UnrecognizedOption) as cm: self._spec_class(_COMPONENT, **self._kwargs) - self.assertEqual(str(cm.exception), ( - 'Unrecognized options were found in test_component.static_vms[0]: ' - 'fake_option.')) + self.assertEqual( + str(cm.exception), + ( + 'Unrecognized options were found in test_component.static_vms[0]: ' + 'fake_option.' + ), + ) def testInvalidVmCount(self): self._kwargs['vm_count'] = None with self.assertRaises(errors.Config.InvalidValue) as cm: self._spec_class(_COMPONENT, **self._kwargs) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.vm_count value: "None" (of type "NoneType"). ' - 'Value must be one of the following types: int.')) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.vm_count value: "None" (of type' + ' "NoneType"). Value must be one of the following types: int.' + ), + ) self._kwargs['vm_count'] = -1 with self.assertRaises(errors.Config.InvalidValue) as cm: self._spec_class(_COMPONENT, **self._kwargs) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.vm_count value: "-1". ' - 'Value must be at least 0.')) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.vm_count value: "-1". ' + 'Value must be at least 0.' + ), + ) def testInvalidVmSpec(self): self._kwargs['vm_spec'] = {'GCP': None} with self.assertRaises(errors.Config.InvalidValue) as cm: self._spec_class(_COMPONENT, **self._kwargs) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.vm_spec.GCP value: "None" (of type ' - '"NoneType"). Value must be one of the following types: dict.')) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.vm_spec.GCP value: "None" (of type ' + '"NoneType"). Value must be one of the following types: dict.' + ), + ) def testValidInput(self): result = self._spec_class( - _COMPONENT, cloud=provider_info.AWS, disk_count=0, - disk_spec=_GCP_AWS_DISK_CONFIG, os_type=os_types.AMAZONLINUX2, - static_vms=[{}], vm_count=0, vm_spec=_GCP_AWS_VM_CONFIG) + _COMPONENT, + cloud=provider_info.AWS, + disk_count=0, + disk_spec=_GCP_AWS_DISK_CONFIG, + os_type=os_types.AMAZONLINUX2, + static_vms=[{}], + vm_count=0, + vm_spec=_GCP_AWS_VM_CONFIG, + ) self.assertIsInstance(result, vm_group_decoders.VmGroupSpec) self.assertEqual(result.cloud, 'AWS') self.assertEqual(result.disk_count, 0) @@ -269,8 +318,9 @@ def testValidInput(self): self.assertEqual(result.os_type, 'amazonlinux2') self.assertIsInstance(result.static_vms, list) self.assertEqual(len(result.static_vms), 1) - self.assertIsInstance(result.static_vms[0], - static_virtual_machine.StaticVmSpec) + self.assertIsInstance( + result.static_vms[0], static_virtual_machine.StaticVmSpec + ) self.assertEqual(result.vm_count, 0) self.assertIsInstance(result.vm_spec, virtual_machine.BaseVmSpec) @@ -281,11 +331,15 @@ def testMissingCloudDiskConfig(self): cloud=provider_info.GCP, os_type=os_types.DEFAULT, disk_spec={}, - vm_spec=_GCP_AWS_VM_CONFIG) + vm_spec=_GCP_AWS_VM_CONFIG, + ) self.assertEqual( str(cm.exception), - ('test_component.cloud is "GCP", but test_component.disk_spec does not ' - 'contain a configuration for "GCP".')) + ( + 'test_component.cloud is "GCP", but test_component.disk_spec does' + ' not contain a configuration for "GCP".' + ), + ) def testMissingCloudVmConfig(self): with self.assertRaises(errors.Config.MissingOption) as cm: @@ -293,11 +347,15 @@ def testMissingCloudVmConfig(self): _COMPONENT, cloud=provider_info.GCP, os_type=os_types.DEFAULT, - vm_spec={}) + vm_spec={}, + ) self.assertEqual( str(cm.exception), - ('test_component.cloud is "GCP", but test_component.vm_spec does not ' - 'contain a configuration for "GCP".')) + ( + 'test_component.cloud is "GCP", but test_component.vm_spec does not' + ' contain a configuration for "GCP".' + ), + ) def createNonPresentFlags(self): FLAGS.cloud = provider_info.AWS @@ -313,7 +371,8 @@ def createPresentFlags(self): def testPresentFlagsAndPresentConfigValues(self): self.createPresentFlags() result = self._spec_class( - _COMPONENT, flag_values=FLAGS, vm_count=2, **self._kwargs) + _COMPONENT, flag_values=FLAGS, vm_count=2, **self._kwargs + ) self.assertEqual(result.cloud, 'AWS') self.assertEqual(result.os_type, 'windows2019_core') self.assertEqual(result.vm_count, 2) @@ -321,7 +380,8 @@ def testPresentFlagsAndPresentConfigValues(self): def testPresentFlagsAndNonPresentConfigValues(self): self.createPresentFlags() result = self._spec_class( - _COMPONENT, flag_values=FLAGS, vm_spec=_GCP_AWS_VM_CONFIG) + _COMPONENT, flag_values=FLAGS, vm_spec=_GCP_AWS_VM_CONFIG + ) self.assertEqual(result.cloud, 'AWS') self.assertEqual(result.os_type, 'windows2019_core') self.assertEqual(result.vm_count, 1) @@ -329,8 +389,11 @@ def testPresentFlagsAndNonPresentConfigValues(self): def testNonPresentFlagsAndPresentConfigValues(self): self.createNonPresentFlags() result = self._spec_class( - _COMPONENT, flag_values=self.createNonPresentFlags(), vm_count=2, - **self._kwargs) + _COMPONENT, + flag_values=self.createNonPresentFlags(), + vm_count=2, + **self._kwargs + ) self.assertEqual(result.cloud, 'GCP') self.assertEqual(result.os_type, 'ubuntu2004') self.assertEqual(result.vm_count, 2) @@ -338,7 +401,8 @@ def testNonPresentFlagsAndPresentConfigValues(self): def testVmCountNone(self): self.createNonPresentFlags() result = self._spec_class( - _COMPONENT, vm_count=None, flag_values=FLAGS, **self._kwargs) + _COMPONENT, vm_count=None, flag_values=FLAGS, **self._kwargs + ) self.assertEqual(result.vm_count, 3) def testCallsLoadProviderAndChecksRequirements(self): @@ -367,16 +431,25 @@ def testNone(self): self._decoder.Decode(None, _COMPONENT, {}) def testValidInput(self): - result = self._decoder.Decode({ - 'default': {'cloud': provider_info.GCP, 'os_type': os_types.DEFAULT, - 'vm_spec': _GCP_AWS_VM_CONFIG}}, _COMPONENT, {}) + result = self._decoder.Decode( + { + 'default': { + 'cloud': provider_info.GCP, + 'os_type': os_types.DEFAULT, + 'vm_spec': _GCP_AWS_VM_CONFIG, + } + }, + _COMPONENT, + {}, + ) self.assertIsInstance(result, dict) self.assertEqual(len(result), 1) self.assertIsInstance(result['default'], vm_group_decoders.VmGroupSpec) self.assertEqual(result['default'].cloud, 'GCP') self.assertEqual(result['default'].os_type, 'ubuntu2004') - self.assertIsInstance(result['default'].vm_spec, - gce_virtual_machine.GceVmSpec) + self.assertIsInstance( + result['default'].vm_spec, gce_virtual_machine.GceVmSpec + ) def testInvalidInput(self): with self.assertRaises(errors.Config.UnrecognizedOption) as cm: @@ -392,9 +465,13 @@ def testInvalidInput(self): _COMPONENT, {}, ) - self.assertEqual(str(cm.exception), ( - 'Unrecognized options were found in ' - 'test_component.default.static_vms[1]: fake_option.')) + self.assertEqual( + str(cm.exception), + ( + 'Unrecognized options were found in ' + 'test_component.default.static_vms[1]: fake_option.' + ), + ) class CloudRedisDecoderTestCase(pkb_common_test_case.PkbCommonTestCase): @@ -410,18 +487,19 @@ def testNone(self): self._decoder.Decode(None, _COMPONENT, {}) def testValidInput(self): - result = self._decoder.Decode({ - 'redis_version': 'redis_3_2' - }, _COMPONENT, FLAGS) + result = self._decoder.Decode( + {'redis_version': 'redis_3_2'}, _COMPONENT, FLAGS + ) self.assertIsInstance(result, benchmark_config_spec._CloudRedisSpec) self.assertEqual(result.redis_version, 'redis_3_2') def testInvalidInput(self): with self.assertRaises(errors.Config.UnrecognizedOption) as cm: self._decoder.Decode({'foo': 'bar'}, _COMPONENT, FLAGS) - self.assertEqual(str(cm.exception), ( - 'Unrecognized options were found in ' - 'test_component: foo.')) + self.assertEqual( + str(cm.exception), + 'Unrecognized options were found in test_component: foo.', + ) class CloudRedisSpecTestCase(pkb_common_test_case.PkbCommonTestCase): @@ -433,8 +511,10 @@ def setUp(self): def testMissingValues(self): with self.assertRaises(errors.Config.MissingOption) as cm: self._spec_class(_COMPONENT) - self.assertEqual(str(cm.exception), ( - 'Required options were missing from test_component: cloud.')) + self.assertEqual( + str(cm.exception), + 'Required options were missing from test_component: cloud.', + ) def testDefaults(self): result = self._spec_class(_COMPONENT, flag_values=FLAGS) @@ -449,11 +529,17 @@ def setUp(self): self._spec_class = benchmark_config_spec.BenchmarkConfigSpec self._description = 'Test description.' - self._vm_groups = {'default': {'cloud': provider_info.GCP, - 'os_type': os_types.DEFAULT, - 'vm_spec': _GCP_AWS_VM_CONFIG}} - self._kwargs = {'description': self._description, - 'vm_groups': self._vm_groups} + self._vm_groups = { + 'default': { + 'cloud': provider_info.GCP, + 'os_type': os_types.DEFAULT, + 'vm_spec': _GCP_AWS_VM_CONFIG, + } + } + self._kwargs = { + 'description': self._description, + 'vm_groups': self._vm_groups, + } def testValidInput(self): result = self._spec_class(_COMPONENT, flag_values=FLAGS, **self._kwargs) @@ -462,46 +548,65 @@ def testValidInput(self): self.assertIsNot(result.flags, _GetFlagDict(flags.FLAGS)) self.assertIsInstance(result.vm_groups, dict) self.assertEqual(len(result.vm_groups), 1) - self.assertIsInstance(result.vm_groups['default'], - vm_group_decoders.VmGroupSpec) + self.assertIsInstance( + result.vm_groups['default'], vm_group_decoders.VmGroupSpec + ) self.assertEqual(result.vm_groups['default'].cloud, 'GCP') self.assertEqual(result.vm_groups['default'].os_type, 'ubuntu2004') - self.assertIsInstance(result.vm_groups['default'].vm_spec, - gce_virtual_machine.GceVmSpec) + self.assertIsInstance( + result.vm_groups['default'].vm_spec, gce_virtual_machine.GceVmSpec + ) def testInvalidVmGroups(self): - self._kwargs['vm_groups']['default']['static_vms'] = [{'disk_specs': [{ - 'disk_size': 0.5}]}] + self._kwargs['vm_groups']['default']['static_vms'] = [ + {'disk_specs': [{'disk_size': 0.5}]} + ] with self.assertRaises(errors.Config.InvalidValue) as cm: self._spec_class(_COMPONENT, flag_values=FLAGS, **self._kwargs) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.vm_groups.default.static_vms[0].disk_specs[0]' - '.disk_size value: "0.5" (of type "float"). Value must be one of the ' - 'following types: NoneType, int.')) + self.assertEqual( + str(cm.exception), + ( + 'Invalid' + ' test_component.vm_groups.default.static_vms[0].disk_specs[0].disk_size' + ' value: "0.5" (of type "float"). Value must be one of the' + ' following types: NoneType, int.' + ), + ) def testMismatchedOsTypes(self): self._kwargs['vm_groups'] = { os_type + '_group': {'os_type': os_type, 'vm_spec': _GCP_AWS_VM_CONFIG} - for os_type in (os_types.DEFAULT, os_types.RHEL8, - os_types.WINDOWS2019_CORE)} + for os_type in ( + os_types.DEFAULT, + os_types.RHEL8, + os_types.WINDOWS2019_CORE, + ) + } expected_os_types = os_types.JUJU, os_types.WINDOWS2019_CORE with self.assertRaises(errors.Config.InvalidValue) as cm: self._spec_class( _COMPONENT, expected_os_types=expected_os_types, flag_values=FLAGS, - **self._kwargs) - self.assertEqual(str(cm.exception), ( - "VM groups in test_component may only have the following OS types: " - "'juju', 'windows2019_core'. The following VM group options are " - "invalid:{sep}" - "test_component.vm_groups['rhel8_group'].os_type: 'rhel8'{sep}" - "test_component.vm_groups['ubuntu2004_group'].os_type: 'ubuntu2004'" - .format(sep=os.linesep))) + **self._kwargs + ) + self.assertEqual( + str(cm.exception), + ( + 'VM groups in test_component may only have the following OS types: ' + "'juju', 'windows2019_core'. The following VM group options are " + 'invalid:{sep}' + "test_component.vm_groups['rhel8_group'].os_type: 'rhel8'{sep}" + "test_component.vm_groups['ubuntu2004_group'].os_type: 'ubuntu2004'" + .format(sep=os.linesep) + ), + ) def testFlagOverridesPropagate(self): - self._kwargs['flags'] = {'cloud': provider_info.AWS, - 'ignore_package_requirements': True} + self._kwargs['flags'] = { + 'cloud': provider_info.AWS, + 'ignore_package_requirements': True, + } result = self._spec_class(_COMPONENT, flag_values=FLAGS, **self._kwargs) self.assertIsInstance(result, benchmark_config_spec.BenchmarkConfigSpec) self.assertEqual(result.description, 'Test description.') @@ -511,12 +616,14 @@ def testFlagOverridesPropagate(self): self.assertEqual(FLAGS['cloud'].value, 'GCP') self.assertIsInstance(result.vm_groups, dict) self.assertEqual(len(result.vm_groups), 1) - self.assertIsInstance(result.vm_groups['default'], - vm_group_decoders.VmGroupSpec) + self.assertIsInstance( + result.vm_groups['default'], vm_group_decoders.VmGroupSpec + ) self.assertEqual(result.vm_groups['default'].cloud, 'AWS') self.assertEqual(result.vm_groups['default'].os_type, 'ubuntu2004') - self.assertIsInstance(result.vm_groups['default'].vm_spec, - virtual_machine.BaseVmSpec) + self.assertIsInstance( + result.vm_groups['default'].vm_spec, virtual_machine.BaseVmSpec + ) if __name__ == '__main__': diff --git a/tests/configs/option_decoders_test.py b/tests/configs/option_decoders_test.py index 41f74e4a13..687378f68d 100644 --- a/tests/configs/option_decoders_test.py +++ b/tests/configs/option_decoders_test.py @@ -41,9 +41,13 @@ def testNoDefault(self): self.assertIs(decoder.required, True) with self.assertRaises(AssertionError) as cm: decoder.default - self.assertEqual(str(cm.exception), ( - 'Attempted to get the default value of required config option ' - '"test_option".')) + self.assertEqual( + str(cm.exception), + ( + 'Attempted to get the default value of required config option ' + '"test_option".' + ), + ) def testDefaultValue(self): decoder = _PassThroughDecoder(default=None, option=_OPTION) @@ -58,6 +62,7 @@ def testDefaultCallable(self): def testIncompleteDerivedClass(self): class IncompleteDerivedClass(option_decoders.ConfigOptionDecoder): pass + with self.assertRaises(TypeError): IncompleteDerivedClass(option=_OPTION) # pytype: disable=not-instantiable @@ -65,26 +70,36 @@ class IncompleteDerivedClass(option_decoders.ConfigOptionDecoder): class TypeVerifierTestCase(unittest.TestCase): def testRejectNone(self): - decoder = option_decoders.TypeVerifier((int, float), default=None, - option=_OPTION) + decoder = option_decoders.TypeVerifier( + (int, float), default=None, option=_OPTION + ) self.assertIs(decoder.required, False) self.assertIsNone(decoder.default) self.assertIs(decoder.Decode(5, _COMPONENT, _FLAGS), 5) self.assertIs(decoder.Decode(5.5, _COMPONENT, _FLAGS), 5.5) with self.assertRaises(errors.Config.InvalidValue) as cm: decoder.Decode(None, _COMPONENT, _FLAGS) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.test_option value: "None" (of type ' - '"NoneType"). Value must be one of the following types: int, float.')) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.test_option value: "None" (of type ' + '"NoneType"). Value must be one of the following types: int, float.' + ), + ) with self.assertRaises(errors.Config.InvalidValue) as cm: decoder.Decode('red', _COMPONENT, _FLAGS) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.test_option value: "red" (of type "str"). ' - 'Value must be one of the following types: int, float.')) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.test_option value: "red" (of type "str"). ' + 'Value must be one of the following types: int, float.' + ), + ) def testAcceptNone(self): - decoder = option_decoders.TypeVerifier((int, float), default=None, - none_ok=True, option=_OPTION) + decoder = option_decoders.TypeVerifier( + (int, float), default=None, none_ok=True, option=_OPTION + ) self.assertIs(decoder.required, False) self.assertIsNone(decoder.default) self.assertIs(decoder.Decode(5, _COMPONENT, _FLAGS), 5) @@ -92,9 +107,13 @@ def testAcceptNone(self): self.assertIsNone(decoder.Decode(None, _COMPONENT, _FLAGS)) with self.assertRaises(errors.Config.InvalidValue) as cm: decoder.Decode('red', _COMPONENT, _FLAGS) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.test_option value: "red" (of type "str"). ' - 'Value must be one of the following types: NoneType, int, float.')) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.test_option value: "red" (of type "str"). ' + 'Value must be one of the following types: NoneType, int, float.' + ), + ) class BooleanDecoderTestCase(unittest.TestCase): @@ -115,9 +134,13 @@ def testNonBoolean(self): decoder = option_decoders.BooleanDecoder(option=_OPTION) with self.assertRaises(errors.Config.InvalidValue) as cm: decoder.Decode(5, _COMPONENT, _FLAGS) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.test_option value: "5" (of type "int"). ' - 'Value must be one of the following types: bool.')) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.test_option value: "5" (of type "int"). ' + 'Value must be one of the following types: bool.' + ), + ) def testValidBoolean(self): decoder = option_decoders.BooleanDecoder(option=_OPTION) @@ -142,9 +165,13 @@ def testNonInt(self): decoder = option_decoders.IntDecoder(option=_OPTION) with self.assertRaises(errors.Config.InvalidValue) as cm: decoder.Decode('5', _COMPONENT, _FLAGS) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.test_option value: "5" (of type "str"). ' - 'Value must be one of the following types: int.')) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.test_option value: "5" (of type "str"). ' + 'Value must be one of the following types: int.' + ), + ) def testValidInt(self): decoder = option_decoders.IntDecoder(option=_OPTION) @@ -154,9 +181,13 @@ def testMax(self): decoder = option_decoders.IntDecoder(max=2, option=_OPTION) with self.assertRaises(errors.Config.InvalidValue) as cm: decoder.Decode(5, _COMPONENT, _FLAGS) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.test_option value: "5". Value must be at ' - 'most 2.')) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.test_option value: "5". Value must be at ' + 'most 2.' + ), + ) self.assertIs(decoder.Decode(2, _COMPONENT, _FLAGS), 2) self.assertIs(decoder.Decode(1, _COMPONENT, _FLAGS), 1) @@ -164,9 +195,13 @@ def testMin(self): decoder = option_decoders.IntDecoder(min=10, option=_OPTION) with self.assertRaises(errors.Config.InvalidValue) as cm: decoder.Decode(5, _COMPONENT, _FLAGS) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.test_option value: "5". Value must be at ' - 'least 10.')) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.test_option value: "5". Value must be at ' + 'least 10.' + ), + ) self.assertIs(decoder.Decode(10, _COMPONENT, _FLAGS), 10) self.assertIs(decoder.Decode(15, _COMPONENT, _FLAGS), 15) @@ -197,10 +232,14 @@ def testNonString(self): decoder = option_decoders.StringDecoder(option=_OPTION) with self.assertRaises(errors.Config.InvalidValue) as cm: decoder.Decode(5, _COMPONENT, _FLAGS) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.test_option value: "5" (of type "int"). ' - 'Value must be one of the following types: %s.' % - six.string_types[0].__name__)) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.test_option value: "5" (of type "int"). ' + 'Value must be one of the following types: %s.' + % six.string_types[0].__name__ + ), + ) def testValidString(self): decoder = option_decoders.StringDecoder(option=_OPTION) @@ -225,9 +264,13 @@ def testNonFloat(self): decoder = option_decoders.FloatDecoder(option=_OPTION) with self.assertRaises(errors.Config.InvalidValue) as cm: decoder.Decode('5', _COMPONENT, _FLAGS) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.test_option value: "5" (of type "str"). ' - 'Value must be one of the following types: float, int.')) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.test_option value: "5" (of type "str"). ' + 'Value must be one of the following types: float, int.' + ), + ) def testValidFloat(self): decoder = option_decoders.FloatDecoder(option=_OPTION) @@ -242,9 +285,13 @@ def testMaxFloat(self): decoder = option_decoders.FloatDecoder(max=MAX, option=_OPTION) with self.assertRaises(errors.Config.InvalidValue) as cm: decoder.Decode(5, _COMPONENT, _FLAGS) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.test_option value: "5". Value must be at ' - 'most %s.' % MAX)) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.test_option value: "5". Value must be at ' + 'most %s.' % MAX + ), + ) self.assertIs(decoder.Decode(MAX, _COMPONENT, _FLAGS), MAX) self.assertIs(decoder.Decode(2, _COMPONENT, _FLAGS), 2) self.assertIs(decoder.Decode(1, _COMPONENT, _FLAGS), 1) @@ -254,9 +301,13 @@ def testMaxInt(self): decoder = option_decoders.FloatDecoder(max=MAX, option=_OPTION) with self.assertRaises(errors.Config.InvalidValue) as cm: decoder.Decode(2.01, _COMPONENT, _FLAGS) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.test_option value: "2.01". Value must be at ' - 'most %s.' % MAX)) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.test_option value: "2.01". Value must be at' + ' most %s.' % MAX + ), + ) self.assertIs(decoder.Decode(MAX, _COMPONENT, _FLAGS), MAX) self.assertIs(decoder.Decode(2.0, _COMPONENT, _FLAGS), 2.0) self.assertIs(decoder.Decode(1, _COMPONENT, _FLAGS), 1) @@ -266,9 +317,13 @@ def testMinFloat(self): decoder = option_decoders.FloatDecoder(min=MIN, option=_OPTION) with self.assertRaises(errors.Config.InvalidValue) as cm: decoder.Decode(0, _COMPONENT, _FLAGS) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.test_option value: "0". Value must be at ' - 'least %s.' % MIN)) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.test_option value: "0". Value must be at ' + 'least %s.' % MIN + ), + ) self.assertIs(decoder.Decode(MIN, _COMPONENT, _FLAGS), MIN) self.assertIs(decoder.Decode(2, _COMPONENT, _FLAGS), 2) self.assertIs(decoder.Decode(5, _COMPONENT, _FLAGS), 5) @@ -278,9 +333,13 @@ def testMinInt(self): decoder = option_decoders.FloatDecoder(min=MIN, option=_OPTION) with self.assertRaises(errors.Config.InvalidValue) as cm: decoder.Decode(0, _COMPONENT, _FLAGS) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.test_option value: "0". Value must be at ' - 'least %s.' % MIN)) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.test_option value: "0". Value must be at ' + 'least %s.' % MIN + ), + ) self.assertIs(decoder.Decode(MIN, _COMPONENT, _FLAGS), MIN) self.assertIs(decoder.Decode(2.0, _COMPONENT, _FLAGS), 2.0) self.assertIs(decoder.Decode(5, _COMPONENT, _FLAGS), 5) @@ -301,27 +360,38 @@ def setUp(self): self._int_decoder = option_decoders.IntDecoder() def testNonListInputType(self): - decoder = option_decoders.ListDecoder(item_decoder=self._int_decoder, - option=_OPTION) + decoder = option_decoders.ListDecoder( + item_decoder=self._int_decoder, option=_OPTION + ) with self.assertRaises(errors.Config.InvalidValue) as cm: decoder.Decode(5, _COMPONENT, _FLAGS) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.test_option value: "5" (of type "int"). ' - 'Value must be one of the following types: list.')) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.test_option value: "5" (of type "int"). ' + 'Value must be one of the following types: list.' + ), + ) def testNone(self): - decoder = option_decoders.ListDecoder(item_decoder=self._int_decoder, - none_ok=True, option=_OPTION) + decoder = option_decoders.ListDecoder( + item_decoder=self._int_decoder, none_ok=True, option=_OPTION + ) self.assertIsNone(decoder.Decode(None, _COMPONENT, _FLAGS)) def testInvalidItem(self): - decoder = option_decoders.ListDecoder(item_decoder=self._int_decoder, - option=_OPTION) + decoder = option_decoders.ListDecoder( + item_decoder=self._int_decoder, option=_OPTION + ) with self.assertRaises(errors.Config.InvalidValue) as cm: decoder.Decode([5, 4, 3.5], _COMPONENT, _FLAGS) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.test_option[2] value: "3.5" (of type "float"). ' - 'Value must be one of the following types: int.')) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.test_option[2] value: "3.5" (of type' + ' "float"). Value must be one of the following types: int.' + ), + ) if __name__ == '__main__': diff --git a/tests/configs/spec_test.py b/tests/configs/spec_test.py index c3f0410565..5c76d22732 100644 --- a/tests/configs/spec_test.py +++ b/tests/configs/spec_test.py @@ -17,7 +17,6 @@ import unittest import mock - from perfkitbenchmarker.configs import option_decoders from perfkitbenchmarker.configs import spec from six.moves import range @@ -51,15 +50,19 @@ def testSuccessfulDecode(self): expected_decode_call_order = sorted(self.config_option_names) observed_decode_call_order = [] callback = observed_decode_call_order.append - config = {config_option_name: callback - for config_option_name in self.config_option_names} + config = { + config_option_name: callback + for config_option_name in self.config_option_names + } _TestFixedDecodeOrderSpec(_COMPONENT, **config) self.assertEqual(observed_decode_call_order, expected_decode_call_order) def testFailedDecode(self): callback = mock.MagicMock(side_effect=ValueError) - config = {config_option_name: callback - for config_option_name in self.config_option_names} + config = { + config_option_name: callback + for config_option_name in self.config_option_names + } # Only the first OptionDecoder's Decode method should be called. with self.assertRaises(ValueError): _TestFixedDecodeOrderSpec(_COMPONENT, **config) diff --git a/tests/context_test.py b/tests/context_test.py index dddc192d9c..1642e6ede1 100644 --- a/tests/context_test.py +++ b/tests/context_test.py @@ -18,7 +18,6 @@ import unittest import mock - from perfkitbenchmarker import background_tasks from perfkitbenchmarker import context from tests import pkb_common_test_case @@ -45,8 +44,7 @@ def _DoWork(_): new_benchmark_spec = mock.MagicMock() context.SetThreadBenchmarkSpec(new_benchmark_spec) self.assertNotEqual(benchmark_spec, context.GetThreadBenchmarkSpec()) - self.assertEqual(new_benchmark_spec, - context.GetThreadBenchmarkSpec()) + self.assertEqual(new_benchmark_spec, context.GetThreadBenchmarkSpec()) background_tasks.RunThreaded(_DoWork, list(range(10))) diff --git a/tests/custom_virtual_machine_spec_test.py b/tests/custom_virtual_machine_spec_test.py index ff6e7b713c..39855ae96b 100644 --- a/tests/custom_virtual_machine_spec_test.py +++ b/tests/custom_virtual_machine_spec_test.py @@ -40,67 +40,96 @@ def testValidStrings(self): def testImproperPattern(self): with self.assertRaises(errors.Config.InvalidValue) as cm: self.decoder.Decode('1280', _COMPONENT, _FLAGS) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.memory value: "1280". Examples of valid ' - 'values: "1280MiB", "7.5GiB".')) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.memory value: "1280". Examples of valid ' + 'values: "1280MiB", "7.5GiB".' + ), + ) def testInvalidFloat(self): with self.assertRaises(errors.Config.InvalidValue) as cm: self.decoder.Decode('1280.9.8MiB', _COMPONENT, _FLAGS) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.memory value: "1280.9.8MiB". "1280.9.8" is not ' - 'a valid float.')) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.memory value: "1280.9.8MiB". "1280.9.8" is' + ' not a valid float.' + ), + ) def testNonIntegerMiB(self): with self.assertRaises(errors.Config.InvalidValue) as cm: self.decoder.Decode('7.6GiB', _COMPONENT, _FLAGS) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.memory value: "7.6GiB". The specified size ' - 'must be an integer number of MiB.')) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.memory value: "7.6GiB". The specified size ' + 'must be an integer number of MiB.' + ), + ) class CustomMachineTypeSpecTestCase(unittest.TestCase): def testValid(self): result = custom_virtual_machine_spec.CustomMachineTypeSpec( - _COMPONENT, cpus=1, memory='7.5GiB') + _COMPONENT, cpus=1, memory='7.5GiB' + ) self.assertEqual(result.cpus, 1) self.assertEqual(result.memory, 7680) def testMissingCpus(self): with self.assertRaises(errors.Config.MissingOption) as cm: custom_virtual_machine_spec.CustomMachineTypeSpec( - _COMPONENT, memory='7.5GiB') - self.assertEqual(str(cm.exception), ( - 'Required options were missing from test_component: cpus.')) + _COMPONENT, memory='7.5GiB' + ) + self.assertEqual( + str(cm.exception), + 'Required options were missing from test_component: cpus.', + ) def testMissingMemory(self): with self.assertRaises(errors.Config.MissingOption) as cm: custom_virtual_machine_spec.CustomMachineTypeSpec(_COMPONENT, cpus=1) - self.assertEqual(str(cm.exception), ( - 'Required options were missing from test_component: memory.')) + self.assertEqual( + str(cm.exception), + 'Required options were missing from test_component: memory.', + ) def testExtraOptions(self): with self.assertRaises(errors.Config.UnrecognizedOption) as cm: custom_virtual_machine_spec.CustomMachineTypeSpec( - _COMPONENT, cpus=1, memory='7.5GiB', extra1='one', extra2=2) - self.assertEqual(str(cm.exception), ( - 'Unrecognized options were found in test_component: extra1, extra2.')) + _COMPONENT, cpus=1, memory='7.5GiB', extra1='one', extra2=2 + ) + self.assertEqual( + str(cm.exception), + 'Unrecognized options were found in test_component: extra1, extra2.', + ) def testInvalidCpus(self): with self.assertRaises(errors.Config.InvalidValue) as cm: - custom_virtual_machine_spec.CustomMachineTypeSpec(_COMPONENT, cpus=0, - memory='7.5GiB') - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.cpus value: "0". Value must be at least 1.')) + custom_virtual_machine_spec.CustomMachineTypeSpec( + _COMPONENT, cpus=0, memory='7.5GiB' + ) + self.assertEqual( + str(cm.exception), + 'Invalid test_component.cpus value: "0". Value must be at least 1.', + ) def testInvalidMemory(self): with self.assertRaises(errors.Config.InvalidValue) as cm: custom_virtual_machine_spec.CustomMachineTypeSpec( - _COMPONENT, cpus=1, memory=None) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.memory value: "None" (of type "NoneType"). ' - 'Value must be one of the following types: %s.' % _STRING_TYPE_NAME)) + _COMPONENT, cpus=1, memory=None + ) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.memory value: "None" (of type "NoneType"). ' + 'Value must be one of the following types: %s.' % _STRING_TYPE_NAME + ), + ) class MachineTypeDecoderTestCase(unittest.TestCase): @@ -108,34 +137,45 @@ class MachineTypeDecoderTestCase(unittest.TestCase): def setUp(self): super(MachineTypeDecoderTestCase, self).setUp() self.decoder = custom_virtual_machine_spec.MachineTypeDecoder( - option='machine_type') + option='machine_type' + ) def testDecodeString(self): result = self.decoder.Decode('n1-standard-8', _COMPONENT, {}) self.assertEqual(result, 'n1-standard-8') def testDecodeCustomVm(self): - result = self.decoder.Decode({'cpus': 1, 'memory': '7.5GiB'}, _COMPONENT, - {}) - self.assertIsInstance(result, - custom_virtual_machine_spec.CustomMachineTypeSpec) + result = self.decoder.Decode( + {'cpus': 1, 'memory': '7.5GiB'}, _COMPONENT, {} + ) + self.assertIsInstance( + result, custom_virtual_machine_spec.CustomMachineTypeSpec + ) self.assertEqual(result.cpus, 1) self.assertEqual(result.memory, 7680) def testDecodeInvalidType(self): with self.assertRaises(errors.Config.InvalidValue) as cm: self.decoder.Decode(None, _COMPONENT, {}) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.machine_type value: "None" (of type ' - '"NoneType"). Value must be one of the following types: %s, ' - 'dict.' % _STRING_TYPE_NAME)) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.machine_type value: "None" (of type ' + '"NoneType"). Value must be one of the following types: %s, ' + 'dict.' % _STRING_TYPE_NAME + ), + ) def testDecodeInvalidValue(self): with self.assertRaises(errors.Config.InvalidValue) as cm: self.decoder.Decode({'cpus': 0, 'memory': '7.5GiB'}, _COMPONENT, {}) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.machine_type.cpus value: "0". Value must be at ' - 'least 1.')) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.machine_type.cpus value: "0". Value must be' + ' at least 1.' + ), + ) if __name__ == '__main__': diff --git a/tests/data_discovery_service_test.py b/tests/data_discovery_service_test.py index bd06c1876b..edd0182d29 100644 --- a/tests/data_discovery_service_test.py +++ b/tests/data_discovery_service_test.py @@ -24,11 +24,14 @@ class DataDiscoveryServiceTest(pkb_common_test_case.PkbCommonTestCase): def setUp(self): super().setUp() self.path_flag_mock = self.enter_context( - mock.patch.object(data_discovery_service, - '_DATA_DISCOVERY_OBJECT_STORE_PATH')) + mock.patch.object( + data_discovery_service, '_DATA_DISCOVERY_OBJECT_STORE_PATH' + ) + ) self.path_flag_mock.value = 's3://foo/bar' self.region_flag_mock = self.enter_context( - mock.patch.object(data_discovery_service, '_DATA_DISCOVERY_REGION')) + mock.patch.object(data_discovery_service, '_DATA_DISCOVERY_REGION') + ) self.region_flag_mock.value = 'us-east-1' self.service = FakeDataDiscoveryService() @@ -38,10 +41,10 @@ def testInitialization(self): self.assertEqual(self.service.region, 'us-east-1') def testGetMetadata(self): - self.assertEqual(self.service.GetMetadata(), { - 'cloud': 'abstract', - 'data_discovery_region': 'us-east-1' - }) + self.assertEqual( + self.service.GetMetadata(), + {'cloud': 'abstract', 'data_discovery_region': 'us-east-1'}, + ) if __name__ == '__main__': diff --git a/tests/data_test.py b/tests/data_test.py index 1e0c299909..d7c0082fcb 100644 --- a/tests/data_test.py +++ b/tests/data_test.py @@ -19,7 +19,6 @@ import unittest import mock - from perfkitbenchmarker import data @@ -40,9 +39,9 @@ def _Create(self, file_name, content=''): def testResourcePath_NonExistantResource(self): self.assertListEqual([], os.listdir(self.temp_dir)) - self.assertRaises(data.ResourceNotFound, - self.instance.ResourcePath, - 'fake.txt') + self.assertRaises( + data.ResourceNotFound, self.instance.ResourcePath, 'fake.txt' + ) def testResourcePath_ExtantResource(self): file_name = 'test.txt' @@ -64,9 +63,9 @@ def setUp(self): self.instance = data.PackageResourceLoader(data.__name__) def testResourcePath_NonExistantResource(self): - self.assertRaises(data.ResourceNotFound, - self.instance.ResourcePath, - 'fake.txt') + self.assertRaises( + data.ResourceNotFound, self.instance.ResourcePath, 'fake.txt' + ) def testResourcePath_ExtantResource(self): file_name = '__init__.py' @@ -92,13 +91,15 @@ def setUp(self): mock_not_found_loader = mock.MagicMock(spec=data.ResourceLoader) mock_not_found_loader.ResourceExists.return_value = False - p = mock.patch(data.__name__ + '._GetResourceLoaders', - return_value=[mock_found_loader]) + p = mock.patch( + data.__name__ + '._GetResourceLoaders', return_value=[mock_found_loader] + ) self.mock_get_resource_loaders = p.start() self.addCleanup(p.stop) - p = mock.patch(data.__name__ + '.DEFAULT_RESOURCE_LOADERS', - [mock_not_found_loader]) + p = mock.patch( + data.__name__ + '.DEFAULT_RESOURCE_LOADERS', [mock_not_found_loader] + ) p.start() self.addCleanup(p.stop) diff --git a/tests/disk_iops_to_capacity_test.py b/tests/disk_iops_to_capacity_test.py index f98906a1de..67dbcc438d 100644 --- a/tests/disk_iops_to_capacity_test.py +++ b/tests/disk_iops_to_capacity_test.py @@ -61,25 +61,48 @@ def testSetStorageSizeGCP(self): self.assertEqual(GCPconfig2.GetSize(), 1000) def testValidateProvider(self): - self.assertRaises(disk_iops_to_capacity.InvalidProviderError, - disk_iops_to_capacity.DiskIOPSToCapacity, 300, - 'NONPROVIDER') + self.assertRaises( + disk_iops_to_capacity.InvalidProviderError, + disk_iops_to_capacity.DiskIOPSToCapacity, + 300, + 'NONPROVIDER', + ) def testValidateIOPS(self): - self.assertRaises(disk_iops_to_capacity.InvalidIOPSError, - disk_iops_to_capacity.DiskIOPSToCapacity, 0, 'AWS') - self.assertRaises(disk_iops_to_capacity.InvalidIOPSError, - disk_iops_to_capacity.DiskIOPSToCapacity, 50000, 'GCP') - self.assertRaises(disk_iops_to_capacity.InvalidIOPSError, - disk_iops_to_capacity.DiskIOPSToCapacity, 90000, 'AWS') + self.assertRaises( + disk_iops_to_capacity.InvalidIOPSError, + disk_iops_to_capacity.DiskIOPSToCapacity, + 0, + 'AWS', + ) + self.assertRaises( + disk_iops_to_capacity.InvalidIOPSError, + disk_iops_to_capacity.DiskIOPSToCapacity, + 50000, + 'GCP', + ) + self.assertRaises( + disk_iops_to_capacity.InvalidIOPSError, + disk_iops_to_capacity.DiskIOPSToCapacity, + 90000, + 'AWS', + ) def testValidateStorageType(self): - self.assertRaises(disk_iops_to_capacity.InvalidStorageTypeError, - disk_iops_to_capacity.DiskIOPSToCapacity, 100, 'AWS', - 'ebs-piops') - self.assertRaises(disk_iops_to_capacity.InvalidStorageTypeError, - disk_iops_to_capacity.DiskIOPSToCapacity, 100, 'GCP', - 'pd-hhd') + self.assertRaises( + disk_iops_to_capacity.InvalidStorageTypeError, + disk_iops_to_capacity.DiskIOPSToCapacity, + 100, + 'AWS', + 'ebs-piops', + ) + self.assertRaises( + disk_iops_to_capacity.InvalidStorageTypeError, + disk_iops_to_capacity.DiskIOPSToCapacity, + 100, + 'GCP', + 'pd-hhd', + ) if __name__ == '__main__': diff --git a/tests/disk_metadata_test.py b/tests/disk_metadata_test.py index ab0ade85af..1b11d998a8 100644 --- a/tests/disk_metadata_test.py +++ b/tests/disk_metadata_test.py @@ -18,7 +18,6 @@ from absl import flags from absl.testing import flagsaver import mock - from perfkitbenchmarker import benchmark_spec from perfkitbenchmarker import context from perfkitbenchmarker import disk @@ -50,90 +49,100 @@ def setUp(self): self.addCleanup(p.stop) config_spec = benchmark_config_spec.BenchmarkConfigSpec( - _BENCHMARK_NAME, flag_values=FLAGS, vm_groups={}) + _BENCHMARK_NAME, flag_values=FLAGS, vm_groups={} + ) self.benchmark_spec = benchmark_spec.BenchmarkSpec( - mock.MagicMock(), config_spec, _BENCHMARK_UID) + mock.MagicMock(), config_spec, _BENCHMARK_UID + ) class GcpDiskMetadataTest(_DiskMetadataTestCase): def testPDStandard(self): disk_spec = gce_disk.GceDiskSpec( - _COMPONENT, disk_size=2, disk_type=gce_disk.PD_STANDARD) + _COMPONENT, disk_size=2, disk_type=gce_disk.PD_STANDARD + ) disk_obj = gce_disk.GceDisk(disk_spec, 'name', 'zone', 'project') self.assertDictContainsSubset( - {disk.MEDIA: disk.HDD, disk.REPLICATION: disk.ZONE}, - disk_obj.metadata + {disk.MEDIA: disk.HDD, disk.REPLICATION: disk.ZONE}, disk_obj.metadata ) def testLocalSSD(self): disk_spec = gce_disk.GceDiskSpec( - _COMPONENT, disk_size=2, disk_type=disk.LOCAL, - interface=gce_disk.NVME) + _COMPONENT, disk_size=2, disk_type=disk.LOCAL, interface=gce_disk.NVME + ) disk_obj = gce_disk.GceLocalDisk(disk_spec, 'name') self.assertDictContainsSubset( - {disk.MEDIA: disk.SSD, disk.REPLICATION: 'none', - 'interface': gce_disk.NVME}, - disk_obj.metadata + { + disk.MEDIA: disk.SSD, + disk.REPLICATION: 'none', + 'interface': gce_disk.NVME, + }, + disk_obj.metadata, ) class AwsDiskMetadataTest(_DiskMetadataTestCase): - def DoAwsDiskTest(self, disk_type, machine_type, - goal_media, goal_replication): - disk_spec = aws_disk.AwsDiskSpec(_COMPONENT, disk_size=2, - disk_type=disk_type) + def DoAwsDiskTest( + self, disk_type, machine_type, goal_media, goal_replication + ): + disk_spec = aws_disk.AwsDiskSpec( + _COMPONENT, disk_size=2, disk_type=disk_type + ) vm_spec = aws_virtual_machine.AwsVmSpec( - 'test_vm_spec.AWS', zone='us-east-1a', machine_type=machine_type) + 'test_vm_spec.AWS', zone='us-east-1a', machine_type=machine_type + ) vm = aws_virtual_machine.Ubuntu2004BasedAwsVirtualMachine(vm_spec) vm.GetNVMEDeviceInfo = mock.Mock() - vm.GetNVMEDeviceInfo.return_value = [ - { - 'DevicePath': '/dev/nvme1n2', - 'SerialNumber': 'vol67890', - 'ModelNumber': 'Amazon EC2 NVMe Instance Storage', - } - ] + vm.GetNVMEDeviceInfo.return_value = [{ + 'DevicePath': '/dev/nvme1n2', + 'SerialNumber': 'vol67890', + 'ModelNumber': 'Amazon EC2 NVMe Instance Storage', + }] vm.LogDeviceByDiskSpecId('0_0', 'foobar_1') vm.LogDeviceByName('foobar_1', 'vol67890', None) vm.CreateScratchDisk(0, disk_spec) self.assertDictContainsSubset( {disk.MEDIA: goal_media, disk.REPLICATION: goal_replication}, - vm.scratch_disks[0].metadata + vm.scratch_disks[0].metadata, ) def testLocalSSD(self): - self.DoAwsDiskTest( - disk.LOCAL, - 'c3.2xlarge', - disk.SSD, - disk.NONE) + self.DoAwsDiskTest(disk.LOCAL, 'c3.2xlarge', disk.SSD, disk.NONE) def testLocalHDD(self): - self.DoAwsDiskTest( - disk.LOCAL, - 'd2.2xlarge', - disk.HDD, - disk.NONE) + self.DoAwsDiskTest(disk.LOCAL, 'd2.2xlarge', disk.HDD, disk.NONE) class AzureDiskMetadataTest(_DiskMetadataTestCase): - def DoAzureDiskTest(self, storage_type, disk_type, machine_type, - goal_media, goal_replication, - goal_host_caching, disk_size=2, - goal_size=2, goal_stripes=1): + def DoAzureDiskTest( + self, + storage_type, + disk_type, + machine_type, + goal_media, + goal_replication, + goal_host_caching, + disk_size=2, + goal_size=2, + goal_stripes=1, + ): FLAGS.azure_storage_type = storage_type FLAGS.azure_host_caching = goal_host_caching - disk_spec = disk.BaseDiskSpec(_COMPONENT, disk_size=disk_size, - disk_type=disk_type, - num_striped_disks=goal_stripes) + disk_spec = disk.BaseDiskSpec( + _COMPONENT, + disk_size=disk_size, + disk_type=disk_type, + num_striped_disks=goal_stripes, + ) vm_spec = azure_virtual_machine.AzureVmSpec( - 'test_vm_spec.AZURE', zone='eastus2', machine_type=machine_type) + 'test_vm_spec.AZURE', zone='eastus2', machine_type=machine_type + ) vm = azure_virtual_machine.Ubuntu2004BasedAzureVirtualMachine(vm_spec) azure_disk.AzureDisk.Create = mock.Mock() @@ -141,61 +150,63 @@ def DoAzureDiskTest(self, storage_type, disk_type, machine_type, vm.StripeDisks = mock.Mock() vm.CreateScratchDisk(0, disk_spec) - expected = {disk.MEDIA: goal_media, - disk.REPLICATION: goal_replication, - 'num_stripes': goal_stripes, - 'size': goal_size} + expected = { + disk.MEDIA: goal_media, + disk.REPLICATION: goal_replication, + 'num_stripes': goal_stripes, + 'size': goal_size, + } if goal_host_caching: expected[azure_disk.HOST_CACHING] = goal_host_caching self.assertDictContainsSubset(expected, vm.scratch_disks[0].metadata) @flagsaver.flagsaver(azure_version_log=False) def testPremiumStorage(self): - self.DoAzureDiskTest(azure_flags.PLRS, - azure_disk.PREMIUM_STORAGE, - 'Standard_D1', - disk.SSD, - disk.ZONE, - azure_flags.READ_ONLY) + self.DoAzureDiskTest( + azure_flags.PLRS, + azure_disk.PREMIUM_STORAGE, + 'Standard_D1', + disk.SSD, + disk.ZONE, + azure_flags.READ_ONLY, + ) @flagsaver.flagsaver(azure_version_log=False) def testStandardDisk(self): - self.DoAzureDiskTest(azure_flags.ZRS, - azure_disk.STANDARD_DISK, - 'Standard_D1', - disk.HDD, - disk.REGION, - azure_flags.NONE) + self.DoAzureDiskTest( + azure_flags.ZRS, + azure_disk.STANDARD_DISK, + 'Standard_D1', + disk.HDD, + disk.REGION, + azure_flags.NONE, + ) @flagsaver.flagsaver(azure_version_log=False) def testLocalHDD(self): - self.DoAzureDiskTest(azure_flags.LRS, - disk.LOCAL, - 'Standard_A1', - disk.HDD, - disk.NONE, - None) + self.DoAzureDiskTest( + azure_flags.LRS, disk.LOCAL, 'Standard_A1', disk.HDD, disk.NONE, None + ) @flagsaver.flagsaver(azure_version_log=False) def testLocalSSD(self): - self.DoAzureDiskTest(azure_flags.LRS, - disk.LOCAL, - 'Standard_DS2', - disk.SSD, - disk.NONE, - None) + self.DoAzureDiskTest( + azure_flags.LRS, disk.LOCAL, 'Standard_DS2', disk.SSD, disk.NONE, None + ) @flagsaver.flagsaver(azure_version_log=False) def testStripedDisk(self): - self.DoAzureDiskTest(azure_flags.LRS, - azure_disk.STANDARD_DISK, - 'Standard_D1', - disk.HDD, - disk.ZONE, - azure_flags.READ_ONLY, - disk_size=5, - goal_size=10, - goal_stripes=2) + self.DoAzureDiskTest( + azure_flags.LRS, + azure_disk.STANDARD_DISK, + 'Standard_D1', + disk.HDD, + disk.ZONE, + azure_flags.READ_ONLY, + disk_size=5, + goal_size=10, + goal_stripes=2, + ) if __name__ == '__main__': diff --git a/tests/disk_test.py b/tests/disk_test.py index 4fc53b8615..2017610b57 100644 --- a/tests/disk_test.py +++ b/tests/disk_test.py @@ -16,7 +16,6 @@ import unittest from absl import flags import mock - from perfkitbenchmarker import disk from perfkitbenchmarker import errors from tests import pkb_common_test_case @@ -46,7 +45,8 @@ def testProvidedValid(self): disk_size=75, disk_type='test_disk_type', mount_point='/mountpoint', - num_striped_disks=2) + num_striped_disks=2, + ) self.assertEqual(spec.device_path, 'test_device_path') self.assertEqual(spec.disk_number, 1) self.assertEqual(spec.disk_size, 75) @@ -61,7 +61,8 @@ def testProvidedNone(self): disk_number=None, disk_size=None, disk_type=None, - mount_point=None) + mount_point=None, + ) self.assertIsNone(spec.device_path) self.assertIsNone(spec.disk_number) self.assertIsNone(spec.disk_size) @@ -74,8 +75,11 @@ def testUnrecognizedOptions(self): disk.BaseDiskSpec(_COMPONENT, color='red', flavor='cherry', texture=None) self.assertEqual( str(cm.exception), - ('Unrecognized options were found in test_component: color, flavor, ' - 'texture.')) + ( + 'Unrecognized options were found in test_component: color, flavor, ' + 'texture.' + ), + ) def testInvalidOptionTypes(self): with self.assertRaises(errors.Config.InvalidValue): @@ -108,7 +112,8 @@ def testNonPresentFlagsDoNotOverrideConfigs(self): disk_size=75, disk_type='config_disk_type', mount_point='/mountpoint', - num_striped_disks=2) + num_striped_disks=2, + ) self.assertEqual(spec.device_path, 'config_device_path') self.assertEqual(spec.disk_number, 1) self.assertEqual(spec.disk_size, 75) @@ -129,7 +134,8 @@ def testPresentFlagsOverrideConfigs(self): disk_size=75, disk_type='config_disk_type', mount_point='/mountpoint', - num_striped_disks=2) + num_striped_disks=2, + ) self.assertEqual(spec.device_path, 'config_device_path') self.assertEqual(spec.disk_number, 1) self.assertEqual(spec.disk_size, 100) @@ -145,8 +151,9 @@ def __init__(self, flags=None, default_nfs_version=None): disk_spec = disk.BaseNFSDiskSpec(_COMPONENT, flags) else: disk_spec = disk.BaseNFSDiskSpec(_COMPONENT) - super(_NfsDisk, self).__init__(disk_spec, 'host1:/volume1', - default_nfs_version) + super(_NfsDisk, self).__init__( + disk_spec, 'host1:/volume1', default_nfs_version + ) class NfsDiskTestCase(pkb_common_test_case.PkbCommonTestCase): @@ -157,7 +164,7 @@ def MountOptions(self, **overrides): 'retrans': 2, 'rsize': 1048576, 'timeo': 600, - 'wsize': 1048576 + 'wsize': 1048576, } mount_options.update(overrides) return mount_options @@ -175,8 +182,9 @@ def MountOptionsAsDict(self, mount_options_str): def testDefaults(self): nfs_disk = _NfsDisk() self.assertEqual('host1:/volume1', nfs_disk.GetDevicePath()) - self.assertEqual(self.MountOptions(), - self.MountOptionsAsDict(nfs_disk.mount_options)) + self.assertEqual( + self.MountOptions(), self.MountOptionsAsDict(nfs_disk.mount_options) + ) self.assertEqual(nfs_disk.mount_options, nfs_disk.fstab_options) disk_meta = {} for key, value in six.iteritems(self.MountOptions()): @@ -201,10 +209,12 @@ def testNfsFlags(self): timeo=30, wsize=2, nfsvers='4.1', - nconnect=5) + nconnect=5, + ) mount_options.pop('hard') - self.assertEqual(mount_options, - self.MountOptionsAsDict(nfs_disk.mount_options)) + self.assertEqual( + mount_options, self.MountOptionsAsDict(nfs_disk.mount_options) + ) def testDefaultNfsVersion(self): nfs_disk = _NfsDisk(default_nfs_version='4.1') @@ -229,10 +239,12 @@ def __init__(self, default_smb_version=None): disk_spec = disk.BaseSMBDiskSpec(_COMPONENT, FLAGS) else: disk_spec = disk.BaseSMBDiskSpec(_COMPONENT) - super(_SmbDisk, self).__init__(disk_spec, 'host1', { - 'user': 'username', - 'pw': 'password' - }, default_smb_version) + super(_SmbDisk, self).__init__( + disk_spec, + 'host1', + {'user': 'username', 'pw': 'password'}, + default_smb_version, + ) class SmbDiskTestCase(pkb_common_test_case.PkbCommonTestCase): @@ -253,7 +265,8 @@ def MountOptions(self, **overrides): def MountOptionsAsDict(self, mount_options_str): options = dict() string_values = set( - ['vers', 'username', 'password', 'dir_mode', 'file_mode']) + ['vers', 'username', 'password', 'dir_mode', 'file_mode'] + ) for entry in mount_options_str.split(','): parts = entry.split('=', 1) key = parts[0] @@ -264,8 +277,9 @@ def MountOptionsAsDict(self, mount_options_str): def testDefaults(self): smb_disk = _SmbDisk() self.assertEqual('host1', smb_disk.GetDevicePath()) - self.assertEqual(self.MountOptions(), - self.MountOptionsAsDict(smb_disk.mount_options)) + self.assertEqual( + self.MountOptions(), self.MountOptionsAsDict(smb_disk.mount_options) + ) self.assertEqual(smb_disk.mount_options, smb_disk.fstab_options) disk_meta = {} disk_meta.update({'num_stripes': 1, 'size': None, 'type': None}) @@ -276,9 +290,11 @@ def testSmbFlags(self): FLAGS['smb_version'].parse('3.0') smb_disk = _SmbDisk(FLAGS) mount_options = self.MountOptions( - vers='3.0', dir_mode='0777', file_mode='0777') - self.assertEqual(mount_options, - self.MountOptionsAsDict(smb_disk.mount_options)) + vers='3.0', dir_mode='0777', file_mode='0777' + ) + self.assertEqual( + mount_options, self.MountOptionsAsDict(smb_disk.mount_options) + ) def testDefaultSmbVersion(self): smb_disk = _SmbDisk(default_smb_version='3.0') @@ -295,5 +311,6 @@ def testAttach(self): smb_disk.Attach(vm) vm.InstallPackages.assert_called_with('cifs-utils') + if __name__ == '__main__': unittest.main() diff --git a/tests/edw_benchmark_results_aggregator_test.py b/tests/edw_benchmark_results_aggregator_test.py index dec35f937b..0e3d4469ff 100644 --- a/tests/edw_benchmark_results_aggregator_test.py +++ b/tests/edw_benchmark_results_aggregator_test.py @@ -42,8 +42,9 @@ class EdwQueryPerformanceTest(pkb_common_test_case.PkbCommonTestCase): def test_get_performance_simple(self): q_p = agg.EdwQueryPerformance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY) actual_sample = q_p.get_performance_sample(METADATA_EMPTY) - expected_sample = sample.Sample('edw_raw_query_time', Q1_PERFORMANCE, SECS, - METADATA_EMPTY) + expected_sample = sample.Sample( + 'edw_raw_query_time', Q1_PERFORMANCE, SECS, METADATA_EMPTY + ) self.assertEqual(actual_sample.metric, expected_sample.metric) self.assertEqual(actual_sample.value, expected_sample.value) self.assertEqual(actual_sample.unit, expected_sample.unit) @@ -66,8 +67,9 @@ def test_is_successful(self): def test_get_performance_failed_query(self): q_p = agg.EdwQueryPerformance(QFAIL_NAME, QFAIL_PERFORMANCE, METADATA_EMPTY) actual_sample = q_p.get_performance_sample(METADATA_EMPTY) - expected_sample = sample.Sample('edw_raw_query_time', QFAIL_PERFORMANCE, - SECS, METADATA_EMPTY) + expected_sample = sample.Sample( + 'edw_raw_query_time', QFAIL_PERFORMANCE, SECS, METADATA_EMPTY + ) self.assertEqual(actual_sample.metric, expected_sample.metric) self.assertEqual(actual_sample.value, expected_sample.value) self.assertEqual(actual_sample.unit, expected_sample.unit) @@ -80,7 +82,7 @@ def test_get_performance_with_no_metadata(self): self.assertEqual(actual_sample.unit, SECS) expected_metadata = { 'query': Q1_NAME, - 'execution_status': agg.EdwQueryExecutionStatus.SUCCESSFUL + 'execution_status': agg.EdwQueryExecutionStatus.SUCCESSFUL, } self.assertDictEqual(actual_sample.metadata, expected_metadata) @@ -93,7 +95,7 @@ def test_get_performance_with_query_metadata(self): expected_metadata = { 'query': QJOB_NAME, 'execution_status': agg.EdwQueryExecutionStatus.SUCCESSFUL, - 'job_id': QJOB_ID + 'job_id': QJOB_ID, } self.assertDictEqual(actual_sample.metadata, expected_metadata) @@ -107,7 +109,7 @@ def test_get_performance_with_iteration_metadata(self): 'query': QJOB_NAME, 'execution_status': agg.EdwQueryExecutionStatus.SUCCESSFUL, 'job_id': QJOB_ID, - 'iteration': '1' + 'iteration': '1', } self.assertDictEqual(actual_sample.metadata, expected_metadata) @@ -122,16 +124,26 @@ def test_add_query_performance(self): i_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY) actual_iteration_performance = i_p.performance expected_iteration_performance = {Q1_NAME: q1_p, Q2_NAME: q2_p} - self.assertSameElements(actual_iteration_performance.keys(), - expected_iteration_performance.keys()) - self.assertEqual(actual_iteration_performance[Q1_NAME].performance, - expected_iteration_performance[Q1_NAME].performance) - self.assertDictEqual(actual_iteration_performance[Q1_NAME].metadata, - expected_iteration_performance[Q1_NAME].metadata) - self.assertEqual(actual_iteration_performance[Q2_NAME].performance, - expected_iteration_performance[Q2_NAME].performance) - self.assertDictEqual(actual_iteration_performance[Q2_NAME].metadata, - expected_iteration_performance[Q2_NAME].metadata) + self.assertSameElements( + actual_iteration_performance.keys(), + expected_iteration_performance.keys(), + ) + self.assertEqual( + actual_iteration_performance[Q1_NAME].performance, + expected_iteration_performance[Q1_NAME].performance, + ) + self.assertDictEqual( + actual_iteration_performance[Q1_NAME].metadata, + expected_iteration_performance[Q1_NAME].metadata, + ) + self.assertEqual( + actual_iteration_performance[Q2_NAME].performance, + expected_iteration_performance[Q2_NAME].performance, + ) + self.assertDictEqual( + actual_iteration_performance[Q2_NAME].metadata, + expected_iteration_performance[Q2_NAME].metadata, + ) self.assertEqual(i_p.total_count, 2) self.assertEqual(i_p.successful_count, 2) @@ -158,44 +170,55 @@ def test_get_query_performance(self): def test_get_query_metadata(self): i_p = agg.EdwPowerIterationPerformance('1', 4) - i_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, - {'job_id': 'q1_i1_job_id'}) + i_p.add_query_performance( + Q1_NAME, Q1_PERFORMANCE, {'job_id': 'q1_i1_job_id'} + ) i_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, {}) - actual_aggregated_query_metadata_q1 = i_p.get_query_metadata( - Q1_NAME) + actual_aggregated_query_metadata_q1 = i_p.get_query_metadata(Q1_NAME) expected_aggregated_query_metadata_q1 = { 'job_id': 'q1_i1_job_id', } - self.assertDictEqual(actual_aggregated_query_metadata_q1, - expected_aggregated_query_metadata_q1) - actual_aggregated_query_metadata_q2 = i_p.get_query_metadata( - Q2_NAME) + self.assertDictEqual( + actual_aggregated_query_metadata_q1, + expected_aggregated_query_metadata_q1, + ) + actual_aggregated_query_metadata_q2 = i_p.get_query_metadata(Q2_NAME) expected_aggregated_query_metadata_q2 = {} - self.assertDictEqual(actual_aggregated_query_metadata_q2, - expected_aggregated_query_metadata_q2) + self.assertDictEqual( + actual_aggregated_query_metadata_q2, + expected_aggregated_query_metadata_q2, + ) def test_get_all_query_performance_samples(self): i_p = agg.EdwPowerIterationPerformance('1', 10) i_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY) i_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY) actual_all_query_performance = i_p.get_all_query_performance_samples( - METADATA_EMPTY) + METADATA_EMPTY + ) self.assertEqual(len(actual_all_query_performance), 2) - self.assertListEqual([x.metric for x in actual_all_query_performance], - ['edw_raw_query_time', 'edw_raw_query_time']) - self.assertSameElements([x.value for x in actual_all_query_performance], - [Q1_PERFORMANCE, Q2_PERFORMANCE]) - self.assertListEqual([x.unit for x in actual_all_query_performance], - [SECS, SECS]) + self.assertListEqual( + [x.metric for x in actual_all_query_performance], + ['edw_raw_query_time', 'edw_raw_query_time'], + ) + self.assertSameElements( + [x.value for x in actual_all_query_performance], + [Q1_PERFORMANCE, Q2_PERFORMANCE], + ) + self.assertListEqual( + [x.unit for x in actual_all_query_performance], [SECS, SECS] + ) self.assertSameElements( [x.metadata['query'] for x in actual_all_query_performance], - [Q1_NAME, Q2_NAME]) + [Q1_NAME, Q2_NAME], + ) self.assertSameElements( [x.metadata['execution_status'] for x in actual_all_query_performance], [ agg.EdwQueryExecutionStatus.SUCCESSFUL, - agg.EdwQueryExecutionStatus.SUCCESSFUL - ]) + agg.EdwQueryExecutionStatus.SUCCESSFUL, + ], + ) def test_is_successful(self): i_p = agg.EdwPowerIterationPerformance('1', 2) @@ -205,28 +228,38 @@ def test_is_successful(self): i2_p = agg.EdwPowerIterationPerformance('2', 2) i_p.add_query_performance(QFAIL_NAME, QFAIL_PERFORMANCE, METADATA_EMPTY) self.assertFalse( - i2_p.is_successful(expected_queries=[Q1_NAME, Q2_NAME, QFAIL_NAME])) + i2_p.is_successful(expected_queries=[Q1_NAME, Q2_NAME, QFAIL_NAME]) + ) def test_get_queries_geomean_performance(self): i_p = agg.EdwPowerIterationPerformance('1', 2) i_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY) i_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY) - iteration_geomean_performance_sample = i_p.get_queries_geomean_performance_sample( - expected_queries=[Q1_NAME, Q2_NAME], metadata=METADATA_EMPTY) - self.assertEqual(iteration_geomean_performance_sample.metric, - 'edw_iteration_geomean_time') - self.assertEqual(iteration_geomean_performance_sample.value, - agg.geometric_mean([Q1_PERFORMANCE, Q2_PERFORMANCE])) + iteration_geomean_performance_sample = ( + i_p.get_queries_geomean_performance_sample( + expected_queries=[Q1_NAME, Q2_NAME], metadata=METADATA_EMPTY + ) + ) + self.assertEqual( + iteration_geomean_performance_sample.metric, + 'edw_iteration_geomean_time', + ) + self.assertEqual( + iteration_geomean_performance_sample.value, + agg.geometric_mean([Q1_PERFORMANCE, Q2_PERFORMANCE]), + ) self.assertEqual(iteration_geomean_performance_sample.unit, SECS) - self.assertDictEqual(iteration_geomean_performance_sample.metadata, - METADATA_EMPTY) + self.assertDictEqual( + iteration_geomean_performance_sample.metadata, METADATA_EMPTY + ) class EdwBenchmarkPerformanceTest(pkb_common_test_case.PkbCommonTestCase): def test_add_iteration_performance(self): b_p = agg.EdwBenchmarkPerformance( - total_iterations=2, expected_queries=[Q1_NAME]) + total_iterations=2, expected_queries=[Q1_NAME] + ) i1_p = agg.EdwPowerIterationPerformance('1', 1) i2_p = agg.EdwPowerIterationPerformance('2', 1) i1_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY) @@ -240,7 +273,8 @@ def test_add_iteration_performance_duplicate_iteration(self): """Testing the scenario where a iteration with missing query is added.""" # Creating the bechmark performance b_p = agg.EdwBenchmarkPerformance( - total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME]) + total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME] + ) i1_p = agg.EdwPowerIterationPerformance('1', 1) i1_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY) i1_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY) @@ -254,7 +288,8 @@ def test_add_iteration_performance_duplicate_iteration(self): def test_is_successful_all_query_success(self): b_p = agg.EdwBenchmarkPerformance( - total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME]) + total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME] + ) i1_p = agg.EdwPowerIterationPerformance('1', 2) i1_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY) i1_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY) @@ -269,7 +304,8 @@ def test_is_successful_all_query_success(self): def test_is_successful_not_all_query_success(self): b_p = agg.EdwBenchmarkPerformance( - total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME]) + total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME] + ) i1_p = agg.EdwPowerIterationPerformance('1', 2) i1_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY) i1_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY) @@ -284,7 +320,8 @@ def test_is_successful_not_all_query_success(self): def test_aggregated_query_status_passing(self): b_p = agg.EdwBenchmarkPerformance( - total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME]) + total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME] + ) i1_p = agg.EdwPowerIterationPerformance('1', 2) i1_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY) i1_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY) @@ -298,7 +335,8 @@ def test_aggregated_query_status_passing(self): def test_aggregated_query_status_look_for_missing_query(self): b_p = agg.EdwBenchmarkPerformance( - total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME]) + total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME] + ) i1_p = agg.EdwPowerIterationPerformance('1', 2) i1_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY) i1_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY) @@ -312,7 +350,8 @@ def test_aggregated_query_status_look_for_missing_query(self): def test_aggregated_query_status_look_for_failing_query(self): b_p = agg.EdwBenchmarkPerformance( - total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME]) + total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME] + ) i1_p = agg.EdwPowerIterationPerformance('1', 2) i1_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY) i1_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY) @@ -326,7 +365,8 @@ def test_aggregated_query_status_look_for_failing_query(self): def test_aggregated_query_execution_time_passing(self): b_p = agg.EdwBenchmarkPerformance( - total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME]) + total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME] + ) i1_p = agg.EdwPowerIterationPerformance('1', 2) i1_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY) i1_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY) @@ -336,13 +376,16 @@ def test_aggregated_query_execution_time_passing(self): i2_p.add_query_performance(Q2_NAME, 4.0, METADATA_EMPTY) b_p.add_iteration_performance(i2_p) self.assertEqual( - b_p.aggregated_query_execution_time(Q1_NAME), (1.0 + 3.0) / 2) + b_p.aggregated_query_execution_time(Q1_NAME), (1.0 + 3.0) / 2 + ) self.assertEqual( - b_p.aggregated_query_execution_time(Q2_NAME), (2.0 + 4.0) / 2) + b_p.aggregated_query_execution_time(Q2_NAME), (2.0 + 4.0) / 2 + ) def test_aggregated_query_execution_time_missing_query(self): b_p = agg.EdwBenchmarkPerformance( - total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME]) + total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME] + ) i1_p = agg.EdwPowerIterationPerformance('1', 2) i1_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY) i1_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY) @@ -356,7 +399,8 @@ def test_aggregated_query_execution_time_missing_query(self): def test_aggregated_query_execution_time_failing_query(self): b_p = agg.EdwBenchmarkPerformance( - total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME]) + total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME] + ) i1_p = agg.EdwPowerIterationPerformance('1', 2) i1_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY) i1_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY) @@ -370,7 +414,8 @@ def test_aggregated_query_execution_time_failing_query(self): def test_aggregated_query_metadata_passing(self): b_p = agg.EdwBenchmarkPerformance( - total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME]) + total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME] + ) i1_p = agg.EdwPowerIterationPerformance('1', 2) i1_p.add_query_performance(Q1_NAME, 1.0, {'job_id': 'q1_i1_job_id'}) i1_p.add_query_performance(Q2_NAME, 2.0, {'job_id': 'q2_i1_job_id'}) @@ -382,20 +427,25 @@ def test_aggregated_query_metadata_passing(self): actual_aggregated_query_metadata_q1 = b_p.aggregated_query_metadata(Q1_NAME) expected_aggregated_query_metadata_q1 = { '1' + '_job_id': 'q1_i1_job_id', - '2' + '_job_id': 'q1_i2_job_id' + '2' + '_job_id': 'q1_i2_job_id', } - self.assertDictEqual(expected_aggregated_query_metadata_q1, - actual_aggregated_query_metadata_q1) + self.assertDictEqual( + expected_aggregated_query_metadata_q1, + actual_aggregated_query_metadata_q1, + ) actual_aggregated_query_metadata_q2 = b_p.aggregated_query_metadata(Q2_NAME) expected_aggregated_query_metadata_q2 = { '1' + '_job_id': 'q2_i1_job_id', } - self.assertDictEqual(expected_aggregated_query_metadata_q2, - actual_aggregated_query_metadata_q2) + self.assertDictEqual( + expected_aggregated_query_metadata_q2, + actual_aggregated_query_metadata_q2, + ) def test_aggregated_query_metadata_missing_query(self): b_p = agg.EdwBenchmarkPerformance( - total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME]) + total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME] + ) i1_p = agg.EdwPowerIterationPerformance('1', 2) i1_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY) i1_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY) @@ -409,7 +459,8 @@ def test_aggregated_query_metadata_missing_query(self): def test_aggregated_query_metadata_failing_query(self): b_p = agg.EdwBenchmarkPerformance( - total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME]) + total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME] + ) i1_p = agg.EdwPowerIterationPerformance('1', 2) i1_p.add_query_performance(Q1_NAME, Q1_PERFORMANCE, METADATA_EMPTY) i1_p.add_query_performance(Q2_NAME, Q2_PERFORMANCE, METADATA_EMPTY) @@ -423,7 +474,8 @@ def test_aggregated_query_metadata_failing_query(self): def test_get_aggregated_query_performance_sample_passing(self): b_p = agg.EdwBenchmarkPerformance( - total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME]) + total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME] + ) i1_p = agg.EdwPowerIterationPerformance('1', 2) i1_p.add_query_performance(Q1_NAME, 1.0, {'job_id': 'q1_i1_job_id'}) i1_p.add_query_performance(Q2_NAME, 2.0, {'job_id': 'q2_i1_job_id'}) @@ -433,7 +485,8 @@ def test_get_aggregated_query_performance_sample_passing(self): i2_p.add_query_performance(Q2_NAME, 4.0, {}) b_p.add_iteration_performance(i2_p) actual_sample_q1 = b_p.get_aggregated_query_performance_sample( - Q1_NAME, {'benchmark_name': 'b_name'}) + Q1_NAME, {'benchmark_name': 'b_name'} + ) self.assertEqual(actual_sample_q1.metric, 'edw_aggregated_query_time') self.assertEqual(actual_sample_q1.value, (1.0 + 3.0) / 2) self.assertEqual(actual_sample_q1.unit, 'seconds') @@ -443,7 +496,7 @@ def test_get_aggregated_query_performance_sample_passing(self): 'query': Q1_NAME, 'aggregation_method': 'mean', 'execution_status': agg.EdwQueryExecutionStatus.SUCCESSFUL, - 'benchmark_name': 'b_name' + 'benchmark_name': 'b_name', } self.assertDictEqual(actual_sample_q1.metadata, expected_metadata_q1) actual_sample_q2 = b_p.get_aggregated_query_performance_sample(Q2_NAME, {}) @@ -454,13 +507,14 @@ def test_get_aggregated_query_performance_sample_passing(self): '1' + '_job_id': 'q2_i1_job_id', 'query': Q2_NAME, 'aggregation_method': 'mean', - 'execution_status': agg.EdwQueryExecutionStatus.SUCCESSFUL + 'execution_status': agg.EdwQueryExecutionStatus.SUCCESSFUL, } self.assertDictEqual(actual_sample_q2.metadata, expected_metadata_q2) def test_get_all_query_performance_samples_passing(self): b_p = agg.EdwBenchmarkPerformance( - total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME]) + total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME] + ) i1_p = agg.EdwPowerIterationPerformance('1', 2) i1_p.add_query_performance(Q1_NAME, 1.0, {'job_id': 'q1_i1_job_id'}) i1_p.add_query_performance(Q2_NAME, 2.0, {'job_id': 'q2_i1_job_id'}) @@ -472,15 +526,22 @@ def test_get_all_query_performance_samples_passing(self): actual_sample_list = b_p.get_all_query_performance_samples({}) self.assertEqual(len(actual_sample_list), 6) # 4 raw query samples and 2 aggregated samples - self.assertSameElements([x.metric for x in actual_sample_list], [ - 'edw_raw_query_time', 'edw_raw_query_time', 'edw_raw_query_time', - 'edw_raw_query_time', 'edw_aggregated_query_time', - 'edw_aggregated_query_time' - ]) + self.assertSameElements( + [x.metric for x in actual_sample_list], + [ + 'edw_raw_query_time', + 'edw_raw_query_time', + 'edw_raw_query_time', + 'edw_raw_query_time', + 'edw_aggregated_query_time', + 'edw_aggregated_query_time', + ], + ) def test_get_aggregated_geomean_performance_sample_passing(self): b_p = agg.EdwBenchmarkPerformance( - total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME]) + total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME] + ) i1_p = agg.EdwPowerIterationPerformance('1', 2) i1_p.add_query_performance(Q1_NAME, 1.0, {'job_id': 'q1_i1_job_id'}) i1_p.add_query_performance(Q2_NAME, 2.0, {'job_id': 'q2_i1_job_id'}) @@ -490,21 +551,25 @@ def test_get_aggregated_geomean_performance_sample_passing(self): i2_p.add_query_performance(Q2_NAME, 4.0, {}) b_p.add_iteration_performance(i2_p) actual_sample = b_p.get_aggregated_geomean_performance_sample( - {'benchmark_name': 'b_name'}) + {'benchmark_name': 'b_name'} + ) self.assertEqual(actual_sample.metric, 'edw_aggregated_geomean') - self.assertEqual(actual_sample.value, - agg.geometric_mean([(1.0 + 3.0) / 2, (2.0 + 4.0) / 2])) + self.assertEqual( + actual_sample.value, + agg.geometric_mean([(1.0 + 3.0) / 2, (2.0 + 4.0) / 2]), + ) self.assertEqual(actual_sample.unit, 'seconds') expected_metadata = { 'benchmark_name': 'b_name', 'intra_query_aggregation_method': 'mean', - 'inter_query_aggregation_method': 'geomean' + 'inter_query_aggregation_method': 'geomean', } self.assertDictEqual(actual_sample.metadata, expected_metadata) def test_get_queries_geomean_performance_samples_passing(self): b_p = agg.EdwBenchmarkPerformance( - total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME]) + total_iterations=2, expected_queries=[Q1_NAME, Q2_NAME] + ) i1_p = agg.EdwPowerIterationPerformance('1', 2) i1_p.add_query_performance(Q1_NAME, 1.0, {'job_id': 'q1_i1_job_id'}) i1_p.add_query_performance(Q2_NAME, 2.0, {'job_id': 'q2_i1_job_id'}) @@ -514,28 +579,41 @@ def test_get_queries_geomean_performance_samples_passing(self): i2_p.add_query_performance(Q2_NAME, 4.0, {}) b_p.add_iteration_performance(i2_p) actual_sample_list = b_p.get_queries_geomean_performance_samples( - {'benchmark_name': 'b_name'}) + {'benchmark_name': 'b_name'} + ) self.assertEqual(len(actual_sample_list), 3) - self.assertSameElements([x.metric for x in actual_sample_list], [ - 'edw_iteration_geomean_time', 'edw_iteration_geomean_time', - 'edw_aggregated_geomean' - ]) + self.assertSameElements( + [x.metric for x in actual_sample_list], + [ + 'edw_iteration_geomean_time', + 'edw_iteration_geomean_time', + 'edw_aggregated_geomean', + ], + ) raw_samples = list( - filter(lambda x: x.metric == 'edw_iteration_geomean_time', - actual_sample_list)) + filter( + lambda x: x.metric == 'edw_iteration_geomean_time', + actual_sample_list, + ) + ) actual_raw_samples_values = [x.value for x in raw_samples] expected_raw_samples_values = [ agg.geometric_mean([1.0, 2.0]), - agg.geometric_mean([3.0, 4.0]) + agg.geometric_mean([3.0, 4.0]), ] - self.assertSameElements(actual_raw_samples_values, - expected_raw_samples_values) + self.assertSameElements( + actual_raw_samples_values, expected_raw_samples_values + ) aggregated_sample = list( - filter(lambda x: x.metric == 'edw_aggregated_geomean', - actual_sample_list))[0] - self.assertEqual(aggregated_sample.value, - agg.geometric_mean([(1.0 + 3.0) / 2, (2.0 + 4.0) / 2])) + filter( + lambda x: x.metric == 'edw_aggregated_geomean', actual_sample_list + ) + )[0] + self.assertEqual( + aggregated_sample.value, + agg.geometric_mean([(1.0 + 3.0) / 2, (2.0 + 4.0) / 2]), + ) def test_geometric_mean_valid_values(self): performance_iterable = [1.0, 2.0, 3.0] diff --git a/tests/edw_service_test.py b/tests/edw_service_test.py index 75fd06d9a1..69a9ccf481 100644 --- a/tests/edw_service_test.py +++ b/tests/edw_service_test.py @@ -16,7 +16,6 @@ import copy import unittest from absl import flags - from perfkitbenchmarker import edw_service from perfkitbenchmarker.configs import benchmark_config_spec from tests import pkb_common_test_case @@ -38,7 +37,7 @@ 'user': _USERNAME, 'password': _PASSWORD, 'node_type': _REDSHIFT_NODE_TYPE, - 'node_count': 1 + 'node_count': 1, } FLAGS = flags.FLAGS @@ -80,10 +79,9 @@ def setUp(self): FLAGS.zones = [_AWS_ZONE_US_EAST_1A] def testIsUserManaged(self): - kwargs = copy.copy({ - 'cluster_identifier': _PKB_CLUSTER, - 'db': _PKB_CLUSTER_DATABASE - }) + kwargs = copy.copy( + {'cluster_identifier': _PKB_CLUSTER, 'db': _PKB_CLUSTER_DATABASE} + ) spec = benchmark_config_spec._EdwServiceSpec('NAME', **kwargs) edw_local = FakeEdwService(spec) self.assertTrue(edw_local.IsUserManaged(spec)) @@ -95,10 +93,9 @@ def testIsPkbManaged(self): self.assertFalse(edw_local.IsUserManaged(spec)) def testUserManagedGetClusterIdentifier(self): - kwargs = copy.copy({ - 'cluster_identifier': _PKB_CLUSTER, - 'db': _PKB_CLUSTER_DATABASE - }) + kwargs = copy.copy( + {'cluster_identifier': _PKB_CLUSTER, 'db': _PKB_CLUSTER_DATABASE} + ) spec = benchmark_config_spec._EdwServiceSpec('NAME', **kwargs) edw_local = FakeEdwService(spec) self.assertEqual(_PKB_CLUSTER, edw_local.GetClusterIdentifier(spec)) @@ -108,8 +105,9 @@ def testPkbManagedGetClusterIdentifier(self): kwargs = copy.copy({'db': _PKB_CLUSTER_DATABASE}) spec = benchmark_config_spec._EdwServiceSpec('NAME', **kwargs) edw_local = FakeEdwService(spec) - self.assertEqual('pkb-' + FLAGS.run_uri, - edw_local.GetClusterIdentifier(spec)) + self.assertEqual( + 'pkb-' + FLAGS.run_uri, edw_local.GetClusterIdentifier(spec) + ) self.assertEqual('pkb-' + FLAGS.run_uri, edw_local.cluster_identifier) diff --git a/tests/flag_alias_test.py b/tests/flag_alias_test.py index f418e343a5..101326cf9a 100644 --- a/tests/flag_alias_test.py +++ b/tests/flag_alias_test.py @@ -30,11 +30,13 @@ class TestFlagAlias(parameterized.TestCase): ('TestValueUnchanged1', ['-ab=a'], ['-ab=a']), ('TestValueUnchanged2', ['-a=-a'], ['-ab=-a']), ('TestNoPrefix1', ['-noa'], ['-noab']), - ('TestNoPrefix2', ['-noab'], ['-noab'])) + ('TestNoPrefix2', ['-noab'], ['-noab']), + ) def testAliasFlagsFromArgs(self, argv, expected_argv): flags = [{'a': 'ab'}] self.assertListEqual( - flag_alias.AliasFlagsFromArgs(argv, flags), expected_argv) + flag_alias.AliasFlagsFromArgs(argv, flags), expected_argv + ) @parameterized.named_parameters( ('BaseCase1', {'a': 'cat'}, {'ab': 'cat'}), @@ -45,7 +47,8 @@ def testAliasFlagsFromArgs(self, argv, expected_argv): def testAliasFlagsFromYaml(self, dic, expected_dic): flags = [{'a': 'ab'}, {'b': 'd'}] self.assertDictEqual( - flag_alias.AliasFlagsFromYaml(dic, flags), expected_dic) + flag_alias.AliasFlagsFromYaml(dic, flags), expected_dic + ) class TestZonesFlagAlias(pkb_common_test_case.PkbCommonTestCase): diff --git a/tests/flag_util_test.py b/tests/flag_util_test.py index 1d3ffd3292..b607bb5233 100644 --- a/tests/flag_util_test.py +++ b/tests/flag_util_test.py @@ -114,20 +114,23 @@ def testIntegerRangeWithStep(self): self.assertEqual(list(self.ilp.parse('-3:1:2')), [-3, -1, 1]) def testIntegerList(self): - self.assertEqual(list(self.ilp.parse('3-5,8,10-12')), - [3, 4, 5, 8, 10, 11, 12]) - self.assertEqual(list(self.ilp.parse('3:5,8,10:12')), - [3, 4, 5, 8, 10, 11, 12]) + self.assertEqual( + list(self.ilp.parse('3-5,8,10-12')), [3, 4, 5, 8, 10, 11, 12] + ) + self.assertEqual( + list(self.ilp.parse('3:5,8,10:12')), [3, 4, 5, 8, 10, 11, 12] + ) def testIntegerListWithRangeAndStep(self): - self.assertEqual(list(self.ilp.parse('3-5,8,10-15-2')), - [3, 4, 5, 8, 10, 12, 14]) - self.assertEqual(list(self.ilp.parse('3:5,8,10:15:2')), - [3, 4, 5, 8, 10, 12, 14]) + self.assertEqual( + list(self.ilp.parse('3-5,8,10-15-2')), [3, 4, 5, 8, 10, 12, 14] + ) + self.assertEqual( + list(self.ilp.parse('3:5,8,10:15:2')), [3, 4, 5, 8, 10, 12, 14] + ) def testIncreasingIntegerLists(self): - self.assertEqual(list(self.ilp.parse('1-5-2,6-8')), - [1, 3, 5, 6, 7, 8]) + self.assertEqual(list(self.ilp.parse('1-5-2,6-8')), [1, 3, 5, 6, 7, 8]) def testAnyNegativeValueRequiresNewFormat(self): for str_range in ('-1-5', '3--5', '3-1--1'): @@ -179,13 +182,15 @@ def testTrailingComma(self): def testNonIncreasingEntries(self): ilp = flag_util.IntegerListParser( - on_nonincreasing=flag_util.IntegerListParser.EXCEPTION) + on_nonincreasing=flag_util.IntegerListParser.EXCEPTION + ) with self.assertRaises(ValueError): ilp.parse('3,2,1') def testNonIncreasingRange(self): ilp = flag_util.IntegerListParser( - on_nonincreasing=flag_util.IntegerListParser.EXCEPTION) + on_nonincreasing=flag_util.IntegerListParser.EXCEPTION + ) with self.assertRaises(ValueError): ilp.parse('3-1') with self.assertRaises(ValueError): @@ -193,7 +198,8 @@ def testNonIncreasingRange(self): def testNonIncreasingRangeWithStep(self): ilp = flag_util.IntegerListParser( - on_nonincreasing=flag_util.IntegerListParser.EXCEPTION) + on_nonincreasing=flag_util.IntegerListParser.EXCEPTION + ) with self.assertRaises(ValueError): ilp.parse('3-1-2') with self.assertRaises(ValueError): @@ -203,11 +209,13 @@ def testNonIncreasingRangeWithStep(self): def testIntegerListsWhichAreNotIncreasing(self): ilp = flag_util.IntegerListParser( - on_nonincreasing=flag_util.IntegerListParser.EXCEPTION) + on_nonincreasing=flag_util.IntegerListParser.EXCEPTION + ) with self.assertRaises(ValueError) as cm: ilp.parse('1-5,3-7') - self.assertEqual('Integer list 1-5,3-7 is not increasing', - str(cm.exception)) + self.assertEqual( + 'Integer list 1-5,3-7 is not increasing', str(cm.exception) + ) class TestIntegerListSerializer(unittest.TestCase): @@ -216,8 +224,7 @@ def testSerialize(self): ser = flag_util.IntegerListSerializer() il = flag_util.IntegerList([1, (2, 5), 9]) - self.assertEqual(ser.serialize(il), - '1,2-5,9') + self.assertEqual(ser.serialize(il), '1,2-5,9') self.assertEqual(str(il), '1,2-5,9') # previously was self.assertEqual(repr(il), 'IntegerList([1,2-5,9])') @@ -253,8 +260,7 @@ def testSerializeWithStep(self): ser = flag_util.IntegerListSerializer() il = flag_util.IntegerList([1, (2, 5, 2), 9]) - self.assertEqual(ser.serialize(il), - '1,2-5-2,9') + self.assertEqual(ser.serialize(il), '1,2-5-2,9') class OverrideFlagsTestCase(unittest.TestCase): @@ -271,8 +277,9 @@ def testFlagAlias(self): flag_values_overrides = {} flag_values_overrides['test_flag_2'] = 1 self.assertFlagState(flag_values, 0, False) - with flag_util.OverrideFlags(flag_values, flag_values_overrides, - [{'test_flag_2': 'test_flag'}]): + with flag_util.OverrideFlags( + flag_values, flag_values_overrides, [{'test_flag_2': 'test_flag'}] + ): self.assertFlagState(flag_values, 1, True) self.assertFlagState(flag_values, 0, False) @@ -363,8 +370,7 @@ def testPercent(self): class TestStringToBytes(unittest.TestCase): def testValidString(self): - self.assertEqual(flag_util.StringToBytes('100KB'), - 100000) + self.assertEqual(flag_util.StringToBytes('100KB'), 100000) def testUnparseableString(self): with self.assertRaises(ValueError) as cm: @@ -387,8 +393,7 @@ def testNegativeBytes(self): class TestStringToRawPct(unittest.TestCase): def testValidPct(self): - self.assertEqual(flag_util.StringToRawPercent('50.5%'), - 50.5) + self.assertEqual(flag_util.StringToRawPercent('50.5%'), 50.5) def testNullString(self): with self.assertRaises(ValueError): @@ -417,12 +422,10 @@ def setUp(self): self.parser = flag_util.YAMLParser() def testValidString(self): - self.assertEqual(self.parser.parse('[1, 2, 3]'), - [1, 2, 3]) + self.assertEqual(self.parser.parse('[1, 2, 3]'), [1, 2, 3]) def testPreParsedYAML(self): - self.assertEqual(self.parser.parse([1, 2, 3]), - [1, 2, 3]) + self.assertEqual(self.parser.parse([1, 2, 3]), [1, 2, 3]) def testBadYAML(self): with self.assertRaises(ValueError) as cm: @@ -430,7 +433,7 @@ def testBadYAML(self): self.assertIn("Couldn't parse YAML string '{a': ", str(cm.exception)) -class MockFlag(): +class MockFlag: def __init__(self, name, value, present): self.name = name @@ -455,11 +458,14 @@ def tearDown(self): flag_util.FLAGS = {} def testGetProvidedCommandLineFlags(self): - self.assertDictEqual({ - 'flag_int': 1, - 'flag_str': 'str', - 'flag_enum': 'BOTH', - }, flag_util.GetProvidedCommandLineFlags()) + self.assertDictEqual( + { + 'flag_int': 1, + 'flag_str': 'str', + 'flag_enum': 'BOTH', + }, + flag_util.GetProvidedCommandLineFlags(), + ) if __name__ == '__main__': diff --git a/tests/gce_nfs_service_test.py b/tests/gce_nfs_service_test.py index fcd997e0f0..54c0060e6e 100644 --- a/tests/gce_nfs_service_test.py +++ b/tests/gce_nfs_service_test.py @@ -50,21 +50,24 @@ def _DescribeResult(tier='STANDARD'): 'networks': [{ 'ipAddresses': ['10.198.13.2'], 'network': 'default2', - 'reservedIpRange': '10.198.13.0/29' + 'reservedIpRange': '10.198.13.0/29', }], 'state': 'READY', 'tier': tier, - 'volumes': [{ - 'capacityGb': '1024', - 'name': 'vol0' - }] + 'volumes': [{'capacityGb': '1024', 'name': 'vol0'}], } def _FullGcloud(args, location): prefix = [ - 'gcloud', '--quiet', '--format', 'json', '--project', _PROJECT, - 'filestore', 'instances' + 'gcloud', + '--quiet', + '--format', + 'json', + '--project', + _PROJECT, + 'filestore', + 'instances', ] postfix = ['--location', location] return prefix + list(args) + postfix @@ -110,7 +113,8 @@ def _SetResponses(self, *responses): def assertCommandCalled(self, *args, location=_ZONE): self.issue_cmd.assert_called_with( - _FullGcloud(args, location), raise_on_failure=False, timeout=1800) + _FullGcloud(args, location), raise_on_failure=False, timeout=1800 + ) def assertMultipleCommands(self, *cmds, location=_ZONE): expected_calls = [] @@ -119,7 +123,9 @@ def assertMultipleCommands(self, *cmds, location=_ZONE): mock.call( _FullGcloud(cmd, location=location), raise_on_failure=False, - timeout=1800)) + timeout=1800, + ) + ) self.assertEqual(expected_calls, self.issue_cmd.call_args_list) def testCreate(self): diff --git a/tests/gce_windows_virtual_machine_test.py b/tests/gce_windows_virtual_machine_test.py index 35cde388f5..0420c2a861 100644 --- a/tests/gce_windows_virtual_machine_test.py +++ b/tests/gce_windows_virtual_machine_test.py @@ -34,27 +34,41 @@ class GceWindowsVirtualMachineTestCase(pkb_common_test_case.PkbCommonTestCase): def setUp(self): super(GceWindowsVirtualMachineTestCase, self).setUp() self.spec = gce_virtual_machine.GceVmSpec( - 'test_component', machine_type='test_machine_type', project='p') - p = mock.patch(gce_virtual_machine.__name__ + - '.gce_network.GceNetwork.GetNetwork') + 'test_component', machine_type='test_machine_type', project='p' + ) + p = mock.patch( + gce_virtual_machine.__name__ + '.gce_network.GceNetwork.GetNetwork' + ) self.mock_get_network = p.start() self.addCleanup(p.stop) - p = mock.patch(gce_virtual_machine.__name__ + - '.gce_network.GceFirewall.GetFirewall') + p = mock.patch( + gce_virtual_machine.__name__ + '.gce_network.GceFirewall.GetFirewall' + ) self.mock_get_firewall = p.start() self.addCleanup(p.stop) get_tmp_dir_mock = mock.patch( - vm_util.__name__ + '.GetTempDir', return_value='TempDir') + vm_util.__name__ + '.GetTempDir', return_value='TempDir' + ) get_tmp_dir_mock.start() self.addCleanup(get_tmp_dir_mock.stop) @parameterized.named_parameters( - ('WINDOWS2022_DESKTOP', os_types.WINDOWS2022_DESKTOP, True, - 'windows-2022', 'windows-cloud'), - ('WINDOWS2022_SQLSERVER_2019_ENTERPRISE', - os_types.WINDOWS2022_SQLSERVER_2019_ENTERPRISE, True, - 'sql-ent-2019-win-2022', 'windows-sql-cloud')) + ( + 'WINDOWS2022_DESKTOP', + os_types.WINDOWS2022_DESKTOP, + True, + 'windows-2022', + 'windows-cloud', + ), + ( + 'WINDOWS2022_SQLSERVER_2019_ENTERPRISE', + os_types.WINDOWS2022_SQLSERVER_2019_ENTERPRISE, + True, + 'sql-ent-2019-win-2022', + 'windows-sql-cloud', + ), + ) def testWindowsConfig(self, os_type, gvnic, family, project): vm_class = virtual_machine.GetVmClass(provider_info.GCP, os_type) vm = vm_class(self.spec) diff --git a/tests/iaas_relational_db_test.py b/tests/iaas_relational_db_test.py index 9ae5541e23..24fd8286af 100644 --- a/tests/iaas_relational_db_test.py +++ b/tests/iaas_relational_db_test.py @@ -109,58 +109,37 @@ def setUp(self): 'cloud': 'GCP', 'engine': 'mysql', 'engine_version': '5.7', - 'db_spec': { - 'GCP': { - 'machine_type': 'n1-standard-1' - } - }, - 'db_disk_spec': { - 'GCP': { - 'disk_size': 500 - } - } + 'db_spec': {'GCP': {'machine_type': 'n1-standard-1'}}, + 'db_disk_spec': {'GCP': {'disk_size': 500}}, } self.min_postgres_spec = { 'cloud': 'GCP', 'engine': 'postgres', 'engine_version': '11', - 'db_spec': { - 'GCP': { - 'machine_type': 'n1-standard-1' - } - }, - 'db_disk_spec': { - 'GCP': { - 'disk_size': 500 - } - } + 'db_spec': {'GCP': {'machine_type': 'n1-standard-1'}}, + 'db_disk_spec': {'GCP': {'disk_size': 500}}, } self.min_sqlserver_spec = { 'cloud': 'GCP', 'engine': 'sqlserver', 'engine_version': '2019', - 'db_spec': { - 'GCP': { - 'machine_type': 'n1-standard-1' - } - }, - 'db_disk_spec': { - 'GCP': { - 'disk_size': 500 - } - } + 'db_spec': {'GCP': {'machine_type': 'n1-standard-1'}}, + 'db_disk_spec': {'GCP': {'disk_size': 500}}, } self.mysql_spec = relational_db_spec.RelationalDbSpec( - _COMPONENT, flag_values=FLAGS, **self.min_mysql_spec) + _COMPONENT, flag_values=FLAGS, **self.min_mysql_spec + ) self.postgres_spec = relational_db_spec.RelationalDbSpec( - _COMPONENT, flag_values=FLAGS, **self.min_postgres_spec) + _COMPONENT, flag_values=FLAGS, **self.min_postgres_spec + ) self.sqlserver_spec = relational_db_spec.RelationalDbSpec( - _COMPONENT, flag_values=FLAGS, **self.min_sqlserver_spec) + _COMPONENT, flag_values=FLAGS, **self.min_sqlserver_spec + ) def testMakePostgresClientCommand(self): FLAGS['use_managed_db'].parse(False) @@ -171,8 +150,10 @@ def testMakePostgresClientCommand(self): db.server_vm = CreateTestLinuxVm() self.assertEqual( db.client_vm_query_tools.MakeSqlCommand( - 'Select 1', database_name='postgresql'), - 'psql \'host=1.1.1.1 user=root password=perfkitbenchmarker dbname=postgresql\' -c "Select 1"' + 'Select 1', database_name='postgresql' + ), + "psql 'host=1.1.1.1 user=root password=perfkitbenchmarker" + ' dbname=postgresql\' -c "Select 1"', ) def testIssuePostgresClientCommand(self): @@ -187,10 +168,11 @@ def testIssuePostgresClientCommand(self): command = [ mock.call( - 'psql \'host=1.1.1.1 user=root password=perfkitbenchmarker' + "psql 'host=1.1.1.1 user=root password=perfkitbenchmarker" ' dbname=abc\' -c "Select 1"', ignore_failure=False, - timeout=None) + timeout=None, + ) ] self.assertCountEqual(remote_command.call_args_list, command) @@ -208,14 +190,16 @@ def testIssuePostgresClientCommandWithSessionVariables(self): session_variables=['Set a=b;'], database_name='abc', ignore_failure=False, - timeout=None) + timeout=None, + ) command = [ mock.call( - 'psql \'host=1.1.1.1 user=root password=perfkitbenchmarker' + "psql 'host=1.1.1.1 user=root password=perfkitbenchmarker" ' dbname=abc\' -c "Set a=b;" -c "Select 1"', ignore_failure=False, - timeout=None) + timeout=None, + ) ] self.assertCountEqual(remote_command.call_args_list, command) @@ -228,8 +212,10 @@ def testMakePostgresServerCommand(self): db.port = db.GetDefaultPort() self.assertEqual( db.server_vm_query_tools.MakeSqlCommand( - 'Select 1', database_name='postgresql'), - 'psql \'host=localhost user=root password=perfkitbenchmarker dbname=postgresql\' -c "Select 1"' + 'Select 1', database_name='postgresql' + ), + "psql 'host=localhost user=root password=perfkitbenchmarker" + ' dbname=postgresql\' -c "Select 1"', ) def testMakeMysqlCientCommand(self): @@ -241,8 +227,8 @@ def testMakeMysqlCientCommand(self): db.port = db.GetDefaultPort() self.assertEqual( db.client_vm_query_tools.MakeSqlCommand('Select 1'), - 'mysql -h 1.1.1.1 -P 3306 -u root' - ' -pperfkitbenchmarker -e "Select 1"') + 'mysql -h 1.1.1.1 -P 3306 -u root -pperfkitbenchmarker -e "Select 1"', + ) def testMakeMysqlCommandWithLocalHost(self): FLAGS['use_managed_db'].parse(False) @@ -253,8 +239,8 @@ def testMakeMysqlCommandWithLocalHost(self): db.port = db.GetDefaultPort() self.assertEqual( db.server_vm_query_tools.MakeSqlCommand('Select 1'), - 'mysql -h localhost -P 3306 -u root ' - '-pperfkitbenchmarker -e "Select 1"') + 'mysql -h localhost -P 3306 -u root -pperfkitbenchmarker -e "Select 1"', + ) def testMakeSqlserverCommand(self): FLAGS['use_managed_db'].parse(False) @@ -265,7 +251,7 @@ def testMakeSqlserverCommand(self): db.port = db.GetDefaultPort() self.assertEqual( db.client_vm_query_tools.MakeSqlCommand('Select 1'), - 'sqlcmd -S 1.1.1.1 -U root -P perfkitbenchmarker -Q "Select 1"' + 'sqlcmd -S 1.1.1.1 -U root -P perfkitbenchmarker -Q "Select 1"', ) def testMakeSqlserverCommandWithLocalHost(self): @@ -277,7 +263,7 @@ def testMakeSqlserverCommandWithLocalHost(self): db.port = db.GetDefaultPort() self.assertEqual( db.server_vm_query_tools.MakeSqlCommand('Select 1'), - 'sqlcmd -S localhost -U root -P perfkitbenchmarker -Q "Select 1"' + 'sqlcmd -S localhost -U root -P perfkitbenchmarker -Q "Select 1"', ) def testInstallMYSQLServer(self): @@ -304,32 +290,47 @@ def testInstallMYSQLServer(self): mock.call('sudo rsync -avzh /tmp/ /scratch/tmp'), mock.call('df'), mock.call( - 'echo "alias /var/lib/mysql -> /scratch/mysql," | sudo tee -a /etc/apparmor.d/tunables/alias' + 'echo "alias /var/lib/mysql -> /scratch/mysql," | sudo tee -a' + ' /etc/apparmor.d/tunables/alias' ), mock.call( - 'echo "alias /tmp -> /scratch/tmp," | sudo tee -a /etc/apparmor.d/tunables/alias' + 'echo "alias /tmp -> /scratch/tmp," | sudo tee -a' + ' /etc/apparmor.d/tunables/alias' ), mock.call( - 'sudo sed -i "s|# Allow data files dir access| /scratch/mysql/ r, /scratch/mysql/** rwk, /scratch/tmp/ r, /scratch/tmp/** rwk, /proc/*/status r, /sys/devices/system/node/ r, /sys/devices/system/node/node*/meminfo r, /sys/devices/system/node/*/* r, /sys/devices/system/node/* r, # Allow data files dir access|g" /etc/apparmor.d/usr.sbin.mysqld' + 'sudo sed -i "s|# Allow data files dir access| /scratch/mysql/ r,' + ' /scratch/mysql/** rwk, /scratch/tmp/ r, /scratch/tmp/** rwk,' + ' /proc/*/status r, /sys/devices/system/node/ r,' + ' /sys/devices/system/node/node*/meminfo r,' + ' /sys/devices/system/node/*/* r, /sys/devices/system/node/* r, #' + ' Allow data files dir access|g" /etc/apparmor.d/usr.sbin.mysqld' ), mock.call('sudo apparmor_parser -r /etc/apparmor.d/usr.sbin.mysqld'), mock.call('sudo systemctl restart apparmor'), mock.call( - 'sudo sed -i "s|datadir\t\t= /var/lib/mysql|datadir\t\t= /scratch/mysql|g" None' + 'sudo sed -i "s|datadir\t\t= /var/lib/mysql|datadir\t\t=' + ' /scratch/mysql|g" None' ), mock.call( - 'sudo sed -i "s|tmpdir\t\t= /tmp|tmpdir\t\t= /scratch/tmp|g" None'), + 'sudo sed -i "s|tmpdir\t\t= /tmp|tmpdir\t\t= /scratch/tmp|g" None' + ), mock.call( - 'echo "\ninnodb_buffer_pool_size = 100G\ninnodb_flush_method = O_DIRECT\ninnodb_flush_neighbors = 0\ninnodb_log_file_size = 1000M" | sudo tee -a None' + 'echo "\ninnodb_buffer_pool_size = 100G\ninnodb_flush_method =' + ' O_DIRECT\ninnodb_flush_neighbors = 0\ninnodb_log_file_size =' + ' 1000M" | sudo tee -a None' ), mock.call( - 'echo "\nskip-name-resolve\nconnect_timeout = 86400\nwait_timeout = 86400\ninteractive_timeout = 86400" | sudo tee -a None' + 'echo "\nskip-name-resolve\nconnect_timeout =' + ' 86400\nwait_timeout = 86400\ninteractive_timeout =' + ' 86400" | sudo tee -a None' ), mock.call('sudo sed -i "s/^bind-address/#bind-address/g" None'), mock.call( - 'sudo sed -i "s/^mysqlx-bind-address/#mysqlx-bind-address/g" None'), + 'sudo sed -i "s/^mysqlx-bind-address/#mysqlx-bind-address/g" None' + ), mock.call( - 'sudo sed -i "s/max_allowed_packet\t= 16M/max_allowed_packet\t= 1024M/g" None' + 'sudo sed -i "s/max_allowed_packet\t= 16M/max_allowed_packet\t=' + ' 1024M/g" None' ), mock.call('echo "\nlog_error_verbosity = 3" | sudo tee -a None'), mock.call('sudo service None restart'), @@ -338,23 +339,27 @@ def testInstallMYSQLServer(self): 'sudo mysql -h localhost -P 3306 -u root -pperfkitbenchmarker ' '-e "SET GLOBAL max_connections=8000;"', ignore_failure=False, - timeout=None), + timeout=None, + ), mock.call( 'sudo mysql -h localhost -P 3306 -u root -pperfkitbenchmarker -e ' - '"CREATE USER \'root\'@\'None\' ' - 'IDENTIFIED BY \'perfkitbenchmarker\';"', + "\"CREATE USER 'root'@'None' " + "IDENTIFIED BY 'perfkitbenchmarker';\"", ignore_failure=True, - timeout=None), + timeout=None, + ), mock.call( 'sudo mysql -h localhost -P 3306 -u root -pperfkitbenchmarker -e ' - '"GRANT ALL PRIVILEGES ON *.* TO \'root\'@\'None\';"', + "\"GRANT ALL PRIVILEGES ON *.* TO 'root'@'None';\"", ignore_failure=True, - timeout=None), + timeout=None, + ), mock.call( 'sudo mysql -h localhost -P 3306 -u root -pperfkitbenchmarker -e ' '"FLUSH PRIVILEGES;"', ignore_failure=True, - timeout=None) + timeout=None, + ), ] self.assertCountEqual(remote_command.call_args_list, command) diff --git a/tests/integration/aws_disk_integration_test.py b/tests/integration/aws_disk_integration_test.py index 46f259d5d7..c03051824b 100644 --- a/tests/integration/aws_disk_integration_test.py +++ b/tests/integration/aws_disk_integration_test.py @@ -23,8 +23,10 @@ MOUNT_POINT = '/scratch' -@unittest.skipUnless('PERFKIT_INTEGRATION' in os.environ, - 'PERFKIT_INTEGRATION not in environment') +@unittest.skipUnless( + 'PERFKIT_INTEGRATION' in os.environ, + 'PERFKIT_INTEGRATION not in environment', +) class AwsScratchDiskIntegrationTest(unittest.TestCase): """Integration tests for AWS disks. @@ -35,92 +37,104 @@ def setUp(self): pkb.SetUpPKB() def testEBSStandard(self): - test_util.assertDiskMounts({ - 'vm_groups': { - 'vm_group_1': { - 'cloud': 'AWS', - 'vm_spec': { - 'AWS': { - 'machine_type': 'm4.large', - 'zone': 'us-east-1a' - } - }, - 'disk_spec': { - 'AWS': { - 'disk_type': 'standard', - 'disk_size': 2, - 'mount_point': MOUNT_POINT - } + test_util.assertDiskMounts( + { + 'vm_groups': { + 'vm_group_1': { + 'cloud': 'AWS', + 'vm_spec': { + 'AWS': { + 'machine_type': 'm4.large', + 'zone': 'us-east-1a', + } + }, + 'disk_spec': { + 'AWS': { + 'disk_type': 'standard', + 'disk_size': 2, + 'mount_point': MOUNT_POINT, + } + }, } } - } - }, MOUNT_POINT) + }, + MOUNT_POINT, + ) def testEBSGP(self): - test_util.assertDiskMounts({ - 'vm_groups': { - 'vm_group_1': { - 'cloud': 'AWS', - 'vm_spec': { - 'AWS': { - 'machine_type': 'm4.large', - 'zone': 'us-east-1a' - } - }, - 'disk_spec': { - 'AWS': { - 'disk_type': 'gp2', - 'disk_size': 2, - 'mount_point': MOUNT_POINT - } + test_util.assertDiskMounts( + { + 'vm_groups': { + 'vm_group_1': { + 'cloud': 'AWS', + 'vm_spec': { + 'AWS': { + 'machine_type': 'm4.large', + 'zone': 'us-east-1a', + } + }, + 'disk_spec': { + 'AWS': { + 'disk_type': 'gp2', + 'disk_size': 2, + 'mount_point': MOUNT_POINT, + } + }, } } - } - }, MOUNT_POINT) + }, + MOUNT_POINT, + ) def testEBSPIOPS(self): - test_util.assertDiskMounts({ - 'vm_groups': { - 'vm_group_1': { - 'cloud': 'AWS', - 'vm_spec': { - 'AWS': { - 'machine_type': 'm4.large', - 'zone': 'us-east-1a' - } - }, - 'disk_spec': { - 'AWS': { - 'disk_type': 'io1', - 'disk_size': 35, # maximum size/IOPS ratio is 30 - 'iops': 100, # minimum value is 100 IOPS - 'mount_point': MOUNT_POINT - } + test_util.assertDiskMounts( + { + 'vm_groups': { + 'vm_group_1': { + 'cloud': 'AWS', + 'vm_spec': { + 'AWS': { + 'machine_type': 'm4.large', + 'zone': 'us-east-1a', + } + }, + 'disk_spec': { + 'AWS': { + 'disk_type': 'io1', + 'disk_size': 35, # maximum size/IOPS ratio is 30 + 'iops': 100, # minimum value is 100 IOPS + 'mount_point': MOUNT_POINT, + } + }, } } - } - }, MOUNT_POINT) + }, + MOUNT_POINT, + ) def testLocalSSD(self): - test_util.assertDiskMounts({ - 'vm_groups': { - 'vm_group_1': { - 'cloud': 'AWS', - 'vm_spec': { - 'AWS': { - 'machine_type': 'm3.medium', - 'zone': 'us-east-1a' - } - }, - 'disk_spec': { - 'AWS': { - 'disk_type': 'local', - 'mount_point': MOUNT_POINT - } + test_util.assertDiskMounts( + { + 'vm_groups': { + 'vm_group_1': { + 'cloud': 'AWS', + 'vm_spec': { + 'AWS': { + 'machine_type': 'm3.medium', + 'zone': 'us-east-1a', + } + }, + 'disk_spec': { + 'AWS': { + 'disk_type': 'local', + 'mount_point': MOUNT_POINT, + } + }, } } - } - }, MOUNT_POINT) + }, + MOUNT_POINT, + ) if __name__ == '__main__': diff --git a/tests/integration/azure_disk_integration_test.py b/tests/integration/azure_disk_integration_test.py index e8b4d0680d..25e762bbe4 100644 --- a/tests/integration/azure_disk_integration_test.py +++ b/tests/integration/azure_disk_integration_test.py @@ -25,8 +25,10 @@ MOUNT_POINT = '/scratch' -@unittest.skipUnless('PERFKIT_INTEGRATION' in os.environ, - 'PERFKIT_INTEGRATION not in environment') +@unittest.skipUnless( + 'PERFKIT_INTEGRATION' in os.environ, + 'PERFKIT_INTEGRATION not in environment', +) class AzureScratchDiskIntegrationTest(unittest.TestCase): """Integration tests for Azure disks. @@ -37,76 +39,81 @@ def setUp(self): pkb.SetUpPKB() def testPremiumStorage(self): - test_util.assertDiskMounts({ - 'flags': { - 'azure_storage_type': azure_flags.PLRS - }, - 'vm_groups': { - 'vm_group_1': { - 'cloud': 'Azure', - 'vm_spec': { - 'Azure': { - 'machine_type': 'Standard_DS2', - 'zone': 'eastus' - } - }, - 'disk_spec': { - 'Azure': { - 'disk_type': azure_disk.PREMIUM_STORAGE, - 'disk_size': 10, # disk size must be between - # 10 and 1024 GB. - 'mount_point': MOUNT_POINT - } + test_util.assertDiskMounts( + { + 'flags': {'azure_storage_type': azure_flags.PLRS}, + 'vm_groups': { + 'vm_group_1': { + 'cloud': 'Azure', + 'vm_spec': { + 'Azure': { + 'machine_type': 'Standard_DS2', + 'zone': 'eastus', + } + }, + 'disk_spec': { + 'Azure': { + 'disk_type': azure_disk.PREMIUM_STORAGE, + 'disk_size': 10, # disk size must be between + # 10 and 1024 GB. + 'mount_point': MOUNT_POINT, + } + }, } - } - } - }, MOUNT_POINT) + }, + }, + MOUNT_POINT, + ) def testStandardDisk(self): - test_util.assertDiskMounts({ - 'flags': { - 'azure_storage_type': azure_flags.LRS - }, - 'vm_groups': { - 'vm_group_1': { - 'cloud': 'Azure', - 'vm_spec': { - 'Azure': { - 'machine_type': 'Standard_D2', - 'zone': 'eastus' - } - }, - 'disk_spec': { - 'Azure': { - 'disk_type': azure_disk.STANDARD_DISK, - 'disk_size': 2, - 'mount_point': MOUNT_POINT - } + test_util.assertDiskMounts( + { + 'flags': {'azure_storage_type': azure_flags.LRS}, + 'vm_groups': { + 'vm_group_1': { + 'cloud': 'Azure', + 'vm_spec': { + 'Azure': { + 'machine_type': 'Standard_D2', + 'zone': 'eastus', + } + }, + 'disk_spec': { + 'Azure': { + 'disk_type': azure_disk.STANDARD_DISK, + 'disk_size': 2, + 'mount_point': MOUNT_POINT, + } + }, } - } - } - }, MOUNT_POINT) + }, + }, + MOUNT_POINT, + ) def testLocalSSD(self): - test_util.assertDiskMounts({ - 'vm_groups': { - 'vm_group_1': { - 'cloud': 'Azure', - 'vm_spec': { - 'Azure': { - 'machine_type': 'Standard_D1', - 'zone': 'eastus' - } - }, - 'disk_spec': { - 'Azure': { - 'disk_type': 'local', - 'mount_point': MOUNT_POINT - } + test_util.assertDiskMounts( + { + 'vm_groups': { + 'vm_group_1': { + 'cloud': 'Azure', + 'vm_spec': { + 'Azure': { + 'machine_type': 'Standard_D1', + 'zone': 'eastus', + } + }, + 'disk_spec': { + 'Azure': { + 'disk_type': 'local', + 'mount_point': MOUNT_POINT, + } + }, } } - } - }, MOUNT_POINT) + }, + MOUNT_POINT, + ) if __name__ == '__main__': diff --git a/tests/integration/gcp_disk_integration_test.py b/tests/integration/gcp_disk_integration_test.py index dc06eb5109..b5a3b76b5f 100644 --- a/tests/integration/gcp_disk_integration_test.py +++ b/tests/integration/gcp_disk_integration_test.py @@ -23,8 +23,10 @@ MOUNT_POINT = '/scratch' -@unittest.skipUnless('PERFKIT_INTEGRATION' in os.environ, - 'PERFKIT_INTEGRATION not in environment') +@unittest.skipUnless( + 'PERFKIT_INTEGRATION' in os.environ, + 'PERFKIT_INTEGRATION not in environment', +) class GcpScratchDiskIntegrationTest(unittest.TestCase): """Integration tests for GCE disks. @@ -35,70 +37,79 @@ def setUp(self): pkb.SetUpPKB() def testPDStandard(self): - test_util.assertDiskMounts({ - 'vm_groups': { - 'vm_group_1': { - 'cloud': 'GCP', - 'vm_spec': { - 'GCP': { - 'machine_type': 'n1-standard-2', - 'zone': 'us-central1-a' - } - }, - 'disk_spec': { - 'GCP': { - 'disk_type': 'pd-standard', - 'disk_size': 2, - 'mount_point': MOUNT_POINT - } + test_util.assertDiskMounts( + { + 'vm_groups': { + 'vm_group_1': { + 'cloud': 'GCP', + 'vm_spec': { + 'GCP': { + 'machine_type': 'n1-standard-2', + 'zone': 'us-central1-a', + } + }, + 'disk_spec': { + 'GCP': { + 'disk_type': 'pd-standard', + 'disk_size': 2, + 'mount_point': MOUNT_POINT, + } + }, } } - } - }, MOUNT_POINT) + }, + MOUNT_POINT, + ) def testPDSSD(self): - test_util.assertDiskMounts({ - 'vm_groups': { - 'vm_group_1': { - 'cloud': 'GCP', - 'vm_spec': { - 'GCP': { - 'machine_type': 'n1-standard-2', - 'zone': 'us-central1-a' - } - }, - 'disk_spec': { - 'GCP': { - 'disk_type': 'pd-ssd', - 'disk_size': 2, - 'mount_point': MOUNT_POINT - } + test_util.assertDiskMounts( + { + 'vm_groups': { + 'vm_group_1': { + 'cloud': 'GCP', + 'vm_spec': { + 'GCP': { + 'machine_type': 'n1-standard-2', + 'zone': 'us-central1-a', + } + }, + 'disk_spec': { + 'GCP': { + 'disk_type': 'pd-ssd', + 'disk_size': 2, + 'mount_point': MOUNT_POINT, + } + }, } } - } - }, MOUNT_POINT) + }, + MOUNT_POINT, + ) def testLocalSSD(self): - test_util.assertDiskMounts({ - 'vm_groups': { - 'vm_group_1': { - 'cloud': 'GCP', - 'vm_spec': { - 'GCP': { - 'machine_type': 'n1-standard-2', - 'zone': 'us-central1-a', - 'num_local_ssds': 1 - } - }, - 'disk_spec': { - 'GCP': { - 'disk_type': 'local', - 'mount_point': MOUNT_POINT - } + test_util.assertDiskMounts( + { + 'vm_groups': { + 'vm_group_1': { + 'cloud': 'GCP', + 'vm_spec': { + 'GCP': { + 'machine_type': 'n1-standard-2', + 'zone': 'us-central1-a', + 'num_local_ssds': 1, + } + }, + 'disk_spec': { + 'GCP': { + 'disk_type': 'local', + 'mount_point': MOUNT_POINT, + } + }, } } - } - }, MOUNT_POINT) + }, + MOUNT_POINT, + ) if __name__ == '__main__': diff --git a/tests/integration/scripts/object_storage_worker_integration_tests.py b/tests/integration/scripts/object_storage_worker_integration_tests.py index a2b021c283..3dd0ef8c0a 100644 --- a/tests/integration/scripts/object_storage_worker_integration_tests.py +++ b/tests/integration/scripts/object_storage_worker_integration_tests.py @@ -20,7 +20,10 @@ import object_storage_interface -class MockObjectStorageService(object_storage_interface.ObjectStorageServiceBase): # noqa +class MockObjectStorageService( + object_storage_interface.ObjectStorageServiceBase +): # noqa + def __init__(self): self.bucket = None self.objects = {} @@ -39,15 +42,18 @@ def _CheckBucket(self, bucket): self.bucket = bucket elif self.bucket != bucket: raise ValueError( - 'MockObjectStorageService passed two bucket names: %s and %s' % - (self.bucket, bucket)) + 'MockObjectStorageService passed two bucket names: %s and %s' + % (self.bucket, bucket) + ) def ListObjects(self, bucket, prefix): self._CheckBucket(bucket) - return [value - for name, value in self.objects.iteritems() - if name.startswith(prefix)] + return [ + value + for name, value in self.objects.iteritems() + if name.startswith(prefix) + ] def DeleteObjects(self, bucket, objects_to_delete, objects_deleted=None): self._CheckBucket(bucket) @@ -96,11 +102,13 @@ def testOneByteRW(self): def testListConsistency(self): object_storage_api_tests.ListConsistencyBenchmark( - MockObjectStorageService()) + MockObjectStorageService() + ) def testSingleStreamThroughput(self): object_storage_api_tests.SingleStreamThroughputBenchmark( - MockObjectStorageService()) + MockObjectStorageService() + ) def testCleanupBucket(self): object_storage_api_tests.CleanupBucket(MockObjectStorageService()) diff --git a/tests/large_scale_boot_listener_server_test.py b/tests/large_scale_boot_listener_server_test.py index 1193bcb5b3..c4de3568e5 100644 --- a/tests/large_scale_boot_listener_server_test.py +++ b/tests/large_scale_boot_listener_server_test.py @@ -16,7 +16,6 @@ import unittest import mock - from perfkitbenchmarker.data.large_scale_boot import listener_server from tests import pkb_common_test_case @@ -44,9 +43,11 @@ def testConfirmIPAccessibleFail(self, mock_subproc_popen): fake_client_host = '1.2.3.4' fake_port = 22 success = listener_server.ConfirmIPAccessible( - fake_client_host, fake_port, timeout=1) + fake_client_host, fake_port, timeout=1 + ) self.assertEqual(success.split(':')[0], 'Fail') self.assertEqual(success.split(':')[1], fake_client_host) + if __name__ == '__main__': unittest.main() diff --git a/tests/linux_benchmarks/apachebench_benchmark_test.py b/tests/linux_benchmarks/apachebench_benchmark_test.py index 20b82bce7f..5f12a053de 100644 --- a/tests/linux_benchmarks/apachebench_benchmark_test.py +++ b/tests/linux_benchmarks/apachebench_benchmark_test.py @@ -44,9 +44,11 @@ def setUp(self): self.apachebench_output = self.populateTestString('apachebench_output.txt') self.apachebench_raw_request_times = self.populateTestString( - 'apachebench_raw_request_times.tsv') + 'apachebench_raw_request_times.tsv' + ) self.apachebench_percentile_data = self.populateTestString( - 'apachebench_percentile_data.csv') + 'apachebench_percentile_data.csv' + ) self.run_config = apachebench_benchmark.ApacheBenchRunConfig( num_requests=1, @@ -56,22 +58,22 @@ def setUp(self): socket_timeout=30, timelimit=None, server_content_size=1, - client_vms=1 + client_vms=1, ) self.ip_config = apachebench_benchmark.ApacheBenchIpConfig( 'internal-ip', 'internal_ip', 'internal_results.txt', 'internal_ip_percentile_output.csv', - 'internal_ip_raw_request_times.tsv') + 'internal_ip_raw_request_times.tsv', + ) - client = mock.MagicMock( - hostname='pkb-mock-0', - id='0123456789876543210') + client = mock.MagicMock(hostname='pkb-mock-0', id='0123456789876543210') server = mock.MagicMock( hostname='pkb-mock-1', id='9876543210123456789', - internal_ip='30.128.0.2') + internal_ip='30.128.0.2', + ) self.vm_spec = mock.MagicMock(spec=benchmark_spec.BenchmarkSpec) setattr(self.vm_spec, 'vm_groups', {'client': [client], 'server': [server]}) @@ -85,7 +87,7 @@ def setUp(self): 0: [4.0, 5.0, 5.0], 1: [4.0, 5.0], 2: [5.0, 6.0], - 3: [6.0, 7.0, 7.0] + 3: [6.0, 7.0, 7.0], } def testApacheBenchRunConfigGetCommand(self): @@ -103,7 +105,8 @@ def testApacheBenchRunConfigGetCommand(self): self.assertIn(f'1> {self.ip_config.output_path}', cmd) @mock.patch.multiple( - apache2_server, SetupServer=mock.DEFAULT, StartServer=mock.DEFAULT) + apache2_server, SetupServer=mock.DEFAULT, StartServer=mock.DEFAULT + ) def testPrepare(self, SetupServer, StartServer): # pylint: disable=invalid-name client = self.vm_spec.vm_groups['client'][0] server = self.vm_spec.vm_groups['server'][0] @@ -119,11 +122,17 @@ def testPrepare(self, SetupServer, StartServer): # pylint: disable=invalid-name @mock.patch.object(apache2_server, 'GetApacheCPUSeconds', autospec=True) @mock.patch.object( - apachebench_benchmark, '_ParseHistogramFromFile', autospec=True) + apachebench_benchmark, '_ParseHistogramFromFile', autospec=True + ) @mock.patch.object( - apachebench_benchmark, '_ParseRawRequestData', autospec=True) - def testApacheBench_Run(self, _ParseRawRequestData, _ParseHistogramFromFile, # pylint: disable=invalid-name - GetApacheCPUSeconds): # pylint: disable=invalid-name + apachebench_benchmark, '_ParseRawRequestData', autospec=True + ) + def testApacheBench_Run( + self, + _ParseRawRequestData, + _ParseHistogramFromFile, # pylint: disable=invalid-name + GetApacheCPUSeconds, + ): # pylint: disable=invalid-name server = self.vm_spec.vm_groups['server'][0] clients = self.vm_spec.vm_groups['client'] for client in clients: @@ -133,8 +142,9 @@ def testApacheBench_Run(self, _ParseRawRequestData, _ParseHistogramFromFile, # _ParseHistogramFromFile.return_value = self.expected_histogram _ParseRawRequestData.return_value = self.expected_raw_request_data - result = apachebench_benchmark._Run(clients, server, self.run_config, - self.ip_config) + result = apachebench_benchmark._Run( + clients, server, self.run_config, self.ip_config + ) expected_attrs = { 'complete_requests': 1, @@ -155,7 +165,7 @@ def testApacheBench_Run(self, _ParseRawRequestData, _ParseHistogramFromFile, # 'html_transferred_unit': 'bytes', 'histogram': self.expected_histogram, 'raw_results': self.expected_raw_request_data, - 'cpu_seconds': 1.0 + 'cpu_seconds': 1.0, } for metric, expected in expected_attrs.items(): @@ -166,7 +176,9 @@ def testApacheBenchGetMetadata(self): for client in clients: client.RemoteCommand.return_value = (self.apachebench_output, '') - FLAGS.apachebench_run_mode = apachebench_benchmark.ApacheBenchRunMode.MAX_THROUGHPUT + FLAGS.apachebench_run_mode = ( + apachebench_benchmark.ApacheBenchRunMode.MAX_THROUGHPUT + ) result = apachebench_benchmark.ApacheBenchResults( complete_requests=1, @@ -187,9 +199,11 @@ def testApacheBenchGetMetadata(self): html_transferred_unit='bytes', histogram=self.expected_histogram, raw_results={0: [1.0]}, - cpu_seconds=1.0) - metadata = apachebench_benchmark.GetMetadata(result, self.run_config, - self.ip_config) + cpu_seconds=1.0, + ) + metadata = apachebench_benchmark.GetMetadata( + result, self.run_config, self.ip_config + ) expected_metadata = { 'apachebench_requests': 1, @@ -203,8 +217,9 @@ def testApacheBenchGetMetadata(self): 'apachebench_client_vms': 1, 'apachebench_complete_requests': 1, 'apachebench_time_taken_for_tests': 1.0, - 'apachebench_run_mode': + 'apachebench_run_mode': ( apachebench_benchmark.ApacheBenchRunMode.MAX_THROUGHPUT + ), } self.assertDictEqual(metadata, expected_metadata) diff --git a/tests/linux_benchmarks/aws_dynamodb_ycsb_benchmark_test.py b/tests/linux_benchmarks/aws_dynamodb_ycsb_benchmark_test.py index 107f9132e7..a96fbebad7 100644 --- a/tests/linux_benchmarks/aws_dynamodb_ycsb_benchmark_test.py +++ b/tests/linux_benchmarks/aws_dynamodb_ycsb_benchmark_test.py @@ -33,8 +33,10 @@ class LoadStageTest(pkb_common_test_case.PkbCommonTestCase): def setUp(self): super().setUp() self.enter_context( - mock.patch.object(aws_dynamodb_ycsb_benchmark, - 'GetRemoteVMCredentialsFullPath')) + mock.patch.object( + aws_dynamodb_ycsb_benchmark, 'GetRemoteVMCredentialsFullPath' + ) + ) self.enter_context(mock.patch.object(background_tasks, 'RunThreaded')) self.enter_context(mock.patch.object(ycsb.YCSBExecutor, 'Load')) self.mock_spec = mock.MagicMock() @@ -44,22 +46,23 @@ def testLoadThroughputIncreases(self): # Arrange instance = aws_dynamodb.AwsDynamoDBInstance(rcu=1000, wcu=1000) mock_set_throughput = self.enter_context( - mock.patch.object(instance, 'SetThroughput')) + mock.patch.object(instance, 'SetThroughput') + ) self.mock_spec.non_relational_db = instance # Act aws_dynamodb_ycsb_benchmark.Prepare(self.mock_spec) # Assert - mock_set_throughput.assert_has_calls( - [mock.call(wcu=10000), mock.call()]) + mock_set_throughput.assert_has_calls([mock.call(wcu=10000), mock.call()]) def testLoadThroughputStaysSame(self): # WCU stays the same during loading if > 10k. # Arrange instance = aws_dynamodb.AwsDynamoDBInstance(rcu=1000, wcu=30000) mock_set_throughput = self.enter_context( - mock.patch.object(instance, 'SetThroughput')) + mock.patch.object(instance, 'SetThroughput') + ) self.mock_spec.non_relational_db = instance # Act @@ -72,11 +75,13 @@ def testLoadThroughputStaysSame(self): aws_dynamodb_autoscaling_target=50, aws_dynamodb_autoscaling_wcu_max=100, aws_dynamodb_autoscaling_rcu_max=200, - aws_dynamodb_ycsb_provision_wcu=10000) + aws_dynamodb_ycsb_provision_wcu=10000, + ) def testLoadPhaseWithAutoscalingDoesNotSetThroughput(self): instance = aws_dynamodb.AwsDynamoDBInstance(rcu=1000, wcu=3000) mock_set_throughput = self.enter_context( - mock.patch.object(instance, 'SetThroughput')) + mock.patch.object(instance, 'SetThroughput') + ) self.mock_spec.non_relational_db = instance aws_dynamodb_ycsb_benchmark.Prepare(self.mock_spec) @@ -85,9 +90,7 @@ def testLoadPhaseWithAutoscalingDoesNotSetThroughput(self): @parameterized.parameters((100, 100, True, 200), (100, 100, False, 300)) @flagsaver.flagsaver() - def testTargetQpsIsCorrect( - self, rcu, wcu, strong_consistency, expected_qps - ): + def testTargetQpsIsCorrect(self, rcu, wcu, strong_consistency, expected_qps): FLAGS.aws_dynamodb_ycsb_consistentReads = strong_consistency instance = aws_dynamodb.AwsDynamoDBInstance(rcu=rcu, wcu=wcu) actual_qps = aws_dynamodb_ycsb_benchmark._GetTargetQps(instance) diff --git a/tests/linux_benchmarks/beam_integration_benchmark_test.py b/tests/linux_benchmarks/beam_integration_benchmark_test.py index d80baca1c7..ffa7bd20c5 100644 --- a/tests/linux_benchmarks/beam_integration_benchmark_test.py +++ b/tests/linux_benchmarks/beam_integration_benchmark_test.py @@ -20,69 +20,80 @@ class BeamArgsOptionsTestCase(unittest.TestCase): + def testNoFlagsPassed(self): options_list = beam_pipeline_options.GenerateAllPipelineOptions( - None, None, [], []) + None, None, [], [] + ) self.assertListEqual(options_list, []) def testAllFlagsPassed(self): options_list = beam_pipeline_options.GenerateAllPipelineOptions( "--itargone=anarg,--itargtwo=anotherarg", - "[\"--project=testProj\"," - "\"--gcpTempLocation=gs://test-bucket/staging\"]", + '["--project=testProj","--gcpTempLocation=gs://test-bucket/staging"]', [{"postgresUsername": "postgres"}, {"postgresPassword": "mypass"}], - [{"name": "aTestVal", "type": "TestValue", "value": "this_is_a_test"}, - {"name": "testier", "type": "TestValue", "value": "another_test"}] + [ + { + "name": "aTestVal", + "type": "TestValue", + "value": "this_is_a_test", + }, + {"name": "testier", "type": "TestValue", "value": "another_test"}, + ], ) - self.assertListEqual(options_list, - ["\"--itargone=anarg\"", - "\"--itargtwo=anotherarg\"", - "\"--project=testProj\"", - "\"--gcpTempLocation=gs://test-bucket/staging\"", - "\"--aTestVal=this_is_a_test\"", - "\"--testier=another_test\"", - "\"--postgresUsername=postgres\"", - "\"--postgresPassword=mypass\""]) + self.assertListEqual( + options_list, + [ + '"--itargone=anarg"', + '"--itargtwo=anotherarg"', + '"--project=testProj"', + '"--gcpTempLocation=gs://test-bucket/staging"', + '"--aTestVal=this_is_a_test"', + '"--testier=another_test"', + '"--postgresUsername=postgres"', + '"--postgresPassword=mypass"', + ], + ) def testItOptionsWithSpaces(self): options_list = beam_pipeline_options.GenerateAllPipelineOptions( None, - "[\"--project=testProj\", " - "\"--gcpTempLocation=gs://test-bucket/staging\"]", + '["--project=testProj", "--gcpTempLocation=gs://test-bucket/staging"]', + [], [], - []) + ) - self.assertListEqual(options_list, - ["\"--project=testProj\"", - "\"--gcpTempLocation=gs://test-bucket/staging\""]) + self.assertListEqual( + options_list, + [ + '"--project=testProj"', + '"--gcpTempLocation=gs://test-bucket/staging"', + ], + ) def testDynamicPipelineOpionsWithFormat(self): - dynamic_options = [ - { - "name": "test_value_A", - "type": "TestValue", - "value": "a_value", - "format": "other representation of {{TestValue}}", - }, - { - "name": "test_value_B", - "type": "TestValue", - "value": "b_value" - } - ] + dynamic_options = [ + { + "name": "test_value_A", + "type": "TestValue", + "value": "a_value", + "format": "other representation of {{TestValue}}", + }, + {"name": "test_value_B", "type": "TestValue", "value": "b_value"}, + ] - self.assertListEqual( - beam_pipeline_options.EvaluateDynamicPipelineOptions(dynamic_options), - [ - ("test_value_A", "other representation of a_value"), - ("test_value_B", "b_value"), - ] - ) + self.assertListEqual( + beam_pipeline_options.EvaluateDynamicPipelineOptions(dynamic_options), + [ + ("test_value_A", "other representation of a_value"), + ("test_value_B", "b_value"), + ], + ) def dynamicPipelineOptions(self): beam_pipeline_options.EvaluateDynamicPipelineOptions() -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/linux_benchmarks/bidirectional_network_benchmark_test.py b/tests/linux_benchmarks/bidirectional_network_benchmark_test.py index 025f180457..96af97a7da 100644 --- a/tests/linux_benchmarks/bidirectional_network_benchmark_test.py +++ b/tests/linux_benchmarks/bidirectional_network_benchmark_test.py @@ -32,21 +32,24 @@ class BidirectionalNetworkBenchmarkTestCase( - pkb_common_test_case.PkbCommonTestCase): + pkb_common_test_case.PkbCommonTestCase +): def setUp(self): super(BidirectionalNetworkBenchmarkTestCase, self).setUp() # Load netperf stdout data - path = os.path.join(os.path.dirname(__file__), - '..', 'data', - 'bidirectional_network_results.json') + path = os.path.join( + os.path.dirname(__file__), + '..', + 'data', + 'bidirectional_network_results.json', + ) with open(path) as fp: remote_stdouts = json.load(fp) - self.expected_stdout = [json.dumps(stdout) - for stdout in remote_stdouts] + self.expected_stdout = [json.dumps(stdout) for stdout in remote_stdouts] def _AssertSamples(self, results, delta, values): - """"Asserts that the samples matching delta correspond to values.""" + """ "Asserts that the samples matching delta correspond to values.""" delta_metric = [r for r in results if r[1] == delta] self.assertEqual(1, len(delta_metric), delta) @@ -55,9 +58,10 @@ def _AssertSamples(self, results, delta, values): # _. This can vary due to the parallel nature of # the tests, so we grab all samples with the header that matches the header # of the delta sample. - delta_header = delta_metric[0][0][:len('0_TCP_STREAM_')] - netperf_run = [r for r in results - if r[0][:len(delta_header)] == delta_header] + delta_header = delta_metric[0][0][: len('0_TCP_STREAM_')] + netperf_run = [ + r for r in results if r[0][: len(delta_header)] == delta_header + ] header = delta_header + 'Throughput_' @@ -66,24 +70,36 @@ def _AssertSamples(self, results, delta, values): values.sort() self._AssertSample(netperf_run, header + 'max', values[7], MBPS) self._AssertSample(netperf_run, header + 'min', values[0], MBPS) - self._AssertSample(netperf_run, header + 'stddev', - numpy.std(values, ddof=1), MBPS) + self._AssertSample( + netperf_run, header + 'stddev', numpy.std(values, ddof=1), MBPS + ) self._AssertSample(netperf_run, header + 'p90', values[7], MBPS) - self._AssertSample(netperf_run, header + 'average', numpy.mean(values), - MBPS) + self._AssertSample( + netperf_run, header + 'average', numpy.mean(values), MBPS + ) self._AssertSample(netperf_run, header + 'p50', values[4], MBPS) self._AssertSample(netperf_run, header + 'total', sum(values), MBPS) self._AssertSample(netperf_run, header + 'p99', values[7], MBPS) def _AssertSample(self, results, metric, value, unit): """Asserts results contains a sample matching metric/value/unit.""" - match = [r for r in results if (r.metric == metric and r.unit == unit and - abs(r.value - value) < TOLERANCE)] + match = [ + r + for r in results + if ( + r.metric == metric + and r.unit == unit + and abs(r.value - value) < TOLERANCE + ) + ] self.assertEqual(1, len(match)) def testRun(self): FLAGS.bidirectional_network_tests = [ - 'TCP_STREAM', 'TCP_MAERTS', 'TCP_MAERTS', 'TCP_MAERTS' + 'TCP_STREAM', + 'TCP_MAERTS', + 'TCP_MAERTS', + 'TCP_MAERTS', ] FLAGS.bidirectional_network_test_length = 60 FLAGS.bidirectional_stream_num_streams = 8 @@ -91,10 +107,7 @@ def testRun(self): # Helper for GetNetperfStdOut whichs holds the last returned index for the # given test. Used to ensure the mocked stdout is returned matching the # requested netperf test and each is returned exactly once. - last_returned = { - 'TCP_STREAM': -1, - 'TCP_MAERTS': -1 - } + last_returned = {'TCP_STREAM': -1, 'TCP_MAERTS': -1} stdout_lock = threading.Lock() @@ -114,8 +127,13 @@ def GetNetperfStdOut(remote_cmd, timeout): return (self.expected_stdout[i], '') vm_spec = mock.MagicMock(spec=benchmark_spec.BenchmarkSpec) - vm_spec.vms = [mock.MagicMock(), mock.MagicMock(), mock.MagicMock(), - mock.MagicMock(), mock.MagicMock()] + vm_spec.vms = [ + mock.MagicMock(), + mock.MagicMock(), + mock.MagicMock(), + mock.MagicMock(), + mock.MagicMock(), + ] vm_spec.vms[0].RemoteCommand.side_effect = GetNetperfStdOut results = bidirectional_network_benchmark.Run(vm_spec) @@ -185,10 +203,18 @@ def GetNetperfStdOut(remote_cmd, timeout): # summary metrics self._AssertSample(results, 'outbound_network_total', sum(samples0), MBPS) - self._AssertSample(results, 'inbound_network_total', - sum(samples1 + samples2 + samples3), MBPS) - self._AssertSample(results, 'all_streams_start_delta', - 1508187617.100678 - 1508187614.93243, 'seconds') + self._AssertSample( + results, + 'inbound_network_total', + sum(samples1 + samples2 + samples3), + MBPS, + ) + self._AssertSample( + results, + 'all_streams_start_delta', + 1508187617.100678 - 1508187614.93243, + 'seconds', + ) metrics_per_test = 9 # 8 throughput samples, 1 delta num_tests = 4 diff --git a/tests/linux_benchmarks/bonnie_benchmark_test.py b/tests/linux_benchmarks/bonnie_benchmark_test.py index cc0a28d741..430df54dc4 100644 --- a/tests/linux_benchmarks/bonnie_benchmark_test.py +++ b/tests/linux_benchmarks/bonnie_benchmark_test.py @@ -25,120 +25,383 @@ class BonnieBenchmarkTestCase(unittest.TestCase, test_util.SamplesTestMixin): def setUp(self): - path = os.path.join(os.path.dirname(__file__), '..', 'data', - 'bonnie-plus-plus-sample.txt') + path = os.path.join( + os.path.dirname(__file__), '..', 'data', 'bonnie-plus-plus-sample.txt' + ) with open(path) as fp: self.contents = fp.read() def testParseCSVResults(self): result = bonnie_benchmark.ParseCSVResults(self.contents) expected_result = [ - ['put_block', 72853.0, 'K/sec', - {'name': 'perfkit-7b22f510-0', 'format_version': '1.96', - 'num_files': '100', 'seed': '1421800799', 'concurrency': '1', - 'file_size': '7423M', 'bonnie_version': '1.96'}], - ['put_block_cpu', 15.0, '%s', - {'name': 'perfkit-7b22f510-0', 'format_version': '1.96', - 'num_files': '100', 'seed': '1421800799', 'concurrency': '1', - 'file_size': '7423M', 'bonnie_version': '1.96'}], - ['rewrite', 47358.0, 'K/sec', - {'name': 'perfkit-7b22f510-0', 'format_version': '1.96', - 'num_files': '100', 'seed': '1421800799', 'concurrency': '1', - 'file_size': '7423M', 'bonnie_version': '1.96'}], - ['rewrite_cpu', 5.0, '%s', - {'name': 'perfkit-7b22f510-0', 'format_version': '1.96', - 'num_files': '100', 'seed': '1421800799', 'concurrency': '1', - 'file_size': '7423M', 'bonnie_version': '1.96'}], - ['get_block', 156821.0, 'K/sec', - {'name': 'perfkit-7b22f510-0', 'format_version': '1.96', - 'num_files': '100', 'seed': '1421800799', 'concurrency': '1', - 'file_size': '7423M', 'bonnie_version': '1.96'}], - ['get_block_cpu', 7.0, '%s', - {'name': 'perfkit-7b22f510-0', 'format_version': '1.96', - 'num_files': '100', 'seed': '1421800799', 'concurrency': '1', - 'file_size': '7423M', 'bonnie_version': '1.96'}], - ['seeks', 537.7, 'K/sec', - {'name': 'perfkit-7b22f510-0', 'format_version': '1.96', - 'num_files': '100', 'seed': '1421800799', 'concurrency': '1', - 'file_size': '7423M', 'bonnie_version': '1.96'}], - ['seeks_cpu', 10.0, '%s', - {'name': 'perfkit-7b22f510-0', 'format_version': '1.96', - 'num_files': '100', 'seed': '1421800799', 'concurrency': '1', - 'file_size': '7423M', 'bonnie_version': '1.96'}], - ['seq_create', 49223.0, 'K/sec', - {'name': 'perfkit-7b22f510-0', 'format_version': '1.96', - 'num_files': '100', 'seed': '1421800799', 'concurrency': '1', - 'file_size': '7423M', 'bonnie_version': '1.96'}], - ['seq_create_cpu', 58.0, '%s', - {'name': 'perfkit-7b22f510-0', 'format_version': '1.96', - 'num_files': '100', 'seed': '1421800799', 'concurrency': '1', - 'file_size': '7423M', 'bonnie_version': '1.96'}], - ['seq_del', 54405.0, 'K/sec', - {'name': 'perfkit-7b22f510-0', 'format_version': '1.96', - 'num_files': '100', 'seed': '1421800799', 'concurrency': '1', - 'file_size': '7423M', 'bonnie_version': '1.96'}], - ['seq_del_cpu', 53.0, '%s', - {'name': 'perfkit-7b22f510-0', 'format_version': '1.96', - 'num_files': '100', 'seed': '1421800799', 'concurrency': '1', - 'file_size': '7423M', 'bonnie_version': '1.96'}], - ['ran_create', 2898.0, 'K/sec', - {'name': 'perfkit-7b22f510-0', 'format_version': '1.96', - 'num_files': '100', 'seed': '1421800799', 'concurrency': '1', - 'file_size': '7423M', 'bonnie_version': '1.96'}], - ['ran_create_cpu', 97.0, '%s', - {'name': 'perfkit-7b22f510-0', 'format_version': '1.96', - 'num_files': '100', 'seed': '1421800799', 'concurrency': '1', - 'file_size': '7423M', 'bonnie_version': '1.96'}], - ['ran_del', 59089.0, 'K/sec', - {'name': 'perfkit-7b22f510-0', 'format_version': '1.96', - 'num_files': '100', 'seed': '1421800799', 'concurrency': '1', - 'file_size': '7423M', 'bonnie_version': '1.96'}], - ['ran_del_cpu', 60.0, '%s', - {'name': 'perfkit-7b22f510-0', 'format_version': '1.96', - 'num_files': '100', 'seed': '1421800799', 'concurrency': '1', - 'file_size': '7423M', 'bonnie_version': '1.96'}], - ['put_block_latency', 512.0, 'ms', - {'name': 'perfkit-7b22f510-0', 'format_version': '1.96', - 'num_files': '100', 'seed': '1421800799', 'concurrency': '1', - 'file_size': '7423M', 'bonnie_version': '1.96'}], - ['rewrite_latency', 670.0, 'ms', - {'name': 'perfkit-7b22f510-0', 'format_version': '1.96', - 'num_files': '100', 'seed': '1421800799', 'concurrency': '1', - 'file_size': '7423M', 'bonnie_version': '1.96'}], - ['get_block_latency', 44660.0, 'us', - {'name': 'perfkit-7b22f510-0', 'format_version': '1.96', - 'num_files': '100', 'seed': '1421800799', 'concurrency': '1', - 'file_size': '7423M', 'bonnie_version': '1.96'}], - ['seeks_latency', 200.0, 'ms', - {'name': 'perfkit-7b22f510-0', 'format_version': '1.96', - 'num_files': '100', 'seed': '1421800799', 'concurrency': '1', - 'file_size': '7423M', 'bonnie_version': '1.96'}], - ['seq_create_latency', 3747.0, 'us', - {'name': 'perfkit-7b22f510-0', 'format_version': '1.96', - 'num_files': '100', 'seed': '1421800799', 'concurrency': '1', - 'file_size': '7423M', 'bonnie_version': '1.96'}], - ['seq_stat_latency', 1759.0, 'us', - {'name': 'perfkit-7b22f510-0', 'format_version': '1.96', - 'num_files': '100', 'seed': '1421800799', 'concurrency': '1', - 'file_size': '7423M', 'bonnie_version': '1.96'}], - ['seq_del_latency', 1643.0, 'us', - {'name': 'perfkit-7b22f510-0', 'format_version': '1.96', - 'num_files': '100', 'seed': '1421800799', 'concurrency': '1', - 'file_size': '7423M', 'bonnie_version': '1.96'}], - ['ran_create_latency', 33518.0, 'us', - {'name': 'perfkit-7b22f510-0', 'format_version': '1.96', - 'num_files': '100', 'seed': '1421800799', 'concurrency': '1', - 'file_size': '7423M', 'bonnie_version': '1.96'}], - ['ran_stat_latency', 192.0, 'us', - {'name': 'perfkit-7b22f510-0', 'format_version': '1.96', - 'num_files': '100', 'seed': '1421800799', 'concurrency': '1', - 'file_size': '7423M', 'bonnie_version': '1.96'}], - ['ran_del_latency', 839.0, 'us', - {'name': 'perfkit-7b22f510-0', 'format_version': '1.96', - 'num_files': '100', 'seed': '1421800799', 'concurrency': '1', - 'file_size': '7423M', 'bonnie_version': '1.96'}]] - expected_result = [sample.Sample(*sample_tuple) - for sample_tuple in expected_result] + [ + 'put_block', + 72853.0, + 'K/sec', + { + 'name': 'perfkit-7b22f510-0', + 'format_version': '1.96', + 'num_files': '100', + 'seed': '1421800799', + 'concurrency': '1', + 'file_size': '7423M', + 'bonnie_version': '1.96', + }, + ], + [ + 'put_block_cpu', + 15.0, + '%s', + { + 'name': 'perfkit-7b22f510-0', + 'format_version': '1.96', + 'num_files': '100', + 'seed': '1421800799', + 'concurrency': '1', + 'file_size': '7423M', + 'bonnie_version': '1.96', + }, + ], + [ + 'rewrite', + 47358.0, + 'K/sec', + { + 'name': 'perfkit-7b22f510-0', + 'format_version': '1.96', + 'num_files': '100', + 'seed': '1421800799', + 'concurrency': '1', + 'file_size': '7423M', + 'bonnie_version': '1.96', + }, + ], + [ + 'rewrite_cpu', + 5.0, + '%s', + { + 'name': 'perfkit-7b22f510-0', + 'format_version': '1.96', + 'num_files': '100', + 'seed': '1421800799', + 'concurrency': '1', + 'file_size': '7423M', + 'bonnie_version': '1.96', + }, + ], + [ + 'get_block', + 156821.0, + 'K/sec', + { + 'name': 'perfkit-7b22f510-0', + 'format_version': '1.96', + 'num_files': '100', + 'seed': '1421800799', + 'concurrency': '1', + 'file_size': '7423M', + 'bonnie_version': '1.96', + }, + ], + [ + 'get_block_cpu', + 7.0, + '%s', + { + 'name': 'perfkit-7b22f510-0', + 'format_version': '1.96', + 'num_files': '100', + 'seed': '1421800799', + 'concurrency': '1', + 'file_size': '7423M', + 'bonnie_version': '1.96', + }, + ], + [ + 'seeks', + 537.7, + 'K/sec', + { + 'name': 'perfkit-7b22f510-0', + 'format_version': '1.96', + 'num_files': '100', + 'seed': '1421800799', + 'concurrency': '1', + 'file_size': '7423M', + 'bonnie_version': '1.96', + }, + ], + [ + 'seeks_cpu', + 10.0, + '%s', + { + 'name': 'perfkit-7b22f510-0', + 'format_version': '1.96', + 'num_files': '100', + 'seed': '1421800799', + 'concurrency': '1', + 'file_size': '7423M', + 'bonnie_version': '1.96', + }, + ], + [ + 'seq_create', + 49223.0, + 'K/sec', + { + 'name': 'perfkit-7b22f510-0', + 'format_version': '1.96', + 'num_files': '100', + 'seed': '1421800799', + 'concurrency': '1', + 'file_size': '7423M', + 'bonnie_version': '1.96', + }, + ], + [ + 'seq_create_cpu', + 58.0, + '%s', + { + 'name': 'perfkit-7b22f510-0', + 'format_version': '1.96', + 'num_files': '100', + 'seed': '1421800799', + 'concurrency': '1', + 'file_size': '7423M', + 'bonnie_version': '1.96', + }, + ], + [ + 'seq_del', + 54405.0, + 'K/sec', + { + 'name': 'perfkit-7b22f510-0', + 'format_version': '1.96', + 'num_files': '100', + 'seed': '1421800799', + 'concurrency': '1', + 'file_size': '7423M', + 'bonnie_version': '1.96', + }, + ], + [ + 'seq_del_cpu', + 53.0, + '%s', + { + 'name': 'perfkit-7b22f510-0', + 'format_version': '1.96', + 'num_files': '100', + 'seed': '1421800799', + 'concurrency': '1', + 'file_size': '7423M', + 'bonnie_version': '1.96', + }, + ], + [ + 'ran_create', + 2898.0, + 'K/sec', + { + 'name': 'perfkit-7b22f510-0', + 'format_version': '1.96', + 'num_files': '100', + 'seed': '1421800799', + 'concurrency': '1', + 'file_size': '7423M', + 'bonnie_version': '1.96', + }, + ], + [ + 'ran_create_cpu', + 97.0, + '%s', + { + 'name': 'perfkit-7b22f510-0', + 'format_version': '1.96', + 'num_files': '100', + 'seed': '1421800799', + 'concurrency': '1', + 'file_size': '7423M', + 'bonnie_version': '1.96', + }, + ], + [ + 'ran_del', + 59089.0, + 'K/sec', + { + 'name': 'perfkit-7b22f510-0', + 'format_version': '1.96', + 'num_files': '100', + 'seed': '1421800799', + 'concurrency': '1', + 'file_size': '7423M', + 'bonnie_version': '1.96', + }, + ], + [ + 'ran_del_cpu', + 60.0, + '%s', + { + 'name': 'perfkit-7b22f510-0', + 'format_version': '1.96', + 'num_files': '100', + 'seed': '1421800799', + 'concurrency': '1', + 'file_size': '7423M', + 'bonnie_version': '1.96', + }, + ], + [ + 'put_block_latency', + 512.0, + 'ms', + { + 'name': 'perfkit-7b22f510-0', + 'format_version': '1.96', + 'num_files': '100', + 'seed': '1421800799', + 'concurrency': '1', + 'file_size': '7423M', + 'bonnie_version': '1.96', + }, + ], + [ + 'rewrite_latency', + 670.0, + 'ms', + { + 'name': 'perfkit-7b22f510-0', + 'format_version': '1.96', + 'num_files': '100', + 'seed': '1421800799', + 'concurrency': '1', + 'file_size': '7423M', + 'bonnie_version': '1.96', + }, + ], + [ + 'get_block_latency', + 44660.0, + 'us', + { + 'name': 'perfkit-7b22f510-0', + 'format_version': '1.96', + 'num_files': '100', + 'seed': '1421800799', + 'concurrency': '1', + 'file_size': '7423M', + 'bonnie_version': '1.96', + }, + ], + [ + 'seeks_latency', + 200.0, + 'ms', + { + 'name': 'perfkit-7b22f510-0', + 'format_version': '1.96', + 'num_files': '100', + 'seed': '1421800799', + 'concurrency': '1', + 'file_size': '7423M', + 'bonnie_version': '1.96', + }, + ], + [ + 'seq_create_latency', + 3747.0, + 'us', + { + 'name': 'perfkit-7b22f510-0', + 'format_version': '1.96', + 'num_files': '100', + 'seed': '1421800799', + 'concurrency': '1', + 'file_size': '7423M', + 'bonnie_version': '1.96', + }, + ], + [ + 'seq_stat_latency', + 1759.0, + 'us', + { + 'name': 'perfkit-7b22f510-0', + 'format_version': '1.96', + 'num_files': '100', + 'seed': '1421800799', + 'concurrency': '1', + 'file_size': '7423M', + 'bonnie_version': '1.96', + }, + ], + [ + 'seq_del_latency', + 1643.0, + 'us', + { + 'name': 'perfkit-7b22f510-0', + 'format_version': '1.96', + 'num_files': '100', + 'seed': '1421800799', + 'concurrency': '1', + 'file_size': '7423M', + 'bonnie_version': '1.96', + }, + ], + [ + 'ran_create_latency', + 33518.0, + 'us', + { + 'name': 'perfkit-7b22f510-0', + 'format_version': '1.96', + 'num_files': '100', + 'seed': '1421800799', + 'concurrency': '1', + 'file_size': '7423M', + 'bonnie_version': '1.96', + }, + ], + [ + 'ran_stat_latency', + 192.0, + 'us', + { + 'name': 'perfkit-7b22f510-0', + 'format_version': '1.96', + 'num_files': '100', + 'seed': '1421800799', + 'concurrency': '1', + 'file_size': '7423M', + 'bonnie_version': '1.96', + }, + ], + [ + 'ran_del_latency', + 839.0, + 'us', + { + 'name': 'perfkit-7b22f510-0', + 'format_version': '1.96', + 'num_files': '100', + 'seed': '1421800799', + 'concurrency': '1', + 'file_size': '7423M', + 'bonnie_version': '1.96', + }, + ], + ] + expected_result = [ + sample.Sample(*sample_tuple) for sample_tuple in expected_result + ] self.assertSampleListsEqualUpToTimestamp(result, expected_result) diff --git a/tests/linux_benchmarks/cloud_redis_memtier_benchmark_test.py b/tests/linux_benchmarks/cloud_redis_memtier_benchmark_test.py index d993e56717..07e34a1f6f 100644 --- a/tests/linux_benchmarks/cloud_redis_memtier_benchmark_test.py +++ b/tests/linux_benchmarks/cloud_redis_memtier_benchmark_test.py @@ -84,18 +84,23 @@ def testPrepare(self): redis_instance = mock.Mock() benchmark_spec.vm_groups = {'clients': [vm]} memory_store_patch = self.enter_context( - mock.patch.object(cloud_redis_memtier_benchmark, - '_GetManagedMemoryStore')) + mock.patch.object( + cloud_redis_memtier_benchmark, '_GetManagedMemoryStore' + ) + ) memory_store_patch.return_value = redis_instance ip_patch = self.enter_context( - mock.patch.object(redis_instance, 'GetMemoryStoreIp')) + mock.patch.object(redis_instance, 'GetMemoryStoreIp') + ) ip_patch.return_value = '0.0.0' port_patch = self.enter_context( - mock.patch.object(redis_instance, 'GetMemoryStorePort')) + mock.patch.object(redis_instance, 'GetMemoryStorePort') + ) port_patch.return_value = '1234' password_patch = self.enter_context( - mock.patch.object(redis_instance, 'GetMemoryStorePassword')) + mock.patch.object(redis_instance, 'GetMemoryStorePassword') + ) password_patch.return_value = 'password' load_patch = self.enter_context(mock.patch.object(memtier, 'Load')) @@ -108,7 +113,8 @@ def testRun(self): benchmark_spec = mock.Mock() benchmark_spec.vm_groups = {'clients': [vm]} samples = self.enter_context( - mock.patch.object(memtier, 'RunOverAllThreadsPipelinesAndClients')) + mock.patch.object(memtier, 'RunOverAllThreadsPipelinesAndClients') + ) samples.return_value = [] self.assertEqual([], cloud_redis_memtier_benchmark.Run(benchmark_spec)) @@ -119,7 +125,8 @@ def testRunLatencyAtGivenCpu(self): benchmark_spec = mock.Mock() benchmark_spec.vm_groups = {'clients': [client_vm, measure_latency_vm]} samples = self.enter_context( - mock.patch.object(memtier, 'RunGetLatencyAtCpu')) + mock.patch.object(memtier, 'RunGetLatencyAtCpu') + ) samples.return_value = [] self.assertEqual([], cloud_redis_memtier_benchmark.Run(benchmark_spec)) diff --git a/tests/linux_benchmarks/cloudharmony_iperf_benchmark_test.py b/tests/linux_benchmarks/cloudharmony_iperf_benchmark_test.py index de9dbd619a..c364de2ecf 100644 --- a/tests/linux_benchmarks/cloudharmony_iperf_benchmark_test.py +++ b/tests/linux_benchmarks/cloudharmony_iperf_benchmark_test.py @@ -1,6 +1,6 @@ """Tests for cloudharmony_iperf_benchmark.""" -import os +import os import unittest from absl import flags import mock @@ -15,30 +15,35 @@ class CloudharmonyIperfBenchmarkTestCase( - pkb_common_test_case.PkbCommonTestCase, test_util.SamplesTestMixin): + pkb_common_test_case.PkbCommonTestCase, test_util.SamplesTestMixin +): def setUp(self): super(CloudharmonyIperfBenchmarkTestCase, self).setUp() - self.path = os.path.join(os.path.dirname(__file__), - '../data/cloudharmony_iperf.csv') + self.path = os.path.join( + os.path.dirname(__file__), '../data/cloudharmony_iperf.csv' + ) with open(self.path, 'r') as fp: self.iperf_csv_text = fp.read() - self.cloud_harmony_metadata = cloud_harmony_util.ParseCsvResultsFromString( - self.iperf_csv_text) + self.cloud_harmony_metadata = ( + cloud_harmony_util.ParseCsvResultsFromString(self.iperf_csv_text) + ) def testRun(self): FLAGS.run_uri = '12345678' FLAGS.zone = ['us-central1-a'] remote_command = mock.PropertyMock(return_value=(self.iperf_csv_text, '')) - client = mock.Mock(RemoteCommand=remote_command, - machine_type='n2-standard-2') - server = mock.Mock(internal_ip='10.0.0.1', - machine_type='n2-standard-4') + client = mock.Mock( + RemoteCommand=remote_command, machine_type='n2-standard-2' + ) + server = mock.Mock(internal_ip='10.0.0.1', machine_type='n2-standard-4') benchmark_module = mock.Mock(BENCHMARK_NAME='cloud_harmony_iperf') - benchmark_config = mock.Mock(vm_groups={}, - relational_db=mock.Mock(vm_groups={})) - spec = benchmark_spec.BenchmarkSpec(benchmark_module, benchmark_config, - 'abcdefg') + benchmark_config = mock.Mock( + vm_groups={}, relational_db=mock.Mock(vm_groups={}) + ) + spec = benchmark_spec.BenchmarkSpec( + benchmark_module, benchmark_config, 'abcdefg' + ) spec.vm_groups = {'client': [client], 'server': [server]} results = cloudharmony_iperf_benchmark._Run(spec) client.RobustRemoteCommand.assert_called_with( @@ -60,186 +65,190 @@ def testRun(self): '--iperf_parallel 1 ' '--tcp_bw_file /tmp/tcpbw ' '--wkhtml_xvfb ' - '--verbose') + '--verbose' + ) self.assertLen(results, 2) - expected_gartner_metadata = [{ - 'bandwidth_direction': 'up', - 'bandwidth_max': 9894.384, - 'bandwidth_mean': 9735.5, - 'bandwidth_median': 9888.76, - 'bandwidth_min': 8349.13, - 'bandwidth_p10': 8349.13, - 'bandwidth_p25': 9888.001, - 'bandwidth_p75': 9889.2, - 'bandwidth_p90': 9889.8, - 'bandwidth_stdev': 487, - 'benchmark_version': 1.0, - 'collectd_rrd': '', - 'cpu_client': 34.346987, - 'cpu_server': 2.880331, - 'iperf_bandwidth': '', - 'iperf_cmd': 'iperf3 -c 10.240.0.14 -J -i 1', - 'iperf_concurrency': 1, - 'iperf_interval': 1, - 'iperf_len': '', - 'iperf_mss': '', - 'iperf_nodelay': '', - 'iperf_num': '', - 'iperf_parallel': 1, - 'iperf_reverse': '', - 'iperf_server': '10.240.0.14', - 'iperf_server_instance_id': 'n1-standard-2', - 'iperf_server_os': 'Ubuntu 18.04.5 LTS', - 'iperf_server_provider': '', - 'iperf_server_provider_id': 'google', - 'iperf_server_region': 'us-central1', - 'iperf_server_service': '', - 'iperf_server_service_id': 'google:compute', - 'iperf_time': 10, - 'iperf_tos': '', - 'iperf_udp': '', - 'iperf_version': '3.1.3', - 'iperf_warmup': 0, - 'iperf_window': '', - 'iperf_zerocopy': '', - 'iteration': 1, - 'jitter_max': '', - 'jitter_mean': '', - 'jitter_median': '', - 'jitter_min': '', - 'jitter_p10': '', - 'jitter_p25': '', - 'jitter_p75': '', - 'jitter_p90': '', - 'jitter_stdev': '', - 'loss_max': '', - 'loss_mean': '', - 'loss_median': '', - 'loss_min': '', - 'loss_p10': '', - 'loss_p25': '', - 'loss_p75': '', - 'loss_p90': '', - 'loss_stdev': '', - 'meta_compute_service': '', - 'meta_compute_service_id': 'google:compute', - 'meta_cpu': 'Intel Xeon 2.30GHz', - 'meta_cpu_cache': '46080 KB', - 'meta_cpu_cores': 2, - 'meta_cpu_speed': 2300.0, - 'meta_hostname': 'pkb-3b246db4-0', - 'meta_instance_id': 'n1-standard-2', - 'meta_memory': '7 GB', - 'meta_memory_gb': 7, - 'meta_memory_mb': 7457, - 'meta_os': 'Ubuntu 18.04.5 LTS', - 'meta_provider': '', - 'meta_provider_id': 'google', - 'meta_region': 'us-central1', - 'meta_resource_id': '', - 'meta_run_group_id': '', - 'meta_run_id': '', - 'meta_test_id': '3b246db4', - 'report_pdf': '', - 'report_zip': '', - 'same_instance_id': 1, - 'same_os': 1, - 'same_provider': 1, - 'same_region': 1, - 'same_service': 1, - 'test_started': '2020-10-30 00:33:51', - 'test_stopped': '2020-10-30 00:34:01', - 'transfer': 11605.6 - }, { - 'bandwidth_direction': 'down', - 'bandwidth_max': 9910.723, - 'bandwidth_mean': 9730.44, - 'bandwidth_median': 9866.29, - 'bandwidth_min': 8349.13, - 'bandwidth_p10': 8349.13, - 'bandwidth_p25': 9888.001, - 'bandwidth_p75': 9889.76, - 'bandwidth_p90': 9889.8, - 'bandwidth_stdev': 487, - 'benchmark_version': 1.0, - 'collectd_rrd': '', - 'cpu_client': 34.346987, - 'cpu_server': 2.880331, - 'iperf_bandwidth': '', - 'iperf_cmd': 'iperf3 -c 10.240.0.14 -J -i 1', - 'iperf_concurrency': 1, - 'iperf_interval': 1, - 'iperf_len': '', - 'iperf_mss': '', - 'iperf_nodelay': '', - 'iperf_num': '', - 'iperf_parallel': 1, - 'iperf_reverse': '', - 'iperf_server': '10.240.0.14', - 'iperf_server_instance_id': 'n1-standard-2', - 'iperf_server_os': 'Ubuntu 18.04.5 LTS', - 'iperf_server_provider': '', - 'iperf_server_provider_id': 'google', - 'iperf_server_region': 'us-central1', - 'iperf_server_service': '', - 'iperf_server_service_id': 'google:compute', - 'iperf_time': 10, - 'iperf_tos': '', - 'iperf_udp': '', - 'iperf_version': '3.1.3', - 'iperf_warmup': 0, - 'iperf_window': '', - 'iperf_zerocopy': '', - 'iteration': 1, - 'jitter_max': '', - 'jitter_mean': '', - 'jitter_median': '', - 'jitter_min': '', - 'jitter_p10': '', - 'jitter_p25': '', - 'jitter_p75': '', - 'jitter_p90': '', - 'jitter_stdev': '', - 'loss_max': '', - 'loss_mean': '', - 'loss_median': '', - 'loss_min': '', - 'loss_p10': '', - 'loss_p25': '', - 'loss_p75': '', - 'loss_p90': '', - 'loss_stdev': '', - 'meta_compute_service': '', - 'meta_compute_service_id': 'google:compute', - 'meta_cpu': 'Intel Xeon 2.30GHz', - 'meta_cpu_cache': '46080 KB', - 'meta_cpu_cores': 2, - 'meta_cpu_speed': 2300.0, - 'meta_hostname': 'pkb-4c137ef3-0', - 'meta_instance_id': 'n1-standard-2', - 'meta_memory': '7 GB', - 'meta_memory_gb': 7, - 'meta_memory_mb': 7457, - 'meta_os': 'Ubuntu 18.04.5 LTS', - 'meta_provider': '', - 'meta_provider_id': 'google', - 'meta_region': 'us-central1', - 'meta_resource_id': '', - 'meta_run_group_id': '', - 'meta_run_id': '', - 'meta_test_id': '4c137ef3', - 'report_pdf': '', - 'report_zip': '', - 'same_instance_id': 1, - 'same_os': 1, - 'same_provider': 1, - 'same_region': 1, - 'same_service': 1, - 'test_started': '2021-02-25 4:20:12', - 'test_stopped': '2021-02-25 4:22:01', - 'transfer': 11605.5 - }] + expected_gartner_metadata = [ + { + 'bandwidth_direction': 'up', + 'bandwidth_max': 9894.384, + 'bandwidth_mean': 9735.5, + 'bandwidth_median': 9888.76, + 'bandwidth_min': 8349.13, + 'bandwidth_p10': 8349.13, + 'bandwidth_p25': 9888.001, + 'bandwidth_p75': 9889.2, + 'bandwidth_p90': 9889.8, + 'bandwidth_stdev': 487, + 'benchmark_version': 1.0, + 'collectd_rrd': '', + 'cpu_client': 34.346987, + 'cpu_server': 2.880331, + 'iperf_bandwidth': '', + 'iperf_cmd': 'iperf3 -c 10.240.0.14 -J -i 1', + 'iperf_concurrency': 1, + 'iperf_interval': 1, + 'iperf_len': '', + 'iperf_mss': '', + 'iperf_nodelay': '', + 'iperf_num': '', + 'iperf_parallel': 1, + 'iperf_reverse': '', + 'iperf_server': '10.240.0.14', + 'iperf_server_instance_id': 'n1-standard-2', + 'iperf_server_os': 'Ubuntu 18.04.5 LTS', + 'iperf_server_provider': '', + 'iperf_server_provider_id': 'google', + 'iperf_server_region': 'us-central1', + 'iperf_server_service': '', + 'iperf_server_service_id': 'google:compute', + 'iperf_time': 10, + 'iperf_tos': '', + 'iperf_udp': '', + 'iperf_version': '3.1.3', + 'iperf_warmup': 0, + 'iperf_window': '', + 'iperf_zerocopy': '', + 'iteration': 1, + 'jitter_max': '', + 'jitter_mean': '', + 'jitter_median': '', + 'jitter_min': '', + 'jitter_p10': '', + 'jitter_p25': '', + 'jitter_p75': '', + 'jitter_p90': '', + 'jitter_stdev': '', + 'loss_max': '', + 'loss_mean': '', + 'loss_median': '', + 'loss_min': '', + 'loss_p10': '', + 'loss_p25': '', + 'loss_p75': '', + 'loss_p90': '', + 'loss_stdev': '', + 'meta_compute_service': '', + 'meta_compute_service_id': 'google:compute', + 'meta_cpu': 'Intel Xeon 2.30GHz', + 'meta_cpu_cache': '46080 KB', + 'meta_cpu_cores': 2, + 'meta_cpu_speed': 2300.0, + 'meta_hostname': 'pkb-3b246db4-0', + 'meta_instance_id': 'n1-standard-2', + 'meta_memory': '7 GB', + 'meta_memory_gb': 7, + 'meta_memory_mb': 7457, + 'meta_os': 'Ubuntu 18.04.5 LTS', + 'meta_provider': '', + 'meta_provider_id': 'google', + 'meta_region': 'us-central1', + 'meta_resource_id': '', + 'meta_run_group_id': '', + 'meta_run_id': '', + 'meta_test_id': '3b246db4', + 'report_pdf': '', + 'report_zip': '', + 'same_instance_id': 1, + 'same_os': 1, + 'same_provider': 1, + 'same_region': 1, + 'same_service': 1, + 'test_started': '2020-10-30 00:33:51', + 'test_stopped': '2020-10-30 00:34:01', + 'transfer': 11605.6, + }, + { + 'bandwidth_direction': 'down', + 'bandwidth_max': 9910.723, + 'bandwidth_mean': 9730.44, + 'bandwidth_median': 9866.29, + 'bandwidth_min': 8349.13, + 'bandwidth_p10': 8349.13, + 'bandwidth_p25': 9888.001, + 'bandwidth_p75': 9889.76, + 'bandwidth_p90': 9889.8, + 'bandwidth_stdev': 487, + 'benchmark_version': 1.0, + 'collectd_rrd': '', + 'cpu_client': 34.346987, + 'cpu_server': 2.880331, + 'iperf_bandwidth': '', + 'iperf_cmd': 'iperf3 -c 10.240.0.14 -J -i 1', + 'iperf_concurrency': 1, + 'iperf_interval': 1, + 'iperf_len': '', + 'iperf_mss': '', + 'iperf_nodelay': '', + 'iperf_num': '', + 'iperf_parallel': 1, + 'iperf_reverse': '', + 'iperf_server': '10.240.0.14', + 'iperf_server_instance_id': 'n1-standard-2', + 'iperf_server_os': 'Ubuntu 18.04.5 LTS', + 'iperf_server_provider': '', + 'iperf_server_provider_id': 'google', + 'iperf_server_region': 'us-central1', + 'iperf_server_service': '', + 'iperf_server_service_id': 'google:compute', + 'iperf_time': 10, + 'iperf_tos': '', + 'iperf_udp': '', + 'iperf_version': '3.1.3', + 'iperf_warmup': 0, + 'iperf_window': '', + 'iperf_zerocopy': '', + 'iteration': 1, + 'jitter_max': '', + 'jitter_mean': '', + 'jitter_median': '', + 'jitter_min': '', + 'jitter_p10': '', + 'jitter_p25': '', + 'jitter_p75': '', + 'jitter_p90': '', + 'jitter_stdev': '', + 'loss_max': '', + 'loss_mean': '', + 'loss_median': '', + 'loss_min': '', + 'loss_p10': '', + 'loss_p25': '', + 'loss_p75': '', + 'loss_p90': '', + 'loss_stdev': '', + 'meta_compute_service': '', + 'meta_compute_service_id': 'google:compute', + 'meta_cpu': 'Intel Xeon 2.30GHz', + 'meta_cpu_cache': '46080 KB', + 'meta_cpu_cores': 2, + 'meta_cpu_speed': 2300.0, + 'meta_hostname': 'pkb-4c137ef3-0', + 'meta_instance_id': 'n1-standard-2', + 'meta_memory': '7 GB', + 'meta_memory_gb': 7, + 'meta_memory_mb': 7457, + 'meta_os': 'Ubuntu 18.04.5 LTS', + 'meta_provider': '', + 'meta_provider_id': 'google', + 'meta_region': 'us-central1', + 'meta_resource_id': '', + 'meta_run_group_id': '', + 'meta_run_id': '', + 'meta_test_id': '4c137ef3', + 'report_pdf': '', + 'report_zip': '', + 'same_instance_id': 1, + 'same_os': 1, + 'same_provider': 1, + 'same_region': 1, + 'same_service': 1, + 'test_started': '2021-02-25 4:20:12', + 'test_stopped': '2021-02-25 4:22:01', + 'transfer': 11605.5, + }, + ] self.assertListEqual(expected_gartner_metadata, results) diff --git a/tests/linux_benchmarks/cloudharmony_network_benchmark_test.py b/tests/linux_benchmarks/cloudharmony_network_benchmark_test.py index 3bbd919279..69310ff0f9 100644 --- a/tests/linux_benchmarks/cloudharmony_network_benchmark_test.py +++ b/tests/linux_benchmarks/cloudharmony_network_benchmark_test.py @@ -1,4 +1,5 @@ """Tests for cloudharmony_network_benchmark.""" + import os import unittest @@ -15,234 +16,246 @@ class CloudharmonyNetworkBenchmarkTestCase( - pkb_common_test_case.PkbCommonTestCase, test_util.SamplesTestMixin): + pkb_common_test_case.PkbCommonTestCase, test_util.SamplesTestMixin +): def setUp(self): super(CloudharmonyNetworkBenchmarkTestCase, self).setUp() - self.path = os.path.join(os.path.dirname(__file__), - '../data/cloudharmony_network.csv') + self.path = os.path.join( + os.path.dirname(__file__), '../data/cloudharmony_network.csv' + ) with open(self.path, 'r') as fp: self.network_csv_text = fp.read() - self.cloud_harmony_metadata = cloud_harmony_util.ParseCsvResultsFromString( - self.network_csv_text) + self.cloud_harmony_metadata = ( + cloud_harmony_util.ParseCsvResultsFromString(self.network_csv_text) + ) def testParseOutput(self): results = cloudharmony_network_benchmark.ParseOutput( - self.cloud_harmony_metadata) + self.cloud_harmony_metadata + ) self.assertLen(results, 2) - expected_gartner_metadata = [{ - 'benchmark_version': 1.0, - 'collectd_rrd': '', - 'dns_recursive': '', - 'dns_servers': '', - 'iteration': 1, - 'meta_compute_service': 'Google Compute Engine', - 'meta_compute_service_id': 'google:compute', - 'meta_cpu': 'Intel Xeon CPU', - 'meta_cpu_cores': 4, - 'meta_geo_region': '', - 'meta_hostname': 'pkb-1348d630-0', - 'meta_instance_id': 'n2-standard-4', - 'meta_location': '', - 'meta_location_country': '', - 'meta_location_state': '', - 'meta_memory': '16 GB', - 'meta_memory_gb': 16, - 'meta_memory_mb': 16012, - 'meta_os_info': 'Ubuntu 18.04.3 LTS', - 'meta_provider': 'Google Cloud Platform', - 'meta_provider_id': 'google', - 'meta_region': 'europe-west4', - 'meta_resource_id': '', - 'meta_run_id': '', - 'meta_test_id': '', - 'metric': 0.164, - 'metric_10': 0.187, - 'metric_25': 0.174, - 'metric_75': 0.159, - 'metric_90': 0.154, - 'metric_fastest': 0.145, - 'metric_max': 1.05, - 'metric_mean': 0.1767, - 'metric_min': 0.145, - 'metric_rstdev': 54.561, - 'metric_slowest': 1.05, - 'metric_stdev': 0.0895, - 'metric_sum': 17.674, - 'metric_sum_squares': 3.9164, - 'metric_timed': '', - 'metric_unit': 'ms', - 'metric_unit_long': 'milliseconds', - 'metrics': - '1.05|0.178|0.172|0.164|0.164|0.179|0.18|0.184|0.161|0.177|0.169|' - '0.167|0.203|0.153|0.164|0.157|0.18|0.159|0.163|0.155|0.161|0.167|' - '0.157|0.159|0.156|0.174|0.163|0.166|0.187|0.18|0.174|0.174|0.191|' - '0.165|0.154|0.156|0.159|0.161|0.154|0.164|0.168|0.16|0.167|0.16|' - '0.169|0.172|0.195|0.185|0.176|0.176|0.171|0.162|0.171|0.151|0.154|' - '0.223|0.161|0.231|0.213|0.165|0.169|0.212|0.154|0.172|0.159|0.156|' - '0.158|0.164|0.164|0.172|0.156|0.174|0.168|0.169|0.149|0.152|0.16|' - '0.159|0.151|0.178|0.159|0.158|0.158|0.163|0.165|0.16|0.187|0.182|' - '0.15|0.145|0.177|0.153|0.156|0.16|0.159|0.162|0.161|0.164|0.168|' - '0.16', - 'samples': 100, - 'status': 'success', - 'tcp_file': '', - 'test': 'latency', - 'test_endpoint': '10.240.0.86', - 'test_geo_region': '', - 'test_instance_id': 'n2-standard-4', - 'test_ip': '10.240.0.86', - 'test_location': '', - 'test_location_country': '', - 'test_location_state': '', - 'test_private_endpoint': '', - 'test_private_network_type': '', - 'test_provider': 'Google Cloud Platform', - 'test_provider_id': 'google', - 'test_region': 'europe-west4', - 'test_service': 'Google Compute Engine', - 'test_service_id': 'google:compute', - 'test_service_type': 'compute', - 'test_started': '2020-09-28 23:29:09', - 'test_stopped': '2020-09-28 23:29:29', - 'tests_failed': 0, - 'tests_success': 100, - 'throughput_custom_cmd': '', - 'throughput_https': '', - 'throughput_keepalive': '', - 'throughput_size': '', - 'throughput_small_file': '', - 'throughput_threads': '', - 'throughput_time': '', - 'throughput_transfer': '', - 'timeout': 3, - 'traceroute': '', - 'server_type': 'nginx', - }, { - 'benchmark_version': 1.0, - 'collectd_rrd': '', - 'dns_recursive': '', - 'dns_servers': '', - 'iteration': 1, - 'meta_compute_service': 'Google Compute Engine', - 'meta_compute_service_id': 'google:compute', - 'meta_cpu': 'Intel Xeon CPU', - 'meta_cpu_cores': 4, - 'meta_geo_region': '', - 'meta_hostname': 'pkb-4c137ef3-0', - 'meta_instance_id': 'n2-standard-8', - 'meta_location': '', - 'meta_location_country': '', - 'meta_location_state': '', - 'meta_memory': '16 GB', - 'meta_memory_gb': 16, - 'meta_memory_mb': 16012, - 'meta_os_info': 'Ubuntu 18.04.3 LTS', - 'meta_provider': 'Google Cloud Platform', - 'meta_provider_id': 'google', - 'meta_region': 'europe-west4', - 'meta_resource_id': '', - 'meta_run_id': '', - 'meta_test_id': '', - 'metric': 0.164, - 'metric_10': 0.187, - 'metric_25': 0.174, - 'metric_75': 0.159, - 'metric_90': 0.154, - 'metric_fastest': 0.145, - 'metric_max': 1.05, - 'metric_mean': 0.1767, - 'metric_min': 0.145, - 'metric_rstdev': 54.561, - 'metric_slowest': 1.05, - 'metric_stdev': 0.0895, - 'metric_sum': 17.674, - 'metric_sum_squares': 3.9164, - 'metric_timed': '', - 'metric_unit': 'ms', - 'metric_unit_long': 'milliseconds', - 'metrics': - '1.05|0.178|0.172|0.164|0.164|0.179|0.18|0.184|0.161|0.177|0.169|' - '0.167|0.203|0.153|0.164|0.157|0.18|0.159|0.163|0.155|0.161|0.167|' - '0.157|0.159|0.156|0.174|0.163|0.166|0.187|0.18|0.174|0.174|0.191|' - '0.165|0.154|0.156|0.159|0.161|0.154|0.164|0.168|0.16|0.167|0.16|' - '0.169|0.172|0.195|0.185|0.176|0.176|0.171|0.162|0.171|0.151|0.154|' - '0.223|0.161|0.231|0.213|0.165|0.169|0.212|0.154|0.172|0.159|0.156|' - '0.158|0.164|0.164|0.172|0.156|0.174|0.168|0.169|0.149|0.152|0.16|' - '0.159|0.151|0.178|0.159|0.158|0.158|0.163|0.165|0.16|0.187|0.182|' - '0.15|0.145|0.177|0.153|0.156|0.16|0.159|0.162|0.161|0.164|0.168|' - '0.16', - 'samples': 100, - 'status': 'success', - 'tcp_file': '', - 'test': 'latency', - 'test_endpoint': '10.240.0.87', - 'test_geo_region': '', - 'test_instance_id': 'n2-standard-8', - 'test_ip': '10.240.0.87', - 'test_location': '', - 'test_location_country': '', - 'test_location_state': '', - 'test_private_endpoint': '', - 'test_private_network_type': '', - 'test_provider': 'Google Cloud Platform', - 'test_provider_id': 'google', - 'test_region': 'europe-west4', - 'test_service': 'Google Compute Engine', - 'test_service_id': 'google:compute', - 'test_service_type': 'compute', - 'test_started': '2021-02-25 4:20:12', - 'test_stopped': '2021-02-25 4:22:01', - 'tests_failed': 0, - 'tests_success': 100, - 'throughput_custom_cmd': '', - 'throughput_https': '', - 'throughput_keepalive': '', - 'throughput_size': '', - 'throughput_small_file': '', - 'throughput_threads': '', - 'throughput_time': '', - 'throughput_transfer': '', - 'timeout': 3, - 'traceroute': '', - 'server_type': 'nginx', - }] + expected_gartner_metadata = [ + { + 'benchmark_version': 1.0, + 'collectd_rrd': '', + 'dns_recursive': '', + 'dns_servers': '', + 'iteration': 1, + 'meta_compute_service': 'Google Compute Engine', + 'meta_compute_service_id': 'google:compute', + 'meta_cpu': 'Intel Xeon CPU', + 'meta_cpu_cores': 4, + 'meta_geo_region': '', + 'meta_hostname': 'pkb-1348d630-0', + 'meta_instance_id': 'n2-standard-4', + 'meta_location': '', + 'meta_location_country': '', + 'meta_location_state': '', + 'meta_memory': '16 GB', + 'meta_memory_gb': 16, + 'meta_memory_mb': 16012, + 'meta_os_info': 'Ubuntu 18.04.3 LTS', + 'meta_provider': 'Google Cloud Platform', + 'meta_provider_id': 'google', + 'meta_region': 'europe-west4', + 'meta_resource_id': '', + 'meta_run_id': '', + 'meta_test_id': '', + 'metric': 0.164, + 'metric_10': 0.187, + 'metric_25': 0.174, + 'metric_75': 0.159, + 'metric_90': 0.154, + 'metric_fastest': 0.145, + 'metric_max': 1.05, + 'metric_mean': 0.1767, + 'metric_min': 0.145, + 'metric_rstdev': 54.561, + 'metric_slowest': 1.05, + 'metric_stdev': 0.0895, + 'metric_sum': 17.674, + 'metric_sum_squares': 3.9164, + 'metric_timed': '', + 'metric_unit': 'ms', + 'metric_unit_long': 'milliseconds', + 'metrics': ( + '1.05|0.178|0.172|0.164|0.164|0.179|0.18|0.184|0.161|0.177|0.169|' + '0.167|0.203|0.153|0.164|0.157|0.18|0.159|0.163|0.155|0.161|0.167|' + '0.157|0.159|0.156|0.174|0.163|0.166|0.187|0.18|0.174|0.174|0.191|' + '0.165|0.154|0.156|0.159|0.161|0.154|0.164|0.168|0.16|0.167|0.16|' + '0.169|0.172|0.195|0.185|0.176|0.176|0.171|0.162|0.171|0.151|0.154|' + '0.223|0.161|0.231|0.213|0.165|0.169|0.212|0.154|0.172|0.159|0.156|' + '0.158|0.164|0.164|0.172|0.156|0.174|0.168|0.169|0.149|0.152|0.16|' + '0.159|0.151|0.178|0.159|0.158|0.158|0.163|0.165|0.16|0.187|0.182|' + '0.15|0.145|0.177|0.153|0.156|0.16|0.159|0.162|0.161|0.164|0.168|' + '0.16' + ), + 'samples': 100, + 'status': 'success', + 'tcp_file': '', + 'test': 'latency', + 'test_endpoint': '10.240.0.86', + 'test_geo_region': '', + 'test_instance_id': 'n2-standard-4', + 'test_ip': '10.240.0.86', + 'test_location': '', + 'test_location_country': '', + 'test_location_state': '', + 'test_private_endpoint': '', + 'test_private_network_type': '', + 'test_provider': 'Google Cloud Platform', + 'test_provider_id': 'google', + 'test_region': 'europe-west4', + 'test_service': 'Google Compute Engine', + 'test_service_id': 'google:compute', + 'test_service_type': 'compute', + 'test_started': '2020-09-28 23:29:09', + 'test_stopped': '2020-09-28 23:29:29', + 'tests_failed': 0, + 'tests_success': 100, + 'throughput_custom_cmd': '', + 'throughput_https': '', + 'throughput_keepalive': '', + 'throughput_size': '', + 'throughput_small_file': '', + 'throughput_threads': '', + 'throughput_time': '', + 'throughput_transfer': '', + 'timeout': 3, + 'traceroute': '', + 'server_type': 'nginx', + }, + { + 'benchmark_version': 1.0, + 'collectd_rrd': '', + 'dns_recursive': '', + 'dns_servers': '', + 'iteration': 1, + 'meta_compute_service': 'Google Compute Engine', + 'meta_compute_service_id': 'google:compute', + 'meta_cpu': 'Intel Xeon CPU', + 'meta_cpu_cores': 4, + 'meta_geo_region': '', + 'meta_hostname': 'pkb-4c137ef3-0', + 'meta_instance_id': 'n2-standard-8', + 'meta_location': '', + 'meta_location_country': '', + 'meta_location_state': '', + 'meta_memory': '16 GB', + 'meta_memory_gb': 16, + 'meta_memory_mb': 16012, + 'meta_os_info': 'Ubuntu 18.04.3 LTS', + 'meta_provider': 'Google Cloud Platform', + 'meta_provider_id': 'google', + 'meta_region': 'europe-west4', + 'meta_resource_id': '', + 'meta_run_id': '', + 'meta_test_id': '', + 'metric': 0.164, + 'metric_10': 0.187, + 'metric_25': 0.174, + 'metric_75': 0.159, + 'metric_90': 0.154, + 'metric_fastest': 0.145, + 'metric_max': 1.05, + 'metric_mean': 0.1767, + 'metric_min': 0.145, + 'metric_rstdev': 54.561, + 'metric_slowest': 1.05, + 'metric_stdev': 0.0895, + 'metric_sum': 17.674, + 'metric_sum_squares': 3.9164, + 'metric_timed': '', + 'metric_unit': 'ms', + 'metric_unit_long': 'milliseconds', + 'metrics': ( + '1.05|0.178|0.172|0.164|0.164|0.179|0.18|0.184|0.161|0.177|0.169|' + '0.167|0.203|0.153|0.164|0.157|0.18|0.159|0.163|0.155|0.161|0.167|' + '0.157|0.159|0.156|0.174|0.163|0.166|0.187|0.18|0.174|0.174|0.191|' + '0.165|0.154|0.156|0.159|0.161|0.154|0.164|0.168|0.16|0.167|0.16|' + '0.169|0.172|0.195|0.185|0.176|0.176|0.171|0.162|0.171|0.151|0.154|' + '0.223|0.161|0.231|0.213|0.165|0.169|0.212|0.154|0.172|0.159|0.156|' + '0.158|0.164|0.164|0.172|0.156|0.174|0.168|0.169|0.149|0.152|0.16|' + '0.159|0.151|0.178|0.159|0.158|0.158|0.163|0.165|0.16|0.187|0.182|' + '0.15|0.145|0.177|0.153|0.156|0.16|0.159|0.162|0.161|0.164|0.168|' + '0.16' + ), + 'samples': 100, + 'status': 'success', + 'tcp_file': '', + 'test': 'latency', + 'test_endpoint': '10.240.0.87', + 'test_geo_region': '', + 'test_instance_id': 'n2-standard-8', + 'test_ip': '10.240.0.87', + 'test_location': '', + 'test_location_country': '', + 'test_location_state': '', + 'test_private_endpoint': '', + 'test_private_network_type': '', + 'test_provider': 'Google Cloud Platform', + 'test_provider_id': 'google', + 'test_region': 'europe-west4', + 'test_service': 'Google Compute Engine', + 'test_service_id': 'google:compute', + 'test_service_type': 'compute', + 'test_started': '2021-02-25 4:20:12', + 'test_stopped': '2021-02-25 4:22:01', + 'tests_failed': 0, + 'tests_success': 100, + 'throughput_custom_cmd': '', + 'throughput_https': '', + 'throughput_keepalive': '', + 'throughput_size': '', + 'throughput_small_file': '', + 'throughput_threads': '', + 'throughput_time': '', + 'throughput_transfer': '', + 'timeout': 3, + 'traceroute': '', + 'server_type': 'nginx', + }, + ] expected_gartner_samples = [] for result in expected_gartner_metadata: expected_gartner_samples.append( sample.Sample( - metric='cloudharmony_output', value='', unit='', - metadata=result)) + metric='cloudharmony_output', value='', unit='', metadata=result + ) + ) self.assertEqual(len(expected_gartner_samples), len(results)) for index in range(len(results)): - self.assertSamplesEqualUpToTimestamp(expected_gartner_samples[index], - results[index]) + self.assertSamplesEqualUpToTimestamp( + expected_gartner_samples[index], results[index] + ) def testRun(self): FLAGS.run_uri = '12345678' FLAGS.zone = ['us-east1-a'] remote_command = mock.PropertyMock(return_value=(self.network_csv_text, '')) - client = mock.Mock(RemoteCommand=remote_command, - machine_type='n2-standard-2', - zone='us-east1-a', - os_info='Ubuntu 18.04.5 LTS') - server1 = mock.Mock(internal_ip='10.0.0.1', machine_type='n2-standard-4', - zone='us-west1-a') - server2 = mock.Mock(internal_ip='10.0.0.2', machine_type='n2-standard-4', - zone='us-west1-a') + client = mock.Mock( + RemoteCommand=remote_command, + machine_type='n2-standard-2', + zone='us-east1-a', + os_info='Ubuntu 18.04.5 LTS', + ) + server1 = mock.Mock( + internal_ip='10.0.0.1', machine_type='n2-standard-4', zone='us-west1-a' + ) + server2 = mock.Mock( + internal_ip='10.0.0.2', machine_type='n2-standard-4', zone='us-west1-a' + ) benchmark_module = mock.Mock(BENCHMARK_NAME='cloud_harmony_network') benchmark_config = mock.Mock( - vm_groups={}, - relational_db=mock.Mock(vm_groups={}) + vm_groups={}, relational_db=mock.Mock(vm_groups={}) ) - spec = benchmark_spec.BenchmarkSpec(benchmark_module, benchmark_config, - 'abcdefg') - spec.vm_groups = { - 'client': [client], - 'server': [server1, server2] - } + spec = benchmark_spec.BenchmarkSpec( + benchmark_module, benchmark_config, 'abcdefg' + ) + spec.vm_groups = {'client': [client], 'server': [server1, server2]} cloudharmony_network_benchmark.Run(spec) client.RobustRemoteCommand.assert_called_with( 'sudo /opt/pkb/network/run.sh --test_endpoint=10.0.0.1 ' @@ -254,7 +267,8 @@ def testRun(self): '--test_service_type compute --test latency --throughput_size 5.0 ' '--throughput_threads 2 --throughput_samples 5 --tcp_samples 10 ' '--test_instance_id n2-standard-4 --test_region us-west1 ' - '--output=/tmp/pkb --verbose') + '--output=/tmp/pkb --verbose' + ) if __name__ == '__main__': diff --git a/tests/linux_benchmarks/cluster_boot_benchmark_test.py b/tests/linux_benchmarks/cluster_boot_benchmark_test.py index cdb5de5070..31e6526be8 100644 --- a/tests/linux_benchmarks/cluster_boot_benchmark_test.py +++ b/tests/linux_benchmarks/cluster_boot_benchmark_test.py @@ -27,13 +27,15 @@ def vm_mock(index: int, timestamp: float) -> mock.Mock: return mock.Mock( delete_start_time=timestamp, delete_end_time=timestamp + index + 5, - OS_TYPE=f'linux{index}') + OS_TYPE=f'linux{index}', + ) def vm_mock_given_delete_times( index: int, delete_start_time: Optional[float] = None, - delete_end_time: Optional[float] = None) -> mock.Mock: + delete_end_time: Optional[float] = None, +) -> mock.Mock: """Creates a mock vm with a provided delete_start_time and delete_end_time. Args: @@ -49,11 +51,13 @@ def vm_mock_given_delete_times( return mock.Mock( delete_start_time=delete_start_time, delete_end_time=delete_end_time, - OS_TYPE=f'linux{index}') + OS_TYPE=f'linux{index}', + ) -class ClusterBootBenchmarkTest(pkb_common_test_case.PkbCommonTestCase, - test_util.SamplesTestMixin): +class ClusterBootBenchmarkTest( + pkb_common_test_case.PkbCommonTestCase, test_util.SamplesTestMixin +): def testMeasureDelete(self): """Unit test for Measuredelete function.""" @@ -77,15 +81,26 @@ def testMeasureDelete(self): expected_cluster_delete_time = 7 for i in range(num_vms): expected_samples.append( - sample.Sample('Delete Time', expected_delete_times[i], 'seconds', { - 'machine_instance': i, - 'num_vms': num_vms, - 'os_type': f'linux{i}', - })) + sample.Sample( + 'Delete Time', + expected_delete_times[i], + 'seconds', + { + 'machine_instance': i, + 'num_vms': num_vms, + 'os_type': f'linux{i}', + }, + ) + ) expected_samples.append( - sample.Sample('Cluster Delete Time', expected_cluster_delete_time, - 'seconds', expected_cluster_delete_metadata)) + sample.Sample( + 'Cluster Delete Time', + expected_cluster_delete_time, + 'seconds', + expected_cluster_delete_metadata, + ) + ) # assert actual and expected samples are equal self.assertSampleListsEqualUpToTimestamp(actual_samples, expected_samples) @@ -95,12 +110,14 @@ def testMeasureDeleteNoValidVMs(self): timestamp = 1625863325.003580 vm_with_neither = vm_mock_given_delete_times(index=0) vm_with_start_only = vm_mock_given_delete_times( - index=1, delete_start_time=timestamp) + index=1, delete_start_time=timestamp + ) # We never expect to see this in production, but it does ensure that # the conditional measurement of delete times works for all combinations # of missing attributes. vm_with_end_only = vm_mock_given_delete_times( - index=2, delete_end_time=timestamp + 5) + index=2, delete_end_time=timestamp + 5 + ) vms_to_test = [vm_with_neither, vm_with_start_only, vm_with_end_only] # invoke MeasureDelete @@ -113,11 +130,12 @@ def testMeasureDeleteAttributeChecks(self): """MeasureDelete test where some VMs do not have valid delete measurements.""" timestamp = 1625863325.003580 vm_with_neither = vm_mock_given_delete_times(index=0) - vm_with_start_only = vm_mock_given_delete_times(index=1, - delete_start_time=timestamp) - vm_with_both = vm_mock_given_delete_times(index=2, - delete_start_time=timestamp, - delete_end_time=timestamp + 5) + vm_with_start_only = vm_mock_given_delete_times( + index=1, delete_start_time=timestamp + ) + vm_with_both = vm_mock_given_delete_times( + index=2, delete_start_time=timestamp, delete_end_time=timestamp + 5 + ) vms_to_test = [vm_with_neither, vm_with_start_only, vm_with_both] # invoke MeasureDelete actual_samples = cluster_boot_benchmark.MeasureDelete(vms_to_test) @@ -130,11 +148,14 @@ def testMeasureDeleteAttributeChecks(self): expected_num_vms = 1 expected_samples = [ sample.Sample( - 'Delete Time', expected_delete_time, 'seconds', { + 'Delete Time', + expected_delete_time, + 'seconds', + { 'machine_instance': 0, 'num_vms': expected_num_vms, 'os_type': f'linux{expected_instance_index}', - } + }, ) ] @@ -144,8 +165,13 @@ def testMeasureDeleteAttributeChecks(self): 'os_type': f'linux{expected_instance_index}', } expected_samples.append( - sample.Sample('Cluster Delete Time', expected_cluster_delete_time, - 'seconds', expected_cluster_delete_metadata)) + sample.Sample( + 'Cluster Delete Time', + expected_cluster_delete_time, + 'seconds', + expected_cluster_delete_metadata, + ) + ) # assert actual and expected samples are equal self.assertSampleListsEqualUpToTimestamp(actual_samples, expected_samples) diff --git a/tests/linux_benchmarks/coremark_benchmark_test.py b/tests/linux_benchmarks/coremark_benchmark_test.py index d35e12bfe2..60a196bc74 100644 --- a/tests/linux_benchmarks/coremark_benchmark_test.py +++ b/tests/linux_benchmarks/coremark_benchmark_test.py @@ -24,7 +24,8 @@ _SAMPLE_OUTPUT_FILE = os.path.join( - os.path.dirname(__file__), '../data/coremark_sample_output.txt') + os.path.dirname(__file__), '../data/coremark_sample_output.txt' +) class CoremarkBenchmarkTest(pkb_common_test_case.PkbCommonTestCase): @@ -36,16 +37,20 @@ def testParseOutput(self): samples = coremark_benchmark._ParseOutputForSamples(output, 2) self.assertEqual('Coremark Score', samples[0].metric) self.assertAlmostEqual(29504.182218, samples[0].value) - self.assertEqual('CoreMark 1.0 : 29504.182218 / GCC7.3.0 -O2 -g -O2 ' - '-DMULTITHREAD=2 -DUSE_PTHREAD -DPERFORMANCE_RUN=1 ' - '-DPERFORMANCE_RUN=1 -lrt -lpthread / Heap / 2:PThreads', - samples[0].metadata['summary']) + self.assertEqual( + 'CoreMark 1.0 : 29504.182218 / GCC7.3.0 -O2 -g -O2 ' + '-DMULTITHREAD=2 -DUSE_PTHREAD -DPERFORMANCE_RUN=1 ' + '-DPERFORMANCE_RUN=1 -lrt -lpthread / Heap / 2:PThreads', + samples[0].metadata['summary'], + ) self.assertEqual(666, samples[0].metadata['size']) self.assertEqual(67787, samples[0].metadata['total_ticks']) self.assertAlmostEqual(67.787000, samples[0].metadata['total_time_sec']) self.assertEqual(2000000, samples[0].metadata['iterations']) - self.assertEqual(coremark_benchmark.ITERATIONS_PER_CPU, - samples[0].metadata['iterations_per_cpu']) + self.assertEqual( + coremark_benchmark.ITERATIONS_PER_CPU, + samples[0].metadata['iterations_per_cpu'], + ) self.assertEqual('PTHREAD', samples[0].metadata['parallelism_method']) self.assertEqual(2, samples[0].metadata['thread_count']) diff --git a/tests/linux_benchmarks/cuda_memcpy_benchmark_test.py b/tests/linux_benchmarks/cuda_memcpy_benchmark_test.py index f8247d18b3..72c97d147e 100644 --- a/tests/linux_benchmarks/cuda_memcpy_benchmark_test.py +++ b/tests/linux_benchmarks/cuda_memcpy_benchmark_test.py @@ -17,7 +17,6 @@ import unittest from absl import flags import mock - from perfkitbenchmarker import sample from perfkitbenchmarker import test_util from perfkitbenchmarker.linux_benchmarks import cuda_memcpy_benchmark @@ -28,19 +27,23 @@ FLAGS = flags.FLAGS -class CudaMemcpyBenchmarkTest(pkb_common_test_case.PkbCommonTestCase, - test_util.SamplesTestMixin): +class CudaMemcpyBenchmarkTest( + pkb_common_test_case.PkbCommonTestCase, test_util.SamplesTestMixin +): def setUp(self) -> None: super(CudaMemcpyBenchmarkTest, self).setUp() - self.enter_context(mock.patch.object( - nvidia_driver, 'QueryNumberOfGpus', return_value=1)) - self.enter_context(mock.patch.object( - cuda_toolkit, 'GetMetadata', return_value={})) + self.enter_context( + mock.patch.object(nvidia_driver, 'QueryNumberOfGpus', return_value=1) + ) + self.enter_context( + mock.patch.object(cuda_toolkit, 'GetMetadata', return_value={}) + ) def CudaOutput(self) -> str: - path = os.path.join(os.path.dirname(__file__), '..', 'data', - 'cuda_memcpy_output.txt') + path = os.path.join( + os.path.dirname(__file__), '..', 'data', 'cuda_memcpy_output.txt' + ) with open(path) as reader: return reader.read() @@ -55,28 +58,33 @@ def testCmd(self, check_nvidia_smi_exists: mock.Mock) -> None: cuda_memcpy_benchmark.Run(mock.Mock(vms=[vm])) vm.RemoteCommandWithReturnCode.assert_called_with( '/usr/local/cuda/extras/demo_suite/bandwidthTest --csv --memory=pinned ' - '--mode=quick --htod --dtoh --dtod --device=0', ignore_failure=True) + '--mode=quick --htod --dtoh --dtod --device=0', + ignore_failure=True, + ) @mock.patch.object(nvidia_driver, 'CheckNvidiaSmiExists', return_value=True) def testSample(self, check_nvidia_smi_exists: mock.Mock) -> None: samples = cuda_memcpy_benchmark.Run(mock.Mock(vms=[self.MockVm()])) expected = sample.Sample( - 'H2D-Pinned', 8494.3, 'MB/s', + 'H2D-Pinned', + 8494.3, + 'MB/s', { 'time': 0.00377, 'size': 33554432, 'NumDevsUsed': '1', 'device': 0, - 'command': + 'command': ( '/usr/local/cuda/extras/demo_suite/bandwidthTest --csv ' - '--memory=pinned --mode=quick --htod --dtoh --dtod --device=0', + '--memory=pinned --mode=quick --htod --dtoh --dtod --device=0' + ), 'memory': 'pinned', 'mode': 'quick', 'htod': True, 'dtoh': True, 'dtod': True, 'wc': False, - } + }, ) self.assertSamplesEqualUpToTimestamp(expected, samples[0]) diff --git a/tests/linux_benchmarks/dino_benchmark_test.py b/tests/linux_benchmarks/dino_benchmark_test.py index 72dc03a8d9..38f2a3abf1 100644 --- a/tests/linux_benchmarks/dino_benchmark_test.py +++ b/tests/linux_benchmarks/dino_benchmark_test.py @@ -1,4 +1,5 @@ """Tests for dino_benchmark.""" + import os import unittest diff --git a/tests/linux_benchmarks/dpb_sparksql_benchmark_test.py b/tests/linux_benchmarks/dpb_sparksql_benchmark_test.py index 7fb2b0639e..185af4b719 100644 --- a/tests/linux_benchmarks/dpb_sparksql_benchmark_test.py +++ b/tests/linux_benchmarks/dpb_sparksql_benchmark_test.py @@ -5,7 +5,6 @@ from absl.testing import flagsaver import freezegun - from perfkitbenchmarker import dpb_sparksql_benchmark_helper from perfkitbenchmarker.linux_benchmarks import dpb_sparksql_benchmark from tests import pkb_common_test_case @@ -23,55 +22,83 @@ def setUp(self): def testRunQueriesPower(self): self.benchmark_spec_mock.query_dir = 'gs://test' self.benchmark_spec_mock.query_streams = ( - dpb_sparksql_benchmark_helper.GetStreams()) + dpb_sparksql_benchmark_helper.GetStreams() + ) dpb_sparksql_benchmark._RunQueries(self.benchmark_spec_mock) self.benchmark_spec_mock.dpb_service.SubmitJob.assert_called_once() _, kwargs = self.benchmark_spec_mock.dpb_service.SubmitJob.call_args self.assertEqual( kwargs['job_arguments'], - ['--sql-scripts-dir', 'gs://test', - '--sql-scripts', '1,2,3', - '--report-dir', 'gs://test2/report-1680048000000', - '--enable-hive', 'True']) + [ + '--sql-scripts-dir', + 'gs://test', + '--sql-scripts', + '1,2,3', + '--report-dir', + 'gs://test2/report-1680048000000', + '--enable-hive', + 'True', + ], + ) @freezegun.freeze_time('2023-03-29') @flagsaver.flagsaver( - dpb_sparksql_order=['1', '2', '3'], - dpb_sparksql_simultaneous=True) + dpb_sparksql_order=['1', '2', '3'], dpb_sparksql_simultaneous=True + ) def testRunQueriesThroughput(self): self.benchmark_spec_mock.query_dir = 'gs://test' self.benchmark_spec_mock.query_streams = ( - dpb_sparksql_benchmark_helper.GetStreams()) + dpb_sparksql_benchmark_helper.GetStreams() + ) dpb_sparksql_benchmark._RunQueries(self.benchmark_spec_mock) self.benchmark_spec_mock.dpb_service.SubmitJob.assert_called_once() _, kwargs = self.benchmark_spec_mock.dpb_service.SubmitJob.call_args self.assertEqual( kwargs['job_arguments'], - ['--sql-scripts-dir', 'gs://test', - '--sql-scripts', '1', - '--sql-scripts', '2', - '--sql-scripts', '3', - '--report-dir', 'gs://test2/report-1680048000000', - '--enable-hive', 'True']) + [ + '--sql-scripts-dir', + 'gs://test', + '--sql-scripts', + '1', + '--sql-scripts', + '2', + '--sql-scripts', + '3', + '--report-dir', + 'gs://test2/report-1680048000000', + '--enable-hive', + 'True', + ], + ) @freezegun.freeze_time('2023-03-29') - @flagsaver.flagsaver( - dpb_sparksql_streams=['1,2,3', '2,1,3', '3,1,2']) + @flagsaver.flagsaver(dpb_sparksql_streams=['1,2,3', '2,1,3', '3,1,2']) def testRunQueriesSimultaneous(self): self.benchmark_spec_mock.query_dir = 'gs://test' self.benchmark_spec_mock.query_streams = ( - dpb_sparksql_benchmark_helper.GetStreams()) + dpb_sparksql_benchmark_helper.GetStreams() + ) dpb_sparksql_benchmark._RunQueries(self.benchmark_spec_mock) self.benchmark_spec_mock.dpb_service.SubmitJob.assert_called_once() _, kwargs = self.benchmark_spec_mock.dpb_service.SubmitJob.call_args self.assertEqual( kwargs['job_arguments'], - ['--sql-scripts-dir', 'gs://test', - '--sql-scripts', '1,2,3', - '--sql-scripts', '2,1,3', - '--sql-scripts', '3,1,2', - '--report-dir', 'gs://test2/report-1680048000000', - '--enable-hive', 'True']) + [ + '--sql-scripts-dir', + 'gs://test', + '--sql-scripts', + '1,2,3', + '--sql-scripts', + '2,1,3', + '--sql-scripts', + '3,1,2', + '--report-dir', + 'gs://test2/report-1680048000000', + '--enable-hive', + 'True', + ], + ) + if __name__ == '__main__': unittest.main() diff --git a/tests/linux_benchmarks/dpdk_benchmark_test.py b/tests/linux_benchmarks/dpdk_benchmark_test.py index a6a882ca0f..3fb4d9d55a 100644 --- a/tests/linux_benchmarks/dpdk_benchmark_test.py +++ b/tests/linux_benchmarks/dpdk_benchmark_test.py @@ -18,7 +18,6 @@ from absl import flags from absl.testing import parameterized import mock - from perfkitbenchmarker import benchmark_spec from perfkitbenchmarker import sample from perfkitbenchmarker.linux_benchmarks import dpdk_benchmark diff --git a/tests/linux_benchmarks/fio_benchmark_test.py b/tests/linux_benchmarks/fio_benchmark_test.py index 9b6e1f7e7d..4b3a9c493e 100644 --- a/tests/linux_benchmarks/fio_benchmark_test.py +++ b/tests/linux_benchmarks/fio_benchmark_test.py @@ -17,7 +17,6 @@ import unittest from absl import flags import mock - from perfkitbenchmarker import temp_dir from perfkitbenchmarker import units from perfkitbenchmarker import vm_util @@ -174,11 +173,10 @@ def testAllScenarios(self): self.assertEqual( fio_benchmark.GenerateJobFileString( - self.filename, - ['all'], - [1], [1], - None, None, 600, 10, True, [], [0]), - expected_jobfile) + self.filename, ['all'], [1], [1], None, None, 600, 10, True, [], [0] + ), + expected_jobfile, + ) def testMultipleScenarios(self): expected_jobfile = """ @@ -219,9 +217,18 @@ def testMultipleScenarios(self): fio_benchmark.GenerateJobFileString( self.filename, ['sequential_read', 'sequential_write'], - [1], [1], - None, None, 600, 10, True, ['randrepeat=0'], [0]), - expected_jobfile) + [1], + [1], + None, + None, + 600, + 10, + True, + ['randrepeat=0'], + [0], + ), + expected_jobfile, + ) def testCustomBlocksize(self): orig_blocksize = fio_benchmark.SCENARIOS['sequential_write']['blocksize'] @@ -229,20 +236,39 @@ def testCustomBlocksize(self): job_file = fio_benchmark.GenerateJobFileString( self.filename, ['sequential_read'], - [1], [1], None, units.Unit('megabyte') * 2, 600, 10, True, [], [0]) + [1], + [1], + None, + units.Unit('megabyte') * 2, + 600, + 10, + True, + [], + [0], + ) self.assertIn('blocksize=2000000B', job_file) # Test that generating a job file doesn't modify the global # SCENARIOS variable. - self.assertEqual(fio_benchmark.SCENARIOS['sequential_write']['blocksize'], - orig_blocksize) + self.assertEqual( + fio_benchmark.SCENARIOS['sequential_write']['blocksize'], orig_blocksize + ) def testIndirectIO(self): job_file = fio_benchmark.GenerateJobFileString( self.filename, ['sequential_read'], - [1], [1], None, units.Unit('megabyte') * 2, 600, 10, False, [], [0]) + [1], + [1], + None, + units.Unit('megabyte') * 2, + 600, + 10, + False, + [], + [0], + ) self.assertIn('direct=0', job_file) def testParseGenerateScenario(self): @@ -348,9 +374,18 @@ def testParseGenerateScenario(self): fio_benchmark.GenerateJobFileString( self.filename, ['seq_64M_read_10TB', 'rand_16k_readwrite_5TB_rwmixread-65'], - [1, 2], [1, 3], - None, None, 600, 10, True, ['randrepeat=0'], [0]), - expected_jobfile) + [1, 2], + [1, 3], + None, + None, + 600, + 10, + True, + ['randrepeat=0'], + [0], + ), + expected_jobfile, + ) def testParseGenerateScenarioWithIodepthNumjobs(self): expected_jobfile = """ @@ -391,11 +426,22 @@ def testParseGenerateScenarioWithIodepthNumjobs(self): self.assertEqual( fio_benchmark.GenerateJobFileString( self.filename, - ['seq_64M_read_10TB_iodepth-1_numjobs-1', - 'rand_16k_readwrite_5TB_iodepth-4_numjobs-4_rwmixread-65'], - [1, 2], [1, 3], - None, None, 600, 10, True, ['randrepeat=0'], [0]), - expected_jobfile) + [ + 'seq_64M_read_10TB_iodepth-1_numjobs-1', + 'rand_16k_readwrite_5TB_iodepth-4_numjobs-4_rwmixread-65', + ], + [1, 2], + [1, 3], + None, + None, + 600, + 10, + True, + ['randrepeat=0'], + [0], + ), + expected_jobfile, + ) class TestProcessedJobFileString(pkb_common_test_case.PkbCommonTestCase): @@ -417,28 +463,40 @@ def testReplaceFilenames(self): self.assertNotIn('zanzibar', jobfile) self.assertNotIn('asdf', jobfile) - def doTargetModeTest(self, mode, - expect_fill_device=None, - expect_against_device=None, - expect_format_disk=None): + def doTargetModeTest( + self, + mode, + expect_fill_device=None, + expect_against_device=None, + expect_format_disk=None, + ): fio_name = fio_benchmark.__name__ vm_name = vm_util.__name__ dir_name = temp_dir.__name__ - with mock.patch(fio_name + '.FillDevice') as mock_fill_device, \ - mock.patch(fio_name + '.GetOrGenerateJobFileString') as mock_get_job_string, \ - mock.patch(builtins.__name__ + '.open'), \ - mock.patch(vm_name + '.GetTempDir', return_value='/tmp/dir'), \ - mock.patch(vm_name + '.PrependTempDir', return_value='/tmp/prepend_dir'), \ - mock.patch(dir_name + '.GetRunDirPath', return_value='/tmp/run_dir'), \ - mock.patch(fio_name + '.fio.ParseResults'), \ - mock.patch(fio_name + '.FLAGS') as mock_fio_flags, \ - mock.patch.object(numactl, 'GetNuma', new=lambda vm: {'0': '0'}): + with mock.patch(fio_name + '.FillDevice') as mock_fill_device, mock.patch( + fio_name + '.GetOrGenerateJobFileString' + ) as mock_get_job_string, mock.patch( + builtins.__name__ + '.open' + ), mock.patch( + vm_name + '.GetTempDir', return_value='/tmp/dir' + ), mock.patch( + vm_name + '.PrependTempDir', return_value='/tmp/prepend_dir' + ), mock.patch( + dir_name + '.GetRunDirPath', return_value='/tmp/run_dir' + ), mock.patch( + fio_name + '.fio.ParseResults' + ), mock.patch( + fio_name + '.FLAGS' + ) as mock_fio_flags, mock.patch.object( + numactl, 'GetNuma', new=lambda vm: {'0': '0'} + ): mock_fio_flags.fio_target_mode = mode benchmark_spec = mock.MagicMock() benchmark_spec.vms = [mock.MagicMock()] - benchmark_spec.vms[0].RobustRemoteCommand = ( - mock.MagicMock(return_value=('"stdout"', '"stderr"'))) + benchmark_spec.vms[0].RobustRemoteCommand = mock.MagicMock( + return_value=('"stdout"', '"stderr"') + ) fio_benchmark.Prepare(benchmark_spec) fio_benchmark.Run(benchmark_spec) @@ -460,28 +518,36 @@ def doTargetModeTest(self, mode, self.assertEqual(benchmark_spec.vms[0].FormatDisk.call_count, 0) def testAgainstFileWithFill(self): - self.doTargetModeTest('against_file_with_fill', - expect_fill_device=True, - expect_against_device=False, - expect_format_disk=True) + self.doTargetModeTest( + 'against_file_with_fill', + expect_fill_device=True, + expect_against_device=False, + expect_format_disk=True, + ) def testAgainstFileWithoutFill(self): - self.doTargetModeTest('against_file_without_fill', - expect_fill_device=False, - expect_against_device=False, - expect_format_disk=False) + self.doTargetModeTest( + 'against_file_without_fill', + expect_fill_device=False, + expect_against_device=False, + expect_format_disk=False, + ) def testAgainstDeviceWithFill(self): - self.doTargetModeTest('against_device_with_fill', - expect_fill_device=True, - expect_against_device=True, - expect_format_disk=False) + self.doTargetModeTest( + 'against_device_with_fill', + expect_fill_device=True, + expect_against_device=True, + expect_format_disk=False, + ) def testAgainstDeviceWithoutFill(self): - self.doTargetModeTest('against_device_without_fill', - expect_fill_device=False, - expect_against_device=True, - expect_format_disk=False) + self.doTargetModeTest( + 'against_device_without_fill', + expect_fill_device=False, + expect_against_device=True, + expect_format_disk=False, + ) if __name__ == '__main__': diff --git a/tests/linux_benchmarks/glibc_benchmark_test.py b/tests/linux_benchmarks/glibc_benchmark_test.py index 33b00eaf97..8efacc2cd6 100644 --- a/tests/linux_benchmarks/glibc_benchmark_test.py +++ b/tests/linux_benchmarks/glibc_benchmark_test.py @@ -18,7 +18,6 @@ import unittest import mock - from perfkitbenchmarker.linux_benchmarks import glibc_benchmark from perfkitbenchmarker.linux_packages import glibc @@ -46,16 +45,14 @@ def CallParseOutput(self, filename, upper_key, results, metadata): """Read sample outputs of glibc_benchmark and call ParseOutput function. Args: - filename: The name of the sample output file - required to run the benchmark. + filename: The name of the sample output file required to run the + benchmark. upper_key: The first dimension key of the glibc_output dict. - results: - A list to which the ParseOutput function will append new samples based - on the glibc output. + results: A list to which the ParseOutput function will append new samples + based on the glibc output. metadata: Common metadata to attach to samples. """ - path = os.path.join(os.path.dirname(__file__), '../data', - filename) + path = os.path.join(os.path.dirname(__file__), '../data', filename) with open(path) as fp: self.contents = fp.read() glibc_benchmark.ParseOutput(self.contents, upper_key, results, metadata) @@ -71,13 +68,14 @@ def testParseOutputAttachesCorrectCommonMetadata(self): results = [] upper_key = 'functions' - self.CallParseOutput( - 'glibc_bench_output.txt', upper_key, results, metadata) + self.CallParseOutput('glibc_bench_output.txt', upper_key, results, metadata) for sample in results: result_metadata = sample.metadata self.assertEqual(result_metadata['gcc'], _TEST_GCC_VERSION) - self.assertEqual(result_metadata['glibc_benchset'], - glibc_benchmark.glibc_default_benchset) + self.assertEqual( + result_metadata['glibc_benchset'], + glibc_benchmark.glibc_default_benchset, + ) self.assertEqual(result_metadata['glibc_version'], glibc.GLIBC_VERSION) self.assertEqual(result_metadata['num_machines'], _TEST_NUM_VMS) @@ -85,15 +83,14 @@ def testParseGlibc(self): results = [] upper_key = 'functions' - self.CallParseOutput( - 'glibc_bench_output.txt', upper_key, results, {}) + self.CallParseOutput('glibc_bench_output.txt', upper_key, results, {}) result = {i.metric: i.metadata for i in results} metadata = result['pthread_once:'] self.assertEqual(63, len(results)) - self.assertAlmostEqual(1.72198e+10, metadata['duration']) - self.assertAlmostEqual(3.20756e+09, metadata['iterations']) + self.assertAlmostEqual(1.72198e10, metadata['duration']) + self.assertAlmostEqual(3.20756e09, metadata['iterations']) self.assertAlmostEqual(9626.89, metadata['max']) self.assertAlmostEqual(5.198, metadata['min']) self.assertAlmostEqual(5.3685, metadata['mean']) @@ -102,14 +99,13 @@ def testParseGlibc2(self): results = [] upper_key = 'math-inlines' - self.CallParseOutput( - 'glibc_benchset_output.txt', upper_key, results, {}) + self.CallParseOutput('glibc_benchset_output.txt', upper_key, results, {}) result = {i.metric: i.metadata for i in results} metadata = result['__isnan:inf/nan'] self.assertEqual(42, len(results)) - self.assertAlmostEqual(8.42329e+06, metadata['duration']) + self.assertAlmostEqual(8.42329e06, metadata['duration']) self.assertAlmostEqual(500, metadata['iterations']) self.assertAlmostEqual(16846, metadata['mean']) @@ -117,16 +113,15 @@ def testParseGlibc3(self): results = [] upper_key = 'functions' - self.CallParseOutput( - 'glibc_malloc_output.txt', upper_key, results, {}) + self.CallParseOutput('glibc_malloc_output.txt', upper_key, results, {}) metadata = results[0].metadata metric = results[0].metric self.assertEqual(1, len(results)) self.assertEqual('malloc:', metric) - self.assertAlmostEqual(1.2e+11, metadata['duration']) - self.assertAlmostEqual(2.82979e+09, metadata['iterations']) + self.assertAlmostEqual(1.2e11, metadata['duration']) + self.assertAlmostEqual(2.82979e09, metadata['iterations']) self.assertAlmostEqual(42.406, metadata['time_per_iteration']) self.assertAlmostEqual(1800, metadata['max_rss']) self.assertAlmostEqual(1, metadata['threads']) diff --git a/tests/linux_benchmarks/gpu_pcie_bandwidth_benchmark_test.py b/tests/linux_benchmarks/gpu_pcie_bandwidth_benchmark_test.py index 0e1b129a28..ad0c69c79c 100644 --- a/tests/linux_benchmarks/gpu_pcie_bandwidth_benchmark_test.py +++ b/tests/linux_benchmarks/gpu_pcie_bandwidth_benchmark_test.py @@ -17,7 +17,6 @@ import unittest import mock - from perfkitbenchmarker.linux_benchmarks import gpu_pcie_bandwidth_benchmark @@ -28,77 +27,109 @@ def setUp(self): p.start() self.addCleanup(p.stop) - path = os.path.join(os.path.dirname(__file__), '../data', - 'cuda_bandwidth_test_results.txt') + path = os.path.join( + os.path.dirname(__file__), '../data', 'cuda_bandwidth_test_results.txt' + ) with open(path) as fp: self.test_output = fp.read() - path = os.path.join(os.path.dirname(__file__), '../data', - 'cuda_bandwidth_test_range_results.txt') + path = os.path.join( + os.path.dirname(__file__), + '../data', + 'cuda_bandwidth_test_range_results.txt', + ) with open(path) as fp: self.range_test_output = fp.read() def testParseDeviceMetadata(self): - actual = gpu_pcie_bandwidth_benchmark.\ - _ParseDeviceInfo(self.test_output) + actual = gpu_pcie_bandwidth_benchmark._ParseDeviceInfo(self.test_output) expected = {'0': 'Tesla K80', '1': 'Tesla K80'} self.assertEqual(expected, actual) def testParseCudaBandwidthTestResults(self): - results = gpu_pcie_bandwidth_benchmark.\ - _ParseOutputFromSingleIteration(self.test_output) + results = gpu_pcie_bandwidth_benchmark._ParseOutputFromSingleIteration( + self.test_output + ) self.assertEqual(3, len(results)) self.assertAlmostEqual(9254.7, results['Host to device bandwidth']) self.assertAlmostEqual(9686.1, results['Device to host bandwidth']) self.assertAlmostEqual(155985.8, results['Device to device bandwidth']) def testParseCudaBandwidthTestRangeResults(self): - results = gpu_pcie_bandwidth_benchmark.\ - _ParseOutputFromSingleIteration(self.range_test_output) + results = gpu_pcie_bandwidth_benchmark._ParseOutputFromSingleIteration( + self.range_test_output + ) self.assertEqual(3, len(results)) - self.assertAlmostEqual(8063.3666667, - results['Host to device bandwidth']) - self.assertAlmostEqual(10518.7666667, - results['Device to host bandwidth']) - self.assertAlmostEqual(157524.4333333, - results['Device to device bandwidth']) + self.assertAlmostEqual(8063.3666667, results['Host to device bandwidth']) + self.assertAlmostEqual(10518.7666667, results['Device to host bandwidth']) + self.assertAlmostEqual( + 157524.4333333, results['Device to device bandwidth'] + ) def testCalculateMetrics(self): - raw_results = [{ - 'Host to device bandwidth': 9250, - 'Device to host bandwidth': 9000, - 'Device to device bandwidth': 155000 - }, { - 'Host to device bandwidth': 8000, - 'Device to host bandwidth': 8500, - 'Device to device bandwidth': 152000 - }] - samples = gpu_pcie_bandwidth_benchmark.\ - _CalculateMetricsOverAllIterations(raw_results) + raw_results = [ + { + 'Host to device bandwidth': 9250, + 'Device to host bandwidth': 9000, + 'Device to device bandwidth': 155000, + }, + { + 'Host to device bandwidth': 8000, + 'Device to host bandwidth': 8500, + 'Device to device bandwidth': 152000, + }, + ] + samples = gpu_pcie_bandwidth_benchmark._CalculateMetricsOverAllIterations( + raw_results + ) metrics = {i[0]: i[1] for i in samples} - sample = next(x for x in samples if x.metadata == {'iteration': 0} - and x.metric == 'Host to device bandwidth') + sample = next( + x + for x in samples + if x.metadata == {'iteration': 0} + and x.metric == 'Host to device bandwidth' + ) self.assertAlmostEqual(9250, sample.value) - sample = next(x for x in samples if x.metadata == {'iteration': 0} - and x.metric == 'Device to host bandwidth') + sample = next( + x + for x in samples + if x.metadata == {'iteration': 0} + and x.metric == 'Device to host bandwidth' + ) self.assertAlmostEqual(9000, sample.value) - sample = next(x for x in samples if x.metadata == {'iteration': 0} - and x.metric == 'Device to device bandwidth') + sample = next( + x + for x in samples + if x.metadata == {'iteration': 0} + and x.metric == 'Device to device bandwidth' + ) self.assertAlmostEqual(155000, sample.value) - sample = next(x for x in samples if x.metadata == {'iteration': 1} - and x.metric == 'Host to device bandwidth') + sample = next( + x + for x in samples + if x.metadata == {'iteration': 1} + and x.metric == 'Host to device bandwidth' + ) self.assertAlmostEqual(8000, sample.value) - sample = next(x for x in samples if x.metadata == {'iteration': 1} - and x.metric == 'Device to host bandwidth') + sample = next( + x + for x in samples + if x.metadata == {'iteration': 1} + and x.metric == 'Device to host bandwidth' + ) self.assertAlmostEqual(8500, sample.value) - sample = next(x for x in samples if x.metadata == {'iteration': 1} - and x.metric == 'Device to device bandwidth') + sample = next( + x + for x in samples + if x.metadata == {'iteration': 1} + and x.metric == 'Device to device bandwidth' + ) self.assertAlmostEqual(152000, sample.value) self.assertAlmostEqual(8000, metrics['Host to device bandwidth, min']) diff --git a/tests/linux_benchmarks/gpu_pingpong_benchmark_test.py b/tests/linux_benchmarks/gpu_pingpong_benchmark_test.py index 383c698679..d59a93b6c2 100644 --- a/tests/linux_benchmarks/gpu_pingpong_benchmark_test.py +++ b/tests/linux_benchmarks/gpu_pingpong_benchmark_test.py @@ -20,7 +20,6 @@ import unittest from absl import flags import mock - from perfkitbenchmarker import sample from perfkitbenchmarker import test_util from perfkitbenchmarker.linux_benchmarks import gpu_pingpong_benchmark @@ -34,36 +33,42 @@ def MockVm() -> mock.Mock: """Mocks test VM and returns sample results.""" vm = mock.Mock() vm.hostname = 'localhost' - path = os.path.join(os.path.dirname(__file__), '..', 'data', - 'gpu_pingpong_output.txt') + path = os.path.join( + os.path.dirname(__file__), '..', 'data', 'gpu_pingpong_output.txt' + ) with open(path) as reader: stderr = reader.read() vm.RemoteCommand.return_value = '', stderr return vm -class GpuPingpongBenchmarkTest(pkb_common_test_case.PkbCommonTestCase, - test_util.SamplesTestMixin): +class GpuPingpongBenchmarkTest( + pkb_common_test_case.PkbCommonTestCase, test_util.SamplesTestMixin +): def setUp(self) -> None: super().setUp() - self.enter_context(mock.patch.object( - cuda_toolkit, 'GetMetadata', return_value={})) + self.enter_context( + mock.patch.object(cuda_toolkit, 'GetMetadata', return_value={}) + ) def testCmd(self) -> None: """Tests GPU PingPong test command on the remote VM.""" vm = MockVm() gpu_pingpong_benchmark.Run(mock.Mock(vms=[vm, MockVm()])) vm.RemoteCommand.assert_called_with( - 'PATH=/opt/conda/bin:$PATH python gpu_pingpong_test.py localhost:2000') + 'PATH=/opt/conda/bin:$PATH python gpu_pingpong_test.py localhost:2000' + ) def testSample(self) -> None: """Tests GPU PingPong test parsing code.""" samples = gpu_pingpong_benchmark.Run(mock.Mock(vms=[MockVm(), MockVm()])) - expected = sample.Sample(metric='latency', value=618.9995654384239, - unit='microseconds', - metadata={'ping': 305.00009531252977, - 'pong': 313.99947012589416}) + expected = sample.Sample( + metric='latency', + value=618.9995654384239, + unit='microseconds', + metadata={'ping': 305.00009531252977, 'pong': 313.99947012589416}, + ) self.assertSamplesEqualUpToTimestamp(expected, samples[0]) self.assertLen(samples, 19) diff --git a/tests/linux_benchmarks/horovod_benchmark_test.py b/tests/linux_benchmarks/horovod_benchmark_test.py index b7584c55a2..b04d0365c6 100644 --- a/tests/linux_benchmarks/horovod_benchmark_test.py +++ b/tests/linux_benchmarks/horovod_benchmark_test.py @@ -36,12 +36,14 @@ def setUp(self): def testExtractResNetThroughput(self): throughput, _ = horovod_benchmark._ExtractResNetThroughput( - self.test_output['horovod_output_resnet.txt']) + self.test_output['horovod_output_resnet.txt'] + ) self.assertEqual(6638.0, throughput) def testExtractBertThroughput(self): throughput, _ = horovod_benchmark._ExtractBertThroughput( - self.test_output['horovod_output_bert.txt']) + self.test_output['horovod_output_bert.txt'] + ) self.assertEqual(52.3, throughput) diff --git a/tests/linux_benchmarks/hpcc_benchmark_test.py b/tests/linux_benchmarks/hpcc_benchmark_test.py index 4fbb2eb88c..2cb945444c 100644 --- a/tests/linux_benchmarks/hpcc_benchmark_test.py +++ b/tests/linux_benchmarks/hpcc_benchmark_test.py @@ -13,16 +13,15 @@ # limitations under the License. """Tests for HPCC benchmark.""" +import dataclasses import inspect import os from typing import Optional import unittest from absl import flags from absl.testing import parameterized -import dataclasses import jinja2 import mock - from perfkitbenchmarker import data from perfkitbenchmarker import errors from perfkitbenchmarker import vm_util @@ -44,8 +43,9 @@ def ReadTestDataFile(file_name: str) -> str: return fp.read() -def DefaultHpccDimensions(problem_size: int, num_rows: int, - num_columns: int) -> hpcc_benchmark.HpccDimensions: +def DefaultHpccDimensions( + problem_size: int, num_rows: int, num_columns: int +) -> hpcc_benchmark.HpccDimensions: return hpcc_benchmark.HpccDimensions( problem_size=problem_size, block_size=hpcc_benchmark.BLOCK_SIZE, @@ -59,16 +59,18 @@ def DefaultHpccDimensions(problem_size: int, num_rows: int, swap=2, l1=0, u=0, - equilibration=1) + equilibration=1, + ) + # the HPL.out Intel HPL output file contents -_INTEL_MPIRUN_FILE_OUT = ''' +_INTEL_MPIRUN_FILE_OUT = """ T/V N NB P Q Time Gflops --------------------------------------------------------- WR11C2R4 81792 192 1 2 621.92 5.86567e+02 -''' +""" -_INTEL_MPIRUN_STDOUT = ''' +_INTEL_MPIRUN_STDOUT = """ I_MPI_ROOT=/opt/intel/compilers_and_libraries_2018.5.274/linux/mpi I_MPI_HYDRA_UUID=9d260000-7c26-e807-8fb4-050000430af0 this should be ignored key=value @@ -77,11 +79,12 @@ def DefaultHpccDimensions(problem_size: int, num_rows: int, pkb-123-0 : Column=000576 Fraction=0.005 Kernel= 0.58 Mflops=1265648.19 pkb-123-0 : Column=001152 Fraction=0.010 Kernel=969908.14 Mflops=1081059.81 pkb-123-0 : Column=001728 Fraction=0.015 Kernel=956391.64 Mflops=1040609.60 -''' +""" ENV_METADATA = ( 'I_MPI_PIN_MAPPING=4:0 0,1 1,2 2,3 3,4 4;' - 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2018.5.274/linux/mpi') + 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2018.5.274/linux/mpi' +) class HPCCTestCase(pkb_common_test_case.PkbCommonTestCase): @@ -171,38 +174,49 @@ def testParseHpccMetadata(self): # Spot check a few benchmark-specific metrics. self.assertContainsSubDict( - results['PTRANS_time'], { + results['PTRANS_time'], + { 'PTRANS_n': 19776.0, 'PTRANS_nb': 192.0, 'PTRANS_npcol': 2.0, 'PTRANS_nprow': 2.0, - 'PTRANS_residual': 0.0 - }) - self.assertContainsSubDict(results['SingleRandomAccess_GUPs'], - {'RandomAccess_N': 268435456.0}) + 'PTRANS_residual': 0.0, + }, + ) + self.assertContainsSubDict( + results['SingleRandomAccess_GUPs'], {'RandomAccess_N': 268435456.0} + ) self.assertContainsSubDict( - results['MPIRandomAccess_LCG_GUPs'], { + results['MPIRandomAccess_LCG_GUPs'], + { 'MPIRandomAccess_LCG_Algorithm': 0.0, 'MPIRandomAccess_LCG_Errors': 0.0, 'MPIRandomAccess_LCG_ErrorsFraction': 0.0, 'MPIRandomAccess_LCG_N': 1073741824.0, - 'MPIRandomAccess_LCG_TimeBound': 60.0 - }) - self.assertContainsSubDict(results['StarRandomAccess_LCG_GUPs'], - {'RandomAccess_LCG_N': 268435456.0}) - self.assertContainsSubDict(results['MPIFFT_time6'], { - 'MPIFFT_N': 134217728.0, - 'MPIFFT_Procs': 4.0, - 'MPIFFT_maxErr': 2.31089e-15 - }) + 'MPIRandomAccess_LCG_TimeBound': 60.0, + }, + ) + self.assertContainsSubDict( + results['StarRandomAccess_LCG_GUPs'], + {'RandomAccess_LCG_N': 268435456.0}, + ) + self.assertContainsSubDict( + results['MPIFFT_time6'], + { + 'MPIFFT_N': 134217728.0, + 'MPIFFT_Procs': 4.0, + 'MPIFFT_maxErr': 2.31089e-15, + }, + ) self.assertContainsSubDict(results['StarFFT_Gflops'], {'FFT_N': 67108864.0}) - self.assertContainsSubDict(results['SingleSTREAM_Copy'], { - 'STREAM_Threads': 1.0, - 'STREAM_VectorSize': 130363392.0 - }) + self.assertContainsSubDict( + results['SingleSTREAM_Copy'], + {'STREAM_Threads': 1.0, 'STREAM_VectorSize': 130363392.0}, + ) self.assertContainsSubDict(results['AvgPingPongLatency_usec'], {}) - self.assertContainsSubDict(results['StarRandomAccess_GUPs'], - {'RandomAccess_N': 268435456.0}) + self.assertContainsSubDict( + results['StarRandomAccess_GUPs'], {'RandomAccess_N': 268435456.0} + ) def testCreateHpccConfig(self): vm = mock.Mock(total_free_memory_kb=526536216) @@ -225,10 +239,12 @@ def testCreateHpccConfig(self): 'equilibration': 1, } vm.RenderTemplate.assert_called_with( - mock.ANY, remote_path='hpccinf.txt', context=context) + mock.ANY, remote_path='hpccinf.txt', context=context + ) # Test that the template_path file name is correct - self.assertEqual('hpccinf.j2', - os.path.basename(vm.RenderTemplate.call_args[0][0])) + self.assertEqual( + 'hpccinf.j2', os.path.basename(vm.RenderTemplate.call_args[0][0]) + ) def testMpiRunErrorsOut(self): vm = mock.Mock() @@ -240,17 +256,24 @@ def testMpiRunErrorsOut(self): @parameterized.named_parameters( ('nomem_set_large', 2, 32, 74, None, 124416, 8, 8), ('mem_set', 2, 32, 74, 36000, 86784, 8, 8), - ('nomem_set', 1, 48, 48, None, 70656, 6, 8)) - def testCreateHpccDimensions(self, num_vms: int, num_vcpus: int, - memory_size_gb: int, - flag_memory_size_mb: Optional[int], - problem_size: int, num_rows: int, - num_columns: int) -> None: + ('nomem_set', 1, 48, 48, None, 70656, 6, 8), + ) + def testCreateHpccDimensions( + self, + num_vms: int, + num_vcpus: int, + memory_size_gb: int, + flag_memory_size_mb: Optional[int], + problem_size: int, + num_rows: int, + num_columns: int, + ) -> None: if flag_memory_size_mb: FLAGS.memory_size_mb = flag_memory_size_mb expected = DefaultHpccDimensions(problem_size, num_rows, num_columns) actual = hpcc_benchmark._CalculateHpccDimensions( - num_vms, num_vcpus, memory_size_gb * 1000 * 1024) + num_vms, num_vcpus, memory_size_gb * 1000 * 1024 + ) self.assertEqual(dataclasses.asdict(expected), dataclasses.asdict(actual)) def testRenderHpcConfig(self): @@ -262,7 +285,8 @@ def testRenderHpcConfig(self): def testParseIntelLinpackStdout(self): tflops, metadata = hpcc_benchmark._ParseIntelLinpackStdout( - _INTEL_MPIRUN_STDOUT) + _INTEL_MPIRUN_STDOUT + ) expected_metadata = { 'fractions': '0.01,0.015', 'kernel_tflops': '0.96990814,0.95639164', @@ -275,22 +299,24 @@ def testParseIntelLinpackStdout(self): def testCreateIntelMpiRunCommand(self): mock_run_file = self.enter_context( - mock.patch.object(vm_util, 'CreateRemoteFile')) + mock.patch.object(vm_util, 'CreateRemoteFile') + ) mpivars = '/opt/intel/compilers_and_libraries/linux/mpi/bin64/mpivars.sh' vm = mock.Mock(internal_ip='10.0.0.2', numa_node_count=2) vm.RemoteCommand.return_value = mpivars, '' num_rows, num_cols = 3, 4 dim = DefaultHpccDimensions(100, num_rows, num_cols) mpi_cmd, num_processes = hpcc_benchmark._CreateIntelMpiRunCommand([vm], dim) - expected_mpi_cmd = (f'. {mpivars}; ' - 'mpirun -perhost 2 -np 12 -host 10.0.0.2 ./hpl_run') + expected_mpi_cmd = ( + f'. {mpivars}; mpirun -perhost 2 -np 12 -host 10.0.0.2 ./hpl_run' + ) self.assertEqual(1 * num_rows * num_cols, num_processes) self.assertEqual(expected_mpi_cmd, mpi_cmd) - run_file_text = inspect.cleandoc(''' + run_file_text = inspect.cleandoc(""" #!/bin/bash export HPL_HOST_NODE=$((PMI_RANK % 2)) /opt/intel/mkl/benchmarks/mp_linpack/xhpl_intel64_static - ''') + """) mock_run_file.assert_called_with(vm, run_file_text + '\n', './hpl_run') def testRunIntelLinpack(self): @@ -311,8 +337,10 @@ def testRunIntelLinpack(self): 'intel_mpi_env': ENV_METADATA, 'kernel_tflops': '0.96990814,0.95639164', 'last_fraction_completed': 0.015, - 'mpi_cmd': f'. {mpi_dir}/bin64/mpivars.sh; ' - 'mpirun -perhost 2 -np 12 -host 10.0.0.2 ./hpl_run', + 'mpi_cmd': ( + f'. {mpi_dir}/bin64/mpivars.sh; ' + 'mpirun -perhost 2 -np 12 -host 10.0.0.2 ./hpl_run' + ), 'num_processes': 12, 'per_host': 2, 'tflops': '1.08105981,1.0406096', diff --git a/tests/linux_benchmarks/hpcg_benchmark_test.py b/tests/linux_benchmarks/hpcg_benchmark_test.py index 3e738b4cb6..92fd73a11b 100644 --- a/tests/linux_benchmarks/hpcg_benchmark_test.py +++ b/tests/linux_benchmarks/hpcg_benchmark_test.py @@ -17,7 +17,6 @@ import unittest import mock - from perfkitbenchmarker.linux_benchmarks import hpcg_benchmark @@ -28,13 +27,15 @@ def setUp(self): p.start() self.addCleanup(p.stop) - path = os.path.join(os.path.dirname(__file__), '../data', - 'hpcg_results.txt') + path = os.path.join( + os.path.dirname(__file__), '../data', 'hpcg_results.txt' + ) with open(path) as fp: self.test_output = fp.read() - path = os.path.join(os.path.dirname(__file__), '../data', - 'hpcg_results2.txt') + path = os.path.join( + os.path.dirname(__file__), '../data', 'hpcg_results2.txt' + ) with open(path) as fp: self.test_output2 = fp.read() @@ -47,8 +48,9 @@ def testExtractThroughput2(self): self.assertEqual(62.3, throughput) def testExtractProblemSize(self): - self.assertEqual([64, 128, 256], - hpcg_benchmark._ExtractProblemSize(self.test_output)) + self.assertEqual( + [64, 128, 256], hpcg_benchmark._ExtractProblemSize(self.test_output) + ) if __name__ == '__main__': diff --git a/tests/linux_benchmarks/huggingface_bert_pretraining_benchmark_test.py b/tests/linux_benchmarks/huggingface_bert_pretraining_benchmark_test.py index db4c9735a7..852ab58867 100644 --- a/tests/linux_benchmarks/huggingface_bert_pretraining_benchmark_test.py +++ b/tests/linux_benchmarks/huggingface_bert_pretraining_benchmark_test.py @@ -20,20 +20,25 @@ from tests import pkb_common_test_case -class HuggingfaceBertPretrainingTestCase(pkb_common_test_case.PkbCommonTestCase, - test_util.SamplesTestMixin): +class HuggingfaceBertPretrainingTestCase( + pkb_common_test_case.PkbCommonTestCase, test_util.SamplesTestMixin +): def setUp(self): super(HuggingfaceBertPretrainingTestCase, self).setUp() path = os.path.join( - os.path.dirname(__file__), '..', 'data', - 'huggingface_bert_pretraining_output.txt') + os.path.dirname(__file__), + '..', + 'data', + 'huggingface_bert_pretraining_output.txt', + ) with open(path) as fp: self.contents = fp.read() def testTrainResults(self): samples = huggingface_bert_pretraining_benchmark.MakeSamplesFromOutput( - self.contents) + self.contents + ) self.assertLen(samples, 24) self.assertEqual(samples[0].metric, 'throughput') self.assertAlmostEqual(samples[0].value, 1.78) diff --git a/tests/linux_benchmarks/inception3_benchmark_test.py b/tests/linux_benchmarks/inception3_benchmark_test.py index d355a55cf3..c3c0cc747a 100644 --- a/tests/linux_benchmarks/inception3_benchmark_test.py +++ b/tests/linux_benchmarks/inception3_benchmark_test.py @@ -15,37 +15,53 @@ import os import unittest import mock - from perfkitbenchmarker import test_util from perfkitbenchmarker.linux_benchmarks import mnist_benchmark from perfkitbenchmarker.sample import Sample -class Inception3BenchmarkTestCase(unittest.TestCase, - test_util.SamplesTestMixin): +class Inception3BenchmarkTestCase( + unittest.TestCase, test_util.SamplesTestMixin +): def setUp(self): - path = os.path.join(os.path.dirname(__file__), '..', 'data', - 'inception3_output.txt') + path = os.path.join( + os.path.dirname(__file__), '..', 'data', 'inception3_output.txt' + ) with open(path) as fp: self.contents = fp.read() - self.metadata_input = {'num_examples_per_epoch': 1251.1, - 'train_batch_size': 1024} - self.metadata_output = {'num_examples_per_epoch': 1251.1, - 'train_batch_size': 1024, 'step': 4000, - 'epoch': 3.197186475901207, 'elapsed_seconds': 0} + self.metadata_input = { + 'num_examples_per_epoch': 1251.1, + 'train_batch_size': 1024, + } + self.metadata_output = { + 'num_examples_per_epoch': 1251.1, + 'train_batch_size': 1024, + 'step': 4000, + 'epoch': 3.197186475901207, + 'elapsed_seconds': 0, + } @mock.patch('time.time', mock.MagicMock(return_value=0)) def testTrainResults(self): samples = mnist_benchmark.MakeSamplesFromTrainOutput( - self.metadata_input, self.contents, 0, 4000) + self.metadata_input, self.contents, 0, 4000 + ) golden = [ Sample('Loss', 5.7193503, '', self.metadata_output), - Sample('Global Steps Per Second', 1.4384171428571428, - 'global_steps/sec', self.metadata_output), - Sample('Examples Per Second', 1472.9414285714283, - 'examples/sec', self.metadata_output) + Sample( + 'Global Steps Per Second', + 1.4384171428571428, + 'global_steps/sec', + self.metadata_output, + ), + Sample( + 'Examples Per Second', + 1472.9414285714283, + 'examples/sec', + self.metadata_output, + ), ] self.assertEqual(samples, golden) diff --git a/tests/linux_benchmarks/ior_benchmark_test.py b/tests/linux_benchmarks/ior_benchmark_test.py index d5f3c1523a..82f1b93daa 100644 --- a/tests/linux_benchmarks/ior_benchmark_test.py +++ b/tests/linux_benchmarks/ior_benchmark_test.py @@ -37,8 +37,9 @@ def MockVm(): runmdtest_sample = sample.Sample('runmdtest', 0, '') -class IorBenchmarkTestCase(pkb_common_test_case.PkbCommonTestCase, - test_util.SamplesTestMixin): +class IorBenchmarkTestCase( + pkb_common_test_case.PkbCommonTestCase, test_util.SamplesTestMixin +): def setUp(self): super().setUp() @@ -47,7 +48,8 @@ def setUp(self): self.spec = mock.Mock(vms=self.vms) self.mock_runior = self.enter_context(mock.patch.object(ior, 'RunIOR')) self.mock_runmdtest = self.enter_context( - mock.patch.object(ior, 'RunMdtest')) + mock.patch.object(ior, 'RunMdtest') + ) self.mock_runior.return_value = [runior_sample] self.mock_runmdtest.return_value = [runmdtest_sample] @@ -65,18 +67,23 @@ def testRunDefaults(self): self.headnode.PushDataFile.assert_called_with( 'default_ior_script', '/ior/default_ior_script', - should_double_copy=False) - self.mock_runior.assert_called_with(self.headnode, 256, - '/ior/default_ior_script') + should_double_copy=False, + ) + self.mock_runior.assert_called_with( + self.headnode, 256, '/ior/default_ior_script' + ) # one RunIOR and 3 MdTest based on mdtest_args expected_samples = [ - runior_sample, runmdtest_sample, runmdtest_sample, runmdtest_sample + runior_sample, + runmdtest_sample, + runmdtest_sample, + runmdtest_sample, ] self.assertSampleListsEqualUpToTimestamp(expected_samples, samples) self.mock_runmdtest.assert_has_calls([ mock.call(self.headnode, 32, '-n 1000 -u -C'), mock.call(self.headnode, 32, '-n 1000 -u -T'), - mock.call(self.headnode, 32, '-n 1000 -u -r') + mock.call(self.headnode, 32, '-n 1000 -u -r'), ]) for vm in self.vms: vm.DropCaches.assert_called() @@ -106,12 +113,14 @@ def testNoIorScript(self): ior_num_procs=4, mdtest_num_procs=24, mdtest_args=['a', 'b'], - data_disk_type=disk.SMB) + data_disk_type=disk.SMB, + ) def testNonDefaultFlags(self): samples = ior_benchmark.Run(self.spec) self.headnode.PushDataFile.assert_called_with( - 'myior.sh', '/ior/myior.sh', should_double_copy=True) + 'myior.sh', '/ior/myior.sh', should_double_copy=True + ) expected_mdtest_args = ['a -C', 'a -T', 'a -r', 'b -C', 'b -T', 'b -r'] expected_calls = [ diff --git a/tests/linux_benchmarks/iperf_benchmark_test.py b/tests/linux_benchmarks/iperf_benchmark_test.py index ed996bcf0a..be05cb6a8a 100644 --- a/tests/linux_benchmarks/iperf_benchmark_test.py +++ b/tests/linux_benchmarks/iperf_benchmark_test.py @@ -16,7 +16,6 @@ import unittest from absl import flags import mock - from perfkitbenchmarker import benchmark_spec from perfkitbenchmarker.linux_benchmarks import iperf_benchmark diff --git a/tests/linux_benchmarks/large_scale_boot_benchmark_test.py b/tests/linux_benchmarks/large_scale_boot_benchmark_test.py index 9e583f2a03..ede15693c1 100644 --- a/tests/linux_benchmarks/large_scale_boot_benchmark_test.py +++ b/tests/linux_benchmarks/large_scale_boot_benchmark_test.py @@ -17,7 +17,6 @@ import unittest from absl import flags import mock - from perfkitbenchmarker import errors from perfkitbenchmarker import sample from perfkitbenchmarker.linux_benchmarks import large_scale_boot_benchmark @@ -30,7 +29,8 @@ class LargeScaleBootBenchmarkTest(pkb_common_test_case.PkbCommonTestCase): @mock.patch.object( - large_scale_boot_benchmark, '_GetExpectedBoots', return_value=4) + large_scale_boot_benchmark, '_GetExpectedBoots', return_value=4 + ) def testWaitForResponsesSuccess(self, mock_func): vm1 = mock.Mock() vm1.RemoteCommand.side_effect = [('', ''), ('2', '')] @@ -38,12 +38,15 @@ def testWaitForResponsesSuccess(self, mock_func): vm2.RemoteCommand.side_effect = [('', ''), ('2', '')] large_scale_boot_benchmark._WaitForResponses([vm1, vm2]) vm1.RemoteCommand.assert_called_with( - 'grep -c Pass /tmp/pkb/results', ignore_failure=True) + 'grep -c Pass /tmp/pkb/results', ignore_failure=True + ) vm2.RemoteCommand.assert_called_with( - 'grep -c Pass /tmp/pkb/results', ignore_failure=True) + 'grep -c Pass /tmp/pkb/results', ignore_failure=True + ) @mock.patch.object( - large_scale_boot_benchmark, '_GetExpectedBoots', return_value=4) + large_scale_boot_benchmark, '_GetExpectedBoots', return_value=4 + ) def testWaitForResponsesDeadServer(self, mock_func): vm1 = mock.Mock() vm1.RemoteCommand.side_effect = [('Error: Failed', ''), ('2', '')] @@ -51,11 +54,11 @@ def testWaitForResponsesDeadServer(self, mock_func): large_scale_boot_benchmark._WaitForResponses([vm1]) @mock.patch.object( - large_scale_boot_benchmark, '_GetExpectedBoots', return_value=5) + large_scale_boot_benchmark, '_GetExpectedBoots', return_value=5 + ) def testWaitForResponsesTwice(self, mock_func): vm1 = mock.Mock() - vm1.RemoteCommand.side_effect = [ - ('', ''), ('2', ''), ('', ''), ('5', '')] + vm1.RemoteCommand.side_effect = [('', ''), ('2', ''), ('', ''), ('5', '')] large_scale_boot_benchmark._WaitForResponses([vm1]) self.assertEqual(vm1.RemoteCommand.call_count, 4) @@ -65,12 +68,16 @@ def testParseResult(self): FLAGS['zone'].value = 'zone' vm1 = mock.Mock() - vm1.RemoteCommand.side_effect = [('6', ''), - ('Pass:a:8\nPass:b:9\nPass:c:13', '')] + vm1.RemoteCommand.side_effect = [ + ('6', ''), + ('Pass:a:8\nPass:b:9\nPass:c:13', ''), + ] vm1.zone = 'zone' vm2 = mock.Mock() - vm2.RemoteCommand.side_effect = [('2', ''), - ('Pass:d:4\nFail:e:5\nPass:f:6', '')] + vm2.RemoteCommand.side_effect = [ + ('2', ''), + ('Pass:d:4\nFail:e:5\nPass:f:6', ''), + ] vm2.zone = 'zone' results = large_scale_boot_benchmark._ParseResult([vm1, vm2]) @@ -90,65 +97,81 @@ def testParseResult(self): 'zone': 'zone', 'launcher_successes': 3, 'launcher_boot_durations_ns': [2, 3, 7], - 'launcher_closed_incoming': 0 + 'launcher_closed_incoming': 0, }) metadata2 = copy.deepcopy(common_metadata) metadata2.update({ 'zone': 'zone', 'launcher_successes': 2, 'launcher_boot_durations_ns': [2, 4], - 'launcher_closed_incoming': 1 + 'launcher_closed_incoming': 1, }) expected = [ sample.Sample( metric='Launcher Boot Details', value=-1, unit='', - metadata=metadata1), + metadata=metadata1, + ), sample.Sample( metric='Launcher Boot Details', value=-1, unit='', - metadata=metadata2), + metadata=metadata2, + ), sample.Sample( metric='Cluster Max Boot Time', value=7, unit='nanoseconds', - metadata=common_metadata), + metadata=common_metadata, + ), sample.Sample( metric='Cluster Max Boot Sec', value=0.000000007, unit='seconds', - metadata=common_metadata), + metadata=common_metadata, + ), sample.Sample( metric='Cluster Mean Boot Sec', value=0.000000003, unit='seconds', - metadata=common_metadata), + metadata=common_metadata, + ), sample.Sample( metric='Cluster Median Boot Sec', value=0.000000003, unit='seconds', - metadata=common_metadata), + metadata=common_metadata, + ), sample.Sample( metric='Cluster Expected Boots', value=6, unit='', - metadata=common_metadata), + metadata=common_metadata, + ), sample.Sample( - metric='Cluster Success Boots', value=5, unit='', - metadata=common_metadata) + metric='Cluster Success Boots', + value=5, + unit='', + metadata=common_metadata, + ), ] for result, expected in zip(results, expected): - self.assertEqual(result.metric, expected.metric, - 'Metric name for {} is not equal.'.format( - expected.metric)) - self.assertEqual(result.value, expected.value, - 'Metric value for {} is not equal.'.format( - expected.metric)) - self.assertDictEqual(result.metadata, expected.metadata, - 'Metadata for {} is not equal'.format( - expected.metric)) + self.assertEqual( + result.metric, + expected.metric, + 'Metric name for {} is not equal.'.format(expected.metric), + ) + self.assertEqual( + result.value, + expected.value, + 'Metric value for {} is not equal.'.format(expected.metric), + ) + self.assertDictEqual( + result.metadata, + expected.metadata, + 'Metadata for {} is not equal'.format(expected.metric), + ) if __name__ == '__main__': diff --git a/tests/linux_benchmarks/lmbench_benchmark_test.py b/tests/linux_benchmarks/lmbench_benchmark_test.py index 0bd695926a..d67cd5ba8f 100644 --- a/tests/linux_benchmarks/lmbench_benchmark_test.py +++ b/tests/linux_benchmarks/lmbench_benchmark_test.py @@ -16,7 +16,6 @@ import unittest import mock - from perfkitbenchmarker.linux_benchmarks import lmbench_benchmark @@ -28,7 +27,8 @@ def setUp(self): self.addCleanup(p.stop) path = os.path.join( - os.path.dirname(__file__), '../data', 'lmbench_output.txt') + os.path.dirname(__file__), '../data', 'lmbench_output.txt' + ) with open(path) as fp: self.contents = fp.read() @@ -52,44 +52,70 @@ def testParseLmbench(self): self.assertAlmostEqual(0.6888, processor_results['stat']) self.assertAlmostEqual(0.3669, processor_results['fstat']) self.assertAlmostEqual(1.5541, processor_results['open/close']) - self.assertAlmostEqual(0.3226, - processor_results['Signal handler installation']) + self.assertAlmostEqual( + 0.3226, processor_results['Signal handler installation'] + ) self.assertAlmostEqual(1.1736, processor_results['Signal handler overhead']) self.assertAlmostEqual(0.7491, processor_results['Protection fault']) self.assertAlmostEqual(25.5437, processor_results['Pipe latency']) self.assertAlmostEqual(121.7399, processor_results['Process fork+exit']) self.assertAlmostEqual(318.6445, processor_results['Process fork+execve']) - self.assertAlmostEqual(800.2188, - processor_results['Process fork+/bin/sh -c']) - - sample = next(x for x in samples if x.metric == 'context_switching_time' and - x.metadata['num_of_processes'] == 96 and - x.metadata['memory_size'] == '64k') + self.assertAlmostEqual( + 800.2188, processor_results['Process fork+/bin/sh -c'] + ) + + sample = next( + x + for x in samples + if x.metric == 'context_switching_time' + and x.metadata['num_of_processes'] == 96 + and x.metadata['memory_size'] == '64k' + ) self.assertAlmostEqual(15.45, sample.value) - sample = next(x for x in samples if x.metric == 'context_switching_time' and - x.metadata['num_of_processes'] == 4 and - x.metadata['memory_size'] == '32k') + sample = next( + x + for x in samples + if x.metric == 'context_switching_time' + and x.metadata['num_of_processes'] == 4 + and x.metadata['memory_size'] == '32k' + ) self.assertAlmostEqual(13.96, sample.value) - sample = next(x for x in samples if x.metric == 'context_switching_time' and - x.metadata['num_of_processes'] == 2 and - x.metadata['memory_size'] == '16k') + sample = next( + x + for x in samples + if x.metric == 'context_switching_time' + and x.metadata['num_of_processes'] == 2 + and x.metadata['memory_size'] == '16k' + ) self.assertAlmostEqual(14.21, sample.value) - sample = next(x for x in samples if x.metric == 'context_switching_time' and - x.metadata['num_of_processes'] == 16 and - x.metadata['memory_size'] == '8k') + sample = next( + x + for x in samples + if x.metric == 'context_switching_time' + and x.metadata['num_of_processes'] == 16 + and x.metadata['memory_size'] == '8k' + ) self.assertAlmostEqual(13.02, sample.value) - sample = next(x for x in samples if x.metric == 'context_switching_time' and - x.metadata['num_of_processes'] == 8 and - x.metadata['memory_size'] == '4k') + sample = next( + x + for x in samples + if x.metric == 'context_switching_time' + and x.metadata['num_of_processes'] == 8 + and x.metadata['memory_size'] == '4k' + ) self.assertAlmostEqual(12.40, sample.value) - sample = next(x for x in samples if x.metric == 'context_switching_time' and - x.metadata['num_of_processes'] == 32 and - x.metadata['memory_size'] == '0k') + sample = next( + x + for x in samples + if x.metric == 'context_switching_time' + and x.metadata['num_of_processes'] == 32 + and x.metadata['memory_size'] == '0k' + ) self.assertAlmostEqual(12.63, sample.value) diff --git a/tests/linux_benchmarks/messaging_service_benchmark_test.py b/tests/linux_benchmarks/messaging_service_benchmark_test.py index 561b3ffa38..255ee0597a 100644 --- a/tests/linux_benchmarks/messaging_service_benchmark_test.py +++ b/tests/linux_benchmarks/messaging_service_benchmark_test.py @@ -24,16 +24,23 @@ timestamp=1623628800.0, metadata={ 'samples': [ - 0.20989608764648438, 0.2431643009185791, 0.14051604270935059, - 0.08317422866821289, 0.11351299285888672, 0.17781305313110352, - 0.037261247634887695, 0.030757904052734375, - 0.042165279388427734, 0.036507606506347656 + 0.20989608764648438, + 0.2431643009185791, + 0.14051604270935059, + 0.08317422866821289, + 0.11351299285888672, + 0.17781305313110352, + 0.037261247634887695, + 0.030757904052734375, + 0.042165279388427734, + 0.036507606506347656, ], 'number_of_messages': NUMBER_OF_MESSAGES, 'message_size': MESSAGE_SIZE, 'cloud': CLOUD, 'streaming_pull': False, - }), + }, + ), sample.Sample( metric='publish_latency_mean_without_cold_start', value=0.06490101814270019, @@ -44,7 +51,8 @@ 'message_size': MESSAGE_SIZE, 'cloud': CLOUD, 'streaming_pull': False, - }), + }, + ), sample.Sample( metric='publish_latency_p50', value=0.0983436107635498, @@ -55,7 +63,8 @@ 'message_size': MESSAGE_SIZE, 'cloud': CLOUD, 'streaming_pull': False, - }), + }, + ), sample.Sample( metric='publish_latency_p99', value=0.2401701617240906, @@ -66,7 +75,8 @@ 'message_size': MESSAGE_SIZE, 'cloud': CLOUD, 'streaming_pull': False, - }), + }, + ), sample.Sample( metric='publish_latency_p999', value=0.2428648869991303, @@ -77,7 +87,8 @@ 'message_size': MESSAGE_SIZE, 'cloud': CLOUD, 'streaming_pull': False, - }), + }, + ), sample.Sample( metric='publish_latency_percentage_received', value=100.0, @@ -88,7 +99,8 @@ 'message_size': MESSAGE_SIZE, 'cloud': CLOUD, 'streaming_pull': False, - }), + }, + ), sample.Sample( metric='pull_latency_mean', value=1.384722399711609, @@ -105,13 +117,14 @@ 1.1416656970977783, 1.032665729522705, 1.032970666885376, - 2.331076145172119 + 2.331076145172119, ], 'number_of_messages': NUMBER_OF_MESSAGES, 'message_size': MESSAGE_SIZE, 'cloud': CLOUD, 'streaming_pull': False, - }), + }, + ), sample.Sample( metric='pull_latency_mean_without_cold_start', value=1.4439927577972411, @@ -122,7 +135,8 @@ 'message_size': MESSAGE_SIZE, 'cloud': CLOUD, 'streaming_pull': False, - }), + }, + ), sample.Sample( metric='pull_latency_p50', value=1.044317603111267, @@ -133,7 +147,8 @@ 'message_size': MESSAGE_SIZE, 'cloud': CLOUD, 'streaming_pull': False, - }), + }, + ), sample.Sample( metric='pull_latency_p99', value=2.4716687655448912, @@ -144,7 +159,8 @@ 'message_size': MESSAGE_SIZE, 'cloud': CLOUD, 'streaming_pull': False, - }), + }, + ), sample.Sample( metric='pull_latency_p999', value=2.4841830537319187, @@ -155,7 +171,8 @@ 'message_size': MESSAGE_SIZE, 'cloud': CLOUD, 'streaming_pull': False, - }), + }, + ), sample.Sample( metric='pull_latency_percentage_received', value=100.0, @@ -166,7 +183,8 @@ 'message_size': MESSAGE_SIZE, 'cloud': CLOUD, 'streaming_pull': False, - }), + }, + ), ] @@ -175,22 +193,26 @@ class MessagingServiceBenchmarkTest(parameterized.TestCase): def setUp(self): super().setUp() - path = os.path.join(os.path.dirname(__file__), '..', 'data', - 'messaging_service_output.json') + path = os.path.join( + os.path.dirname(__file__), '..', 'data', 'messaging_service_output.json' + ) with open(path) as fp: self.contents = json.loads(fp.read()) @parameterized.named_parameters(('aggregate_samples', _AGGREGATE_SAMPLES)) def testCreateSamples(self, expected_samples): actual_samples = messaging_service_benchmark._CreateSamples( - self.contents, NUMBER_OF_MESSAGES, MESSAGE_SIZE, CLOUD, False) + self.contents, NUMBER_OF_MESSAGES, MESSAGE_SIZE, CLOUD, False + ) for expected_sample in expected_samples: if expected_sample not in actual_samples: sample_not_found_message = ( f'Expected sample:\n{expected_sample}\nnot found in actual samples:' - f'\n{actual_samples}') + f'\n{actual_samples}' + ) raise Exception(sample_not_found_message) + if __name__ == '__main__': unittest.main() diff --git a/tests/linux_benchmarks/mlperf_benchmark_test.py b/tests/linux_benchmarks/mlperf_benchmark_test.py index a6b1e6910b..2f1d253ce3 100644 --- a/tests/linux_benchmarks/mlperf_benchmark_test.py +++ b/tests/linux_benchmarks/mlperf_benchmark_test.py @@ -21,28 +21,32 @@ from tests import pkb_common_test_case -class MlperfBenchmarkTestCase(pkb_common_test_case.PkbCommonTestCase, - test_util.SamplesTestMixin): +class MlperfBenchmarkTestCase( + pkb_common_test_case.PkbCommonTestCase, test_util.SamplesTestMixin +): def setUp(self): super(MlperfBenchmarkTestCase, self).setUp() - path = os.path.join(os.path.dirname(__file__), '..', 'data', - 'mlperf_output.txt') + path = os.path.join( + os.path.dirname(__file__), '..', 'data', 'mlperf_output.txt' + ) with open(path) as fp: self.contents = fp.read() def testTrainResults(self): - samples = mlperf_benchmark.MakeSamplesFromOutput({'version': 'v1.0'}, - self.contents, - use_tpu=False, - model='resnet') + samples = mlperf_benchmark.MakeSamplesFromOutput( + {'version': 'v1.0'}, self.contents, use_tpu=False, model='resnet' + ) self.assertLen(samples, 1123) self.assertSamplesEqualUpToTimestamp( Sample( metric='speed', value=17651.66, unit='samples/sec', - metadata={'version': 'v1.0'}), samples[0]) + metadata={'version': 'v1.0'}, + ), + samples[0], + ) print(samples[-1]) self.assertSamplesEqualUpToTimestamp( Sample( @@ -56,9 +60,12 @@ def testTrainResults(self): 'version': 'v1.0', 'namespace': '', 'event_type': 'INTERVAL_END', - 'value': None + 'value': None, }, - timestamp=1627332194930), samples[-1]) + timestamp=1627332194930, + ), + samples[-1], + ) if __name__ == '__main__': diff --git a/tests/linux_benchmarks/mlperf_inference_benchmark_test.py b/tests/linux_benchmarks/mlperf_inference_benchmark_test.py index 88e2b3fa47..3569b21cac 100644 --- a/tests/linux_benchmarks/mlperf_inference_benchmark_test.py +++ b/tests/linux_benchmarks/mlperf_inference_benchmark_test.py @@ -21,8 +21,9 @@ from tests import pkb_common_test_case -class MlperfInferenceBenchmarkTestCase(pkb_common_test_case.PkbCommonTestCase, - test_util.SamplesTestMixin): +class MlperfInferenceBenchmarkTestCase( + pkb_common_test_case.PkbCommonTestCase, test_util.SamplesTestMixin +): def setUp(self): super(MlperfInferenceBenchmarkTestCase, self).setUp() diff --git a/tests/linux_benchmarks/mlperf_multiworkers_benchmark_test.py b/tests/linux_benchmarks/mlperf_multiworkers_benchmark_test.py index 5865f9c418..fdb79f005c 100644 --- a/tests/linux_benchmarks/mlperf_multiworkers_benchmark_test.py +++ b/tests/linux_benchmarks/mlperf_multiworkers_benchmark_test.py @@ -22,25 +22,34 @@ class MlperfMultiworkersBenchmarkTestCase( - pkb_common_test_case.PkbCommonTestCase, test_util.SamplesTestMixin): + pkb_common_test_case.PkbCommonTestCase, test_util.SamplesTestMixin +): def setUp(self): super().setUp() - path = os.path.join(os.path.dirname(__file__), '..', 'data', - 'mlperf_multiworkers_output.txt') + path = os.path.join( + os.path.dirname(__file__), + '..', + 'data', + 'mlperf_multiworkers_output.txt', + ) with open(path) as fp: self.contents = fp.read() def testTrainResults(self): samples = mlperf_multiworkers_benchmark.MakeSamplesFromOutput( - {'version': 'v2.0'}, self.contents, model='bert') + {'version': 'v2.0'}, self.contents, model='bert' + ) self.assertLen(samples, 458) self.assertSamplesEqualUpToTimestamp( Sample( metric='speed', value=2427.4356684047016, unit='samples/sec', - metadata={'version': 'v2.0'}), samples[0]) + metadata={'version': 'v2.0'}, + ), + samples[0], + ) self.assertSamplesEqualUpToTimestamp( Sample( metric='run_stop', @@ -53,8 +62,11 @@ def testTrainResults(self): 'version': 'v2.0', 'namespace': '', 'event_type': 'INTERVAL_END', - 'value': None - }), samples[-1]) + 'value': None, + }, + ), + samples[-1], + ) if __name__ == '__main__': diff --git a/tests/linux_benchmarks/mnist_benchmark_test.py b/tests/linux_benchmarks/mnist_benchmark_test.py index 7be74e0915..6f00e93ad5 100644 --- a/tests/linux_benchmarks/mnist_benchmark_test.py +++ b/tests/linux_benchmarks/mnist_benchmark_test.py @@ -15,49 +15,65 @@ import os import unittest import mock - from perfkitbenchmarker import test_util from perfkitbenchmarker.linux_benchmarks import mnist_benchmark from perfkitbenchmarker.sample import Sample -class MnistBenchmarkTestCase(unittest.TestCase, - test_util.SamplesTestMixin): +class MnistBenchmarkTestCase(unittest.TestCase, test_util.SamplesTestMixin): def setUp(self): - path = os.path.join(os.path.dirname(__file__), '..', 'data', - 'mnist_output.txt') + path = os.path.join( + os.path.dirname(__file__), '..', 'data', 'mnist_output.txt' + ) with open(path) as fp: self.contents = fp.read() - self.metadata_input = {'num_examples_per_epoch': 1251.1, - 'train_batch_size': 1024} - self.metadata_output = {'num_examples_per_epoch': 1251.1, - 'train_batch_size': 1024, 'step': 2000, - 'elapsed_seconds': 0, 'epoch': 1.5985932379506036} + self.metadata_input = { + 'num_examples_per_epoch': 1251.1, + 'train_batch_size': 1024, + } + self.metadata_output = { + 'num_examples_per_epoch': 1251.1, + 'train_batch_size': 1024, + 'step': 2000, + 'elapsed_seconds': 0, + 'epoch': 1.5985932379506036, + } @mock.patch('time.time', mock.MagicMock(return_value=0)) def testTrainResults(self): samples = mnist_benchmark.MakeSamplesFromTrainOutput( - self.metadata_input, self.contents, 0, 2000) + self.metadata_input, self.contents, 0, 2000 + ) golden = [ Sample('Loss', 0.09562386, '', self.metadata_output), - Sample('Global Steps Per Second', 217.69966666666664, - 'global_steps/sec', self.metadata_output), - Sample('Examples Per Second', 222924.33333333334, - 'examples/sec', self.metadata_output) + Sample( + 'Global Steps Per Second', + 217.69966666666664, + 'global_steps/sec', + self.metadata_output, + ), + Sample( + 'Examples Per Second', + 222924.33333333334, + 'examples/sec', + self.metadata_output, + ), ] self.assertEqual(samples, golden) @mock.patch('time.time', mock.MagicMock(return_value=0)) def testEvalResults(self): samples = mnist_benchmark.MakeSamplesFromEvalOutput( - self.metadata_input, self.contents, 0) + self.metadata_input, self.contents, 0 + ) golden = [ Sample('Eval Loss', 0.03615343, '', self.metadata_output), - Sample('Accuracy', 98.77387, '%', self.metadata_output) + Sample('Accuracy', 98.77387, '%', self.metadata_output), ] self.assertEqual(samples, golden) + if __name__ == '__main__': unittest.main() diff --git a/tests/linux_benchmarks/mpi_benchmark_test.py b/tests/linux_benchmarks/mpi_benchmark_test.py index a404fb0da8..01614030cf 100644 --- a/tests/linux_benchmarks/mpi_benchmark_test.py +++ b/tests/linux_benchmarks/mpi_benchmark_test.py @@ -24,13 +24,16 @@ # Histogram results from reading MPI output file histogram1 = {'12.5': 50000, '10.0': 50000} histogram2 = {'6.0': 50000, '50.0': 50000} -histogram_text = """\ +histogram_text = ( + """\ 1024 12.51 1024 10.01 -""" * 50000 + """\ +""" * 50000 + + """\ 2048 6.00 2048 50.0 """ * 50000 +) MPI_VARS = '/opt/intel/compilers_and_libraries/linux/mpi/intel64/bin/mpivars.sh' @@ -38,34 +41,37 @@ # All VMs have num_cpus=32 class Vm(pkb_common_test_case.TestLinuxVirtualMachine): - def __init__(self, - smt_enabled=True, - ip='10.0.0.2', - robust_remote_command_text=None) -> None: + def __init__( + self, smt_enabled=True, ip='10.0.0.2', robust_remote_command_text=None + ) -> None: super(Vm, self).__init__(vm_spec=pkb_common_test_case.CreateTestVmSpec()) self.internal_ip = ip self.num_cpus = 32 # pylint: disable=invalid-name self.IsSmtEnabled = mock.PropertyMock(return_value=smt_enabled) self.RemoteCommand = mock.PropertyMock( - return_value=('Version 2019 Update 2 Build 2019.2-057', '')) + return_value=('Version 2019 Update 2 Build 2019.2-057', '') + ) self.RobustRemoteCommand = mock.PropertyMock( - return_value=((mpi_test.ReadMpiOutput('mpi_pingpong_output.txt'), ''))) + return_value=((mpi_test.ReadMpiOutput('mpi_pingpong_output.txt'), '')) + ) def MpiRun(vms) -> List[sample.Sample]: benchmark_module = mock.Mock(BENCHMARK_NAME='mpi') benchmark_config = mock.Mock( - vm_groups={}, relational_db=mock.Mock(vm_groups={})) - spec = benchmark_spec.BenchmarkSpec(benchmark_module, benchmark_config, - 'abcdefg') + vm_groups={}, relational_db=mock.Mock(vm_groups={}) + ) + spec = benchmark_spec.BenchmarkSpec( + benchmark_module, benchmark_config, 'abcdefg' + ) spec.vms = vms return mpi_benchmark.Run(spec) -class MpiBenchmarkTestCase(pkb_common_test_case.PkbCommonTestCase, - test_util.SamplesTestMixin): - +class MpiBenchmarkTestCase( + pkb_common_test_case.PkbCommonTestCase, test_util.SamplesTestMixin +): _METRIC_ERR = 'Metric values should be equal %s != %s' _VALUE_ERR = 'Values should be equal %s != %s' _UNIT_ERR = 'Unit values should be equal %s != %s' @@ -77,10 +83,12 @@ def setUp(self) -> None: FLAGS.mpi_benchmarks = ['PingPong'] FLAGS.intelmpi_version = '2019.2-057' self.mock_histo = self.enter_context( - mock.patch.object(mpi, '_GroupLatencyLines')) + mock.patch.object(mpi, '_GroupLatencyLines') + ) self.mock_histo.return_value = [histogram_text.splitlines()] self.enter_context( - mock.patch.object(intelmpi, 'MpiVars', return_value=MPI_VARS)) + mock.patch.object(intelmpi, 'MpiVars', return_value=MPI_VARS) + ) @mock.patch.object(uuid, 'uuid4', side_effect=_MOCK_UUIDS) def testRun(self, mock_uuid) -> None: @@ -100,16 +108,8 @@ def testRun(self, mock_uuid) -> None: self.assertEqual(2, self.mock_histo.call_count) @parameterized.parameters( - { - 'threads': [0], - 'num_vms': 1, - 'expected_threads': [16] - }, - { - 'threads': [2, 6, 18], - 'num_vms': 2, - 'expected_threads': [4, 12, 36] - }, + {'threads': [0], 'num_vms': 1, 'expected_threads': [16]}, + {'threads': [2, 6, 18], 'num_vms': 2, 'expected_threads': [4, 12, 36]}, { 'threads': [0], 'num_vms': 1, @@ -118,22 +118,27 @@ def testRun(self, mock_uuid) -> None: }, ) @mock.patch.object(mpi_benchmark, '_RunTest') - def testRunTestCommand(self, - mock_run: mock.Mock, - num_vms: int, - expected_threads: List[int], - threads: List[int], - smt_enabled: bool = True) -> None: + def testRunTestCommand( + self, + mock_run: mock.Mock, + num_vms: int, + expected_threads: List[int], + threads: List[int], + smt_enabled: bool = True, + ) -> None: FLAGS.mpi_threads = threads MpiRun([Vm(smt_enabled) for _ in range(num_vms)]) - for total_processes, found in zip(expected_threads, - mock_run.call_args_list): + for total_processes, found in zip( + expected_threads, mock_run.call_args_list + ): _, found_total_processes, found_ppn, _ = found[0] self.assertEqual(total_processes, found_total_processes) self.assertEqual(0, found_ppn) self.assertLen( - mock_run.call_args_list, len(expected_threads), - 'Missing / extra calls in {}'.format(mock_run.call_args_list)) + mock_run.call_args_list, + len(expected_threads), + 'Missing / extra calls in {}'.format(mock_run.call_args_list), + ) self.mock_histo.assert_not_called() @mock.patch.object(mpi, 'RunMpiStats') @@ -167,7 +172,9 @@ def testRunMpiStatsCall(self, mock_mpistats: mock.Mock) -> None: compile_from_source=True, record_latencies=True, environment=['I_MPI_DEBUG=6'], - multi=True)) + multi=True, + ), + ) self.mock_histo.assert_not_called() @parameterized.parameters((True, 16), (False, 32)) @@ -192,14 +199,14 @@ def testHistoResults(self) -> None: 'bytes': 1024, 'mpi_groups': 2, 'mpi_processes_per_group': 2, - 'histogram': histogram1 + 'histogram': histogram1, } self.assertDictContainsSubset(meta1, histogram_data[0].metadata) meta2 = { 'bytes': 2048, 'mpi_groups': 2, 'mpi_processes_per_group': 2, - 'histogram': histogram2 + 'histogram': histogram2, } self.assertDictContainsSubset(meta2, histogram_data[1].metadata) self.assertEqual(4, self.mock_histo.call_count) @@ -207,8 +214,9 @@ def testHistoResults(self) -> None: @flagsaver.flagsaver(mpi_benchmarks=['Qubert', 'Broadcast', 'allTOaLL']) def testGetConfigBadBenchmark(self): # Alltoall is a valid benchmark - with self.assertRaisesRegex(errors.Setup.InvalidFlagConfigurationError, - '"broadcast,qubert"'): + with self.assertRaisesRegex( + errors.Setup.InvalidFlagConfigurationError, '"broadcast,qubert"' + ): mpi_benchmark.GetConfig({}) @flagsaver.flagsaver(mpi_benchmarks=['Bcast'], mpi_msglog_sizes=[20]) @@ -229,7 +237,8 @@ def testRunTestWithSuites(self): # Mock response with no results as not testing that functionality response = mpi.MpiResponse('a', 'b', 'c', 'd', [], [], {}) mpirun_mock = self.enter_context( - mock.patch.object(mpi, 'RunMpiStats', return_value=response)) + mock.patch.object(mpi, 'RunMpiStats', return_value=response) + ) vm = Vm() mpi_benchmark._RunTest([vm], 2, 1, True) @@ -253,7 +262,8 @@ def testRunTestWithSuites(self): record_latencies=True, npmin=None, tune=False, - multi=True) + multi=True, + ) # Test the last one called mpirun_mock.assert_called_with(vm, expected_request) # It was called len(IMB-MT suite tests) times diff --git a/tests/linux_benchmarks/multichase_benchmark_test.py b/tests/linux_benchmarks/multichase_benchmark_test.py index 4e96171189..66d73f60a7 100644 --- a/tests/linux_benchmarks/multichase_benchmark_test.py +++ b/tests/linux_benchmarks/multichase_benchmark_test.py @@ -62,8 +62,9 @@ def testPercent(self): class IterMemorySizesTestCase(unittest.TestCase): def setUp(self): - self._func = functools.partial(multichase_benchmark._IterMemorySizes, - lambda: 1024) + self._func = functools.partial( + multichase_benchmark._IterMemorySizes, lambda: 1024 + ) def testHitsUpperBound(self): result = list(self._func(1 * units.byte, 32 * units.byte)) diff --git a/tests/linux_benchmarks/mxnet_benchmark_test.py b/tests/linux_benchmarks/mxnet_benchmark_test.py index f8afcf8373..4c3a654cb1 100644 --- a/tests/linux_benchmarks/mxnet_benchmark_test.py +++ b/tests/linux_benchmarks/mxnet_benchmark_test.py @@ -19,12 +19,12 @@ from perfkitbenchmarker.linux_benchmarks import mxnet_benchmark -class MxnetBenchmarkTestCase(unittest.TestCase, - test_util.SamplesTestMixin): +class MxnetBenchmarkTestCase(unittest.TestCase, test_util.SamplesTestMixin): def setUp(self): - path = os.path.join(os.path.dirname(__file__), '..', 'data', - 'mxnet_output.txt') + path = os.path.join( + os.path.dirname(__file__), '..', 'data', 'mxnet_output.txt' + ) with open(path, 'r') as fp: self.contents = fp.read() diff --git a/tests/linux_benchmarks/nccl_benchmark_test.py b/tests/linux_benchmarks/nccl_benchmark_test.py index a745c7e8b9..d7fd8b252e 100644 --- a/tests/linux_benchmarks/nccl_benchmark_test.py +++ b/tests/linux_benchmarks/nccl_benchmark_test.py @@ -16,7 +16,6 @@ import os import unittest import mock - from perfkitbenchmarker import test_util from perfkitbenchmarker.linux_benchmarks import nccl_benchmark from perfkitbenchmarker.sample import Sample @@ -26,8 +25,9 @@ class NcclBenchmarkTest(unittest.TestCase, test_util.SamplesTestMixin): def setUp(self): super(NcclBenchmarkTest, self).setUp() - path = os.path.join(os.path.dirname(__file__), '..', 'data', - 'nccl_output.txt') + path = os.path.join( + os.path.dirname(__file__), '..', 'data', 'nccl_output.txt' + ) with open(path) as fp: self.contents = fp.read() @@ -57,13 +57,14 @@ def testNcclResults(self): 'validation': '0', 'graph': '0', 'Rank 0': 'Pid 9192 on pkb-2b6a393d-2 device 0 [0x00] Tesla K80', - 'Rank 1': 'Pid 9178 on pkb-2b6a393d-3 device 0 [0x00] Tesla K80' + 'Rank 1': 'Pid 9178 on pkb-2b6a393d-3 device 0 [0x00] Tesla K80', } golden = Sample( metric='In place algorithm bandwidth', value=0.39, unit='GB/s', - metadata=metadata) + metadata=metadata, + ) self.assertIn(golden, samples) self.assertAlmostEqual(bandwidth, 0.63) diff --git a/tests/linux_benchmarks/netperf_benchmark_test.py b/tests/linux_benchmarks/netperf_benchmark_test.py index 6719e3e914..74b1ab1771 100644 --- a/tests/linux_benchmarks/netperf_benchmark_test.py +++ b/tests/linux_benchmarks/netperf_benchmark_test.py @@ -31,20 +31,20 @@ class NetperfBenchmarkTestCase(parameterized.TestCase, unittest.TestCase): - maxDiff = None def setUp(self): super(NetperfBenchmarkTestCase, self).setUp() # Load data - path = os.path.join(os.path.dirname(__file__), - '..', 'data', - 'netperf_results.json') + path = os.path.join( + os.path.dirname(__file__), '..', 'data', 'netperf_results.json' + ) with open(path) as fp: stdouts = ['\n'.join(i) for i in json.load(fp)] - self.expected_stdout = [json.dumps(([stdout], [''], [0])) - for stdout in stdouts] + self.expected_stdout = [ + json.dumps(([stdout], [''], [0])) for stdout in stdouts + ] p = mock.patch(vm_util.__name__ + '.ShouldRunOnExternalIpAddress') self.should_run_external = p.start() @@ -62,7 +62,8 @@ def _ConfigureIpTypes(self, run_external=True, run_internal=True): def testHistogramStatsCalculator(self): histogram = {1: 5, 2: 10, 5: 5} stats = netperf_benchmark._HistogramStatsCalculator( - histogram, [0, 20, 30, 74, 80, 100]) + histogram, [0, 20, 30, 74, 80, 100] + ) self.assertEqual(stats['p0'], 1) self.assertEqual(stats['p20'], 1) self.assertEqual(stats['p30'], 2) @@ -77,7 +78,8 @@ def testExternalAndInternal(self): vm_spec = mock.MagicMock(spec=benchmark_spec.BenchmarkSpec) vm_spec.vms = [mock.MagicMock(), mock.MagicMock()] vm_spec.vms[0].RobustRemoteCommand.side_effect = [ - (i, '') for i in self.expected_stdout] + (i, '') for i in self.expected_stdout + ] vm_spec.vms[1].GetInternalIPs.return_value = ['test_ip'] vm_spec.vms[0].GetInternalIPs.return_value = ['test_ip'] run_result = netperf_benchmark.Run(vm_spec) @@ -156,20 +158,30 @@ def testExternalAndInternal(self): self.assertDictContainsSubset(meta, result[i][3]) @parameterized.named_parameters( - ('no_times_up', - 'MIGRATED TCP STREAM TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET to ' - '10.0.0.137 () port 20157 AF_INET : histogram\nrecv_response_timed_n: no' - ' response received. errno 110 counter 0\n'), - ('has_times_up', - 'MIGRATED TCP STREAM TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET to ' - '10.0.0.172 () port 20169 AF_INET : histogram\ncatcher: timer popped ' - 'with times_up != 0\nrecv_response_timed_n: no response received. errno ' - '4 counter -1\n')) + ( + 'no_times_up', + ( + 'MIGRATED TCP STREAM TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET' + ' to 10.0.0.137 () port 20157 AF_INET :' + ' histogram\nrecv_response_timed_n: no response received. errno' + ' 110 counter 0\n' + ), + ), + ( + 'has_times_up', + ( + 'MIGRATED TCP STREAM TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET' + ' to 10.0.0.172 () port 20169 AF_INET : histogram\ncatcher: timer' + ' popped with times_up != 0\nrecv_response_timed_n: no response' + ' received. errno 4 counter -1\n' + ), + ), + ) def testParseNetperfOutputError(self, output): - with self.assertRaises( - errors.Benchmarks.KnownIntermittentError) as e: - netperf_benchmark.ParseNetperfOutput(output, {}, 'fake_benchmark_name', - False) + with self.assertRaises(errors.Benchmarks.KnownIntermittentError) as e: + netperf_benchmark.ParseNetperfOutput( + output, {}, 'fake_benchmark_name', False + ) self.assertIn('Failed to parse stdout', str(e.exception)) @flagsaver.flagsaver(netperf_benchmarks=[netperf_benchmark.TCP_STREAM]) diff --git a/tests/linux_benchmarks/object_storage_service_benchmark_test.py b/tests/linux_benchmarks/object_storage_service_benchmark_test.py index a4af91df0f..4967d1b256 100644 --- a/tests/linux_benchmarks/object_storage_service_benchmark_test.py +++ b/tests/linux_benchmarks/object_storage_service_benchmark_test.py @@ -19,7 +19,6 @@ import unittest from absl import flags import mock - from perfkitbenchmarker.linux_benchmarks import object_storage_service_benchmark from tests import pkb_common_test_case @@ -44,36 +43,47 @@ def testBuildCommands(self): service = mock.MagicMock() with mock.patch(time.__name__ + '.time', return_value=1.0): - with mock.patch(object_storage_service_benchmark.__name__ + - '.ProcessMultiStreamResults'): - with mock.patch(object_storage_service_benchmark.__name__ + - '.LoadWorkerOutput', return_value=(None, None, None)): + with mock.patch( + object_storage_service_benchmark.__name__ + + '.ProcessMultiStreamResults' + ): + with mock.patch( + object_storage_service_benchmark.__name__ + '.LoadWorkerOutput', + return_value=(None, None, None), + ): object_storage_service_benchmark.MultiStreamRWBenchmark( - [], {}, [vm], command_builder, service, 'bucket') + [], {}, [vm], command_builder, service, 'bucket' + ) self.assertEqual( command_builder.BuildCommand.call_args_list[0], - mock.call(['--bucket=bucket', - '--objects_per_stream=100', - '--num_streams=1', - '--start_time=16.1', - '--objects_written_file=/tmp/pkb/pkb-objects-written', - '--object_sizes="{1000: 100.0}"', - '--object_naming_scheme=sequential_by_stream', - '--scenario=MultiStreamWrite', - '--stream_num_start=0', - '--vm_id=0'])) + mock.call([ + '--bucket=bucket', + '--objects_per_stream=100', + '--num_streams=1', + '--start_time=16.1', + '--objects_written_file=/tmp/pkb/pkb-objects-written', + '--object_sizes="{1000: 100.0}"', + '--object_naming_scheme=sequential_by_stream', + '--scenario=MultiStreamWrite', + '--stream_num_start=0', + '--vm_id=0', + ]), + ) self.assertEqual( command_builder.BuildCommand.call_args_list[1], - mock.call(['--bucket=bucket', - '--objects_per_stream=100', - '--num_streams=1', - '--start_time=16.1', - '--objects_written_file=/tmp/pkb/pkb-objects-written', - '--scenario=MultiStreamRead', - '--stream_num_start=0', - '--vm_id=0'])) + mock.call([ + '--bucket=bucket', + '--objects_per_stream=100', + '--num_streams=1', + '--start_time=16.1', + '--objects_written_file=/tmp/pkb/pkb-objects-written', + '--scenario=MultiStreamRead', + '--stream_num_start=0', + '--vm_id=0', + ]), + ) class TestDistributionToBackendFormat(pkb_common_test_case.PkbCommonTestCase): @@ -83,25 +93,24 @@ def testPointDistribution(self): self.assertEqual( object_storage_service_benchmark._DistributionToBackendFormat(dist), - {100000: 100.0}) + {100000: 100.0}, + ) def testMultiplePointsDistribution(self): - dist = {'1B': '10%', - '2B': '20%', - '4B': '70%'} + dist = {'1B': '10%', '2B': '20%', '4B': '70%'} self.assertEqual( object_storage_service_benchmark._DistributionToBackendFormat(dist), - {1: 10.0, - 2: 20.0, - 4: 70.0}) + {1: 10.0, 2: 20.0, 4: 70.0}, + ) def testAbbreviatedPointDistribution(self): dist = '100KB' self.assertEqual( object_storage_service_benchmark._DistributionToBackendFormat(dist), - {100000: 100.0}) + {100000: 100.0}, + ) def testBadPercentages(self): dist = {'1B': '50%'} @@ -120,16 +129,16 @@ def testFilename(self): with mock.patch.object( object_storage_service_benchmark, '_DatetimeNow', - return_value=write_time): - filename = ( - object_storage_service_benchmark._ColdObjectsWrittenFilename()) + return_value=write_time, + ): + filename = object_storage_service_benchmark._ColdObjectsWrittenFilename() read_time = datetime.datetime.now() with mock.patch.object( - object_storage_service_benchmark, - '_DatetimeNow', - return_value=read_time): + object_storage_service_benchmark, '_DatetimeNow', return_value=read_time + ): age = object_storage_service_benchmark._ColdObjectsWrittenFileAgeHours( - filename) + filename + ) # Verify that the age is between 72 and 73 hours. self.assertLessEqual(72, age) self.assertLessEqual(age, 73) diff --git a/tests/linux_benchmarks/openfoam_benchmark_test.py b/tests/linux_benchmarks/openfoam_benchmark_test.py index fbcaccb36a..92e12d204f 100644 --- a/tests/linux_benchmarks/openfoam_benchmark_test.py +++ b/tests/linux_benchmarks/openfoam_benchmark_test.py @@ -31,34 +31,41 @@ FLAGS = flags.FLAGS -class OpenfoamBenchmarkTest(pkb_common_test_case.PkbCommonTestCase, - test_util.SamplesTestMixin): +class OpenfoamBenchmarkTest( + pkb_common_test_case.PkbCommonTestCase, test_util.SamplesTestMixin +): def setUp(self): super(OpenfoamBenchmarkTest, self).setUp() self.mock_vm = mock.Mock() self.mock_benchmark_spec = mock.Mock(vms=[self.mock_vm]) self.enter_context( - mock.patch.object(openmpi, 'GetMpiVersion', return_value='1.10.2')) + mock.patch.object(openmpi, 'GetMpiVersion', return_value='1.10.2') + ) self.enter_context( mock.patch.object( - openfoam_benchmark, '_GetOpenfoamVersion', return_value='7')) - - @mock.patch.object(openfoam_benchmark, '_ParseRunCommands', - return_value=['mpirun $(getApplication)']) + openfoam_benchmark, '_GetOpenfoamVersion', return_value='7' + ) + ) + + @mock.patch.object( + openfoam_benchmark, + '_ParseRunCommands', + return_value=['mpirun $(getApplication)'], + ) @flagsaver.flagsaver(openfoam_dimensions=['80_32_32']) - def testRunCaseReturnsCorrectlyParsedSamples(self, - mock_parseruncommands): + def testRunCaseReturnsCorrectlyParsedSamples(self, mock_parseruncommands): # Run with mocked output data self.mock_vm.RemoteCommand.return_value = None, '\n'.join( - ['real 131.64', 'user 327.05', 'sys 137.04']) + ['real 131.64', 'user 327.05', 'sys 137.04'] + ) self.mock_vm.NumCpusForBenchmark.return_value = 8 samples = openfoam_benchmark.Run(self.mock_benchmark_spec) # Verify command is what we expected to run run_cmd = [ 'cd $HOME/OpenFOAM/run/motorBike', - 'time -p mpirun $(getApplication)' + 'time -p mpirun $(getApplication)', ] self.mock_vm.RemoteCommand.assert_called_with(' && '.join(run_cmd)) @@ -78,16 +85,20 @@ def testRunCaseReturnsCorrectlyParsedSamples(self, } unit = 'seconds' self.assertSamplesEqualUpToTimestamp( - sample.Sample('time_real', 131, unit, expected_metadata), samples[0]) + sample.Sample('time_real', 131, unit, expected_metadata), samples[0] + ) self.assertSamplesEqualUpToTimestamp( - sample.Sample('time_user', 327, unit, expected_metadata), samples[1]) + sample.Sample('time_user', 327, unit, expected_metadata), samples[1] + ) self.assertSamplesEqualUpToTimestamp( - sample.Sample('time_sys', 137, unit, expected_metadata), samples[2]) + sample.Sample('time_sys', 137, unit, expected_metadata), samples[2] + ) def testYumInstallRaisesNotImplementedError(self): static_vm_spec = static_virtual_machine.StaticVmSpec('test_static_vm_spec') self.mock_vm = static_virtual_machine.Rhel7BasedStaticVirtualMachine( - static_vm_spec) + static_vm_spec + ) self.mock_vm.install_packages = True with self.assertRaises(NotImplementedError): self.mock_vm.Install('openfoam') @@ -97,7 +108,8 @@ def testYumInstallRaisesNotImplementedError(self): openfoam_num_threads_per_vm=8, openfoam_mpi_mapping='hwthread', openfoam_decomp_method='simple', - openfoam_max_global_cells=1e9) + openfoam_max_global_cells=1e9, + ) @mock.patch.object(openfoam_benchmark, '_UseMpi') @mock.patch.object(openfoam_benchmark, '_SetDictEntry') @mock.patch.object(openfoam_benchmark, '_RunCase') @@ -114,8 +126,11 @@ def testRunWithMoreFlags(self, mock_runcase, mock_setdict, mock_usempi): ('method', 'simple', 'system/decomposeParDict'), ('numberOfSubdomains', 8, 'system/decomposeParDict'), ('hierarchicalCoeffs.n', '(8 1 1)', 'system/decomposeParDict'), - ('castellatedMeshControls.maxGlobalCells', float(1e9), - 'system/snappyHexMeshDict'), + ( + 'castellatedMeshControls.maxGlobalCells', + float(1e9), + 'system/snappyHexMeshDict', + ), ] mock_setdict.assert_has_calls([ mock.call(self.mock_vm, key, value, dict_file_name) diff --git a/tests/linux_benchmarks/openssl_speed_benchmark_test.py b/tests/linux_benchmarks/openssl_speed_benchmark_test.py index 9397d04416..41da0c8a98 100644 --- a/tests/linux_benchmarks/openssl_speed_benchmark_test.py +++ b/tests/linux_benchmarks/openssl_speed_benchmark_test.py @@ -24,8 +24,9 @@ flags.FLAGS.mark_as_parsed() -class TestParseOpenSSLOutput(pkb_common_test_case.PkbCommonTestCase, - test_util.SamplesTestMixin): +class TestParseOpenSSLOutput( + pkb_common_test_case.PkbCommonTestCase, test_util.SamplesTestMixin +): def testParseOpenSSLOutput(self): raw_results = """ @@ -36,38 +37,92 @@ def testParseOpenSSLOutput(self): evp 7928414.04k 24973982.48k 64151343.18k 100449384.11k 117555948.61k 118711951.91k """ results = openssl_speed_benchmark.ParseOpenSSLOutput( - raw_results, 'fake_version', 10) + raw_results, 'fake_version', 10 + ) print(results) - self.assertSampleListsEqualUpToTimestamp(results, [ - sample.Sample(metric='Throughput', value=7928414.04, unit='k', - metadata={'duration': 60, 'algorithm': 'aes-256-ctr', - 'parallelism': 10, 'version': 'fake_version', - 'blocksize': 16}, timestamp=1627597373.5422955), - sample.Sample(metric='Throughput', value=24973982.48, unit='k', - metadata={'duration': 60, 'algorithm': 'aes-256-ctr', - 'parallelism': 10, 'version': 'fake_version', - 'blocksize': 64}, timestamp=1627597909.9372606), - sample.Sample(metric='Throughput', value=64151343.18, unit='k', - metadata={'duration': 60, 'algorithm': 'aes-256-ctr', - 'parallelism': 10, 'version': 'fake_version', - 'blocksize': 256}, - timestamp=1627597909.9372814), - sample.Sample(metric='Throughput', value=100449384.11, unit='k', - metadata={'duration': 60, 'algorithm': 'aes-256-ctr', - 'parallelism': 10, 'version': 'fake_version', - 'blocksize': 1024}, - timestamp=1627597909.9372985), - sample.Sample(metric='Throughput', value=117555948.61, unit='k', - metadata={'duration': 60, 'algorithm': 'aes-256-ctr', - 'parallelism': 10, 'version': 'fake_version', - 'blocksize': 8192}, - timestamp=1627597909.937315), - sample.Sample(metric='Throughput', value=118711951.91, unit='k', - metadata={'duration': 60, 'algorithm': 'aes-256-ctr', - 'parallelism': 10, 'version': 'fake_version', - 'blocksize': 16384}, - timestamp=1627597909.9373317) - ]) + self.assertSampleListsEqualUpToTimestamp( + results, + [ + sample.Sample( + metric='Throughput', + value=7928414.04, + unit='k', + metadata={ + 'duration': 60, + 'algorithm': 'aes-256-ctr', + 'parallelism': 10, + 'version': 'fake_version', + 'blocksize': 16, + }, + timestamp=1627597373.5422955, + ), + sample.Sample( + metric='Throughput', + value=24973982.48, + unit='k', + metadata={ + 'duration': 60, + 'algorithm': 'aes-256-ctr', + 'parallelism': 10, + 'version': 'fake_version', + 'blocksize': 64, + }, + timestamp=1627597909.9372606, + ), + sample.Sample( + metric='Throughput', + value=64151343.18, + unit='k', + metadata={ + 'duration': 60, + 'algorithm': 'aes-256-ctr', + 'parallelism': 10, + 'version': 'fake_version', + 'blocksize': 256, + }, + timestamp=1627597909.9372814, + ), + sample.Sample( + metric='Throughput', + value=100449384.11, + unit='k', + metadata={ + 'duration': 60, + 'algorithm': 'aes-256-ctr', + 'parallelism': 10, + 'version': 'fake_version', + 'blocksize': 1024, + }, + timestamp=1627597909.9372985, + ), + sample.Sample( + metric='Throughput', + value=117555948.61, + unit='k', + metadata={ + 'duration': 60, + 'algorithm': 'aes-256-ctr', + 'parallelism': 10, + 'version': 'fake_version', + 'blocksize': 8192, + }, + timestamp=1627597909.937315, + ), + sample.Sample( + metric='Throughput', + value=118711951.91, + unit='k', + metadata={ + 'duration': 60, + 'algorithm': 'aes-256-ctr', + 'parallelism': 10, + 'version': 'fake_version', + 'blocksize': 16384, + }, + timestamp=1627597909.9373317, + ), + ], + ) if __name__ == '__main__': diff --git a/tests/linux_benchmarks/pgbench_benchmark_test.py b/tests/linux_benchmarks/pgbench_benchmark_test.py index caa640bc9b..aed1550b66 100644 --- a/tests/linux_benchmarks/pgbench_benchmark_test.py +++ b/tests/linux_benchmarks/pgbench_benchmark_test.py @@ -16,7 +16,6 @@ import os import unittest import mock - from perfkitbenchmarker.linux_benchmarks import pgbench_benchmark from perfkitbenchmarker.linux_packages import pgbench @@ -28,8 +27,9 @@ def setUp(self): p.start() self.addCleanup(p.stop) - path = os.path.join(os.path.dirname(__file__), '../data', - 'pgbench.stderr.txt') + path = os.path.join( + os.path.dirname(__file__), '../data', 'pgbench.stderr.txt' + ) with open(path) as fp: self.stderr_output = fp.read() @@ -51,7 +51,8 @@ def testMakeSamplesFromOutput(self): }) actual = pgbench.MakeSamplesFromOutput( - self.stderr_output, num_clients, num_jobs, testMetadata) + self.stderr_output, num_clients, num_jobs, testMetadata + ) self.assertEqual(2, len(actual)) tps_sample = [x for x in actual if x.metric == 'tps_array'][0] diff --git a/tests/linux_benchmarks/resnet_benchmark_test.py b/tests/linux_benchmarks/resnet_benchmark_test.py index f54a19ae49..f4ded633d0 100644 --- a/tests/linux_benchmarks/resnet_benchmark_test.py +++ b/tests/linux_benchmarks/resnet_benchmark_test.py @@ -15,7 +15,6 @@ import os import unittest import mock - from perfkitbenchmarker import test_util from perfkitbenchmarker.linux_benchmarks import mnist_benchmark from perfkitbenchmarker.linux_benchmarks import resnet_benchmark @@ -25,40 +24,58 @@ class ResNetBenchmarkTestCase(unittest.TestCase, test_util.SamplesTestMixin): def setUp(self): - path = os.path.join(os.path.dirname(__file__), '..', 'data', - 'resnet_output.txt') + path = os.path.join( + os.path.dirname(__file__), '..', 'data', 'resnet_output.txt' + ) with open(path) as fp: self.contents = fp.read() - self.metadata_input = {'num_examples_per_epoch': 1251.1, - 'train_batch_size': 1024} - self.metadata_output = {'epoch': 4.000479577971386, 'elapsed_seconds': 0, - 'train_batch_size': 1024, - 'num_examples_per_epoch': 1251.1, 'step': 5005} + self.metadata_input = { + 'num_examples_per_epoch': 1251.1, + 'train_batch_size': 1024, + } + self.metadata_output = { + 'epoch': 4.000479577971386, + 'elapsed_seconds': 0, + 'train_batch_size': 1024, + 'num_examples_per_epoch': 1251.1, + 'step': 5005, + } @mock.patch('time.time', mock.MagicMock(return_value=0)) def testTrainResults(self): samples = mnist_benchmark.MakeSamplesFromTrainOutput( - self.metadata_input, self.contents, 0, 5005) + self.metadata_input, self.contents, 0, 5005 + ) golden = [ Sample('Loss', 3.6859958, '', self.metadata_output), - Sample('Global Steps Per Second', 3.6699466666666667, - 'global_steps/sec', self.metadata_output), - Sample('Examples Per Second', 3758.023333333333, - 'examples/sec', self.metadata_output) + Sample( + 'Global Steps Per Second', + 3.6699466666666667, + 'global_steps/sec', + self.metadata_output, + ), + Sample( + 'Examples Per Second', + 3758.023333333333, + 'examples/sec', + self.metadata_output, + ), ] self.assertEqual(samples, golden) @mock.patch('time.time', mock.MagicMock(return_value=0)) def testEvalResults(self): samples = resnet_benchmark.MakeSamplesFromEvalOutput( - self.metadata_input, self.contents, 0) + self.metadata_input, self.contents, 0 + ) golden = [ Sample('Eval Loss', 3.86324, '', self.metadata_output), Sample('Top 1 Accuracy', 32.751465, '%', self.metadata_output), - Sample('Top 5 Accuracy', 58.825684, '%', self.metadata_output) + Sample('Top 5 Accuracy', 58.825684, '%', self.metadata_output), ] self.assertEqual(samples, golden) + if __name__ == '__main__': unittest.main() diff --git a/tests/linux_benchmarks/roberta_mmlm_benchmark_test.py b/tests/linux_benchmarks/roberta_mmlm_benchmark_test.py index 7876ae9335..b232b665b2 100644 --- a/tests/linux_benchmarks/roberta_mmlm_benchmark_test.py +++ b/tests/linux_benchmarks/roberta_mmlm_benchmark_test.py @@ -15,26 +15,28 @@ import os import unittest import mock - from perfkitbenchmarker import test_util from perfkitbenchmarker.linux_benchmarks import roberta_mmlm_benchmark from perfkitbenchmarker.sample import Sample -class RobertaMmlmBenchmarkTestCase(unittest.TestCase, - test_util.SamplesTestMixin): +class RobertaMmlmBenchmarkTestCase( + unittest.TestCase, test_util.SamplesTestMixin +): def setUp(self): super(RobertaMmlmBenchmarkTestCase, self).setUp() - path = os.path.join(os.path.dirname(__file__), '..', 'data', - 'roberta_mmlm_output.txt') + path = os.path.join( + os.path.dirname(__file__), '..', 'data', 'roberta_mmlm_output.txt' + ) with open(path) as fp: self.contents = fp.read() @mock.patch('time.time', mock.MagicMock(return_value=1550279509.59)) def testTrainResults(self): samples = roberta_mmlm_benchmark.MakeSamplesFromOutput( - {'num_accelerators': 16}, self.contents) + {'num_accelerators': 16}, self.contents + ) self.assertEqual(436, len(samples)) golden = Sample( metric='wps', @@ -61,7 +63,8 @@ def testTrainResults(self): 'wall': '28', 'train_wall': '27', }, - timestamp=1550279509.59) + timestamp=1550279509.59, + ) print(samples[0]) print(golden) self.assertEqual(golden, samples[0]) diff --git a/tests/linux_benchmarks/scimark2_benchmark_test.py b/tests/linux_benchmarks/scimark2_benchmark_test.py index fae0d8da58..b1b98b0fd2 100644 --- a/tests/linux_benchmarks/scimark2_benchmark_test.py +++ b/tests/linux_benchmarks/scimark2_benchmark_test.py @@ -69,34 +69,81 @@ } EXPECTED_RESULT_C = [ - sample.Sample(metric='Composite Score', value=1596.04, - unit='Mflops', metadata=EXPECTED_C_METADATA), - sample.Sample(metric='FFT (N=1024)', value=1568.64, - unit='Mflops', metadata=EXPECTED_C_METADATA), - sample.Sample(metric='SOR (100 x 100)', value=1039.98, - unit='Mflops', metadata=EXPECTED_C_METADATA), - sample.Sample(metric='MonteCarlo', value=497.64, - unit='Mflops', metadata=EXPECTED_C_METADATA), - sample.Sample(metric='Sparse matmult (N=1000, nz=5000)', value=1974.39, - unit='Mflops', metadata=EXPECTED_C_METADATA), - sample.Sample(metric='LU (M=100, N=100)', value=2899.56, - unit='Mflops', metadata=EXPECTED_C_METADATA), + sample.Sample( + metric='Composite Score', + value=1596.04, + unit='Mflops', + metadata=EXPECTED_C_METADATA, + ), + sample.Sample( + metric='FFT (N=1024)', + value=1568.64, + unit='Mflops', + metadata=EXPECTED_C_METADATA, + ), + sample.Sample( + metric='SOR (100 x 100)', + value=1039.98, + unit='Mflops', + metadata=EXPECTED_C_METADATA, + ), + sample.Sample( + metric='MonteCarlo', + value=497.64, + unit='Mflops', + metadata=EXPECTED_C_METADATA, + ), + sample.Sample( + metric='Sparse matmult (N=1000, nz=5000)', + value=1974.39, + unit='Mflops', + metadata=EXPECTED_C_METADATA, + ), + sample.Sample( + metric='LU (M=100, N=100)', + value=2899.56, + unit='Mflops', + metadata=EXPECTED_C_METADATA, + ), ] EXPECTED_RESULT_JAVA = [ - sample.Sample(metric='Composite Score', value=1716.3662351463677, - unit='Mflops', metadata=EXPECTED_JAVA_METADATA), - sample.Sample(metric='FFT (1024)', value=1000.1380057152871, - unit='Mflops', metadata=EXPECTED_JAVA_METADATA), - sample.Sample(metric='SOR (100x100)', value=1353.1987180103354, - unit='Mflops', metadata=EXPECTED_JAVA_METADATA), - sample.Sample(metric='Monte Carlo', value=727.7138820888014, - unit='Mflops', metadata=EXPECTED_JAVA_METADATA), - sample.Sample(metric='Sparse matmult (N=1000, nz=5000)', - value=1495.40225150659, unit='Mflops', - metadata=EXPECTED_JAVA_METADATA), - sample.Sample(metric='LU (100x100)', value=4005.3783184108247, - unit='Mflops', metadata=EXPECTED_JAVA_METADATA), + sample.Sample( + metric='Composite Score', + value=1716.3662351463677, + unit='Mflops', + metadata=EXPECTED_JAVA_METADATA, + ), + sample.Sample( + metric='FFT (1024)', + value=1000.1380057152871, + unit='Mflops', + metadata=EXPECTED_JAVA_METADATA, + ), + sample.Sample( + metric='SOR (100x100)', + value=1353.1987180103354, + unit='Mflops', + metadata=EXPECTED_JAVA_METADATA, + ), + sample.Sample( + metric='Monte Carlo', + value=727.7138820888014, + unit='Mflops', + metadata=EXPECTED_JAVA_METADATA, + ), + sample.Sample( + metric='Sparse matmult (N=1000, nz=5000)', + value=1495.40225150659, + unit='Mflops', + metadata=EXPECTED_JAVA_METADATA, + ), + sample.Sample( + metric='LU (100x100)', + value=4005.3783184108247, + unit='Mflops', + metadata=EXPECTED_JAVA_METADATA, + ), ] @@ -113,8 +160,8 @@ def testParseResultsJava(self): def testParseResultsCombined(self): samples = scimark2_benchmark.ParseResults(TEST_OUTPUT_C + TEST_OUTPUT_JAVA) self.assertSampleListsEqualUpToTimestamp( - samples, - EXPECTED_RESULT_C + EXPECTED_RESULT_JAVA) + samples, EXPECTED_RESULT_C + EXPECTED_RESULT_JAVA + ) if __name__ == '__main__': diff --git a/tests/linux_benchmarks/specsfs2014_test.py b/tests/linux_benchmarks/specsfs2014_test.py index e30834b2b9..2de443632d 100644 --- a/tests/linux_benchmarks/specsfs2014_test.py +++ b/tests/linux_benchmarks/specsfs2014_test.py @@ -22,43 +22,46 @@ from perfkitbenchmarker.linux_benchmarks import specsfs2014_benchmark -class SpecSfs2014BenchmarkTestCase(unittest.TestCase, - test_util.SamplesTestMixin): +class SpecSfs2014BenchmarkTestCase( + unittest.TestCase, test_util.SamplesTestMixin +): def setUp(self): self.maxDiff = None # Load data - path = os.path.join(os.path.dirname(__file__), - '..', 'data', - 'specsfs2014_results.xml') + path = os.path.join( + os.path.dirname(__file__), '..', 'data', 'specsfs2014_results.xml' + ) with open(path) as fp: self.specsfs2014_xml_results = fp.read() def testSpecSfs2014Parsing(self): samples = specsfs2014_benchmark._ParseSpecSfsOutput( - self.specsfs2014_xml_results) + self.specsfs2014_xml_results + ) expected_metadata = { 'client data set size (MiB)': '4296', 'starting data set size (MiB)': '4296', - 'maximum file space (MiB)': '4687', 'file size (KB)': '16', - 'run time (seconds)': '300', 'benchmark': 'SWBUILD', - 'business_metric': '1', 'op rate (ops/s)': '500.00', - 'processes per client': '5', 'initial file space (MiB)': '4296', - 'valid_run': True + 'maximum file space (MiB)': '4687', + 'file size (KB)': '16', + 'run time (seconds)': '300', + 'benchmark': 'SWBUILD', + 'business_metric': '1', + 'op rate (ops/s)': '500.00', + 'processes per client': '5', + 'initial file space (MiB)': '4296', + 'valid_run': True, } expected_samples = [ + sample.Sample('achieved rate', 500.05, 'ops/s', expected_metadata), sample.Sample( - 'achieved rate', 500.05, 'ops/s', expected_metadata), - sample.Sample( - 'average latency', 1.48, 'milliseconds', expected_metadata), - sample.Sample( - 'overall throughput', 6463.95, 'KB/s', expected_metadata), - sample.Sample( - 'read throughput', 3204.78, 'KB/s', expected_metadata), - sample.Sample( - 'write throughput', 3259.17, 'KB/s', expected_metadata) + 'average latency', 1.48, 'milliseconds', expected_metadata + ), + sample.Sample('overall throughput', 6463.95, 'KB/s', expected_metadata), + sample.Sample('read throughput', 3204.78, 'KB/s', expected_metadata), + sample.Sample('write throughput', 3259.17, 'KB/s', expected_metadata), ] self.assertSampleListsEqualUpToTimestamp(expected_samples, samples) diff --git a/tests/linux_benchmarks/stencil2d_benchmark_test.py b/tests/linux_benchmarks/stencil2d_benchmark_test.py index e57c8b6892..10eac34501 100644 --- a/tests/linux_benchmarks/stencil2d_benchmark_test.py +++ b/tests/linux_benchmarks/stencil2d_benchmark_test.py @@ -17,7 +17,6 @@ import unittest from absl import flags import mock - from perfkitbenchmarker import flag_util from perfkitbenchmarker.linux_benchmarks import stencil2d_benchmark @@ -28,15 +27,17 @@ class Stencil2DBenchmarkTestCase(unittest.TestCase): def setUp(self): - path = os.path.join(os.path.dirname(__file__), '../data', - 'stencil2d_output.txt') + path = os.path.join( + os.path.dirname(__file__), '../data', 'stencil2d_output.txt' + ) with open(path) as fp: self.test_output = fp.read() def testMakeSampleFromOutput(self): testMetadata = {'foo': 'bar'} actual = stencil2d_benchmark._MakeSamplesFromStencilOutput( - self.test_output, testMetadata) + self.test_output, testMetadata + ) results_dict = {x.metric: x for x in actual} stencil_dp_median_results = results_dict['Stencil2D DP median'] @@ -63,28 +64,36 @@ def testMakeSampleFromOutput(self): self.assertEqual('GFLOPS', stencil_sp_stddev_results.unit) self.assertEqual(testMetadata, stencil_sp_stddev_results.metadata) - @mock.patch(('perfkitbenchmarker.linux_packages.' - 'nvidia_driver.GetPeerToPeerTopology')) - @mock.patch(('perfkitbenchmarker.linux_packages.' - 'nvidia_driver.GetGpuType')) - @mock.patch(('perfkitbenchmarker.linux_packages.' - 'nvidia_driver.GetDriverVersion')) - @mock.patch(('perfkitbenchmarker.linux_packages.' - 'nvidia_driver.QueryNumberOfGpus')) - @mock.patch(('perfkitbenchmarker.linux_packages.' - 'nvidia_driver.QueryGpuClockSpeed')) - @mock.patch(('perfkitbenchmarker.linux_packages.' - 'nvidia_driver.QueryAutoboostPolicy')) - @mock.patch(('perfkitbenchmarker.linux_benchmarks.' - 'stencil2d_benchmark._RunSingleIteration')) - def testRun(self, - run_single_iteration_mock, - query_autoboost_policy_mock, - query_gpu_clock_speed_mock, - query_number_of_gpus_mock, - get_driver_version_mock, - get_gpu_type, - get_peer_to_peer_topology): + @mock.patch( + 'perfkitbenchmarker.linux_packages.nvidia_driver.GetPeerToPeerTopology' + ) + @mock.patch('perfkitbenchmarker.linux_packages.nvidia_driver.GetGpuType') + @mock.patch( + 'perfkitbenchmarker.linux_packages.nvidia_driver.GetDriverVersion' + ) + @mock.patch( + 'perfkitbenchmarker.linux_packages.nvidia_driver.QueryNumberOfGpus' + ) + @mock.patch( + 'perfkitbenchmarker.linux_packages.nvidia_driver.QueryGpuClockSpeed' + ) + @mock.patch( + 'perfkitbenchmarker.linux_packages.nvidia_driver.QueryAutoboostPolicy' + ) + @mock.patch(( + 'perfkitbenchmarker.linux_benchmarks.' + 'stencil2d_benchmark._RunSingleIteration' + )) + def testRun( + self, + run_single_iteration_mock, + query_autoboost_policy_mock, + query_gpu_clock_speed_mock, + query_number_of_gpus_mock, + get_driver_version_mock, + get_gpu_type, + get_peer_to_peer_topology, + ): get_gpu_type.return_value = 'k80' get_driver_version_mock.return_value = '123.45' query_number_of_gpus_mock.return_value = 8 @@ -95,8 +104,9 @@ def testRun(self, } benchmark_spec = mock.MagicMock() problem_sizes = [2, 3, 4] - stencil2d_benchmark.FLAGS.stencil2d_problem_sizes = ( - flag_util.IntegerList(problem_sizes)) + stencil2d_benchmark.FLAGS.stencil2d_problem_sizes = flag_util.IntegerList( + problem_sizes + ) expected_calls = [ mock.call(mock.ANY, size, mock.ANY, mock.ANY, mock.ANY) diff --git a/tests/linux_benchmarks/stress_ng_benchmark_test.py b/tests/linux_benchmarks/stress_ng_benchmark_test.py index 085500bbcd..fce7925332 100644 --- a/tests/linux_benchmarks/stress_ng_benchmark_test.py +++ b/tests/linux_benchmarks/stress_ng_benchmark_test.py @@ -16,7 +16,6 @@ import unittest import mock - from perfkitbenchmarker.linux_benchmarks import stress_ng_benchmark @@ -29,19 +28,16 @@ def setUp(self): self.addCleanup(p.stop) path = os.path.join( - os.path.dirname(__file__), '../data', 'stress_ng_output.txt') + os.path.dirname(__file__), '../data', 'stress_ng_output.txt' + ) with open(path) as fp: self.contents = fp.read() def testParseStressngResult(self): - metadata = { - 'duration_sec': 10, - 'threads': 16 - } + metadata = {'duration_sec': 10, 'threads': 16} samples = [] - result = stress_ng_benchmark._ParseStressngResult( - metadata, self.contents) + result = stress_ng_benchmark._ParseStressngResult(metadata, self.contents) self.assertIsNotNone(result) samples.append(result) @@ -49,9 +45,7 @@ def testParseStressngResult(self): metadata = samples[0].metadata # Test metric and value - expected = { - 'context': 4485.820000 - } + expected = {'context': 4485.820000} for sample in samples: self.assertEqual(expected[sample.metric], sample.value) @@ -61,7 +55,10 @@ def testParseStressngResult(self): def testGeoMean(self): floats = [1.0, 3.0, 5.0] - self.assertAlmostEqual(stress_ng_benchmark._GeoMeanOverflow(floats), - 2.466212074) + self.assertAlmostEqual( + stress_ng_benchmark._GeoMeanOverflow(floats), 2.466212074 + ) + + if __name__ == '__main__': unittest.main() diff --git a/tests/linux_benchmarks/suspend_resume_benchmark_test.py b/tests/linux_benchmarks/suspend_resume_benchmark_test.py index af14650422..0b434bc019 100644 --- a/tests/linux_benchmarks/suspend_resume_benchmark_test.py +++ b/tests/linux_benchmarks/suspend_resume_benchmark_test.py @@ -1,9 +1,9 @@ """Tests for suspend_resume_benchmark.""" + import itertools import time import unittest from absl.testing import parameterized - import mock from perfkitbenchmarker import sample from perfkitbenchmarker import test_util @@ -30,18 +30,21 @@ def create_mock_vm( return vm -class SuspendResumeBenchmarkTest(pkb_common_test_case.PkbCommonTestCase, - test_util.SamplesTestMixin): +class SuspendResumeBenchmarkTest( + pkb_common_test_case.PkbCommonTestCase, test_util.SamplesTestMixin +): def setUp(self): super(SuspendResumeBenchmarkTest, self).setUp() time.time() self.enter_context( - mock.patch.object(time, 'time', side_effect=itertools.cycle([0, 52]))) + mock.patch.object(time, 'time', side_effect=itertools.cycle([0, 52])) + ) @parameterized.named_parameters( dict(testcase_name='get_time_to_suspend', operation='Suspend'), - dict(testcase_name='get_time_to_resume', operation='Resume')) + dict(testcase_name='get_time_to_resume', operation='Resume'), + ) def testMeasureOperation(self, operation: str): """Unit test for Suspend and Resume functions.""" num_vms = 3 @@ -51,7 +54,8 @@ def testMeasureOperation(self, operation: str): # Call suspend function on mock vms - generates actual samples for the test. actual_samples = getattr(suspend_resume_benchmark, f'GetTimeTo{operation}')( - vms_to_test) + vms_to_test + ) # Create expected samples from mock vms. expected_times = [50, 51, 52] @@ -65,23 +69,34 @@ def testMeasureOperation(self, operation: str): for i in range(num_vms): expected_samples.append( - sample.Sample(f'{operation} Time', expected_times[i], 'seconds', { - 'machine_instance': i, - 'num_vms': num_vms, - 'os_type': f'linux{i}', - })) + sample.Sample( + f'{operation} Time', + expected_times[i], + 'seconds', + { + 'machine_instance': i, + 'num_vms': num_vms, + 'os_type': f'linux{i}', + }, + ) + ) expected_samples.append( - sample.Sample(f'Cluster {operation} Time', expected_cluster_time, - 'seconds', expected_cluster_metadata)) + sample.Sample( + f'Cluster {operation} Time', + expected_cluster_time, + 'seconds', + expected_cluster_metadata, + ) + ) # Assert Suspend() function is called on vms. for vm in vms_to_test: getattr(vm, f'{operation}').assert_called() # Assert actual and expected samples are equal. - self.assertSampleListsEqualUpToTimestamp(actual_samples, - expected_samples) + self.assertSampleListsEqualUpToTimestamp(actual_samples, expected_samples) + if __name__ == '__main__': unittest.main() diff --git a/tests/linux_benchmarks/sysbench_benchmark_test.py b/tests/linux_benchmarks/sysbench_benchmark_test.py index 899f1ad277..631c35877a 100644 --- a/tests/linux_benchmarks/sysbench_benchmark_test.py +++ b/tests/linux_benchmarks/sysbench_benchmark_test.py @@ -94,32 +94,98 @@ def testScaleUpClient(self): ) -class MySQLServiceBenchmarkTestCase(unittest.TestCase, - test_util.SamplesTestMixin): +class MySQLServiceBenchmarkTestCase( + unittest.TestCase, test_util.SamplesTestMixin +): def setUp(self): super().setUp() - path = os.path.join(os.path.dirname(__file__), '..', 'data', - 'sysbench-output-sample.txt') + path = os.path.join( + os.path.dirname(__file__), '..', 'data', 'sysbench-output-sample.txt' + ) with open(path) as fp: self.contents = fp.read() def testParseSysbenchResult(self): metadata = {} results = sysbench_benchmark._ParseSysbenchTimeSeries( - self.contents, metadata) + self.contents, metadata + ) logging.info('results are, %s', results) expected_results = [ - sample.Sample('tps_array', -1, 'tps', {'tps': [ - 1012.86, 1006.64, 1022.3, 1016.16, 1009.03, 1016.99, 1010.0, 1018.0, - 1002.01, 998.49, 959.52, 913.49, 936.98, 916.01, 957.96]}), - sample.Sample('latency_array', -1, 'ms', {'latency': [ - 28.67, 64.47, 38.94, 44.98, 89.16, 29.72, 106.75, 46.63, 116.8, - 41.85, 27.17, 104.84, 58.92, 75.82, 73.13]}), - sample.Sample('qps_array', -1, 'qps', {'qps': [ - 20333.18, 20156.38, 20448.49, 20334.15, 20194.07, 20331.31, - 20207.00, 20348.96, 20047.11, 19972.86, 19203.97, 18221.83, - 18689.14, 18409.68, 19155.63]})] + sample.Sample( + 'tps_array', + -1, + 'tps', + { + 'tps': [ + 1012.86, + 1006.64, + 1022.3, + 1016.16, + 1009.03, + 1016.99, + 1010.0, + 1018.0, + 1002.01, + 998.49, + 959.52, + 913.49, + 936.98, + 916.01, + 957.96, + ] + }, + ), + sample.Sample( + 'latency_array', + -1, + 'ms', + { + 'latency': [ + 28.67, + 64.47, + 38.94, + 44.98, + 89.16, + 29.72, + 106.75, + 46.63, + 116.8, + 41.85, + 27.17, + 104.84, + 58.92, + 75.82, + 73.13, + ] + }, + ), + sample.Sample( + 'qps_array', + -1, + 'qps', + { + 'qps': [ + 20333.18, + 20156.38, + 20448.49, + 20334.15, + 20194.07, + 20331.31, + 20207.00, + 20348.96, + 20047.11, + 19972.86, + 19203.97, + 18221.83, + 18689.14, + 18409.68, + 19155.63, + ] + }, + ), + ] self.assertSampleListsEqualUpToTimestamp(results, expected_results) diff --git a/tests/linux_benchmarks/sysbench_memory_benchmark_test.py b/tests/linux_benchmarks/sysbench_memory_benchmark_test.py index ae0b2c7970..723293b8c3 100644 --- a/tests/linux_benchmarks/sysbench_memory_benchmark_test.py +++ b/tests/linux_benchmarks/sysbench_memory_benchmark_test.py @@ -21,20 +21,26 @@ from perfkitbenchmarker.linux_benchmarks import sysbench_memory_benchmark -class SysBenchMemoryBenchmarkTestCase(unittest.TestCase, - test_util.SamplesTestMixin): +class SysBenchMemoryBenchmarkTestCase( + unittest.TestCase, test_util.SamplesTestMixin +): def setUp(self): super(SysBenchMemoryBenchmarkTestCase, self).setUp() - path = os.path.join(os.path.dirname(__file__), '..', 'data', - 'sysbench-memory-output-sample.txt') + path = os.path.join( + os.path.dirname(__file__), + '..', + 'data', + 'sysbench-memory-output-sample.txt', + ) with open(path) as fp: self.contents = fp.read() def testParseSysbenchMemoryResult(self): metadata = {} results = sysbench_memory_benchmark.GenerateMetricsForSysbenchMemoryOutput( - self.contents, metadata) + self.contents, metadata + ) logging.info('results are, %s', results) expected_results = [ sample.Sample('Minimum_throughput', 3803.18, 'MiB/sec', {}), @@ -43,8 +49,8 @@ def testParseSysbenchMemoryResult(self): sample.Sample('vcpu_variation', 774.26, 'MiB/sec', {}), sample.Sample('Range_throughput', 1549.88, 'MiB/sec', {}), sample.Sample('Difference_from_all_max', 3101.12, 'MiB/sec', {}), - sample.Sample('Difference_from_all_median', -1.36, 'MiB/sec', {}) - ] + sample.Sample('Difference_from_all_median', -1.36, 'MiB/sec', {}), + ] self.assertSampleListsEqualUpToTimestamp(results, expected_results) diff --git a/tests/linux_benchmarks/t2t_benchmark_test.py b/tests/linux_benchmarks/t2t_benchmark_test.py index d5c19a83f7..97c4f90aae 100644 --- a/tests/linux_benchmarks/t2t_benchmark_test.py +++ b/tests/linux_benchmarks/t2t_benchmark_test.py @@ -20,148 +20,122 @@ from perfkitbenchmarker.sample import Sample -class Tensor2TensorBenchmarkTestCase(unittest.TestCase, - test_util.SamplesTestMixin): +class Tensor2TensorBenchmarkTestCase( + unittest.TestCase, test_util.SamplesTestMixin +): @mock.patch('time.time', mock.MagicMock(return_value=0)) def testT2TTpuOutput(self): self.maxDiff = None path = os.path.join( - os.path.dirname(__file__), '..', 'data', 't2t_tpu_output.txt') + os.path.dirname(__file__), '..', 'data', 't2t_tpu_output.txt' + ) with open(path) as fp: t2t_contents = fp.read() - samples = t2t_benchmark._MakeSamplesFromOutput({ - 'use_tpu': True - }, t2t_contents) + samples = t2t_benchmark._MakeSamplesFromOutput( + {'use_tpu': True}, t2t_contents + ) golden = [ Sample( metric='Global Steps Per Second', value=1.85777, unit='global_steps/sec', - metadata={ - 'use_tpu': True, - 'index': 0 - }, - timestamp=0), + metadata={'use_tpu': True, 'index': 0}, + timestamp=0, + ), Sample( metric='Global Steps Per Second', value=5.06989, unit='global_steps/sec', - metadata={ - 'use_tpu': True, - 'index': 1 - }, - timestamp=0), + metadata={'use_tpu': True, 'index': 1}, + timestamp=0, + ), Sample( metric='Examples Per Second', value=118.897, unit='examples/sec', - metadata={ - 'use_tpu': True, - 'index': 0 - }, - timestamp=0), + metadata={'use_tpu': True, 'index': 0}, + timestamp=0, + ), Sample( metric='Examples Per Second', value=324.473, unit='examples/sec', - metadata={ - 'use_tpu': True, - 'index': 1 - }, - timestamp=0), + metadata={'use_tpu': True, 'index': 1}, + timestamp=0, + ), Sample( metric='Eval Loss', value=3.9047337, unit='', - metadata={ - 'use_tpu': True, - 'step': 1000 - }, - timestamp=0), + metadata={'use_tpu': True, 'step': 1000}, + timestamp=0, + ), Sample( metric='Accuracy', value=32.064167, unit='%', - metadata={ - 'use_tpu': True, - 'step': 1000 - }, - timestamp=0), + metadata={'use_tpu': True, 'step': 1000}, + timestamp=0, + ), Sample( metric='Accuracy Per Sequence', value=0.0, unit='%', - metadata={ - 'use_tpu': True, - 'step': 1000 - }, - timestamp=0), + metadata={'use_tpu': True, 'step': 1000}, + timestamp=0, + ), Sample( metric='Negative Log Perplexity', value=-4.501835, unit='perplexity', - metadata={ - 'use_tpu': True, - 'step': 1000 - }, - timestamp=0), + metadata={'use_tpu': True, 'step': 1000}, + timestamp=0, + ), Sample( metric='Top 5 Accuracy', value=50.96436, unit='%', - metadata={ - 'use_tpu': True, - 'step': 1000 - }, - timestamp=0), + metadata={'use_tpu': True, 'step': 1000}, + timestamp=0, + ), Sample( metric='Eval Loss', value=3.7047337, unit='', - metadata={ - 'use_tpu': True, - 'step': 1200 - }, - timestamp=0), + metadata={'use_tpu': True, 'step': 1200}, + timestamp=0, + ), Sample( metric='Accuracy', value=33.064167, unit='%', - metadata={ - 'use_tpu': True, - 'step': 1200 - }, - timestamp=0), + metadata={'use_tpu': True, 'step': 1200}, + timestamp=0, + ), Sample( metric='Accuracy Per Sequence', value=0.0, unit='%', - metadata={ - 'use_tpu': True, - 'step': 1200 - }, - timestamp=0), + metadata={'use_tpu': True, 'step': 1200}, + timestamp=0, + ), Sample( metric='Negative Log Perplexity', value=-4.101835, unit='perplexity', - metadata={ - 'use_tpu': True, - 'step': 1200 - }, - timestamp=0), + metadata={'use_tpu': True, 'step': 1200}, + timestamp=0, + ), Sample( metric='Top 5 Accuracy', value=55.96436, unit='%', - metadata={ - 'use_tpu': True, - 'step': 1200 - }, - timestamp=0) + metadata={'use_tpu': True, 'step': 1200}, + timestamp=0, + ), ] self.assertEqual(samples, golden) @@ -169,141 +143,114 @@ def testT2TTpuOutput(self): def testT2TGpuOutput(self): self.maxDiff = None path = os.path.join( - os.path.dirname(__file__), '..', 'data', 't2t_gpu_output.txt') + os.path.dirname(__file__), '..', 'data', 't2t_gpu_output.txt' + ) with open(path) as fp: t2t_contents = fp.read() - samples = t2t_benchmark._MakeSamplesFromOutput({ - 'use_tpu': False - }, t2t_contents) + samples = t2t_benchmark._MakeSamplesFromOutput( + {'use_tpu': False}, t2t_contents + ) golden = [ Sample( metric='Global Steps Per Second', value=3.04983, unit='global_steps/sec', - metadata={ - 'index': 0, - 'use_tpu': False - }, - timestamp=0), + metadata={'index': 0, 'use_tpu': False}, + timestamp=0, + ), Sample( metric='Global Steps Per Second', value=4.12771, unit='global_steps/sec', - metadata={ - 'index': 1, - 'use_tpu': False - }, - timestamp=0), + metadata={'index': 1, 'use_tpu': False}, + timestamp=0, + ), Sample( metric='Global Steps Per Second', value=4.11027, unit='global_steps/sec', - metadata={ - 'index': 2, - 'use_tpu': False - }, - timestamp=0), + metadata={'index': 2, 'use_tpu': False}, + timestamp=0, + ), Sample( metric='Global Steps Per Second', value=4.10924, unit='global_steps/sec', - metadata={ - 'index': 3, - 'use_tpu': False - }, - timestamp=0), + metadata={'index': 3, 'use_tpu': False}, + timestamp=0, + ), Sample( metric='Global Steps Per Second', value=4.12186, unit='global_steps/sec', - metadata={ - 'index': 4, - 'use_tpu': False - }, - timestamp=0), + metadata={'index': 4, 'use_tpu': False}, + timestamp=0, + ), Sample( metric='Global Steps Per Second', value=4.08434, unit='global_steps/sec', - metadata={ - 'index': 5, - 'use_tpu': False - }, - timestamp=0), + metadata={'index': 5, 'use_tpu': False}, + timestamp=0, + ), Sample( metric='Global Steps Per Second', value=4.10174, unit='global_steps/sec', - metadata={ - 'index': 6, - 'use_tpu': False - }, - timestamp=0), + metadata={'index': 6, 'use_tpu': False}, + timestamp=0, + ), Sample( metric='Global Steps Per Second', value=4.11809, unit='global_steps/sec', - metadata={ - 'index': 7, - 'use_tpu': False - }, - timestamp=0), + metadata={'index': 7, 'use_tpu': False}, + timestamp=0, + ), Sample( metric='Global Steps Per Second', value=4.10496, unit='global_steps/sec', - metadata={ - 'index': 8, - 'use_tpu': False - }, - timestamp=0), + metadata={'index': 8, 'use_tpu': False}, + timestamp=0, + ), Sample( metric='Eval Loss', value=7.2263174, unit='', - metadata={ - 'use_tpu': False, - 'step': 1000 - }, - timestamp=0), + metadata={'use_tpu': False, 'step': 1000}, + timestamp=0, + ), Sample( metric='Accuracy', value=13.972055999999998, unit='%', - metadata={ - 'use_tpu': False, - 'step': 1000 - }, - timestamp=0), + metadata={'use_tpu': False, 'step': 1000}, + timestamp=0, + ), Sample( metric='Accuracy Per Sequence', value=0.0, unit='%', - metadata={ - 'use_tpu': False, - 'step': 1000 - }, - timestamp=0), + metadata={'use_tpu': False, 'step': 1000}, + timestamp=0, + ), Sample( metric='Negative Log Perplexity', value=-7.2263174, unit='perplexity', - metadata={ - 'use_tpu': False, - 'step': 1000 - }, - timestamp=0), + metadata={'use_tpu': False, 'step': 1000}, + timestamp=0, + ), Sample( metric='Top 5 Accuracy', value=24.800399000000002, unit='%', - metadata={ - 'use_tpu': False, - 'step': 1000 - }, - timestamp=0) + metadata={'use_tpu': False, 'step': 1000}, + timestamp=0, + ), ] self.assertEqual(samples, golden) diff --git a/tests/linux_benchmarks/tensorflow_benchmark_test.py b/tests/linux_benchmarks/tensorflow_benchmark_test.py index 7de28a0525..b3dfe40fb6 100644 --- a/tests/linux_benchmarks/tensorflow_benchmark_test.py +++ b/tests/linux_benchmarks/tensorflow_benchmark_test.py @@ -21,7 +21,8 @@ class TensorflowBenchmarkBatchSizesTestCase( - pkb_common_test_case.PkbCommonTestCase): + pkb_common_test_case.PkbCommonTestCase +): def setUp(self): super(TensorflowBenchmarkBatchSizesTestCase, self).setUp() @@ -29,121 +30,145 @@ def setUp(self): def testFlagOverridesDefaultBatchSize(self): batch_size = tensorflow_benchmark._GetBatchSizes( - 'alexnet', tensorflow_benchmark.CPU) + 'alexnet', tensorflow_benchmark.CPU + ) self.assertEqual([99], batch_size) class TensorflowBenchmarkDefaultBatchSizesTestCase( - pkb_common_test_case.PkbCommonTestCase): + pkb_common_test_case.PkbCommonTestCase +): def testUnknownGpuTypeReturnsDefaultBatchSize(self): batch_size = tensorflow_benchmark._GetBatchSizes( - 'alexnet', 'unknown_gpu_type') + 'alexnet', 'unknown_gpu_type' + ) self.assertEqual([64], batch_size) def testUnknownModelReturnsDefaultBatchSize(self): batch_size = tensorflow_benchmark._GetBatchSizes( - 'unknown_model', tensorflow_benchmark.CPU) + 'unknown_model', tensorflow_benchmark.CPU + ) self.assertEqual([64], batch_size) def testCpuAlexnetDefault(self): batch_size = tensorflow_benchmark._GetBatchSizes( - 'alexnet', tensorflow_benchmark.CPU) + 'alexnet', tensorflow_benchmark.CPU + ) self.assertEqual([512], batch_size) def testCpuInception3Default(self): batch_size = tensorflow_benchmark._GetBatchSizes( - 'inception3', tensorflow_benchmark.CPU) + 'inception3', tensorflow_benchmark.CPU + ) self.assertEqual([64], batch_size) def testCpuResnet50Default(self): batch_size = tensorflow_benchmark._GetBatchSizes( - 'resnet50', tensorflow_benchmark.CPU) + 'resnet50', tensorflow_benchmark.CPU + ) self.assertEqual([64], batch_size) def testCpuResnet152Default(self): batch_size = tensorflow_benchmark._GetBatchSizes( - 'resnet152', tensorflow_benchmark.CPU) + 'resnet152', tensorflow_benchmark.CPU + ) self.assertEqual([32], batch_size) def testCpuVgg16Default(self): batch_size = tensorflow_benchmark._GetBatchSizes( - 'vgg16', tensorflow_benchmark.CPU) + 'vgg16', tensorflow_benchmark.CPU + ) self.assertEqual([32], batch_size) def testK80AlexnetDefault(self): batch_size = tensorflow_benchmark._GetBatchSizes( - 'alexnet', tensorflow_benchmark.NVIDIA_TESLA_K80) + 'alexnet', tensorflow_benchmark.NVIDIA_TESLA_K80 + ) self.assertEqual([512], batch_size) def testK80Inception3Default(self): batch_size = tensorflow_benchmark._GetBatchSizes( - 'inception3', tensorflow_benchmark.NVIDIA_TESLA_K80) + 'inception3', tensorflow_benchmark.NVIDIA_TESLA_K80 + ) self.assertEqual([64], batch_size) def testK80Resnet50Default(self): batch_size = tensorflow_benchmark._GetBatchSizes( - 'resnet50', tensorflow_benchmark.NVIDIA_TESLA_K80) + 'resnet50', tensorflow_benchmark.NVIDIA_TESLA_K80 + ) self.assertEqual([64], batch_size) def testK80Resnet152Default(self): batch_size = tensorflow_benchmark._GetBatchSizes( - 'resnet152', tensorflow_benchmark.NVIDIA_TESLA_K80) + 'resnet152', tensorflow_benchmark.NVIDIA_TESLA_K80 + ) self.assertEqual([32], batch_size) def testK80Vgg16Default(self): batch_size = tensorflow_benchmark._GetBatchSizes( - 'vgg16', tensorflow_benchmark.NVIDIA_TESLA_K80) + 'vgg16', tensorflow_benchmark.NVIDIA_TESLA_K80 + ) self.assertEqual([32], batch_size) def testP100AlexnetDefault(self): batch_size = tensorflow_benchmark._GetBatchSizes( - 'alexnet', tensorflow_benchmark.NVIDIA_TESLA_P100) + 'alexnet', tensorflow_benchmark.NVIDIA_TESLA_P100 + ) self.assertEqual([512], batch_size) def testP100Inception3Default(self): batch_size = tensorflow_benchmark._GetBatchSizes( - 'inception3', tensorflow_benchmark.NVIDIA_TESLA_P100) + 'inception3', tensorflow_benchmark.NVIDIA_TESLA_P100 + ) self.assertEqual([256], batch_size) def testP100Resnet50Default(self): batch_size = tensorflow_benchmark._GetBatchSizes( - 'resnet50', tensorflow_benchmark.NVIDIA_TESLA_P100) + 'resnet50', tensorflow_benchmark.NVIDIA_TESLA_P100 + ) self.assertEqual([256], batch_size) def testP100Resnet152Default(self): batch_size = tensorflow_benchmark._GetBatchSizes( - 'resnet152', tensorflow_benchmark.NVIDIA_TESLA_P100) + 'resnet152', tensorflow_benchmark.NVIDIA_TESLA_P100 + ) self.assertEqual([128], batch_size) def testP100Vgg16Default(self): batch_size = tensorflow_benchmark._GetBatchSizes( - 'vgg16', tensorflow_benchmark.NVIDIA_TESLA_P100) + 'vgg16', tensorflow_benchmark.NVIDIA_TESLA_P100 + ) self.assertEqual([128], batch_size) def testV100AlexnetDefault(self): batch_size = tensorflow_benchmark._GetBatchSizes( - 'alexnet', tensorflow_benchmark.NVIDIA_TESLA_V100) + 'alexnet', tensorflow_benchmark.NVIDIA_TESLA_V100 + ) self.assertEqual([512], batch_size) def testV100Inception3Default(self): batch_size = tensorflow_benchmark._GetBatchSizes( - 'inception3', tensorflow_benchmark.NVIDIA_TESLA_V100) + 'inception3', tensorflow_benchmark.NVIDIA_TESLA_V100 + ) self.assertEqual([256], batch_size) def testV100Resnet50Default(self): batch_size = tensorflow_benchmark._GetBatchSizes( - 'resnet50', tensorflow_benchmark.NVIDIA_TESLA_V100) + 'resnet50', tensorflow_benchmark.NVIDIA_TESLA_V100 + ) self.assertEqual([256], batch_size) def testV100Resnet152Default(self): batch_size = tensorflow_benchmark._GetBatchSizes( - 'resnet152', tensorflow_benchmark.NVIDIA_TESLA_V100) + 'resnet152', tensorflow_benchmark.NVIDIA_TESLA_V100 + ) self.assertEqual([128], batch_size) def testV100Vgg16Default(self): batch_size = tensorflow_benchmark._GetBatchSizes( - 'vgg16', tensorflow_benchmark.NVIDIA_TESLA_V100) + 'vgg16', tensorflow_benchmark.NVIDIA_TESLA_V100 + ) self.assertEqual([128], batch_size) diff --git a/tests/linux_benchmarks/tensorflow_serving_benchmark_test.py b/tests/linux_benchmarks/tensorflow_serving_benchmark_test.py index 95912054cd..4a1e5c5147 100644 --- a/tests/linux_benchmarks/tensorflow_serving_benchmark_test.py +++ b/tests/linux_benchmarks/tensorflow_serving_benchmark_test.py @@ -41,7 +41,8 @@ def testParseStdoutFromClientScript(self): benchmark_spec = mock.MagicMock() num_client_threads = 12 samples = tensorflow_serving_benchmark._MakeSamplesFromClientOutput( - benchmark_spec, self.test_output, num_client_threads) + benchmark_spec, self.test_output, num_client_threads + ) expected_metadata = { 'client_thread_count': num_client_threads, diff --git a/tests/linux_benchmarks/unixbench_benchmark_test.py b/tests/linux_benchmarks/unixbench_benchmark_test.py index e5c073e7ad..1815214a6b 100644 --- a/tests/linux_benchmarks/unixbench_benchmark_test.py +++ b/tests/linux_benchmarks/unixbench_benchmark_test.py @@ -22,12 +22,12 @@ class UnixBenchBenchmarkTestCase(unittest.TestCase, test_util.SamplesTestMixin): - maxDiff = None def setUp(self): - path = os.path.join(os.path.dirname(__file__), '..', 'data', - 'unix-bench-sample-result.txt') + path = os.path.join( + os.path.dirname(__file__), '..', 'data', 'unix-bench-sample-result.txt' + ) with open(path) as fp: self.contents = fp.read() @@ -37,106 +37,307 @@ def tearDown(self): def testParseUnixBench(self): result = unixbench_benchmark.ParseResults(self.contents) expected_result = [ - ['Dhrystone 2 using register variables', 34872897.7, 'lps', - {'num_parallel_copies': 1, 'samples': 7, 'time': '10.0s'}], - ['Double-Precision Whetstone', 4022.0, 'MWIPS', - {'num_parallel_copies': 1, 'samples': 7, 'time': '9.9s'}], - ['Execl Throughput', 4735.8, 'lps', - {'num_parallel_copies': 1, 'samples': 2, 'time': '29.8s'}], - ['File Copy 1024 bufsize 2000 maxblocks', 1294367.0, 'KBps', - {'num_parallel_copies': 1, 'samples': 2, 'time': '30.0s'}], - ['File Copy 256 bufsize 500 maxblocks', 396912.9, 'KBps', - {'num_parallel_copies': 1, 'samples': 2, 'time': '30.0s'}], - ['File Copy 4096 bufsize 8000 maxblocks', 2513158.7, 'KBps', - {'num_parallel_copies': 1, 'samples': 2, 'time': '30.0s'}], - ['Pipe Throughput', 2221775.6, 'lps', - {'num_parallel_copies': 1, 'samples': 7, 'time': '10.0s'}], - ['Pipe-based Context Switching', 369000.7, 'lps', - {'num_parallel_copies': 1, 'samples': 7, 'time': '10.0s'}], - ['Process Creation', 12587.7, 'lps', - {'num_parallel_copies': 1, 'samples': 2, 'time': '30.0s'}], - ['Shell Scripts (1 concurrent)', 8234.3, 'lpm', - {'num_parallel_copies': 1, 'samples': 2, 'time': '60.0s'}], - ['Shell Scripts (8 concurrent)', 1064.5, 'lpm', - {'num_parallel_copies': 1, 'samples': 2, 'time': '60.0s'}], - ['System Call Overhead', 4439274.5, 'lps', - {'num_parallel_copies': 1, 'samples': 7, 'time': '10.0s'}], - ['Dhrystone 2 using register variables:score', 34872897.7, '', - {'index': 2988.3, 'baseline': 116700.0, 'num_parallel_copies': 1}], - ['Double-Precision Whetstone:score', 4022.0, '', - {'index': 731.3, 'baseline': 55.0, 'num_parallel_copies': 1}], - ['Execl Throughput:score', 4735.8, '', - {'index': 1101.4, 'baseline': 43.0, 'num_parallel_copies': 1}], - ['File Copy 1024 bufsize 2000 maxblocks:score', 1294367.0, '', - {'index': 3268.6, 'baseline': 3960.0, 'num_parallel_copies': 1}], - ['File Copy 256 bufsize 500 maxblocks:score', 396912.9, '', - {'index': 2398.3, 'baseline': 1655.0, 'num_parallel_copies': 1}], - ['File Copy 4096 bufsize 8000 maxblocks:score', 2513158.7, '', - {'index': 4333.0, 'baseline': 5800.0, 'num_parallel_copies': 1}], - ['Pipe Throughput:score', 2221775.6, '', - {'index': 1786.0, 'baseline': 12440.0, 'num_parallel_copies': 1}], - ['Pipe-based Context Switching:score', 369000.7, '', - {'index': 922.5, 'baseline': 4000.0, 'num_parallel_copies': 1}], - ['Process Creation:score', 12587.7, '', - {'index': 999.0, 'baseline': 126.0, 'num_parallel_copies': 1}], - ['Shell Scripts (1 concurrent):score', 8234.3, '', - {'index': 1942.1, 'baseline': 42.4, 'num_parallel_copies': 1}], - ['Shell Scripts (8 concurrent):score', 1064.5, '', - {'index': 1774.2, 'baseline': 6.0, 'num_parallel_copies': 1}], - ['System Call Overhead:score', 4439274.5, '', - {'index': 2959.5, 'baseline': 15000.0, 'num_parallel_copies': 1}], - ['System Benchmarks Index Score', 1825.8, '', - {'num_parallel_copies': 1}], - ['Dhrystone 2 using register variables', 155391896.7, 'lps', - {'num_parallel_copies': 8, 'samples': 7, 'time': '10.0s'}], - ['Double-Precision Whetstone', 28632.5, 'MWIPS', - {'num_parallel_copies': 8, 'samples': 7, 'time': '9.8s'}], - ['Execl Throughput', 15184.0, 'lps', - {'num_parallel_copies': 8, 'samples': 2, 'time': '30.0s'}], - ['File Copy 1024 bufsize 2000 maxblocks', 985484.8, 'KBps', - {'num_parallel_copies': 8, 'samples': 2, 'time': '30.0s'}], - ['File Copy 256 bufsize 500 maxblocks', 269732.2, 'KBps', - {'num_parallel_copies': 8, 'samples': 2, 'time': '30.0s'}], - ['File Copy 4096 bufsize 8000 maxblocks', 2706156.4, 'KBps', - {'num_parallel_copies': 8, 'samples': 2, 'time': '30.0s'}], - ['Pipe Throughput', 8525928.8, 'lps', - {'num_parallel_copies': 8, 'samples': 7, 'time': '10.0s'}], - ['Pipe-based Context Switching', 1017270.4, 'lps', - {'num_parallel_copies': 8, 'samples': 7, 'time': '10.0s'}], - ['Process Creation', 31563.7, 'lps', - {'num_parallel_copies': 8, 'samples': 2, 'time': '30.0s'}], - ['Shell Scripts (1 concurrent)', 32516.3, 'lpm', - {'num_parallel_copies': 8, 'samples': 2, 'time': '60.0s'}], - ['Shell Scripts (8 concurrent)', 5012.2, 'lpm', - {'num_parallel_copies': 8, 'samples': 2, 'time': '60.0s'}], - ['System Call Overhead', 10288762.3, 'lps', - {'num_parallel_copies': 8, 'samples': 7, 'time': '10.0s'}], - ['Dhrystone 2 using register variables:score', 155391896.7, '', - {'index': 13315.5, 'baseline': 116700.0, 'num_parallel_copies': 8}], - ['Double-Precision Whetstone:score', 28632.5, '', - {'index': 5205.9, 'baseline': 55.0, 'num_parallel_copies': 8}], - ['Execl Throughput:score', 15184.0, '', - {'index': 3531.2, 'baseline': 43.0, 'num_parallel_copies': 8}], - ['File Copy 1024 bufsize 2000 maxblocks:score', 985484.8, '', - {'index': 2488.6, 'baseline': 3960.0, 'num_parallel_copies': 8}], - ['File Copy 256 bufsize 500 maxblocks:score', 269732.2, '', - {'index': 1629.8, 'baseline': 1655.0, 'num_parallel_copies': 8}], - ['File Copy 4096 bufsize 8000 maxblocks:score', 2706156.4, '', - {'index': 4665.8, 'baseline': 5800.0, 'num_parallel_copies': 8}], - ['Pipe Throughput:score', 8525928.8, '', - {'index': 6853.6, 'baseline': 12440.0, 'num_parallel_copies': 8}], - ['Pipe-based Context Switching:score', 1017270.4, '', - {'index': 2543.2, 'baseline': 4000.0, 'num_parallel_copies': 8}], - ['Process Creation:score', 31563.7, '', - {'index': 2505.1, 'baseline': 126.0, 'num_parallel_copies': 8}], - ['Shell Scripts (1 concurrent):score', 32516.3, '', - {'index': 7668.9, 'baseline': 42.4, 'num_parallel_copies': 8}], - ['Shell Scripts (8 concurrent):score', 5012.2, '', - {'index': 8353.6, 'baseline': 6.0, 'num_parallel_copies': 8}], - ['System Call Overhead:score', 10288762.3, '', - {'index': 6859.2, 'baseline': 15000.0, 'num_parallel_copies': 8}], - ['System Benchmarks Index Score', 4596.2, '', - {'num_parallel_copies': 8}]] + [ + 'Dhrystone 2 using register variables', + 34872897.7, + 'lps', + {'num_parallel_copies': 1, 'samples': 7, 'time': '10.0s'}, + ], + [ + 'Double-Precision Whetstone', + 4022.0, + 'MWIPS', + {'num_parallel_copies': 1, 'samples': 7, 'time': '9.9s'}, + ], + [ + 'Execl Throughput', + 4735.8, + 'lps', + {'num_parallel_copies': 1, 'samples': 2, 'time': '29.8s'}, + ], + [ + 'File Copy 1024 bufsize 2000 maxblocks', + 1294367.0, + 'KBps', + {'num_parallel_copies': 1, 'samples': 2, 'time': '30.0s'}, + ], + [ + 'File Copy 256 bufsize 500 maxblocks', + 396912.9, + 'KBps', + {'num_parallel_copies': 1, 'samples': 2, 'time': '30.0s'}, + ], + [ + 'File Copy 4096 bufsize 8000 maxblocks', + 2513158.7, + 'KBps', + {'num_parallel_copies': 1, 'samples': 2, 'time': '30.0s'}, + ], + [ + 'Pipe Throughput', + 2221775.6, + 'lps', + {'num_parallel_copies': 1, 'samples': 7, 'time': '10.0s'}, + ], + [ + 'Pipe-based Context Switching', + 369000.7, + 'lps', + {'num_parallel_copies': 1, 'samples': 7, 'time': '10.0s'}, + ], + [ + 'Process Creation', + 12587.7, + 'lps', + {'num_parallel_copies': 1, 'samples': 2, 'time': '30.0s'}, + ], + [ + 'Shell Scripts (1 concurrent)', + 8234.3, + 'lpm', + {'num_parallel_copies': 1, 'samples': 2, 'time': '60.0s'}, + ], + [ + 'Shell Scripts (8 concurrent)', + 1064.5, + 'lpm', + {'num_parallel_copies': 1, 'samples': 2, 'time': '60.0s'}, + ], + [ + 'System Call Overhead', + 4439274.5, + 'lps', + {'num_parallel_copies': 1, 'samples': 7, 'time': '10.0s'}, + ], + [ + 'Dhrystone 2 using register variables:score', + 34872897.7, + '', + {'index': 2988.3, 'baseline': 116700.0, 'num_parallel_copies': 1}, + ], + [ + 'Double-Precision Whetstone:score', + 4022.0, + '', + {'index': 731.3, 'baseline': 55.0, 'num_parallel_copies': 1}, + ], + [ + 'Execl Throughput:score', + 4735.8, + '', + {'index': 1101.4, 'baseline': 43.0, 'num_parallel_copies': 1}, + ], + [ + 'File Copy 1024 bufsize 2000 maxblocks:score', + 1294367.0, + '', + {'index': 3268.6, 'baseline': 3960.0, 'num_parallel_copies': 1}, + ], + [ + 'File Copy 256 bufsize 500 maxblocks:score', + 396912.9, + '', + {'index': 2398.3, 'baseline': 1655.0, 'num_parallel_copies': 1}, + ], + [ + 'File Copy 4096 bufsize 8000 maxblocks:score', + 2513158.7, + '', + {'index': 4333.0, 'baseline': 5800.0, 'num_parallel_copies': 1}, + ], + [ + 'Pipe Throughput:score', + 2221775.6, + '', + {'index': 1786.0, 'baseline': 12440.0, 'num_parallel_copies': 1}, + ], + [ + 'Pipe-based Context Switching:score', + 369000.7, + '', + {'index': 922.5, 'baseline': 4000.0, 'num_parallel_copies': 1}, + ], + [ + 'Process Creation:score', + 12587.7, + '', + {'index': 999.0, 'baseline': 126.0, 'num_parallel_copies': 1}, + ], + [ + 'Shell Scripts (1 concurrent):score', + 8234.3, + '', + {'index': 1942.1, 'baseline': 42.4, 'num_parallel_copies': 1}, + ], + [ + 'Shell Scripts (8 concurrent):score', + 1064.5, + '', + {'index': 1774.2, 'baseline': 6.0, 'num_parallel_copies': 1}, + ], + [ + 'System Call Overhead:score', + 4439274.5, + '', + {'index': 2959.5, 'baseline': 15000.0, 'num_parallel_copies': 1}, + ], + [ + 'System Benchmarks Index Score', + 1825.8, + '', + {'num_parallel_copies': 1}, + ], + [ + 'Dhrystone 2 using register variables', + 155391896.7, + 'lps', + {'num_parallel_copies': 8, 'samples': 7, 'time': '10.0s'}, + ], + [ + 'Double-Precision Whetstone', + 28632.5, + 'MWIPS', + {'num_parallel_copies': 8, 'samples': 7, 'time': '9.8s'}, + ], + [ + 'Execl Throughput', + 15184.0, + 'lps', + {'num_parallel_copies': 8, 'samples': 2, 'time': '30.0s'}, + ], + [ + 'File Copy 1024 bufsize 2000 maxblocks', + 985484.8, + 'KBps', + {'num_parallel_copies': 8, 'samples': 2, 'time': '30.0s'}, + ], + [ + 'File Copy 256 bufsize 500 maxblocks', + 269732.2, + 'KBps', + {'num_parallel_copies': 8, 'samples': 2, 'time': '30.0s'}, + ], + [ + 'File Copy 4096 bufsize 8000 maxblocks', + 2706156.4, + 'KBps', + {'num_parallel_copies': 8, 'samples': 2, 'time': '30.0s'}, + ], + [ + 'Pipe Throughput', + 8525928.8, + 'lps', + {'num_parallel_copies': 8, 'samples': 7, 'time': '10.0s'}, + ], + [ + 'Pipe-based Context Switching', + 1017270.4, + 'lps', + {'num_parallel_copies': 8, 'samples': 7, 'time': '10.0s'}, + ], + [ + 'Process Creation', + 31563.7, + 'lps', + {'num_parallel_copies': 8, 'samples': 2, 'time': '30.0s'}, + ], + [ + 'Shell Scripts (1 concurrent)', + 32516.3, + 'lpm', + {'num_parallel_copies': 8, 'samples': 2, 'time': '60.0s'}, + ], + [ + 'Shell Scripts (8 concurrent)', + 5012.2, + 'lpm', + {'num_parallel_copies': 8, 'samples': 2, 'time': '60.0s'}, + ], + [ + 'System Call Overhead', + 10288762.3, + 'lps', + {'num_parallel_copies': 8, 'samples': 7, 'time': '10.0s'}, + ], + [ + 'Dhrystone 2 using register variables:score', + 155391896.7, + '', + {'index': 13315.5, 'baseline': 116700.0, 'num_parallel_copies': 8}, + ], + [ + 'Double-Precision Whetstone:score', + 28632.5, + '', + {'index': 5205.9, 'baseline': 55.0, 'num_parallel_copies': 8}, + ], + [ + 'Execl Throughput:score', + 15184.0, + '', + {'index': 3531.2, 'baseline': 43.0, 'num_parallel_copies': 8}, + ], + [ + 'File Copy 1024 bufsize 2000 maxblocks:score', + 985484.8, + '', + {'index': 2488.6, 'baseline': 3960.0, 'num_parallel_copies': 8}, + ], + [ + 'File Copy 256 bufsize 500 maxblocks:score', + 269732.2, + '', + {'index': 1629.8, 'baseline': 1655.0, 'num_parallel_copies': 8}, + ], + [ + 'File Copy 4096 bufsize 8000 maxblocks:score', + 2706156.4, + '', + {'index': 4665.8, 'baseline': 5800.0, 'num_parallel_copies': 8}, + ], + [ + 'Pipe Throughput:score', + 8525928.8, + '', + {'index': 6853.6, 'baseline': 12440.0, 'num_parallel_copies': 8}, + ], + [ + 'Pipe-based Context Switching:score', + 1017270.4, + '', + {'index': 2543.2, 'baseline': 4000.0, 'num_parallel_copies': 8}, + ], + [ + 'Process Creation:score', + 31563.7, + '', + {'index': 2505.1, 'baseline': 126.0, 'num_parallel_copies': 8}, + ], + [ + 'Shell Scripts (1 concurrent):score', + 32516.3, + '', + {'index': 7668.9, 'baseline': 42.4, 'num_parallel_copies': 8}, + ], + [ + 'Shell Scripts (8 concurrent):score', + 5012.2, + '', + {'index': 8353.6, 'baseline': 6.0, 'num_parallel_copies': 8}, + ], + [ + 'System Call Overhead:score', + 10288762.3, + '', + {'index': 6859.2, 'baseline': 15000.0, 'num_parallel_copies': 8}, + ], + [ + 'System Benchmarks Index Score', + 4596.2, + '', + {'num_parallel_copies': 8}, + ], + ] expected_result = [sample.Sample(*exp) for exp in expected_result] self.assertSampleListsEqualUpToTimestamp(result, expected_result) diff --git a/tests/linux_benchmarks/vm_stop_start_benchmark_test.py b/tests/linux_benchmarks/vm_stop_start_benchmark_test.py index 878fe5451f..e711c13e4e 100644 --- a/tests/linux_benchmarks/vm_stop_start_benchmark_test.py +++ b/tests/linux_benchmarks/vm_stop_start_benchmark_test.py @@ -21,28 +21,32 @@ def vm_mock(index: int, operation_return_value: int) -> mock.Mock: Returns: A mock vm. """ - mock_vm = mock.Mock( - OS_TYPE=f'linux{index}') + mock_vm = mock.Mock(OS_TYPE=f'linux{index}') mock_vm.Start.return_value = operation_return_value mock_vm.Stop.return_value = operation_return_value return mock_vm -class StopStartBenchmarkTest(pkb_common_test_case.PkbCommonTestCase, - test_util.SamplesTestMixin): +class StopStartBenchmarkTest( + pkb_common_test_case.PkbCommonTestCase, test_util.SamplesTestMixin +): def setUp(self): super(StopStartBenchmarkTest, self).setUp() - self.enter_context(mock.patch.object( - time, - 'time', - # one pair of timestamps used for Measure and each sample.append call - side_effect=itertools.cycle([0, 7]))) + self.enter_context( + mock.patch.object( + time, + 'time', + # one pair of timestamps used for Measure and each sample.append call + side_effect=itertools.cycle([0, 7]), + ) + ) # operation should be capitalized @parameterized.named_parameters( dict(testcase_name='measure_stop', operation='Stop'), - dict(testcase_name='measure_start', operation='Start')) + dict(testcase_name='measure_start', operation='Start'), + ) def testMeasureOperation(self, operation: str): """Unit test for Stop and Start functions.""" num_vms = 3 @@ -51,8 +55,9 @@ def testMeasureOperation(self, operation: str): vms_to_test = [vm_mock(0, 5), vm_mock(1, 6), vm_mock(2, 7)] # call Stop on vms - actual_samples = getattr(vm_stop_start_benchmark, - f'_Measure{operation}')(vms_to_test) + actual_samples = getattr(vm_stop_start_benchmark, f'_Measure{operation}')( + vms_to_test + ) # for all vms create mock samples ie the expected samples expected_operation_times = [5, 6, 7] @@ -65,17 +70,27 @@ def testMeasureOperation(self, operation: str): for i in range(num_vms): expected_samples.append( - sample.Sample(f'{operation} Time', expected_operation_times[i], - 'seconds', { - 'machine_instance': i, - 'num_vms': num_vms, - 'os_type': f'linux{i}', - })) + sample.Sample( + f'{operation} Time', + expected_operation_times[i], + 'seconds', + { + 'machine_instance': i, + 'num_vms': num_vms, + 'os_type': f'linux{i}', + }, + ) + ) expected_cluster_time = 7 expected_samples.append( - sample.Sample(f'Cluster {operation} Time', expected_cluster_time, - 'seconds', expected_cluster_metadata)) + sample.Sample( + f'Cluster {operation} Time', + expected_cluster_time, + 'seconds', + expected_cluster_metadata, + ) + ) # assert the function is called on vms for vm in vms_to_test: @@ -84,5 +99,6 @@ def testMeasureOperation(self, operation: str): # assert actual and expected samples are equal self.assertSampleListsEqualUpToTimestamp(actual_samples, expected_samples) + if __name__ == '__main__': unittest.main() diff --git a/tests/linux_benchmarks/xgboost_benchmark_test.py b/tests/linux_benchmarks/xgboost_benchmark_test.py index 3ee6f37dbc..ffb855b2b5 100644 --- a/tests/linux_benchmarks/xgboost_benchmark_test.py +++ b/tests/linux_benchmarks/xgboost_benchmark_test.py @@ -17,7 +17,6 @@ import unittest from absl import flags import mock - from perfkitbenchmarker import sample from perfkitbenchmarker import test_util from perfkitbenchmarker.linux_benchmarks import xgboost_benchmark @@ -28,19 +27,23 @@ FLAGS = flags.FLAGS -class XgboostBenchmarkTest(pkb_common_test_case.PkbCommonTestCase, - test_util.SamplesTestMixin): +class XgboostBenchmarkTest( + pkb_common_test_case.PkbCommonTestCase, test_util.SamplesTestMixin +): def setUp(self) -> None: super(XgboostBenchmarkTest, self).setUp() - self.enter_context(mock.patch.object( - nvidia_driver, 'QueryNumberOfGpus', return_value=1)) - self.enter_context(mock.patch.object( - cuda_toolkit, 'GetMetadata', return_value={})) + self.enter_context( + mock.patch.object(nvidia_driver, 'QueryNumberOfGpus', return_value=1) + ) + self.enter_context( + mock.patch.object(cuda_toolkit, 'GetMetadata', return_value={}) + ) def CudaOutput(self) -> str: - path = os.path.join(os.path.dirname(__file__), '..', 'data', - 'xgboost_output.txt') + path = os.path.join( + os.path.dirname(__file__), '..', 'data', 'xgboost_output.txt' + ) with open(path) as reader: return reader.read() diff --git a/tests/linux_packages/aerospike_client_test.py b/tests/linux_packages/aerospike_client_test.py index ab3877f30c..0a630166d0 100644 --- a/tests/linux_packages/aerospike_client_test.py +++ b/tests/linux_packages/aerospike_client_test.py @@ -86,7 +86,6 @@ def _ReadFileToString(filename): class AerospikeClientTestCase(pkb_common_test_case.PkbCommonTestCase): - maxDiff = None # pylint: disable=invalid-name def setUp(self): @@ -96,96 +95,106 @@ def setUp(self): def testParseAsbenchStdout(self): actual_samples = aerospike_client.ParseAsbenchStdout( - ASBENCH_OUTPUT_HEADER + ASBENCH_OUTPUT_RESULT_1) + ASBENCH_OUTPUT_HEADER + ASBENCH_OUTPUT_RESULT_1 + ) - self.assertEqual(actual_samples, [ - sample.Sample( - metric='throughput', - value=157442.0, - unit='transaction_per_second', - metadata={ - 'tps': 157442.0, - 'timeouts': 0.0, - 'errors': 0.0, - 'start_timestamp': 1658945954.0, - 'window': 1, - 'write_min': 70.0, - 'write_max': 16735.0, - 'write_p50': 726.0, - 'write_p90': 1248.0, - 'write_p99': 2008.0, - 'write_p99.9': 3577.0, - 'write_p99.99': 13183.0, - 'read_min': 61.0, - 'read_max': 68863.0, - 'read_p50': 685.0, - 'read_p90': 1201.0, - 'read_p99': 2008.0, - 'read_p99.9': 3315.0, - 'read_p99.99': 45471.0 - }, - timestamp=actual_samples[0].timestamp), - sample.Sample( - metric='throughput', - value=155885.0, - unit='transaction_per_second', - metadata={ - 'tps': 155885.0, - 'timeouts': 0.0, - 'errors': 0.0, - 'start_timestamp': 1658945954.0, - 'window': 2, - 'write_min': 70.0, - 'write_max': 16735.0, - 'write_p50': 750.0, - 'write_p90': 1278.0, - 'write_p99': 2071.0, - 'write_p99.9': 3137.0, - 'write_p99.99': 8575.0, - 'read_min': 61.0, - 'read_max': 68863.0, - 'read_p50': 712.0, - 'read_p90': 1230.0, - 'read_p99': 2026.0, - 'read_p99.9': 3095.0, - 'read_p99.99': 18607.0 - }, - timestamp=actual_samples[1].timestamp), - sample.Sample( - metric='throughput', - value=157480.0, - unit='transaction_per_second', - metadata={ - 'tps': 157480.0, - 'timeouts': 0.0, - 'errors': 0.0, - 'start_timestamp': 1658945954.0, - 'window': 3, - 'write_min': 70.0, - 'write_max': 16735.0, - 'write_p50': 755.0, - 'write_p90': 1280.0, - 'write_p99': 2063.0, - 'write_p99.9': 3013.0, - 'write_p99.99': 8511.0, - 'read_min': 61.0, - 'read_max': 68863.0, - 'read_p50': 718.0, - 'read_p90': 1236.0, - 'read_p99': 2031.0, - 'read_p99.9': 2981.0, - 'read_p99.99': 10303.0 - }, - timestamp=actual_samples[2].timestamp) - ]) + self.assertEqual( + actual_samples, + [ + sample.Sample( + metric='throughput', + value=157442.0, + unit='transaction_per_second', + metadata={ + 'tps': 157442.0, + 'timeouts': 0.0, + 'errors': 0.0, + 'start_timestamp': 1658945954.0, + 'window': 1, + 'write_min': 70.0, + 'write_max': 16735.0, + 'write_p50': 726.0, + 'write_p90': 1248.0, + 'write_p99': 2008.0, + 'write_p99.9': 3577.0, + 'write_p99.99': 13183.0, + 'read_min': 61.0, + 'read_max': 68863.0, + 'read_p50': 685.0, + 'read_p90': 1201.0, + 'read_p99': 2008.0, + 'read_p99.9': 3315.0, + 'read_p99.99': 45471.0, + }, + timestamp=actual_samples[0].timestamp, + ), + sample.Sample( + metric='throughput', + value=155885.0, + unit='transaction_per_second', + metadata={ + 'tps': 155885.0, + 'timeouts': 0.0, + 'errors': 0.0, + 'start_timestamp': 1658945954.0, + 'window': 2, + 'write_min': 70.0, + 'write_max': 16735.0, + 'write_p50': 750.0, + 'write_p90': 1278.0, + 'write_p99': 2071.0, + 'write_p99.9': 3137.0, + 'write_p99.99': 8575.0, + 'read_min': 61.0, + 'read_max': 68863.0, + 'read_p50': 712.0, + 'read_p90': 1230.0, + 'read_p99': 2026.0, + 'read_p99.9': 3095.0, + 'read_p99.99': 18607.0, + }, + timestamp=actual_samples[1].timestamp, + ), + sample.Sample( + metric='throughput', + value=157480.0, + unit='transaction_per_second', + metadata={ + 'tps': 157480.0, + 'timeouts': 0.0, + 'errors': 0.0, + 'start_timestamp': 1658945954.0, + 'window': 3, + 'write_min': 70.0, + 'write_max': 16735.0, + 'write_p50': 755.0, + 'write_p90': 1280.0, + 'write_p99': 2063.0, + 'write_p99.9': 3013.0, + 'write_p99.99': 8511.0, + 'read_min': 61.0, + 'read_max': 68863.0, + 'read_p50': 718.0, + 'read_p90': 1236.0, + 'read_p99': 2031.0, + 'read_p99.9': 2981.0, + 'read_p99.99': 10303.0, + }, + timestamp=actual_samples[2].timestamp, + ), + ], + ) @mock.patch('time.time', mock.MagicMock(return_value=0)) def testAggregateAsbenchSamples(self): agg_samples = aerospike_client.AggregateAsbenchSamples( - aerospike_client.ParseAsbenchStdout(ASBENCH_OUTPUT_HEADER + - ASBENCH_OUTPUT_RESULT_1) + - aerospike_client.ParseAsbenchStdout(ASBENCH_OUTPUT_HEADER + - ASBENCH_OUTPUT_RESULT_2)) + aerospike_client.ParseAsbenchStdout( + ASBENCH_OUTPUT_HEADER + ASBENCH_OUTPUT_RESULT_1 + ) + + aerospike_client.ParseAsbenchStdout( + ASBENCH_OUTPUT_HEADER + ASBENCH_OUTPUT_RESULT_2 + ) + ) self.assertEqual( agg_samples, [ @@ -453,96 +462,126 @@ def testParseHistogramFile(self): HISTOGRAM_OUTPUT_2, histograms, timestamps ) self.assertEqual( - histograms, { - (1, 'read'): - collections.OrderedDict([(0, 4152), (100, 6850), (200, 2373)]), - (1, 'write'): - collections.OrderedDict([(0, 2830), (100, 739), (200, 253)]), - (2, 'read'): - collections.OrderedDict([(0, 7), (100, 18), (200, 195), - (300, 2681)]), - (2, 'write'): - collections.OrderedDict([(100, 3), (200, 9), (300, 176)]), - (3, 'read'): - collections.OrderedDict([(0, 8), (100, 21), (200, 162), - (300, 2857)]), - (3, 'write'): - collections.OrderedDict([(0, 1), (100, 2), (200, 10), - (300, 186), (400, 1357)]) - }) + histograms, + { + (1, 'read'): collections.OrderedDict( + [(0, 4152), (100, 6850), (200, 2373)] + ), + (1, 'write'): collections.OrderedDict( + [(0, 2830), (100, 739), (200, 253)] + ), + (2, 'read'): collections.OrderedDict( + [(0, 7), (100, 18), (200, 195), (300, 2681)] + ), + (2, 'write'): collections.OrderedDict( + [(100, 3), (200, 9), (300, 176)] + ), + (3, 'read'): collections.OrderedDict( + [(0, 8), (100, 21), (200, 162), (300, 2857)] + ), + (3, 'write'): collections.OrderedDict( + [(0, 1), (100, 2), (200, 10), (300, 186), (400, 1357)] + ), + }, + ) def testCreateTimeSeriesSample(self): - raw_samples = aerospike_client.ParseAsbenchStdout(ASBENCH_OUTPUT_HEADER + - ASBENCH_OUTPUT_RESULT_1) + raw_samples = aerospike_client.ParseAsbenchStdout( + ASBENCH_OUTPUT_HEADER + ASBENCH_OUTPUT_RESULT_1 + ) ts_samples = aerospike_client.CreateTimeSeriesSample(raw_samples) - self.assertEqual(ts_samples, [ - sample.Sample( - metric='OPS_time_series', - value=0.0, - unit='ops', - metadata={ - 'values': [157442.0, 155885.0, 157480.0], - 'timestamps': - [1658945954000.0, 1658945955000.0, 1658945956000.0], - 'interval': 1, - 'ramp_up_ends': 1658946014000.0 - }, - timestamp=ts_samples[0].timestamp), - sample.Sample( - metric='Read_Min_Latency_time_series', - value=0.0, - unit='us', - metadata={ - 'values': [61.0, 61.0, 61.0], - 'timestamps': - [1658945954000.0, 1658945955000.0, 1658945956000.0], - 'interval': 1, - 'ramp_up_ends': 1658946014000.0 - }, - timestamp=ts_samples[1].timestamp), - sample.Sample( - metric='Read_Max_Latency_time_series', - value=0.0, - unit='us', - metadata={ - 'values': [68863.0, 68863.0, 68863.0], - 'timestamps': - [1658945954000.0, 1658945955000.0, 1658945956000.0], - 'interval': 1, - 'ramp_up_ends': 1658946014000.0 - }, - timestamp=ts_samples[2].timestamp), - sample.Sample( - metric='Write_Min_Latency_time_series', - value=0.0, - unit='us', - metadata={ - 'values': [70.0, 70.0, 70.0], - 'timestamps': - [1658945954000.0, 1658945955000.0, 1658945956000.0], - 'interval': 1, - 'ramp_up_ends': 1658946014000.0 - }, - timestamp=ts_samples[3].timestamp), - sample.Sample( - metric='Write_Max_Latency_time_series', - value=0.0, - unit='us', - metadata={ - 'values': [16735.0, 16735.0, 16735.0], - 'timestamps': - [1658945954000.0, 1658945955000.0, 1658945956000.0], - 'interval': 1, - 'ramp_up_ends': 1658946014000.0 - }, - timestamp=ts_samples[4].timestamp), - sample.Sample( - metric='total_ops', - value=156935.66666666666, - unit='ops', - metadata={}, - timestamp=ts_samples[5].timestamp) - ]) + self.assertEqual( + ts_samples, + [ + sample.Sample( + metric='OPS_time_series', + value=0.0, + unit='ops', + metadata={ + 'values': [157442.0, 155885.0, 157480.0], + 'timestamps': [ + 1658945954000.0, + 1658945955000.0, + 1658945956000.0, + ], + 'interval': 1, + 'ramp_up_ends': 1658946014000.0, + }, + timestamp=ts_samples[0].timestamp, + ), + sample.Sample( + metric='Read_Min_Latency_time_series', + value=0.0, + unit='us', + metadata={ + 'values': [61.0, 61.0, 61.0], + 'timestamps': [ + 1658945954000.0, + 1658945955000.0, + 1658945956000.0, + ], + 'interval': 1, + 'ramp_up_ends': 1658946014000.0, + }, + timestamp=ts_samples[1].timestamp, + ), + sample.Sample( + metric='Read_Max_Latency_time_series', + value=0.0, + unit='us', + metadata={ + 'values': [68863.0, 68863.0, 68863.0], + 'timestamps': [ + 1658945954000.0, + 1658945955000.0, + 1658945956000.0, + ], + 'interval': 1, + 'ramp_up_ends': 1658946014000.0, + }, + timestamp=ts_samples[2].timestamp, + ), + sample.Sample( + metric='Write_Min_Latency_time_series', + value=0.0, + unit='us', + metadata={ + 'values': [70.0, 70.0, 70.0], + 'timestamps': [ + 1658945954000.0, + 1658945955000.0, + 1658945956000.0, + ], + 'interval': 1, + 'ramp_up_ends': 1658946014000.0, + }, + timestamp=ts_samples[3].timestamp, + ), + sample.Sample( + metric='Write_Max_Latency_time_series', + value=0.0, + unit='us', + metadata={ + 'values': [16735.0, 16735.0, 16735.0], + 'timestamps': [ + 1658945954000.0, + 1658945955000.0, + 1658945956000.0, + ], + 'interval': 1, + 'ramp_up_ends': 1658946014000.0, + }, + timestamp=ts_samples[4].timestamp, + ), + sample.Sample( + metric='total_ops', + value=156935.66666666666, + unit='ops', + metadata={}, + timestamp=ts_samples[5].timestamp, + ), + ], + ) if __name__ == '__main__': diff --git a/tests/linux_packages/azure_cli_test.py b/tests/linux_packages/azure_cli_test.py index 5decffd6cc..a008265d50 100644 --- a/tests/linux_packages/azure_cli_test.py +++ b/tests/linux_packages/azure_cli_test.py @@ -33,9 +33,9 @@ def assertCallArgsEqual(self, call_args_singles, mock_method): expects. Args: - call_args_singles: List of single arguments sent to the mock_method, - ie ['x', 'y'] is for when mock_method was called twice: once with - x and then with y. + call_args_singles: List of single arguments sent to the mock_method, ie + ['x', 'y'] is for when mock_method was called twice: once with x and + then with y. mock_method: Method that was mocked and called with call_args_singles. """ # convert from ['a', 'b'] into [(('a',),), (('b',),)] @@ -73,14 +73,16 @@ def testShowHowMockingWorks(self): def testYumInstall(self): azure_cli.YumInstall(self.vm) self.assertRemoteCommandsEqual([ - 'echo "[azure-cli]\n' - 'name=Azure CLI\n' - 'baseurl=https://packages.microsoft.com/yumrepos/azure-cli\n' - 'enabled=1\n' - 'gpgcheck=1\n' - 'gpgkey=https://packages.microsoft.com/keys/microsoft.asc\n"' - ' | sudo tee /etc/yum.repos.d/azure-cli.repo', - 'sudo rpm --import https://packages.microsoft.com/keys/microsoft.asc' + ( + 'echo "[azure-cli]\n' + 'name=Azure CLI\n' + 'baseurl=https://packages.microsoft.com/yumrepos/azure-cli\n' + 'enabled=1\n' + 'gpgcheck=1\n' + 'gpgkey=https://packages.microsoft.com/keys/microsoft.asc\n"' + ' | sudo tee /etc/yum.repos.d/azure-cli.repo' + ), + 'sudo rpm --import https://packages.microsoft.com/keys/microsoft.asc', ]) self.assertInstallPackageCommandsEqual(['azure-cli']) self.assertOnlyKnownMethodsCalled('RemoteCommand', 'InstallPackages') @@ -94,17 +96,23 @@ def testAptInstall(self): ) return self.assertRemoteCommandsEqual([ - 'lsb_release -cs', 'echo "deb [arch=amd64] ' - 'https://packages.microsoft.com/repos/azure-cli/ wheezy main" | sudo ' - 'tee /etc/apt/sources.list.d/azure-cli.list', - 'curl -L https://packages.microsoft.com/keys/microsoft.asc | sudo ' - 'apt-key add -', - 'sudo apt-get update' + 'lsb_release -cs', + ( + 'echo "deb [arch=amd64]' + ' https://packages.microsoft.com/repos/azure-cli/ wheezy main"' + ' | sudo tee /etc/apt/sources.list.d/azure-cli.list' + ), + ( + 'curl -L https://packages.microsoft.com/keys/microsoft.asc |' + ' sudo apt-key add -' + ), + 'sudo apt-get update', ]) self.assertInstallPackageCommandsEqual(['apt-transport-https', 'azure-cli']) self.assertVmInstallCommandsEqual(['python', 'lsb_release', 'curl']) - self.assertOnlyKnownMethodsCalled('RemoteCommand', 'Install', - 'InstallPackages') + self.assertOnlyKnownMethodsCalled( + 'RemoteCommand', 'Install', 'InstallPackages' + ) if __name__ == '__main__': diff --git a/tests/linux_packages/blazemark_test.py b/tests/linux_packages/blazemark_test.py index 133af785be..1d1c537117 100644 --- a/tests/linux_packages/blazemark_test.py +++ b/tests/linux_packages/blazemark_test.py @@ -21,7 +21,6 @@ class BlazemarkTestCase(unittest.TestCase, test_util.SamplesTestMixin): - maxDiff = None def setUp(self): diff --git a/tests/linux_packages/cmake_test.py b/tests/linux_packages/cmake_test.py index be695e8747..49dced64b0 100644 --- a/tests/linux_packages/cmake_test.py +++ b/tests/linux_packages/cmake_test.py @@ -24,10 +24,16 @@ def testAptInstallViaKitware(self): vm = MockVm('ubuntu2004') cmake.AptInstall(vm) expected_cmds = [ - 'curl --silent https://apt.kitware.com/keys/kitware-archive-latest.asc ' - '| gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/kitware.gpg ' - '>/dev/null', 'sudo apt-add-repository ' - '"deb https://apt.kitware.com/ubuntu/ focal main"' + ( + 'curl --silent' + ' https://apt.kitware.com/keys/kitware-archive-latest.asc | gpg' + ' --dearmor | sudo tee /etc/apt/trusted.gpg.d/kitware.gpg' + ' >/dev/null' + ), + ( + 'sudo apt-add-repository ' + '"deb https://apt.kitware.com/ubuntu/ focal main"' + ), ] vm.RemoteCommand.assert_has_calls([mock.call(cmd) for cmd in expected_cmds]) diff --git a/tests/linux_packages/dstat_test.py b/tests/linux_packages/dstat_test.py index 5e8336fcc6..b3967be3f5 100644 --- a/tests/linux_packages/dstat_test.py +++ b/tests/linux_packages/dstat_test.py @@ -16,39 +16,87 @@ import os import unittest - from perfkitbenchmarker.linux_packages import dstat class DstatTestCase(unittest.TestCase): def testParseDstatFile(self): - path = os.path.join(os.path.dirname(__file__), '..', 'data', - 'dstat-result.csv') + path = os.path.join( + os.path.dirname(__file__), '..', 'data', 'dstat-result.csv' + ) with open(path) as f: labels, out = dstat.ParseCsvFile(f) self.assertEqual(len(labels), len(out[0])) self.assertEqual(out.shape, (383, 62)) - self.assertEqual([ - 'epoch__epoch', 'usr__total cpu usage', 'sys__total cpu usage', - 'idl__total cpu usage', 'wai__total cpu usage', 'hiq__total cpu usage', - 'siq__total cpu usage', '1m__load avg', '5m__load avg', '15m__load avg', - 'read__io/total', 'writ__io/total', 'read__io/sda', 'writ__io/sda', - 'read__dsk/total', 'writ__dsk/total', 'read__dsk/sda', 'writ__dsk/sda', - 'recv__net/total', 'send__net/total', 'in__paging', 'out__paging', - 'int__system', 'csw__system', '12__interrupts', '25__interrupts', - '30__interrupts', 'run__procs', 'blk__procs', 'new__procs', - 'used__memory usage', 'buff__memory usage', 'cach__memory usage', - 'free__memory usage', 'used__swap', 'free__swap', 'files__filesystem', - 'inodes__filesystem', 'msg__sysv ipc', 'sem__sysv ipc', 'shm__sysv ipc', - 'lis__tcp sockets', 'act__tcp sockets', 'syn__tcp sockets', - 'tim__tcp sockets', 'clo__tcp sockets', 'lis__udp', 'act__udp', - 'raw__raw', 'tot__sockets', 'tcp__sockets', 'udp__sockets', - 'raw__sockets', 'frg__sockets', 'dgm__unix sockets', - 'str__unix sockets', 'lis__unix sockets', 'act__unix sockets', - 'majpf__virtual memory', 'minpf__virtual memory', - 'alloc__virtual memory', 'free__virtual memory'], labels) + self.assertEqual( + [ + 'epoch__epoch', + 'usr__total cpu usage', + 'sys__total cpu usage', + 'idl__total cpu usage', + 'wai__total cpu usage', + 'hiq__total cpu usage', + 'siq__total cpu usage', + '1m__load avg', + '5m__load avg', + '15m__load avg', + 'read__io/total', + 'writ__io/total', + 'read__io/sda', + 'writ__io/sda', + 'read__dsk/total', + 'writ__dsk/total', + 'read__dsk/sda', + 'writ__dsk/sda', + 'recv__net/total', + 'send__net/total', + 'in__paging', + 'out__paging', + 'int__system', + 'csw__system', + '12__interrupts', + '25__interrupts', + '30__interrupts', + 'run__procs', + 'blk__procs', + 'new__procs', + 'used__memory usage', + 'buff__memory usage', + 'cach__memory usage', + 'free__memory usage', + 'used__swap', + 'free__swap', + 'files__filesystem', + 'inodes__filesystem', + 'msg__sysv ipc', + 'sem__sysv ipc', + 'shm__sysv ipc', + 'lis__tcp sockets', + 'act__tcp sockets', + 'syn__tcp sockets', + 'tim__tcp sockets', + 'clo__tcp sockets', + 'lis__udp', + 'act__udp', + 'raw__raw', + 'tot__sockets', + 'tcp__sockets', + 'udp__sockets', + 'raw__sockets', + 'frg__sockets', + 'dgm__unix sockets', + 'str__unix sockets', + 'lis__unix sockets', + 'act__unix sockets', + 'majpf__virtual memory', + 'minpf__virtual memory', + 'alloc__virtual memory', + 'free__virtual memory', + ], + labels, + ) if __name__ == '__main__': diff --git a/tests/linux_packages/fio_test.py b/tests/linux_packages/fio_test.py index 27e1a39ee3..d18c572d2b 100644 --- a/tests/linux_packages/fio_test.py +++ b/tests/linux_packages/fio_test.py @@ -18,7 +18,6 @@ import unittest import mock - from perfkitbenchmarker import sample from perfkitbenchmarker import test_util from perfkitbenchmarker import vm_util @@ -42,7 +41,6 @@ def _ExtractHistogramFromMetric(results, sample_name): class FioTestCase(unittest.TestCase, test_util.SamplesTestMixin): - maxDiff = None def setUp(self): @@ -53,7 +51,8 @@ def setUp(self): self.job_contents = _ReadFileToString(job_file_path) get_tmp_dir_mock = mock.patch( - vm_util.__name__ + '.GetTempDir', return_value='tmp') + vm_util.__name__ + '.GetTempDir', return_value='tmp' + ) get_tmp_dir_mock.start() self.addCleanup(get_tmp_dir_mock.stop) @@ -61,32 +60,68 @@ def testParseFioJobFile(self): parameter_dict = fio.ParseJobFile(self.job_contents) expected_result = { 'random_write_test': { - 'rw': 'randwrite', 'blocksize': '4k', 'direct': '1', - 'filename': 'fio_test_file', 'filesize': '10*10*1000*$mb_memory', - 'ioengine': 'libaio', 'iodepth': '1', 'directory': '/scratch0', - 'overwrite': '1', 'size': '10*1000*$mb_memory'}, + 'rw': 'randwrite', + 'blocksize': '4k', + 'direct': '1', + 'filename': 'fio_test_file', + 'filesize': '10*10*1000*$mb_memory', + 'ioengine': 'libaio', + 'iodepth': '1', + 'directory': '/scratch0', + 'overwrite': '1', + 'size': '10*1000*$mb_memory', + }, 'sequential_read': { - 'invalidate': '1', 'rw': 'read', 'blocksize': '512k', - 'direct': '1', 'filename': 'fio_test_file', - 'filesize': '10*10*1000*$mb_memory', 'ioengine': 'libaio', - 'iodepth': '64', 'overwrite': '0', 'directory': '/scratch0', - 'size': '10*10*1000*$mb_memory'}, + 'invalidate': '1', + 'rw': 'read', + 'blocksize': '512k', + 'direct': '1', + 'filename': 'fio_test_file', + 'filesize': '10*10*1000*$mb_memory', + 'ioengine': 'libaio', + 'iodepth': '64', + 'overwrite': '0', + 'directory': '/scratch0', + 'size': '10*10*1000*$mb_memory', + }, 'sequential_write': { - 'rw': 'write', 'end_fsync': '1', 'blocksize': '512k', 'direct': '1', - 'filename': 'fio_test_file', 'filesize': '10*10*1000*$mb_memory', - 'ioengine': 'libaio', 'iodepth': '64', 'overwrite': '0', - 'size': '10*10*1000*$mb_memory', 'directory': '/scratch0'}, + 'rw': 'write', + 'end_fsync': '1', + 'blocksize': '512k', + 'direct': '1', + 'filename': 'fio_test_file', + 'filesize': '10*10*1000*$mb_memory', + 'ioengine': 'libaio', + 'iodepth': '64', + 'overwrite': '0', + 'size': '10*10*1000*$mb_memory', + 'directory': '/scratch0', + }, 'random_read_test': { - 'invalidate': '1', 'rw': 'randread', 'blocksize': '4k', - 'direct': '1', 'filename': 'fio_test_file', - 'filesize': '10*10*1000*$mb_memory', 'directory': '/scratch0', - 'ioengine': 'libaio', 'iodepth': '1', 'size': '10*1000*$mb_memory'}, + 'invalidate': '1', + 'rw': 'randread', + 'blocksize': '4k', + 'direct': '1', + 'filename': 'fio_test_file', + 'filesize': '10*10*1000*$mb_memory', + 'directory': '/scratch0', + 'ioengine': 'libaio', + 'iodepth': '1', + 'size': '10*1000*$mb_memory', + }, 'random_read_test_parallel': { - 'invalidate': '1', 'rw': 'randread', 'blocksize': '4k', - 'direct': '1', 'filename': 'fio_test_file', - 'directory': '/scratch0', 'filesize': '10*10*1000*$mb_memory', - 'ioengine': 'libaio', 'iodepth': '64', - 'size': '10*1000*$mb_memory'}} + 'invalidate': '1', + 'rw': 'randread', + 'blocksize': '4k', + 'direct': '1', + 'filename': 'fio_test_file', + 'directory': '/scratch0', + 'filesize': '10*10*1000*$mb_memory', + 'ioengine': 'libaio', + 'iodepth': '64', + 'size': '10*1000*$mb_memory', + }, + } self.assertDictEqual(parameter_dict, expected_result) def testParseFioResults(self): @@ -97,22 +132,30 @@ def testParseFioResults(self): 'sequential_read': {}, 'random_write_test': {}, 'random_read_test': {}, - 'random_read_test_parallel': {}}): + 'random_read_test_parallel': {}, + }, + ): result = fio.ParseResults('', self.result_contents) expected_result = [ [ - 'sequential_write:write:bandwidth', 68118, 'KB/s', { + 'sequential_write:write:bandwidth', + 68118, + 'KB/s', + { 'bw_max': 74454, 'bw_agg': 63936.8, 'bw_min': 19225, 'bw_dev': 20346.28, 'bw_mean': 63936.8, 'fio_job': 'sequential_write', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'sequential_write:write:latency', 477734.84, 'usec', { + 'sequential_write:write:latency', + 477734.84, + 'usec', + { 'max': 869891, 'stddev': 92609.34, 'min': 189263, @@ -135,154 +178,160 @@ def testParseFioResults(self): 'p50': 444416, 'p40': 444416, 'fio_job': 'sequential_write', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'sequential_write:write:latency:min', 189263, 'usec', { - 'fio_job': 'sequential_write', - 'fio_version': 'fio-3.27' - } + 'sequential_write:write:latency:min', + 189263, + 'usec', + {'fio_job': 'sequential_write', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_write:write:latency:max', 869891, 'usec', { - 'fio_job': 'sequential_write', - 'fio_version': 'fio-3.27' - } + 'sequential_write:write:latency:max', + 869891, + 'usec', + {'fio_job': 'sequential_write', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_write:write:latency:mean', 477734.84, 'usec', { - 'fio_job': 'sequential_write', - 'fio_version': 'fio-3.27' - } + 'sequential_write:write:latency:mean', + 477734.84, + 'usec', + {'fio_job': 'sequential_write', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_write:write:latency:stddev', 92609.34, 'usec', { - 'fio_job': 'sequential_write', - 'fio_version': 'fio-3.27' - } + 'sequential_write:write:latency:stddev', + 92609.34, + 'usec', + {'fio_job': 'sequential_write', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_write:write:latency:p1', 387072, 'usec', { - 'fio_job': 'sequential_write', - 'fio_version': 'fio-3.27' - } + 'sequential_write:write:latency:p1', + 387072, + 'usec', + {'fio_job': 'sequential_write', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_write:write:latency:p5', 440320, 'usec', { - 'fio_job': 'sequential_write', - 'fio_version': 'fio-3.27' - } + 'sequential_write:write:latency:p5', + 440320, + 'usec', + {'fio_job': 'sequential_write', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_write:write:latency:p10', 440320, 'usec', { - 'fio_job': 'sequential_write', - 'fio_version': 'fio-3.27' - } + 'sequential_write:write:latency:p10', + 440320, + 'usec', + {'fio_job': 'sequential_write', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_write:write:latency:p20', 440320, 'usec', { - 'fio_job': 'sequential_write', - 'fio_version': 'fio-3.27' - } + 'sequential_write:write:latency:p20', + 440320, + 'usec', + {'fio_job': 'sequential_write', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_write:write:latency:p30', 444416, 'usec', { - 'fio_job': 'sequential_write', - 'fio_version': 'fio-3.27' - } + 'sequential_write:write:latency:p30', + 444416, + 'usec', + {'fio_job': 'sequential_write', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_write:write:latency:p40', 444416, 'usec', { - 'fio_job': 'sequential_write', - 'fio_version': 'fio-3.27' - } + 'sequential_write:write:latency:p40', + 444416, + 'usec', + {'fio_job': 'sequential_write', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_write:write:latency:p50', 444416, 'usec', { - 'fio_job': 'sequential_write', - 'fio_version': 'fio-3.27' - } + 'sequential_write:write:latency:p50', + 444416, + 'usec', + {'fio_job': 'sequential_write', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_write:write:latency:p60', 444416, 'usec', { - 'fio_job': 'sequential_write', - 'fio_version': 'fio-3.27' - } + 'sequential_write:write:latency:p60', + 444416, + 'usec', + {'fio_job': 'sequential_write', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_write:write:latency:p70', 448512, 'usec', { - 'fio_job': 'sequential_write', - 'fio_version': 'fio-3.27' - } + 'sequential_write:write:latency:p70', + 448512, + 'usec', + {'fio_job': 'sequential_write', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_write:write:latency:p80', 452608, 'usec', { - 'fio_job': 'sequential_write', - 'fio_version': 'fio-3.27' - } + 'sequential_write:write:latency:p80', + 452608, + 'usec', + {'fio_job': 'sequential_write', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_write:write:latency:p90', 610304, 'usec', { - 'fio_job': 'sequential_write', - 'fio_version': 'fio-3.27' - } + 'sequential_write:write:latency:p90', + 610304, + 'usec', + {'fio_job': 'sequential_write', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_write:write:latency:p95', 724992, 'usec', { - 'fio_job': 'sequential_write', - 'fio_version': 'fio-3.27' - } + 'sequential_write:write:latency:p95', + 724992, + 'usec', + {'fio_job': 'sequential_write', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_write:write:latency:p99', 823296, 'usec', { - 'fio_job': 'sequential_write', - 'fio_version': 'fio-3.27' - } + 'sequential_write:write:latency:p99', + 823296, + 'usec', + {'fio_job': 'sequential_write', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_write:write:latency:p99.5', 847872, 'usec', { - 'fio_job': 'sequential_write', - 'fio_version': 'fio-3.27' - } + 'sequential_write:write:latency:p99.5', + 847872, + 'usec', + {'fio_job': 'sequential_write', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_write:write:latency:p99.9', 872448, 'usec', { - 'fio_job': 'sequential_write', - 'fio_version': 'fio-3.27' - } + 'sequential_write:write:latency:p99.9', + 872448, + 'usec', + {'fio_job': 'sequential_write', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_write:write:latency:p99.95', 872448, 'usec', { - 'fio_job': 'sequential_write', - 'fio_version': 'fio-3.27' - } + 'sequential_write:write:latency:p99.95', + 872448, + 'usec', + {'fio_job': 'sequential_write', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_write:write:latency:p99.99', 872448, 'usec', { - 'fio_job': 'sequential_write', - 'fio_version': 'fio-3.27' - } + 'sequential_write:write:latency:p99.99', + 872448, + 'usec', + {'fio_job': 'sequential_write', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_write:write:iops', 133, '', { - 'fio_job': 'sequential_write', - 'fio_version': 'fio-3.27' - } + 'sequential_write:write:iops', + 133, + '', + {'fio_job': 'sequential_write', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_read:read:bandwidth', 129836, 'KB/s', { + 'sequential_read:read:bandwidth', + 129836, + 'KB/s', + { 'bw_max': 162491, 'bw_agg': 130255.2, 'bw_min': 115250, 'bw_dev': 18551.37, 'bw_mean': 130255.2, 'fio_job': 'sequential_read', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'sequential_read:read:latency', 250667.06, 'usec', { + 'sequential_read:read:latency', + 250667.06, + 'usec', + { 'max': 528542, 'stddev': 70403.40, 'min': 24198, @@ -305,154 +354,160 @@ def testParseFioResults(self): 'p50': 264192, 'p40': 257024, 'fio_job': 'sequential_read', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'sequential_read:read:latency:min', 24198, 'usec', { - 'fio_job': 'sequential_read', - 'fio_version': 'fio-3.27' - } + 'sequential_read:read:latency:min', + 24198, + 'usec', + {'fio_job': 'sequential_read', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_read:read:latency:max', 528542, 'usec', { - 'fio_job': 'sequential_read', - 'fio_version': 'fio-3.27' - } + 'sequential_read:read:latency:max', + 528542, + 'usec', + {'fio_job': 'sequential_read', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_read:read:latency:mean', 250667.06, 'usec', { - 'fio_job': 'sequential_read', - 'fio_version': 'fio-3.27' - } + 'sequential_read:read:latency:mean', + 250667.06, + 'usec', + {'fio_job': 'sequential_read', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_read:read:latency:stddev', 70403.40, 'usec', { - 'fio_job': 'sequential_read', - 'fio_version': 'fio-3.27' - } + 'sequential_read:read:latency:stddev', + 70403.40, + 'usec', + {'fio_job': 'sequential_read', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_read:read:latency:p1', 59136, 'usec', { - 'fio_job': 'sequential_read', - 'fio_version': 'fio-3.27' - } + 'sequential_read:read:latency:p1', + 59136, + 'usec', + {'fio_job': 'sequential_read', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_read:read:latency:p5', 116224, 'usec', { - 'fio_job': 'sequential_read', - 'fio_version': 'fio-3.27' - } + 'sequential_read:read:latency:p5', + 116224, + 'usec', + {'fio_job': 'sequential_read', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_read:read:latency:p10', 164864, 'usec', { - 'fio_job': 'sequential_read', - 'fio_version': 'fio-3.27' - } + 'sequential_read:read:latency:p10', + 164864, + 'usec', + {'fio_job': 'sequential_read', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_read:read:latency:p20', 199680, 'usec', { - 'fio_job': 'sequential_read', - 'fio_version': 'fio-3.27' - } + 'sequential_read:read:latency:p20', + 199680, + 'usec', + {'fio_job': 'sequential_read', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_read:read:latency:p30', 246784, 'usec', { - 'fio_job': 'sequential_read', - 'fio_version': 'fio-3.27' - } + 'sequential_read:read:latency:p30', + 246784, + 'usec', + {'fio_job': 'sequential_read', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_read:read:latency:p40', 257024, 'usec', { - 'fio_job': 'sequential_read', - 'fio_version': 'fio-3.27' - } + 'sequential_read:read:latency:p40', + 257024, + 'usec', + {'fio_job': 'sequential_read', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_read:read:latency:p50', 264192, 'usec', { - 'fio_job': 'sequential_read', - 'fio_version': 'fio-3.27' - } + 'sequential_read:read:latency:p50', + 264192, + 'usec', + {'fio_job': 'sequential_read', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_read:read:latency:p60', 268288, 'usec', { - 'fio_job': 'sequential_read', - 'fio_version': 'fio-3.27' - } + 'sequential_read:read:latency:p60', + 268288, + 'usec', + {'fio_job': 'sequential_read', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_read:read:latency:p70', 272384, 'usec', { - 'fio_job': 'sequential_read', - 'fio_version': 'fio-3.27' - } + 'sequential_read:read:latency:p70', + 272384, + 'usec', + {'fio_job': 'sequential_read', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_read:read:latency:p80', 280576, 'usec', { - 'fio_job': 'sequential_read', - 'fio_version': 'fio-3.27' - } + 'sequential_read:read:latency:p80', + 280576, + 'usec', + {'fio_job': 'sequential_read', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_read:read:latency:p90', 292864, 'usec', { - 'fio_job': 'sequential_read', - 'fio_version': 'fio-3.27' - } + 'sequential_read:read:latency:p90', + 292864, + 'usec', + {'fio_job': 'sequential_read', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_read:read:latency:p95', 366592, 'usec', { - 'fio_job': 'sequential_read', - 'fio_version': 'fio-3.27' - } + 'sequential_read:read:latency:p95', + 366592, + 'usec', + {'fio_job': 'sequential_read', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_read:read:latency:p99', 473088, 'usec', { - 'fio_job': 'sequential_read', - 'fio_version': 'fio-3.27' - } + 'sequential_read:read:latency:p99', + 473088, + 'usec', + {'fio_job': 'sequential_read', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_read:read:latency:p99.5', 489472, 'usec', { - 'fio_job': 'sequential_read', - 'fio_version': 'fio-3.27' - } + 'sequential_read:read:latency:p99.5', + 489472, + 'usec', + {'fio_job': 'sequential_read', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_read:read:latency:p99.9', 528384, 'usec', { - 'fio_job': 'sequential_read', - 'fio_version': 'fio-3.27' - } + 'sequential_read:read:latency:p99.9', + 528384, + 'usec', + {'fio_job': 'sequential_read', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_read:read:latency:p99.95', 528384, 'usec', { - 'fio_job': 'sequential_read', - 'fio_version': 'fio-3.27' - } + 'sequential_read:read:latency:p99.95', + 528384, + 'usec', + {'fio_job': 'sequential_read', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_read:read:latency:p99.99', 528384, 'usec', { - 'fio_job': 'sequential_read', - 'fio_version': 'fio-3.27' - } + 'sequential_read:read:latency:p99.99', + 528384, + 'usec', + {'fio_job': 'sequential_read', 'fio_version': 'fio-3.27'}, ], [ - 'sequential_read:read:iops', 253, '', { - 'fio_job': 'sequential_read', - 'fio_version': 'fio-3.27' - } + 'sequential_read:read:iops', + 253, + '', + {'fio_job': 'sequential_read', 'fio_version': 'fio-3.27'}, ], [ - 'random_write_test:write:bandwidth', 6443, 'KB/s', { + 'random_write_test:write:bandwidth', + 6443, + 'KB/s', + { 'bw_max': 7104, 'bw_agg': 6446.55, 'bw_min': 5896, 'bw_dev': 336.21, 'bw_mean': 6446.55, 'fio_job': 'random_write_test', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'random_write_test:write:latency', 587.02, 'usec', { + 'random_write_test:write:latency', + 587.02, + 'usec', + { 'max': 81806, 'stddev': 897.93, 'min': 1, @@ -475,154 +530,160 @@ def testParseFioResults(self): 'p50': 510, 'p40': 502, 'fio_job': 'random_write_test', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'random_write_test:write:latency:min', 1, 'usec', { - 'fio_job': 'random_write_test', - 'fio_version': 'fio-3.27' - } + 'random_write_test:write:latency:min', + 1, + 'usec', + {'fio_job': 'random_write_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_write_test:write:latency:max', 81806, 'usec', { - 'fio_job': 'random_write_test', - 'fio_version': 'fio-3.27' - } + 'random_write_test:write:latency:max', + 81806, + 'usec', + {'fio_job': 'random_write_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_write_test:write:latency:mean', 587.02, 'usec', { - 'fio_job': 'random_write_test', - 'fio_version': 'fio-3.27' - } + 'random_write_test:write:latency:mean', + 587.02, + 'usec', + {'fio_job': 'random_write_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_write_test:write:latency:stddev', 897.93, 'usec', { - 'fio_job': 'random_write_test', - 'fio_version': 'fio-3.27' - } + 'random_write_test:write:latency:stddev', + 897.93, + 'usec', + {'fio_job': 'random_write_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_write_test:write:latency:p1', 446, 'usec', { - 'fio_job': 'random_write_test', - 'fio_version': 'fio-3.27' - } + 'random_write_test:write:latency:p1', + 446, + 'usec', + {'fio_job': 'random_write_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_write_test:write:latency:p5', 462, 'usec', { - 'fio_job': 'random_write_test', - 'fio_version': 'fio-3.27' - } + 'random_write_test:write:latency:p5', + 462, + 'usec', + {'fio_job': 'random_write_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_write_test:write:latency:p10', 470, 'usec', { - 'fio_job': 'random_write_test', - 'fio_version': 'fio-3.27' - } + 'random_write_test:write:latency:p10', + 470, + 'usec', + {'fio_job': 'random_write_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_write_test:write:latency:p20', 482, 'usec', { - 'fio_job': 'random_write_test', - 'fio_version': 'fio-3.27' - } + 'random_write_test:write:latency:p20', + 482, + 'usec', + {'fio_job': 'random_write_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_write_test:write:latency:p30', 494, 'usec', { - 'fio_job': 'random_write_test', - 'fio_version': 'fio-3.27' - } + 'random_write_test:write:latency:p30', + 494, + 'usec', + {'fio_job': 'random_write_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_write_test:write:latency:p40', 502, 'usec', { - 'fio_job': 'random_write_test', - 'fio_version': 'fio-3.27' - } + 'random_write_test:write:latency:p40', + 502, + 'usec', + {'fio_job': 'random_write_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_write_test:write:latency:p50', 510, 'usec', { - 'fio_job': 'random_write_test', - 'fio_version': 'fio-3.27' - } + 'random_write_test:write:latency:p50', + 510, + 'usec', + {'fio_job': 'random_write_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_write_test:write:latency:p60', 524, 'usec', { - 'fio_job': 'random_write_test', - 'fio_version': 'fio-3.27' - } + 'random_write_test:write:latency:p60', + 524, + 'usec', + {'fio_job': 'random_write_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_write_test:write:latency:p70', 532, 'usec', { - 'fio_job': 'random_write_test', - 'fio_version': 'fio-3.27' - } + 'random_write_test:write:latency:p70', + 532, + 'usec', + {'fio_job': 'random_write_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_write_test:write:latency:p80', 564, 'usec', { - 'fio_job': 'random_write_test', - 'fio_version': 'fio-3.27' - } + 'random_write_test:write:latency:p80', + 564, + 'usec', + {'fio_job': 'random_write_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_write_test:write:latency:p90', 636, 'usec', { - 'fio_job': 'random_write_test', - 'fio_version': 'fio-3.27' - } + 'random_write_test:write:latency:p90', + 636, + 'usec', + {'fio_job': 'random_write_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_write_test:write:latency:p95', 1064, 'usec', { - 'fio_job': 'random_write_test', - 'fio_version': 'fio-3.27' - } + 'random_write_test:write:latency:p95', + 1064, + 'usec', + {'fio_job': 'random_write_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_write_test:write:latency:p99', 1688, 'usec', { - 'fio_job': 'random_write_test', - 'fio_version': 'fio-3.27' - } + 'random_write_test:write:latency:p99', + 1688, + 'usec', + {'fio_job': 'random_write_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_write_test:write:latency:p99.5', 1736, 'usec', { - 'fio_job': 'random_write_test', - 'fio_version': 'fio-3.27' - } + 'random_write_test:write:latency:p99.5', + 1736, + 'usec', + {'fio_job': 'random_write_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_write_test:write:latency:p99.9', 3216, 'usec', { - 'fio_job': 'random_write_test', - 'fio_version': 'fio-3.27' - } + 'random_write_test:write:latency:p99.9', + 3216, + 'usec', + {'fio_job': 'random_write_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_write_test:write:latency:p99.95', 4128, 'usec', { - 'fio_job': 'random_write_test', - 'fio_version': 'fio-3.27' - } + 'random_write_test:write:latency:p99.95', + 4128, + 'usec', + {'fio_job': 'random_write_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_write_test:write:latency:p99.99', 81408, 'usec', { - 'fio_job': 'random_write_test', - 'fio_version': 'fio-3.27' - } + 'random_write_test:write:latency:p99.99', + 81408, + 'usec', + {'fio_job': 'random_write_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_write_test:write:iops', 1610, '', { - 'fio_job': 'random_write_test', - 'fio_version': 'fio-3.27' - } + 'random_write_test:write:iops', + 1610, + '', + {'fio_job': 'random_write_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_read_test:read:bandwidth', 1269, 'KB/s', { + 'random_read_test:read:bandwidth', + 1269, + 'KB/s', + { 'bw_max': 1745, 'bw_agg': 1275.52, 'bw_min': 330, 'bw_dev': 201.59, 'bw_mean': 1275.52, 'fio_job': 'random_read_test', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'random_read_test:read:latency', 3117.62, 'usec', { + 'random_read_test:read:latency', + 3117.62, + 'usec', + { 'max': 352736, 'stddev': 5114.37, 'min': 0, @@ -645,154 +706,160 @@ def testParseFioResults(self): 'p50': 3280, 'p40': 3248, 'fio_job': 'random_read_test', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'random_read_test:read:latency:min', 0, 'usec', { - 'fio_job': 'random_read_test', - 'fio_version': 'fio-3.27' - } + 'random_read_test:read:latency:min', + 0, + 'usec', + {'fio_job': 'random_read_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_read_test:read:latency:max', 352736, 'usec', { - 'fio_job': 'random_read_test', - 'fio_version': 'fio-3.27' - } + 'random_read_test:read:latency:max', + 352736, + 'usec', + {'fio_job': 'random_read_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_read_test:read:latency:mean', 3117.62, 'usec', { - 'fio_job': 'random_read_test', - 'fio_version': 'fio-3.27' - } + 'random_read_test:read:latency:mean', + 3117.62, + 'usec', + {'fio_job': 'random_read_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_read_test:read:latency:stddev', 5114.37, 'usec', { - 'fio_job': 'random_read_test', - 'fio_version': 'fio-3.27' - } + 'random_read_test:read:latency:stddev', + 5114.37, + 'usec', + {'fio_job': 'random_read_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_read_test:read:latency:p1', 524, 'usec', { - 'fio_job': 'random_read_test', - 'fio_version': 'fio-3.27' - } + 'random_read_test:read:latency:p1', + 524, + 'usec', + {'fio_job': 'random_read_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_read_test:read:latency:p5', 588, 'usec', { - 'fio_job': 'random_read_test', - 'fio_version': 'fio-3.27' - } + 'random_read_test:read:latency:p5', + 588, + 'usec', + {'fio_job': 'random_read_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_read_test:read:latency:p10', 2544, 'usec', { - 'fio_job': 'random_read_test', - 'fio_version': 'fio-3.27' - } + 'random_read_test:read:latency:p10', + 2544, + 'usec', + {'fio_job': 'random_read_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_read_test:read:latency:p20', 3152, 'usec', { - 'fio_job': 'random_read_test', - 'fio_version': 'fio-3.27' - } + 'random_read_test:read:latency:p20', + 3152, + 'usec', + {'fio_job': 'random_read_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_read_test:read:latency:p30', 3216, 'usec', { - 'fio_job': 'random_read_test', - 'fio_version': 'fio-3.27' - } + 'random_read_test:read:latency:p30', + 3216, + 'usec', + {'fio_job': 'random_read_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_read_test:read:latency:p40', 3248, 'usec', { - 'fio_job': 'random_read_test', - 'fio_version': 'fio-3.27' - } + 'random_read_test:read:latency:p40', + 3248, + 'usec', + {'fio_job': 'random_read_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_read_test:read:latency:p50', 3280, 'usec', { - 'fio_job': 'random_read_test', - 'fio_version': 'fio-3.27' - } + 'random_read_test:read:latency:p50', + 3280, + 'usec', + {'fio_job': 'random_read_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_read_test:read:latency:p60', 3312, 'usec', { - 'fio_job': 'random_read_test', - 'fio_version': 'fio-3.27' - } + 'random_read_test:read:latency:p60', + 3312, + 'usec', + {'fio_job': 'random_read_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_read_test:read:latency:p70', 3344, 'usec', { - 'fio_job': 'random_read_test', - 'fio_version': 'fio-3.27' - } + 'random_read_test:read:latency:p70', + 3344, + 'usec', + {'fio_job': 'random_read_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_read_test:read:latency:p80', 3376, 'usec', { - 'fio_job': 'random_read_test', - 'fio_version': 'fio-3.27' - } + 'random_read_test:read:latency:p80', + 3376, + 'usec', + {'fio_job': 'random_read_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_read_test:read:latency:p90', 3408, 'usec', { - 'fio_job': 'random_read_test', - 'fio_version': 'fio-3.27' - } + 'random_read_test:read:latency:p90', + 3408, + 'usec', + {'fio_job': 'random_read_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_read_test:read:latency:p95', 3440, 'usec', { - 'fio_job': 'random_read_test', - 'fio_version': 'fio-3.27' - } + 'random_read_test:read:latency:p95', + 3440, + 'usec', + {'fio_job': 'random_read_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_read_test:read:latency:p99', 3728, 'usec', { - 'fio_job': 'random_read_test', - 'fio_version': 'fio-3.27' - } + 'random_read_test:read:latency:p99', + 3728, + 'usec', + {'fio_job': 'random_read_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_read_test:read:latency:p99.5', 4128, 'usec', { - 'fio_job': 'random_read_test', - 'fio_version': 'fio-3.27' - } + 'random_read_test:read:latency:p99.5', + 4128, + 'usec', + {'fio_job': 'random_read_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_read_test:read:latency:p99.9', 6880, 'usec', { - 'fio_job': 'random_read_test', - 'fio_version': 'fio-3.27' - } + 'random_read_test:read:latency:p99.9', + 6880, + 'usec', + {'fio_job': 'random_read_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_read_test:read:latency:p99.95', 11840, 'usec', { - 'fio_job': 'random_read_test', - 'fio_version': 'fio-3.27' - } + 'random_read_test:read:latency:p99.95', + 11840, + 'usec', + {'fio_job': 'random_read_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_read_test:read:latency:p99.99', 354304, 'usec', { - 'fio_job': 'random_read_test', - 'fio_version': 'fio-3.27' - } + 'random_read_test:read:latency:p99.99', + 354304, + 'usec', + {'fio_job': 'random_read_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_read_test:read:iops', 317, '', { - 'fio_job': 'random_read_test', - 'fio_version': 'fio-3.27' - } + 'random_read_test:read:iops', + 317, + '', + {'fio_job': 'random_read_test', 'fio_version': 'fio-3.27'}, ], [ - 'random_read_test_parallel:read:bandwidth', 1292, 'KB/s', { + 'random_read_test_parallel:read:bandwidth', + 1292, + 'KB/s', + { 'bw_max': 1693, 'bw_agg': 1284.71, 'bw_min': 795, 'bw_dev': 88.67, 'bw_mean': 1284.71, 'fio_job': 'random_read_test_parallel', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'random_read_test_parallel:read:latency', 198030.44, 'usec', { + 'random_read_test_parallel:read:latency', + 198030.44, + 'usec', + { 'max': 400078, 'stddev': 21709.40, 'min': 0, @@ -815,146 +882,211 @@ def testParseFioResults(self): 'p50': 199680, 'p40': 197632, 'fio_job': 'random_read_test_parallel', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'random_read_test_parallel:read:latency:min', 0, 'usec', { + 'random_read_test_parallel:read:latency:min', + 0, + 'usec', + { 'fio_job': 'random_read_test_parallel', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'random_read_test_parallel:read:latency:max', 400078, 'usec', { + 'random_read_test_parallel:read:latency:max', + 400078, + 'usec', + { 'fio_job': 'random_read_test_parallel', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'random_read_test_parallel:read:latency:mean', 198030.44, 'usec', + 'random_read_test_parallel:read:latency:mean', + 198030.44, + 'usec', { 'fio_job': 'random_read_test_parallel', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'random_read_test_parallel:read:latency:stddev', 21709.40, 'usec', + 'random_read_test_parallel:read:latency:stddev', + 21709.40, + 'usec', { 'fio_job': 'random_read_test_parallel', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'random_read_test_parallel:read:latency:p1', 65280, 'usec', { + 'random_read_test_parallel:read:latency:p1', + 65280, + 'usec', + { 'fio_job': 'random_read_test_parallel', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'random_read_test_parallel:read:latency:p5', 189440, 'usec', { + 'random_read_test_parallel:read:latency:p5', + 189440, + 'usec', + { 'fio_job': 'random_read_test_parallel', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'random_read_test_parallel:read:latency:p10', 189440, 'usec', { + 'random_read_test_parallel:read:latency:p10', + 189440, + 'usec', + { 'fio_job': 'random_read_test_parallel', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'random_read_test_parallel:read:latency:p20', 193536, 'usec', { + 'random_read_test_parallel:read:latency:p20', + 193536, + 'usec', + { 'fio_job': 'random_read_test_parallel', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'random_read_test_parallel:read:latency:p30', 197632, 'usec', { + 'random_read_test_parallel:read:latency:p30', + 197632, + 'usec', + { 'fio_job': 'random_read_test_parallel', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'random_read_test_parallel:read:latency:p40', 197632, 'usec', { + 'random_read_test_parallel:read:latency:p40', + 197632, + 'usec', + { 'fio_job': 'random_read_test_parallel', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'random_read_test_parallel:read:latency:p50', 199680, 'usec', { + 'random_read_test_parallel:read:latency:p50', + 199680, + 'usec', + { 'fio_job': 'random_read_test_parallel', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'random_read_test_parallel:read:latency:p60', 199680, 'usec', { + 'random_read_test_parallel:read:latency:p60', + 199680, + 'usec', + { 'fio_job': 'random_read_test_parallel', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'random_read_test_parallel:read:latency:p70', 203776, 'usec', { + 'random_read_test_parallel:read:latency:p70', + 203776, + 'usec', + { 'fio_job': 'random_read_test_parallel', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'random_read_test_parallel:read:latency:p80', 203776, 'usec', { + 'random_read_test_parallel:read:latency:p80', + 203776, + 'usec', + { 'fio_job': 'random_read_test_parallel', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'random_read_test_parallel:read:latency:p90', 205824, 'usec', { + 'random_read_test_parallel:read:latency:p90', + 205824, + 'usec', + { 'fio_job': 'random_read_test_parallel', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'random_read_test_parallel:read:latency:p95', 209920, 'usec', { + 'random_read_test_parallel:read:latency:p95', + 209920, + 'usec', + { 'fio_job': 'random_read_test_parallel', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'random_read_test_parallel:read:latency:p99', 209920, 'usec', { + 'random_read_test_parallel:read:latency:p99', + 209920, + 'usec', + { 'fio_job': 'random_read_test_parallel', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'random_read_test_parallel:read:latency:p99.5', 257024, 'usec', { + 'random_read_test_parallel:read:latency:p99.5', + 257024, + 'usec', + { 'fio_job': 'random_read_test_parallel', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'random_read_test_parallel:read:latency:p99.9', 370688, 'usec', { + 'random_read_test_parallel:read:latency:p99.9', + 370688, + 'usec', + { 'fio_job': 'random_read_test_parallel', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'random_read_test_parallel:read:latency:p99.95', 387072, 'usec', { + 'random_read_test_parallel:read:latency:p99.95', + 387072, + 'usec', + { 'fio_job': 'random_read_test_parallel', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'random_read_test_parallel:read:latency:p99.99', 399360, 'usec', { + 'random_read_test_parallel:read:latency:p99.99', + 399360, + 'usec', + { 'fio_job': 'random_read_test_parallel', - 'fio_version': 'fio-3.27' - } + 'fio_version': 'fio-3.27', + }, ], [ - 'random_read_test_parallel:read:iops', 323, '', { + 'random_read_test_parallel:read:iops', + 323, + '', + { 'fio_job': 'random_read_test_parallel', - 'fio_version': 'fio-3.27' - } - ] + 'fio_version': 'fio-3.27', + }, + ], + ] + expected_result = [ + sample.Sample(*sample_tuple) for sample_tuple in expected_result ] - expected_result = [sample.Sample(*sample_tuple) - for sample_tuple in expected_result] self.assertSampleListsEqualUpToTimestamp(result, expected_result) def testParseResultsBaseMetadata(self): @@ -965,9 +1097,12 @@ def testParseResultsBaseMetadata(self): 'sequential_read': {}, 'random_write_test': {}, 'random_read_test': {}, - 'random_read_test_parallel': {}}): - results = fio.ParseResults('', self.result_contents, - base_metadata=BASE_METADATA) + 'random_read_test_parallel': {}, + }, + ): + results = fio.ParseResults( + '', self.result_contents, base_metadata=BASE_METADATA + ) for result in results: self.assertDictContainsSubset(BASE_METADATA, result.metadata) @@ -979,7 +1114,8 @@ def testFioCommandToJob(self): '--direct=0 --size=3790088k --iodepth=8 ' '--name=sequential_write --overwrite=0 --rw=write --end_fsync=1 ' '--name=random_read --size=379008k --stonewall --rw=randread ' - '--name=sequential_read --stonewall --rw=read ') + '--name=sequential_read --stonewall --rw=read ' + ) expected_result = ( '[global]\n' 'filesize=10g\n' @@ -1001,61 +1137,72 @@ def testFioCommandToJob(self): 'rw=randread\n' '[sequential_read]\n' 'stonewall\n' - 'rw=read\n') + 'rw=read\n' + ) result = fio.FioParametersToJob(fio_parameters) self.assertEqual(expected_result, result) def testDeleteParameterFromJobFile(self): - original_job_file = ( - '[global]\n' - 'directory=/dev/sdb\n' - 'filename=foo12_3\n' - '...') - expected_job_file = ( - '[global]\n' - '...') + original_job_file = '[global]\ndirectory=/dev/sdb\nfilename=foo12_3\n...' + expected_job_file = '[global]\n...' self.assertEqual( expected_job_file, fio.DeleteParameterFromJobFile( fio.DeleteParameterFromJobFile(original_job_file, 'directory'), - 'filename')) + 'filename', + ), + ) def testParseHistogramMultipleJobs(self): hist_dir = os.path.join(self.data_dir, 'hist') job_file = _ReadFileToString( - os.path.join(hist_dir, 'pkb-7fb0c9d8-0_fio.job')) - fio_json_result = json.loads(_ReadFileToString( - os.path.join(hist_dir, 'pkb-7fb0c9d8-0_fio.json'))) + os.path.join(hist_dir, 'pkb-7fb0c9d8-0_fio.job') + ) + fio_json_result = json.loads( + _ReadFileToString(os.path.join(hist_dir, 'pkb-7fb0c9d8-0_fio.json')) + ) log_file_base = 'pkb_fio_avg_1506559526.49' - single_bin_vals = [float(f) for f in _ReadFileToString( - os.path.join(hist_dir, 'bin_vals')).split()] + single_bin_vals = [ + float(f) + for f in _ReadFileToString(os.path.join(hist_dir, 'bin_vals')).split() + ] # each hist file has its own bin_vals, but they're all the same - bin_vals = [single_bin_vals, single_bin_vals, - single_bin_vals, single_bin_vals] + bin_vals = [ + single_bin_vals, + single_bin_vals, + single_bin_vals, + single_bin_vals, + ] # redirect open to the hist subdirectory def OpenTestFile(filename): return open(os.path.join(hist_dir, os.path.basename(filename))) - with mock.patch(fio.__name__ + '.open', - new=mock.MagicMock(side_effect=OpenTestFile), - create=True): - results = fio.ParseResults(job_file, fio_json_result, None, - log_file_base, bin_vals) + with mock.patch( + fio.__name__ + '.open', + new=mock.MagicMock(side_effect=OpenTestFile), + create=True, + ): + results = fio.ParseResults( + job_file, fio_json_result, None, log_file_base, bin_vals + ) actual_read_hist = _ExtractHistogramFromMetric( - results, - 'rand_16k_read_100%-io-depth-1-num-jobs-2:16384:read:histogram') + results, 'rand_16k_read_100%-io-depth-1-num-jobs-2:16384:read:histogram' + ) expected_read_hist = json.loads( - _ReadFileToString(os.path.join(hist_dir, 'expected_read.json'))) + _ReadFileToString(os.path.join(hist_dir, 'expected_read.json')) + ) self.assertEqual(expected_read_hist, actual_read_hist) actual_write_hist = _ExtractHistogramFromMetric( results, - 'rand_16k_write_100%-io-depth-1-num-jobs-2:16384:write:histogram') + 'rand_16k_write_100%-io-depth-1-num-jobs-2:16384:write:histogram', + ) expected_write_hist = json.loads( - _ReadFileToString(os.path.join(hist_dir, 'expected_write.json'))) + _ReadFileToString(os.path.join(hist_dir, 'expected_write.json')) + ) self.assertEqual(expected_write_hist, actual_write_hist) diff --git a/tests/linux_packages/fortran_test.py b/tests/linux_packages/fortran_test.py index 8334d9f099..c91a26d7f5 100644 --- a/tests/linux_packages/fortran_test.py +++ b/tests/linux_packages/fortran_test.py @@ -17,8 +17,9 @@ class FortranRepoTestCase(pkb_common_test_case.PkbCommonTestCase): def assertRemoteCommandsCalled(self, expected_calls, vm): # call_args_list.assert.... hard to read - for command, call_arg_entry in zip(expected_calls, - vm.RemoteCommand.call_args_list): + for command, call_arg_entry in zip( + expected_calls, vm.RemoteCommand.call_args_list + ): self.assertEqual(command, call_arg_entry[0][0]) self.assertLen(vm.RemoteCommand.call_args_list, len(expected_calls)) @@ -50,14 +51,16 @@ def testYumInstallVersion(self) -> None: fortran._YumInstallVersion(vm, 9) vm.InstallPackages.assert_has_calls([ mock.call('centos-release-scl-rh'), - mock.call('devtoolset-9-gcc-gfortran') + mock.call('devtoolset-9-gcc-gfortran'), ]) scl_bin = '/opt/rh/devtoolset-9/root/usr/bin' expected_commands = [ 'echo "source scl_source enable devtoolset-9" >> .bashrc', f'sudo rm {scl_bin}/sudo', - 'sudo alternatives --install /usr/bin/gfortran-9 fortran ' - f'{scl_bin}/gfortran 100' + ( + 'sudo alternatives --install /usr/bin/gfortran-9 fortran ' + f'{scl_bin}/gfortran 100' + ), ] self.assertRemoteCommandsCalled(expected_commands, vm) diff --git a/tests/linux_packages/gce_hpc_tools_test.py b/tests/linux_packages/gce_hpc_tools_test.py index 85be34c932..533e2b4b87 100644 --- a/tests/linux_packages/gce_hpc_tools_test.py +++ b/tests/linux_packages/gce_hpc_tools_test.py @@ -17,7 +17,6 @@ from absl import flags from absl.testing import parameterized import mock - from perfkitbenchmarker.linux_packages import gce_hpc_tools from tests import pkb_common_test_case @@ -49,8 +48,10 @@ def testRebootFlag(self, hpc_tools_turning_flag, wait_for_reboot_called): def testMetadataRecorded(self): vm = _YumInstall() - hpc_tools_tuning_str = ('hpcprofile,limits,nofirewalld,nomitigation,' - 'noselinux,nosmt,reboot,tcpmem') + hpc_tools_tuning_str = ( + 'hpcprofile,limits,nofirewalld,nomitigation,' + 'noselinux,nosmt,reboot,tcpmem' + ) expected_metadata = { 'hpc_tools': True, 'hpc_tools_tag': 'head', @@ -71,10 +72,13 @@ def testSetGitCommitTag(self, git_commit_tag, metadata_tag): def testBashCommandCalled(self): vm = _YumInstall() base_command = 'cd /tmp/pkb/hpc-tools; sudo bash mpi-tuning.sh' - command_flags = ('--hpcprofile --limits --nofirewalld --nomitigation ' - '--noselinux --nosmt --reboot --tcpmem') + command_flags = ( + '--hpcprofile --limits --nofirewalld --nomitigation ' + '--noselinux --nosmt --reboot --tcpmem' + ) vm.RemoteCommand.assert_called_with( - f'{base_command} {command_flags}', ignore_failure=True) + f'{base_command} {command_flags}', ignore_failure=True + ) if __name__ == '__main__': diff --git a/tests/linux_packages/hadoop_test.py b/tests/linux_packages/hadoop_test.py index ac0e88b112..358dce823a 100644 --- a/tests/linux_packages/hadoop_test.py +++ b/tests/linux_packages/hadoop_test.py @@ -51,7 +51,8 @@ def setUp(self): def testDefaultHadoopVersion(self, mock_requests): mock_requests.get( 'https://downloads.apache.org/hadoop/common/stable', - text=HADOOP_STABLE_DIR) + text=HADOOP_STABLE_DIR, + ) for _ in range(5): observed = hadoop.HadoopVersion() self.assertEqual(version.Version('3.3.3'), observed) @@ -61,21 +62,25 @@ def testDefaultHadoopVersion(self, mock_requests): @requests_mock.Mocker() def testHadoopVersionConnectionError(self, mock_requests): mock_requests.get( - 'https://downloads.apache.org/hadoop/common/stable', status_code=404) + 'https://downloads.apache.org/hadoop/common/stable', status_code=404 + ) with self.assertRaisesRegex( errors.Setup.MissingExecutableError, - 'Could not load https://downloads.apache.org/hadoop/common/stable'): + 'Could not load https://downloads.apache.org/hadoop/common/stable', + ): hadoop.HadoopVersion() @requests_mock.Mocker() def testHadoopVersionParsingError(self, mock_requests): mock_requests.get( 'https://downloads.apache.org/hadoop/common/stable', - text='bar') + text='bar', + ) with self.assertRaisesRegex( errors.Setup.MissingExecutableError, 'Could not find valid hadoop version at ' - 'https://downloads.apache.org/hadoop/common/stable'): + 'https://downloads.apache.org/hadoop/common/stable', + ): hadoop.HadoopVersion() @requests_mock.Mocker() @@ -92,5 +97,6 @@ def testHadoopVersionUrlOverride(self, mock_requests): self.assertFalse(mock_requests.called) self.assertEqual(version.Version('4.2.0'), observed) + if __name__ == '__main__': unittest.main() diff --git a/tests/linux_packages/hammerdb_test.py b/tests/linux_packages/hammerdb_test.py index 35fbeafcb6..eefbb0957b 100644 --- a/tests/linux_packages/hammerdb_test.py +++ b/tests/linux_packages/hammerdb_test.py @@ -44,25 +44,32 @@ def _ReadFileToString(filename): TEST_DATA_DIR = os.path.join( - os.path.dirname(__file__), '..', 'data', 'hammerdb') + os.path.dirname(__file__), '..', 'data', 'hammerdb' +) TPCC_TIMEPROFILE_LOG = _ReadFileToString( - os.path.join(TEST_DATA_DIR, 'hammerdbcli_tpcc_timeprofile.log')) + os.path.join(TEST_DATA_DIR, 'hammerdbcli_tpcc_timeprofile.log') +) TPM_PER_SECOND_4_3_LOG = _ReadFileToString( - os.path.join(TEST_DATA_DIR, 'hammerdbcli_tpcc_tpm_per_second_4_3.log')) + os.path.join(TEST_DATA_DIR, 'hammerdbcli_tpcc_tpm_per_second_4_3.log') +) TPCC_TIMEPROFILE_4_3_LOG = _ReadFileToString( - os.path.join(TEST_DATA_DIR, 'hammerdbcli_tpcc_timeprofile_4_3.log')) + os.path.join(TEST_DATA_DIR, 'hammerdbcli_tpcc_timeprofile_4_3.log') +) TPCC_LOG = _ReadFileToString( - os.path.join(TEST_DATA_DIR, 'hammerdbcli_tpcc.log')) + os.path.join(TEST_DATA_DIR, 'hammerdbcli_tpcc.log') +) TPCH_LOG = _ReadFileToString( - os.path.join(TEST_DATA_DIR, 'hammerdbcli_tpch.log')) + os.path.join(TEST_DATA_DIR, 'hammerdbcli_tpch.log') +) -class HammerdbcliTest(pkb_common_test_case.PkbCommonTestCase, - test_util.SamplesTestMixin): +class HammerdbcliTest( + pkb_common_test_case.PkbCommonTestCase, test_util.SamplesTestMixin +): @mock.patch('time.time', mock.MagicMock(return_value=0)) def testParseTpcCTPMResultsFromFile(self): @@ -70,72 +77,224 @@ def testParseTpcCTPMResultsFromFile(self): FLAGS['hammerdbcli_tpcc_time_profile'].parse(True) FLAGS['hammerdbcli_tpcc_rampup'].parse(0) output = ParseTpcCTPMResultsFromFile(TPM_PER_SECOND_4_3_LOG) - self.assertSampleListsEqualUpToTimestamp(output, [ - sample.Sample( - metric='TPM_time_series', - value=0.0, - unit='TPM', - metadata={ - 'values': [ - 0.0, 324540.0, 324300.0, 316260.0, 331020.0, 308700.0, - 305280.0, 303300.0, 328500.0, 312180.0, 453480.0, 322620.0, - 317760.0, 312060.0, 320820.0, 321420.0, 326520.0, 338940.0, - 315780.0, 325560.0, 320400.0, 325860.0, 340320.0, 326160.0, - 328140.0, 331200.0, 323460.0, 334140.0, 312660.0, 325980.0, - 325440.0, 330360.0, 335460.0, 314160.0, 325440.0, 326760.0, - 319320.0, 328740.0, 320700.0, 323340.0, 331200.0, 329700.0, - 320640.0, 349560.0, 331980.0, 322320.0, 337680.0, 319680.0, - 319680.0, 322920.0, 326760.0, 330600.0, 317460.0, 324300.0, - 319500.0, 332340.0, 340980.0, 388500.0, 345120.0, 322980.0, - 366180.0, 318660.0, 324540.0, 329820.0, 349020.0, 324360.0, - 327180.0, 319800.0, 320160.0, 339960.0, 316680.0, 325140.0, - 327480.0, 323640.0, 343920.0, 323460.0, 320880.0, 323400.0, - 327540.0, 327420.0, 327540.0, 327240.0, 316020.0, 332460.0, - 328320.0, 320880.0, 318840.0, 320160.0, 336360.0, 305880.0, - 331800.0, 348780.0, 333000.0, 330960.0, 325440.0, 324240.0, - 326160.0, 333540.0, 325020.0 - ], - 'timestamps': [ - 1656629020000, 1656629021000, 1656629022000, - 1656629023000, 1656629024000, 1656629025000, - 1656629026000, 1656629027000, 1656629028000, - 1656629029000, 1656629030000, 1656629031000, - 1656629032000, 1656629033000, 1656629034000, - 1656629035000, 1656629036000, 1656629037000, - 1656629038000, 1656629039000, 1656629040000, - 1656629041000, 1656629042000, 1656629043000, - 1656629044000, 1656629045000, 1656629046000, - 1656629047000, 1656629048000, 1656629049000, - 1656629050000, 1656629051000, 1656629052000, - 1656629053000, 1656629054000, 1656629055000, - 1656629056000, 1656629057000, 1656629058000, - 1656629059000, 1656629060000, 1656629061000, - 1656629062000, 1656629063000, 1656629064000, - 1656629065000, 1656629066000, 1656629067000, - 1656629068000, 1656629069000, 1656629070000, - 1656629071000, 1656629072000, 1656629073000, - 1656629074000, 1656629075000, 1656629076000, - 1656629077000, 1656629078000, 1656629079000, - 1656629080000, 1656629081000, 1656629082000, - 1656629083000, 1656629084000, 1656629085000, - 1656629086000, 1656629087000, 1656629088000, - 1656629089000, 1656629090000, 1656629091000, - 1656629092000, 1656629093000, 1656629094000, - 1656629095000, 1656629096000, 1656629097000, - 1656629098000, 1656629099000, 1656629100000, - 1656629101000, 1656629102000, 1656629103000, - 1656629104000, 1656629105000, 1656629106000, - 1656629107000, 1656629108000, 1656629109000, - 1656629110000, 1656629111000, 1656629112000, - 1656629114000, 1656629115000, 1656629116000, - 1656629117000, 1656629118000, 1656629119000 - ], - 'interval': 1, - 'ramp_up_ends': 1656629020000, - 'ramp_down_starts': 1656629620000 - }, - timestamp=0) - ]) + self.assertSampleListsEqualUpToTimestamp( + output, + [ + sample.Sample( + metric='TPM_time_series', + value=0.0, + unit='TPM', + metadata={ + 'values': [ + 0.0, + 324540.0, + 324300.0, + 316260.0, + 331020.0, + 308700.0, + 305280.0, + 303300.0, + 328500.0, + 312180.0, + 453480.0, + 322620.0, + 317760.0, + 312060.0, + 320820.0, + 321420.0, + 326520.0, + 338940.0, + 315780.0, + 325560.0, + 320400.0, + 325860.0, + 340320.0, + 326160.0, + 328140.0, + 331200.0, + 323460.0, + 334140.0, + 312660.0, + 325980.0, + 325440.0, + 330360.0, + 335460.0, + 314160.0, + 325440.0, + 326760.0, + 319320.0, + 328740.0, + 320700.0, + 323340.0, + 331200.0, + 329700.0, + 320640.0, + 349560.0, + 331980.0, + 322320.0, + 337680.0, + 319680.0, + 319680.0, + 322920.0, + 326760.0, + 330600.0, + 317460.0, + 324300.0, + 319500.0, + 332340.0, + 340980.0, + 388500.0, + 345120.0, + 322980.0, + 366180.0, + 318660.0, + 324540.0, + 329820.0, + 349020.0, + 324360.0, + 327180.0, + 319800.0, + 320160.0, + 339960.0, + 316680.0, + 325140.0, + 327480.0, + 323640.0, + 343920.0, + 323460.0, + 320880.0, + 323400.0, + 327540.0, + 327420.0, + 327540.0, + 327240.0, + 316020.0, + 332460.0, + 328320.0, + 320880.0, + 318840.0, + 320160.0, + 336360.0, + 305880.0, + 331800.0, + 348780.0, + 333000.0, + 330960.0, + 325440.0, + 324240.0, + 326160.0, + 333540.0, + 325020.0, + ], + 'timestamps': [ + 1656629020000, + 1656629021000, + 1656629022000, + 1656629023000, + 1656629024000, + 1656629025000, + 1656629026000, + 1656629027000, + 1656629028000, + 1656629029000, + 1656629030000, + 1656629031000, + 1656629032000, + 1656629033000, + 1656629034000, + 1656629035000, + 1656629036000, + 1656629037000, + 1656629038000, + 1656629039000, + 1656629040000, + 1656629041000, + 1656629042000, + 1656629043000, + 1656629044000, + 1656629045000, + 1656629046000, + 1656629047000, + 1656629048000, + 1656629049000, + 1656629050000, + 1656629051000, + 1656629052000, + 1656629053000, + 1656629054000, + 1656629055000, + 1656629056000, + 1656629057000, + 1656629058000, + 1656629059000, + 1656629060000, + 1656629061000, + 1656629062000, + 1656629063000, + 1656629064000, + 1656629065000, + 1656629066000, + 1656629067000, + 1656629068000, + 1656629069000, + 1656629070000, + 1656629071000, + 1656629072000, + 1656629073000, + 1656629074000, + 1656629075000, + 1656629076000, + 1656629077000, + 1656629078000, + 1656629079000, + 1656629080000, + 1656629081000, + 1656629082000, + 1656629083000, + 1656629084000, + 1656629085000, + 1656629086000, + 1656629087000, + 1656629088000, + 1656629089000, + 1656629090000, + 1656629091000, + 1656629092000, + 1656629093000, + 1656629094000, + 1656629095000, + 1656629096000, + 1656629097000, + 1656629098000, + 1656629099000, + 1656629100000, + 1656629101000, + 1656629102000, + 1656629103000, + 1656629104000, + 1656629105000, + 1656629106000, + 1656629107000, + 1656629108000, + 1656629109000, + 1656629110000, + 1656629111000, + 1656629112000, + 1656629114000, + 1656629115000, + 1656629116000, + 1656629117000, + 1656629118000, + 1656629119000, + ], + 'interval': 1, + 'ramp_up_ends': 1656629020000, + 'ramp_down_starts': 1656629620000, + }, + timestamp=0, + ) + ], + ) @mock.patch('time.time', mock.MagicMock(return_value=0)) def testParseTpcCTPMResultsFromFileWithRampUp(self): @@ -143,72 +302,224 @@ def testParseTpcCTPMResultsFromFileWithRampUp(self): FLAGS['hammerdbcli_tpcc_time_profile'].parse(True) FLAGS['hammerdbcli_tpcc_rampup'].parse(1) output = ParseTpcCTPMResultsFromFile(TPM_PER_SECOND_4_3_LOG) - self.assertSampleListsEqualUpToTimestamp(output, [ - sample.Sample( - metric='TPM_time_series', - value=0.0, - unit='TPM', - metadata={ - 'values': [ - 0.0, 324540.0, 324300.0, 316260.0, 331020.0, 308700.0, - 305280.0, 303300.0, 328500.0, 312180.0, 453480.0, 322620.0, - 317760.0, 312060.0, 320820.0, 321420.0, 326520.0, 338940.0, - 315780.0, 325560.0, 320400.0, 325860.0, 340320.0, 326160.0, - 328140.0, 331200.0, 323460.0, 334140.0, 312660.0, 325980.0, - 325440.0, 330360.0, 335460.0, 314160.0, 325440.0, 326760.0, - 319320.0, 328740.0, 320700.0, 323340.0, 331200.0, 329700.0, - 320640.0, 349560.0, 331980.0, 322320.0, 337680.0, 319680.0, - 319680.0, 322920.0, 326760.0, 330600.0, 317460.0, 324300.0, - 319500.0, 332340.0, 340980.0, 388500.0, 345120.0, 322980.0, - 366180.0, 318660.0, 324540.0, 329820.0, 349020.0, 324360.0, - 327180.0, 319800.0, 320160.0, 339960.0, 316680.0, 325140.0, - 327480.0, 323640.0, 343920.0, 323460.0, 320880.0, 323400.0, - 327540.0, 327420.0, 327540.0, 327240.0, 316020.0, 332460.0, - 328320.0, 320880.0, 318840.0, 320160.0, 336360.0, 305880.0, - 331800.0, 348780.0, 333000.0, 330960.0, 325440.0, 324240.0, - 326160.0, 333540.0, 325020.0 - ], - 'timestamps': [ - 1656629020000, 1656629021000, 1656629022000, - 1656629023000, 1656629024000, 1656629025000, - 1656629026000, 1656629027000, 1656629028000, - 1656629029000, 1656629030000, 1656629031000, - 1656629032000, 1656629033000, 1656629034000, - 1656629035000, 1656629036000, 1656629037000, - 1656629038000, 1656629039000, 1656629040000, - 1656629041000, 1656629042000, 1656629043000, - 1656629044000, 1656629045000, 1656629046000, - 1656629047000, 1656629048000, 1656629049000, - 1656629050000, 1656629051000, 1656629052000, - 1656629053000, 1656629054000, 1656629055000, - 1656629056000, 1656629057000, 1656629058000, - 1656629059000, 1656629060000, 1656629061000, - 1656629062000, 1656629063000, 1656629064000, - 1656629065000, 1656629066000, 1656629067000, - 1656629068000, 1656629069000, 1656629070000, - 1656629071000, 1656629072000, 1656629073000, - 1656629074000, 1656629075000, 1656629076000, - 1656629077000, 1656629078000, 1656629079000, - 1656629080000, 1656629081000, 1656629082000, - 1656629083000, 1656629084000, 1656629085000, - 1656629086000, 1656629087000, 1656629088000, - 1656629089000, 1656629090000, 1656629091000, - 1656629092000, 1656629093000, 1656629094000, - 1656629095000, 1656629096000, 1656629097000, - 1656629098000, 1656629099000, 1656629100000, - 1656629101000, 1656629102000, 1656629103000, - 1656629104000, 1656629105000, 1656629106000, - 1656629107000, 1656629108000, 1656629109000, - 1656629110000, 1656629111000, 1656629112000, - 1656629114000, 1656629115000, 1656629116000, - 1656629117000, 1656629118000, 1656629119000 - ], - 'interval': 1, - 'ramp_up_ends': 1656629080000, - 'ramp_down_starts': 1656629680000 - }, - timestamp=0) - ]) + self.assertSampleListsEqualUpToTimestamp( + output, + [ + sample.Sample( + metric='TPM_time_series', + value=0.0, + unit='TPM', + metadata={ + 'values': [ + 0.0, + 324540.0, + 324300.0, + 316260.0, + 331020.0, + 308700.0, + 305280.0, + 303300.0, + 328500.0, + 312180.0, + 453480.0, + 322620.0, + 317760.0, + 312060.0, + 320820.0, + 321420.0, + 326520.0, + 338940.0, + 315780.0, + 325560.0, + 320400.0, + 325860.0, + 340320.0, + 326160.0, + 328140.0, + 331200.0, + 323460.0, + 334140.0, + 312660.0, + 325980.0, + 325440.0, + 330360.0, + 335460.0, + 314160.0, + 325440.0, + 326760.0, + 319320.0, + 328740.0, + 320700.0, + 323340.0, + 331200.0, + 329700.0, + 320640.0, + 349560.0, + 331980.0, + 322320.0, + 337680.0, + 319680.0, + 319680.0, + 322920.0, + 326760.0, + 330600.0, + 317460.0, + 324300.0, + 319500.0, + 332340.0, + 340980.0, + 388500.0, + 345120.0, + 322980.0, + 366180.0, + 318660.0, + 324540.0, + 329820.0, + 349020.0, + 324360.0, + 327180.0, + 319800.0, + 320160.0, + 339960.0, + 316680.0, + 325140.0, + 327480.0, + 323640.0, + 343920.0, + 323460.0, + 320880.0, + 323400.0, + 327540.0, + 327420.0, + 327540.0, + 327240.0, + 316020.0, + 332460.0, + 328320.0, + 320880.0, + 318840.0, + 320160.0, + 336360.0, + 305880.0, + 331800.0, + 348780.0, + 333000.0, + 330960.0, + 325440.0, + 324240.0, + 326160.0, + 333540.0, + 325020.0, + ], + 'timestamps': [ + 1656629020000, + 1656629021000, + 1656629022000, + 1656629023000, + 1656629024000, + 1656629025000, + 1656629026000, + 1656629027000, + 1656629028000, + 1656629029000, + 1656629030000, + 1656629031000, + 1656629032000, + 1656629033000, + 1656629034000, + 1656629035000, + 1656629036000, + 1656629037000, + 1656629038000, + 1656629039000, + 1656629040000, + 1656629041000, + 1656629042000, + 1656629043000, + 1656629044000, + 1656629045000, + 1656629046000, + 1656629047000, + 1656629048000, + 1656629049000, + 1656629050000, + 1656629051000, + 1656629052000, + 1656629053000, + 1656629054000, + 1656629055000, + 1656629056000, + 1656629057000, + 1656629058000, + 1656629059000, + 1656629060000, + 1656629061000, + 1656629062000, + 1656629063000, + 1656629064000, + 1656629065000, + 1656629066000, + 1656629067000, + 1656629068000, + 1656629069000, + 1656629070000, + 1656629071000, + 1656629072000, + 1656629073000, + 1656629074000, + 1656629075000, + 1656629076000, + 1656629077000, + 1656629078000, + 1656629079000, + 1656629080000, + 1656629081000, + 1656629082000, + 1656629083000, + 1656629084000, + 1656629085000, + 1656629086000, + 1656629087000, + 1656629088000, + 1656629089000, + 1656629090000, + 1656629091000, + 1656629092000, + 1656629093000, + 1656629094000, + 1656629095000, + 1656629096000, + 1656629097000, + 1656629098000, + 1656629099000, + 1656629100000, + 1656629101000, + 1656629102000, + 1656629103000, + 1656629104000, + 1656629105000, + 1656629106000, + 1656629107000, + 1656629108000, + 1656629109000, + 1656629110000, + 1656629111000, + 1656629112000, + 1656629114000, + 1656629115000, + 1656629116000, + 1656629117000, + 1656629118000, + 1656629119000, + ], + 'interval': 1, + 'ramp_up_ends': 1656629080000, + 'ramp_down_starts': 1656629680000, + }, + timestamp=0, + ) + ], + ) @mock.patch('time.time', mock.MagicMock(return_value=0)) def testParseTPCCWithTimeProfile(self): @@ -217,7 +528,8 @@ def testParseTPCCWithTimeProfile(self): vm = mock.Mock() output = ParseTpcCResults(TPCC_TIMEPROFILE_LOG, vm) expected_result = [ - ('TPM', 17145, 'TPM'), ('NOPM', 5614, 'NOPM'), + ('TPM', 17145, 'TPM'), + ('NOPM', 5614, 'NOPM'), ('neword_MIN', 7.252800000000001, 'milliseconds'), ('neword_P50', 13.401272222222222, 'milliseconds'), ('neword_P95', 18.249016666666666, 'milliseconds'), @@ -247,11 +559,12 @@ def testParseTPCCWithTimeProfile(self): ('gettimestamp_P50', 0.004, 'milliseconds'), ('gettimestamp_P95', 0.005155555555555556, 'milliseconds'), ('gettimestamp_P99', 0.009266666666666668, 'milliseconds'), - ('gettimestamp_MAX', 0.01857777777777778, 'milliseconds') + ('gettimestamp_MAX', 0.01857777777777778, 'milliseconds'), ] self.assertCountEqual( - output, [sample.Sample(i[0], i[1], i[2]) for i in expected_result]) + output, [sample.Sample(i[0], i[1], i[2]) for i in expected_result] + ) @mock.patch('time.time', mock.MagicMock(return_value=0)) def testParseTpcCTimeProfileResultsFromFile(self): @@ -264,185 +577,216 @@ def testParseTpcCTimeProfileResultsFromFile(self): value=322104.0, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='neword_MIN', value=2.269, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='neword_MAX', value=36.366, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='neword_P99', value=14.197, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='neword_P95', value=12.399, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='neword_P50', value=9.087, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='payment_CALLS', value=321305.0, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='payment_MIN', value=3.038, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='payment_MAX', value=26.534, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='payment_P99', value=10.944, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='payment_P95', value=9.492, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='payment_P50', value=6.818, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='delivery_CALLS', value=31967.0, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='delivery_MIN', value=31.27, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='delivery_MAX', value=83.076, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='delivery_P99', value=68.64, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='delivery_P95', value=65.194, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='delivery_P50', value=57.664, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='slev_CALLS', value=32440.0, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='slev_MIN', value=1.349, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='slev_MAX', value=16.625, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='slev_P99', value=5.53, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='slev_P95', value=4.04, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='slev_P50', value=2.305, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='ostat_CALLS', value=32044.0, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='ostat_MIN', value=1.095, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='ostat_MAX', value=17.16, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='ostat_P99', value=5.246, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='ostat_P95', value=3.752, unit='milliseconds', metadata={}, - timestamp=0), + timestamp=0, + ), sample.Sample( metric='ostat_P50', value=2.033, unit='milliseconds', metadata={}, - timestamp=0) + timestamp=0, + ), ] self.assertCountEqual( - output, [sample.Sample(i[0], i[1], i[2]) for i in expected_result]) + output, [sample.Sample(i[0], i[1], i[2]) for i in expected_result] + ) @mock.patch('time.time', mock.MagicMock(return_value=0)) def testParseTPCCWithoutTimeProfile(self): @@ -450,40 +794,46 @@ def testParseTPCCWithoutTimeProfile(self): FLAGS['hammerdbcli_tpcc_time_profile'].parse(False) vm = mock.Mock() output = ParseTpcCResults(TPCC_LOG, vm) - self.assertCountEqual(output, [ - sample.Sample('TPM', 24772, 'TPM'), - sample.Sample('NOPM', 8153, 'NOPM') - ]) + self.assertCountEqual( + output, + [ + sample.Sample('TPM', 24772, 'TPM'), + sample.Sample('NOPM', 8153, 'NOPM'), + ], + ) @mock.patch('time.time', mock.MagicMock(return_value=0)) def testParseTPCH(self): """Tests parsing metadata from the ANSYS benchmark output.""" output = ParseTpcHResults(TPCH_LOG) - expected_result = [('Query_14', 0.795, 'seconds'), - ('Query_2', 0.188, 'seconds'), - ('Query_9', 5.554, 'seconds'), - ('Query_20', 1.381, 'seconds'), - ('Query_6', 3.631, 'seconds'), - ('Query_17', 0.527, 'seconds'), - ('Query_18', 2.804, 'seconds'), - ('Query_8', 0.932, 'seconds'), - ('Query_21', 2.218, 'seconds'), - ('Query_13', 4.656, 'seconds'), - ('Query_3', 2.556, 'seconds'), - ('Query_22', 0.152, 'seconds'), - ('Query_16', 0.478, 'seconds'), - ('Query_4', 1.538, 'seconds'), - ('Query_11', 0.429, 'seconds'), - ('Query_15', 4.128, 'seconds'), - ('Query_1', 17.681, 'seconds'), - ('Query_10', 1.661, 'seconds'), - ('Query_19', 0.19, 'seconds'), - ('Query_5', 1.613, 'seconds'), - ('Query_7', 2.194, 'seconds'), - ('Query_12', 2.803, 'seconds'), - ('query_times_geomean', 1.40067754041747, 'seconds')] + expected_result = [ + ('Query_14', 0.795, 'seconds'), + ('Query_2', 0.188, 'seconds'), + ('Query_9', 5.554, 'seconds'), + ('Query_20', 1.381, 'seconds'), + ('Query_6', 3.631, 'seconds'), + ('Query_17', 0.527, 'seconds'), + ('Query_18', 2.804, 'seconds'), + ('Query_8', 0.932, 'seconds'), + ('Query_21', 2.218, 'seconds'), + ('Query_13', 4.656, 'seconds'), + ('Query_3', 2.556, 'seconds'), + ('Query_22', 0.152, 'seconds'), + ('Query_16', 0.478, 'seconds'), + ('Query_4', 1.538, 'seconds'), + ('Query_11', 0.429, 'seconds'), + ('Query_15', 4.128, 'seconds'), + ('Query_1', 17.681, 'seconds'), + ('Query_10', 1.661, 'seconds'), + ('Query_19', 0.19, 'seconds'), + ('Query_5', 1.613, 'seconds'), + ('Query_7', 2.194, 'seconds'), + ('Query_12', 2.803, 'seconds'), + ('query_times_geomean', 1.40067754041747, 'seconds'), + ] self.assertCountEqual( - output, [sample.Sample(i[0], i[1], i[2]) for i in expected_result]) + output, [sample.Sample(i[0], i[1], i[2]) for i in expected_result] + ) if __name__ == '__main__': diff --git a/tests/linux_packages/hbase_test.py b/tests/linux_packages/hbase_test.py index 1720b03fab..93f8fef12c 100644 --- a/tests/linux_packages/hbase_test.py +++ b/tests/linux_packages/hbase_test.py @@ -38,8 +38,9 @@ def testGetUrlVersion(self): def MakeHbaseUrl(version): - return ('' - 'hbase-{version}-bin.tar.gz').format(version=version) + return ( + 'hbase-{version}-bin.tar.gz' + ).format(version=version) if __name__ == '__main__': diff --git a/tests/linux_packages/hpcc_test.py b/tests/linux_packages/hpcc_test.py index f9aac4d45e..62d3571c04 100644 --- a/tests/linux_packages/hpcc_test.py +++ b/tests/linux_packages/hpcc_test.py @@ -18,7 +18,6 @@ import unittest import mock - from perfkitbenchmarker import vm_util from perfkitbenchmarker.linux_packages import hpcc @@ -122,7 +121,8 @@ def testLimitBenchmarksToRunToLastBenchmark(self): def testLimitBenchmarksToRunToMultipleBenchmarks(self): """Tests limiting the benchmarks to run to the first benchmark.""" self._RunLimitBenchmarksToRun( - set(['StarRandomAccess', 'SingleRandomAccess'])) + set(['StarRandomAccess', 'SingleRandomAccess']) + ) self._ValidateRunLimitBenchmarksToRun([ ' MPI_Init( &argc, &argv );', ' if (redacted)', diff --git a/tests/linux_packages/imb_test.py b/tests/linux_packages/imb_test.py index bba2e38021..a970e1b6da 100644 --- a/tests/linux_packages/imb_test.py +++ b/tests/linux_packages/imb_test.py @@ -16,22 +16,27 @@ def MockVm(): return mock.Mock( - internal_ip='1.2.3.4', NumCpusForBenchmark=8, BASE_OS_TYPE=os_types.RHEL) + internal_ip='1.2.3.4', NumCpusForBenchmark=8, BASE_OS_TYPE=os_types.RHEL + ) class IntelMpiLibTestCase(pkb_common_test_case.PkbCommonTestCase): - - MPIVARS_FILE = ('/opt/intel/compilers_and_libraries/' - 'linux/mpi/intel64/bin/mpivars.sh') - - COMPILE_2019 = ('cd mpi-benchmarks; ' - '. /opt/intel/mkl/bin/mklvars.sh intel64; ' - '. /opt/intel/compilers_and_libraries/' - 'linux/bin/compilervars.sh intel64; ' - 'CC=mpicc CXX=mpicxx make') - COMPILE_2021 = ('cd mpi-benchmarks; ' - '. /opt/intel/oneapi/setvars.sh; ' - 'CC=mpicc CXX=mpicxx make') + MPIVARS_FILE = ( + '/opt/intel/compilers_and_libraries/linux/mpi/intel64/bin/mpivars.sh' + ) + + COMPILE_2019 = ( + 'cd mpi-benchmarks; ' + '. /opt/intel/mkl/bin/mklvars.sh intel64; ' + '. /opt/intel/compilers_and_libraries/' + 'linux/bin/compilervars.sh intel64; ' + 'CC=mpicc CXX=mpicxx make' + ) + COMPILE_2021 = ( + 'cd mpi-benchmarks; ' + '. /opt/intel/oneapi/setvars.sh; ' + 'CC=mpicc CXX=mpicxx make' + ) def setUp(self): super().setUp() @@ -42,8 +47,10 @@ def MockVmWithReturnValues(self): vm = MockVm() vm_returns = [ self.MPIVARS_FILE, - ('Intel(R) MPI Library for Linux* OS, ' - 'Version 2018 Update 4 Build 20180823 (id: 18555)') + ( + 'Intel(R) MPI Library for Linux* OS, ' + 'Version 2018 Update 4 Build 20180823 (id: 18555)' + ), ] vm.RemoteCommand.side_effect = [(txt, '') for txt in vm_returns] return vm @@ -56,10 +63,15 @@ def testInstallCompileSource(self) -> None: # just confirm that the git clone and patch were done cmd = ';'.join([cmd[0][0] for cmd in vm.RemoteCommand.call_args_list]) self.assertRegex( - cmd, 'git clone -n https://github.com/intel/mpi-benchmarks.git', - 'Missing git clone command') - self.assertRegex(cmd, 'patch -d mpi-benchmarks -p3 < ~/intelmpi.patch', - 'Missing patch command') + cmd, + 'git clone -n https://github.com/intel/mpi-benchmarks.git', + 'Missing git clone command', + ) + self.assertRegex( + cmd, + 'patch -d mpi-benchmarks -p3 < ~/intelmpi.patch', + 'Missing patch command', + ) def testMpirunMpiVersion(self): vm = self.MockVmWithReturnValues() @@ -84,22 +96,29 @@ def testPpn(self, total_processes, expected_suffix): mpirun = imb.MpiRunCommand(vm, hosts, total_processes, 0, [], [], False) # '-ppn 1' is only seen when running single threaded tests - expected_mpirun = (f'mpirun -n {total_processes} -hosts 10.0.0.1,10.0.0.2' - f'{expected_suffix}') + expected_mpirun = ( + f'mpirun -n {total_processes} -hosts 10.0.0.1,10.0.0.2{expected_suffix}' + ) self.assertEqual(f'. {self.MPIVARS_FILE}; {expected_mpirun}', mpirun) @parameterized.parameters( ('2019.6', COMPILE_2019, []), - ('2021.2', COMPILE_2021, - ['intel-oneapi-compiler-dpcpp-cpp', 'intel-oneapi-mpi-devel'])) - def testInstall2021(self, intelmpi_version, expected_compile_cmd, - installed_packages): + ( + '2021.2', + COMPILE_2021, + ['intel-oneapi-compiler-dpcpp-cpp', 'intel-oneapi-mpi-devel'], + ), + ) + def testInstall2021( + self, intelmpi_version, expected_compile_cmd, installed_packages + ): vm = MockVm() with flagsaver.flagsaver(intelmpi_version=intelmpi_version): imb.Install(vm) vm.RemoteCommand.assert_any_call(expected_compile_cmd) vm.InstallPackages.assert_has_calls( - [mock.call(pkb) for pkb in installed_packages]) + [mock.call(pkb) for pkb in installed_packages] + ) class OpenMpiLibTestCase(pkb_common_test_case.PkbCommonTestCase): @@ -113,10 +132,15 @@ def testInstallCompileSource(self) -> None: imb.Install(vm) cmd = ';'.join([cmd[0][0] for cmd in vm.RemoteCommand.call_args_list]) self.assertRegex( - cmd, 'git clone -n https://github.com/intel/mpi-benchmarks.git', - 'Missing git clone command') - self.assertRegex(cmd, 'patch -d mpi-benchmarks -p3 < ~/intelmpi.patch', - 'Missing patch command') + cmd, + 'git clone -n https://github.com/intel/mpi-benchmarks.git', + 'Missing git clone command', + ) + self.assertRegex( + cmd, + 'patch -d mpi-benchmarks -p3 < ~/intelmpi.patch', + 'Missing patch command', + ) @flagsaver.flagsaver(imb_compile_from_source=False) def testInstallWithoutImbCompileFromSourceThrows(self) -> None: @@ -125,7 +149,8 @@ def testInstallWithoutImbCompileFromSourceThrows(self) -> None: imb.Install(vm) self.assertEqual( str(e.exception), - '--mpi_vendor=openmpi requires --imb_compile_from_source') + '--mpi_vendor=openmpi requires --imb_compile_from_source', + ) def testMpiRunCommandEnvVarsExported(self): vm = MockVm() @@ -137,14 +162,16 @@ def testMpiRunCommandEnvVarsExported(self): 'OMPI_MCA_rmaps_base_mapping_policy=node:PE=1', ] - mpirun = imb.MpiRunCommand(vm, hosts, total_proc, ppn, environment, [], - False) + mpirun = imb.MpiRunCommand( + vm, hosts, total_proc, ppn, environment, [], False + ) expected_mpirun = ( 'OMPI_MCA_btl=self,tcp OMPI_MCA_rmaps_base_mapping_policy=node:PE=1 ' 'mpirun -x OMPI_MCA_btl -x OMPI_MCA_rmaps_base_mapping_policy ' '-report-bindings -display-map -n 2 -npernode 1 --use-hwthread-cpus ' - '-host 10.0.0.1:slots=2,10.0.0.2:slots=2') + '-host 10.0.0.1:slots=2,10.0.0.2:slots=2' + ) self.assertEqual(expected_mpirun, mpirun) def testMpiRunCommandNoEnvVarsIsFormattedCorrectly(self): @@ -154,12 +181,14 @@ def testMpiRunCommandNoEnvVarsIsFormattedCorrectly(self): hosts = ['10.0.0.1', '10.0.0.2'] environment = [] - mpirun = imb.MpiRunCommand(vm, hosts, total_proc, ppn, environment, [], - False) + mpirun = imb.MpiRunCommand( + vm, hosts, total_proc, ppn, environment, [], False + ) expected_mpirun = ( 'mpirun -report-bindings -display-map -n 2 -npernode 1 ' - '--use-hwthread-cpus -host 10.0.0.1:slots=2,10.0.0.2:slots=2') + '--use-hwthread-cpus -host 10.0.0.1:slots=2,10.0.0.2:slots=2' + ) self.assertEqual(expected_mpirun, mpirun) def testMpiRunCommandNoPpnSpecified(self): @@ -169,12 +198,14 @@ def testMpiRunCommandNoPpnSpecified(self): hosts = ['10.0.0.1', '10.0.0.2', '10.0.0.3', '10.0.0.4'] environment = [] - mpirun = imb.MpiRunCommand(vm, hosts, total_proc, ppn, environment, [], - False) + mpirun = imb.MpiRunCommand( + vm, hosts, total_proc, ppn, environment, [], False + ) expected_mpirun = ( 'mpirun -report-bindings -display-map -n 8 -npernode 2 ' '--use-hwthread-cpus -host ' - '10.0.0.1:slots=8,10.0.0.2:slots=8,10.0.0.3:slots=8,10.0.0.4:slots=8') + '10.0.0.1:slots=8,10.0.0.2:slots=8,10.0.0.3:slots=8,10.0.0.4:slots=8' + ) self.assertEqual(expected_mpirun, mpirun) diff --git a/tests/linux_packages/intel_oneapi_basekit_test.py b/tests/linux_packages/intel_oneapi_basekit_test.py index 278576775a..0ad2be78f6 100644 --- a/tests/linux_packages/intel_oneapi_basekit_test.py +++ b/tests/linux_packages/intel_oneapi_basekit_test.py @@ -17,7 +17,8 @@ def testInstall(self, mock_verify): vm.RemoteCommand.assert_called_with( 'chmod +x 2021.2.0.sh; ' 'sudo ./2021.2.0.sh -a -s --eula accept; ' - 'rm 2021.2.0.sh') + 'rm 2021.2.0.sh' + ) mock_verify.assert_called_with(vm) def testVerifyInstall(self): @@ -26,7 +27,7 @@ def testVerifyInstall(self): env_1 = ['PATH=/usr/bin'] env_2 = [ 'PATH=/usr/bin:/opt/intel/oneapi/vtune/2021.2.0/bin64', - f'I_MPI_ROOT={mpi_root}' + f'I_MPI_ROOT={mpi_root}', ] vm.RemoteCommand.side_effect = [ ('\n'.join(env_1), ''), @@ -38,7 +39,7 @@ def testVerifyInstall(self): vm.RemoteCommand.assert_has_calls([ mock.call('env'), - mock.call('. /opt/intel/oneapi/setvars.sh >/dev/null; env') + mock.call('. /opt/intel/oneapi/setvars.sh >/dev/null; env'), ]) def testVerifyInstallNoMpiRoot(self): @@ -52,7 +53,8 @@ def testGetVariables(self): vm = mock.Mock() input_vars = {'A': 'B', 'XDG_SESSION_ID': '4'} vars_as_text = '\n'.join( - f'{key}={value}' for key, value in sorted(input_vars.items())) + f'{key}={value}' for key, value in sorted(input_vars.items()) + ) vm.RemoteCommand.return_value = vars_as_text, '' env_vars = intel_oneapi_basekit._GetVariables(vm) diff --git a/tests/linux_packages/intel_repo_test.py b/tests/linux_packages/intel_repo_test.py index 911e4bc8f0..59387a203a 100644 --- a/tests/linux_packages/intel_repo_test.py +++ b/tests/linux_packages/intel_repo_test.py @@ -14,20 +14,24 @@ def testAptInstall(self) -> None: vm = mock.Mock() intel_repo.AptInstall(vm) vm.PushDataFile.call_args_list[0].assert_called_with( - 'intel_repo_key.txt', '/tmp/pkb/intel_repo_key.txt') + 'intel_repo_key.txt', '/tmp/pkb/intel_repo_key.txt' + ) vm.PushDataFile.call_args_list[1].assert_called_with( - 'intel_repo_list.txt', '/tmp/pkb/intel.list') + 'intel_repo_list.txt', '/tmp/pkb/intel.list' + ) vm.InstallPackages.assert_called_with('libgomp1') def testYumInstall(self) -> None: vm = mock.Mock() vm.RemoteCommandWithReturnCode.return_value = ('', '', 0) intel_repo.YumInstall(vm) - vm.PushDataFile.assert_called_with('intel_repo_key.txt', - '/tmp/pkb/intel_repo_key.txt') + vm.PushDataFile.assert_called_with( + 'intel_repo_key.txt', '/tmp/pkb/intel_repo_key.txt' + ) vm.InstallPackages.assert_called_with('yum-utils') vm.RemoteCommandWithReturnCode.assert_called_with( - 'diff /tmp/pkb/intel_repo_key.txt /tmp/pkb/mpi.yumkey') + 'diff /tmp/pkb/intel_repo_key.txt /tmp/pkb/mpi.yumkey' + ) def testYumInstallBadKey(self) -> None: vm = mock.Mock() @@ -40,14 +44,16 @@ def testAptInstall2021(self) -> None: vm = mock.Mock() intel_repo.AptInstall(vm) vm.PushDataFile.call_args_list[0].assert_called_with( - 'intel_repo_key.txt', '/tmp/pkb/intel_repo_key.txt') + 'intel_repo_key.txt', '/tmp/pkb/intel_repo_key.txt' + ) self.assertLen(vm.PushDataFile.call_args_list, 1) expected_command = ( 'sudo apt-key add /tmp/pkb/intel_repo_key.txt;' 'rm /tmp/pkb/intel_repo_key.txt;' 'echo "deb https://apt.repos.intel.com/oneapi all main" ' '| sudo tee /etc/apt/sources.list.d/oneAPI.list;' - 'sudo apt-get update') + 'sudo apt-get update' + ) vm.RemoteCommand.assert_called_with(expected_command) diff --git a/tests/linux_packages/linux_boot_test.py b/tests/linux_packages/linux_boot_test.py index 0abb86f5ab..2b315dd988 100644 --- a/tests/linux_packages/linux_boot_test.py +++ b/tests/linux_packages/linux_boot_test.py @@ -20,20 +20,25 @@ import pytz -class LinuxBootTest(pkb_common_test_case.PkbCommonTestCase, - test_util.SamplesTestMixin): +class LinuxBootTest( + pkb_common_test_case.PkbCommonTestCase, test_util.SamplesTestMixin +): def setUp(self): super(LinuxBootTest, self).setUp() - self.data_dir = os.path.join(os.path.dirname(__file__), - '..', 'data', 'linux_boot') + self.data_dir = os.path.join( + os.path.dirname(__file__), '..', 'data', 'linux_boot' + ) vm_spec = pkb_common_test_case.CreateTestVmSpec() self.mock_vm = pkb_common_test_case.TestLinuxVirtualMachine(vm_spec=vm_spec) def testDatetimeToUTCSeconds(self): self.assertEqual( linux_boot.DatetimeToUTCSeconds( - datetime.datetime(2023, 4, 6, 18, 1, 1)), 1680804061) + datetime.datetime(2023, 4, 6, 18, 1, 1) + ), + 1680804061, + ) def testScrapeConsoleLogLines(self): """Test startup script output parsing.""" @@ -43,21 +48,34 @@ def testScrapeConsoleLogLines(self): # Since sample.Sample will set timestamp upon object creation, use # assertSampleListsEqualUpToTimestamp method to ignore timestamp. self.assertSampleListsEqualUpToTimestamp( - [sample.Sample( - metric='startup_script_run', value=7.685235, unit='second', - metadata={}, timestamp=0), - sample.Sample( - metric='linux_booting', value=3.698, unit='second', - metadata={}, timestamp=9)], + [ + sample.Sample( + metric='startup_script_run', + value=7.685235, + unit='second', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='linux_booting', + value=3.698, + unit='second', + metadata={}, + timestamp=9, + ), + ], linux_boot.ScrapeConsoleLogLines( boot_output, datetime.datetime.fromtimestamp(1680741770, pytz.timezone('UTC')), - linux_boot.CONSOLE_FIRST_START_MATCHERS)) + linux_boot.CONSOLE_FIRST_START_MATCHERS, + ), + ) def testUtcTimestampToDatetime(self): - self.assertEqual(linux_boot.UtcTimestampToDatetime('1680741777.685234619'), - datetime_tz.datetime_tz(2023, 4, 6, 0, 42, 57, 685235, - tzinfo='UTC')) + self.assertEqual( + linux_boot.UtcTimestampToDatetime('1680741777.685234619'), + datetime_tz.datetime_tz(2023, 4, 6, 0, 42, 57, 685235, tzinfo='UTC'), + ) def testCollectKernelSamples(self): """Test dmesg parsing.""" @@ -66,37 +84,106 @@ def testCollectKernelSamples(self): self.mock_vm.RemoteCommand = mock.Mock(return_value=(dmesg, '')) # Compare golden samples and parsed dmesg samples self.assertSampleListsEqualUpToTimestamp( - [sample.Sample(metric='check_timer', value=1.163323, - unit='second', metadata={}, timestamp=0), - sample.Sample(metric='aesni_init', value=3.646818, - unit='second', metadata={}, timestamp=0), - sample.Sample(metric='pci_bridge_created', value=1.464983, - unit='second', metadata={}, timestamp=0), - sample.Sample(metric='pci_dma_setup', value=1.788605, - unit='second', metadata={}, timestamp=0), - sample.Sample(metric='scsi_init', value=1.958622, - unit='second', metadata={}, timestamp=0), - sample.Sample(metric='serial_8250_init', value=1.800432, - unit='second', metadata={}, timestamp=0), - sample.Sample(metric='tcp_bind_alloc', value=1.612092, - unit='second', metadata={}, timestamp=0), - sample.Sample(metric='rod_marked', value=1.873822, - unit='second', metadata={}, timestamp=0), - sample.Sample(metric='ps2_controller', value=1.806604, - unit='second', metadata={}, timestamp=0), - sample.Sample(metric='rng_init', value=1.15044, - unit='second', metadata={}, timestamp=0), - sample.Sample(metric='taskstats_reg', value=1.825742, - unit='second', metadata={}, timestamp=0), - sample.Sample(metric='scsi_pd', value=1.96243, - unit='second', metadata={}, timestamp=0)], - linux_boot.CollectKernelSamples(self.mock_vm, 1)) + [ + sample.Sample( + metric='check_timer', + value=1.163323, + unit='second', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='aesni_init', + value=3.646818, + unit='second', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='pci_bridge_created', + value=1.464983, + unit='second', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='pci_dma_setup', + value=1.788605, + unit='second', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='scsi_init', + value=1.958622, + unit='second', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='serial_8250_init', + value=1.800432, + unit='second', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='tcp_bind_alloc', + value=1.612092, + unit='second', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='rod_marked', + value=1.873822, + unit='second', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='ps2_controller', + value=1.806604, + unit='second', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='rng_init', + value=1.15044, + unit='second', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='taskstats_reg', + value=1.825742, + unit='second', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='scsi_pd', + value=1.96243, + unit='second', + metadata={}, + timestamp=0, + ), + ], + linux_boot.CollectKernelSamples(self.mock_vm, 1), + ) def testKernelStartTimestamp(self): - self.mock_vm.RemoteCommand = mock.Mock(return_value=("""169174.74 2653910.92 -1680819951.881177330""", '')) - self.assertEqual(linux_boot.GetKernelStartTimestamp(self.mock_vm), - 1680650777.1411774) + self.mock_vm.RemoteCommand = mock.Mock( + return_value=( + """169174.74 2653910.92 +1680819951.881177330""", + '', + ) + ) + self.assertEqual( + linux_boot.GetKernelStartTimestamp(self.mock_vm), 1680650777.1411774 + ) @parameterized.named_parameters( ('HourLong', '1h 19min 20.378s', 4760.378), @@ -108,25 +195,36 @@ def testParseSeconds(self, raw_str, sec): def testParseUserTotalTimes(self): """Test kernel time, total time parsing.""" - self.assertEqual(linux_boot.ParseUserTotalTimes( - 'Startup finished in 2.2s (kernel) + 1min 12.5s (userspace) ' - '= 1min 14.774s'), (2.2, 74.7)) - self.assertEqual(linux_boot.ParseUserTotalTimes( - 'Startup finished in 448ms (firmware) + 1.913s (loader) + ' - '1.183s (kernel) + 52.438s (initrd) + 30.413s (userspace) ' - '= 1min 26.398s'), (53.621, 84.034)) + self.assertEqual( + linux_boot.ParseUserTotalTimes( + 'Startup finished in 2.2s (kernel) + 1min 12.5s (userspace) ' + '= 1min 14.774s' + ), + (2.2, 74.7), + ) + self.assertEqual( + linux_boot.ParseUserTotalTimes( + 'Startup finished in 448ms (firmware) + 1.913s (loader) + ' + '1.183s (kernel) + 52.438s (initrd) + 30.413s (userspace) ' + '= 1min 26.398s' + ), + (53.621, 84.034), + ) def testParseSystemDCriticalChain(self): """Test systemd critical-chain output parsing.""" with open(os.path.join(self.data_dir, 'systemd1.output')) as f: - self.assertEqual(linux_boot.ParseSystemDCriticalChainOutput(f.read()), - 2.741) + self.assertEqual( + linux_boot.ParseSystemDCriticalChainOutput(f.read()), 2.741 + ) with open(os.path.join(self.data_dir, 'systemd2.output')) as f: output = f.read() - self.assertEqual(linux_boot.ParseSystemDCriticalChainOutput(output), - 0.671103) - self.assertEqual(linux_boot.ParseSystemDCriticalChainServiceTime(output), - 0.000103) + self.assertEqual( + linux_boot.ParseSystemDCriticalChainOutput(output), 0.671103 + ) + self.assertEqual( + linux_boot.ParseSystemDCriticalChainServiceTime(output), 0.000103 + ) def testCollectVmToVmSamples(self): """Test vm to vm networking result parsing.""" @@ -148,11 +246,25 @@ def testCollectVmToVmSamples(self): linux_boot.CollectVmToVmSamples( self.mock_vm, ('10.128.0.2', ''), - datetime.datetime.fromtimestamp(1680741767, pytz.timezone('UTC'))), - [sample.Sample(metric='internal_ingress', value=10.778103, - unit='second', metadata={}, timestamp=0), - sample.Sample(metric='internal_egress', value=10.390749, - unit='second', metadata={}, timestamp=0)]) + datetime.datetime.fromtimestamp(1680741767, pytz.timezone('UTC')), + ), + [ + sample.Sample( + metric='internal_ingress', + value=10.778103, + unit='second', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='internal_egress', + value=10.390749, + unit='second', + metadata={}, + timestamp=0, + ), + ], + ) if __name__ == '__main__': diff --git a/tests/linux_packages/maven_test.py b/tests/linux_packages/maven_test.py index 22ccbf53cc..4b6daeee61 100644 --- a/tests/linux_packages/maven_test.py +++ b/tests/linux_packages/maven_test.py @@ -40,9 +40,9 @@ def assertCallArgsEqual(self, call_args_singles, mock_method): to be converted into the tuple of positional arguments tuple that mock expects. Args: - call_args_singles: List of single arguments sent to the mock_method, - ie ['x', 'y'] is for when mock_method was called twice: once with - x and then with y. + call_args_singles: List of single arguments sent to the mock_method, ie + ['x', 'y'] is for when mock_method was called twice: once with x and + then with y. mock_method: Method that was mocked and called with call_args_singles. """ # convert from ['a', 'b'] into [(('a',),), (('b',),)] @@ -128,9 +128,9 @@ def testAptInstall(self): f'echo "{maven_env}" | sudo tee -a {maven.MVN_ENV_PATH}', ]) self.assertVmInstallCommandsEqual(['openjdk', 'curl']) - self.assertOnlyKnownMethodsCalled('RemoteCommand', - 'InstallPreprovisionedPackageData', - 'Install') + self.assertOnlyKnownMethodsCalled( + 'RemoteCommand', 'InstallPreprovisionedPackageData', 'Install' + ) if __name__ == '__main__': diff --git a/tests/linux_packages/memtier_test.py b/tests/linux_packages/memtier_test.py index 16290e7a8e..82dbbc454b 100644 --- a/tests/linux_packages/memtier_test.py +++ b/tests/linux_packages/memtier_test.py @@ -1,6 +1,5 @@ """Tests for perfkitbenchmarker.linux_packages.memtier.""" - import json import os import unittest @@ -101,8 +100,9 @@ def GetMemtierResult(ops_per_sec, p95_latency): ) -class MemtierTestCase(pkb_common_test_case.PkbCommonTestCase, - test_util.SamplesTestMixin): +class MemtierTestCase( + pkb_common_test_case.PkbCommonTestCase, test_util.SamplesTestMixin +): def setUp(self): super(MemtierTestCase, self).setUp() @@ -116,7 +116,8 @@ def testParseResults(self): get_metadata = { 'histogram': json.dumps([ {'microsec': 0.0, 'count': 4500}, - {'microsec': 2000.0, 'count': 4500}]) + {'microsec': 2000.0, 'count': 4500}, + ]) } get_metadata.update(METADATA) set_metadata = { @@ -130,7 +131,8 @@ def testParseResults(self): {'microsec': 6000.0, 'count': 200}, {'microsec': 7000.0, 'count': 50}, {'microsec': 8000.0, 'count': 40}, - {'microsec': 9000.0, 'count': 10}]) + {'microsec': 9000.0, 'count': 10}, + ]) } set_metadata.update(METADATA) @@ -142,7 +144,7 @@ def testParseResults(self): 'Start_time': 1657947420452, 'Finish_time': 1657947420454, 'Total_duration': 2, - 'Time_unit': 'MILLISECONDS' + 'Time_unit': 'MILLISECONDS', } expected_result = [ @@ -150,29 +152,32 @@ def testParseResults(self): metric='Ops Throughput', value=44006.55, unit='ops/s', - metadata=METADATA), + metadata=METADATA, + ), sample.Sample( - metric='KB Throughput', - value=1828.0, - unit='KB/s', - metadata=METADATA), + metric='KB Throughput', value=1828.0, unit='KB/s', metadata=METADATA + ), sample.Sample( - metric='Latency', value=1.54, unit='ms', metadata=METADATA), + metric='Latency', value=1.54, unit='ms', metadata=METADATA + ), sample.Sample( metric='get latency histogram', value=0, unit='', - metadata=get_metadata), + metadata=get_metadata, + ), sample.Sample( metric='set latency histogram', value=0, unit='', - metadata=set_metadata), + metadata=set_metadata, + ), sample.Sample( metric='Memtier Duration', value=2, unit='ms', - metadata=runtime_info_metadata), + metadata=runtime_info_metadata, + ), ] samples = [] results = memtier.MemtierResult.Parse(TEST_OUTPUT, TIME_SERIES_JSON) @@ -1397,7 +1402,8 @@ def testParseResults_no_time_series(self): get_metadata = { 'histogram': json.dumps([ {'microsec': 0.0, 'count': 4500}, - {'microsec': 2000.0, 'count': 4500}]) + {'microsec': 2000.0, 'count': 4500}, + ]) } get_metadata.update(METADATA) set_metadata = { @@ -1411,7 +1417,8 @@ def testParseResults_no_time_series(self): {'microsec': 6000.0, 'count': 200}, {'microsec': 7000.0, 'count': 50}, {'microsec': 8000.0, 'count': 40}, - {'microsec': 9000.0, 'count': 10}]) + {'microsec': 9000.0, 'count': 10}, + ]) } set_metadata.update(METADATA) @@ -1425,24 +1432,26 @@ def testParseResults_no_time_series(self): metric='Ops Throughput', value=44006.55, unit='ops/s', - metadata=METADATA), + metadata=METADATA, + ), sample.Sample( - metric='KB Throughput', - value=1828.0, - unit='KB/s', - metadata=METADATA), + metric='KB Throughput', value=1828.0, unit='KB/s', metadata=METADATA + ), sample.Sample( - metric='Latency', value=1.54, unit='ms', metadata=METADATA), + metric='Latency', value=1.54, unit='ms', metadata=METADATA + ), sample.Sample( metric='get latency histogram', value=0, unit='', - metadata=get_metadata), + metadata=get_metadata, + ), sample.Sample( metric='set latency histogram', value=0, unit='', - metadata=set_metadata), + metadata=set_metadata, + ), ] samples = [] results = memtier.MemtierResult.Parse(TEST_OUTPUT, None) @@ -1476,7 +1485,8 @@ def testMeasureLatencyCappedThroughput(self): GetMemtierResult(3, 1.4), ] self.enter_context( - mock.patch.object(memtier, '_Run', side_effect=mock_run_results)) + mock.patch.object(memtier, '_Run', side_effect=mock_run_results) + ) mock_vm = pkb_common_test_case.TestLinuxVirtualMachine( pkb_common_test_case.CreateTestVmSpec() diff --git a/tests/linux_packages/mpi_test.py b/tests/linux_packages/mpi_test.py index 87090e693d..18e6029be2 100644 --- a/tests/linux_packages/mpi_test.py +++ b/tests/linux_packages/mpi_test.py @@ -43,7 +43,8 @@ def _CreateMpiDataFromDict(data: Dict[str, Any]) -> mpi.MpiData: number_bytes = data.pop('bytes', None) repetitions = data.pop('repetitions', None) return mpi.MpiData( - bytes=number_bytes, repetitions=repetitions, data=data['data']) + bytes=number_bytes, repetitions=repetitions, data=data['data'] + ) def _CreateMpiResultsFromDict(result_json: Dict[str, Any]) -> mpi.MpiResult: @@ -60,7 +61,8 @@ def _CreateMpiResultsFromDict(result_json: Dict[str, Any]) -> mpi.MpiResult: def _CreateMpiResponseFromDict( - data: List[Dict[str, Any]]) -> List[mpi.MpiResult]: + data: List[Dict[str, Any]] +) -> List[mpi.MpiResult]: return [_CreateMpiResultsFromDict(result) for result in data] @@ -88,22 +90,16 @@ class MpiTestCase(pkb_common_test_case.PkbCommonTestCase): # The latency_data_file summarized in a Dict. LATENCY_DATA: Dict[int, Dict[float, int]] = { - 0: { - 1.12: 1, - 3.0: 1, - 42.0: 1 - }, - 1024: { - 2.0: 1, - 3.0: 2 - } + 0: {1.12: 1, 3.0: 1, 42.0: 1}, + 1024: {2.0: 1, 3.0: 2}, } def setUp(self): super(MpiTestCase, self).setUp() FLAGS.intelmpi_version = self.MPI_VERSION self.enter_context( - mock.patch.object(intelmpi, 'MpiVars', return_value=MPI_VARS)) + mock.patch.object(intelmpi, 'MpiVars', return_value=MPI_VARS) + ) @parameterized.parameters( # mpirun -n 120 -hosts a,b,c,d -ppn 1 mpi-benchmarks/.... @@ -114,10 +110,12 @@ def setUp(self): ('mpi_latencies_output.txt', 'mpi_latencies_parsed.json'), ('mpi_one_put_all_output.txt', 'mpi_one_put_all_parsed.json'), ) - def testParseMpiOutput(self, mpi_output_file: str, - mpi_parsed_file: str) -> None: + def testParseMpiOutput( + self, mpi_output_file: str, mpi_parsed_file: str + ) -> None: found = list( - mpi.MpiResultParser(ReadMpiOutput(mpi_output_file).splitlines())) + mpi.MpiResultParser(ReadMpiOutput(mpi_output_file).splitlines()) + ) expected = ReadParsedOutput(mpi_parsed_file) self.assertEqual(expected, found) @@ -125,14 +123,16 @@ def testVerifyInstall(self) -> None: vms = [_MockVm(ip) for ip in ('a', 'b')] vms[0].RobustRemoteCommand.return_value = '', '' mpi.VerifyInstall(vms) - mpirun_cmd = ('mpirun -n 8 -hosts a,b -ppn 8 mpi-benchmarks/IMB-MPI1 ' - '-msglog 10:11 -multi 0 -time 20 -off_cache -1 -iter 100 ' - '-iter_policy off -zero_size off -show_tail yes PingPong') + mpirun_cmd = ( + 'mpirun -n 8 -hosts a,b -ppn 8 mpi-benchmarks/IMB-MPI1 ' + '-msglog 10:11 -multi 0 -time 20 -off_cache -1 -iter 100 ' + '-iter_policy off -zero_size off -show_tail yes PingPong' + ) vms[0].RobustRemoteCommand.assert_called_with(RUN_PREFIX + ' ' + mpirun_cmd) - def _CreateMpiRequest(self, - record_latencies: bool, - iterations: int = 100000) -> mpi.MpiRequest: + def _CreateMpiRequest( + self, record_latencies: bool, iterations: int = 100000 + ) -> mpi.MpiRequest: return mpi.MpiRequest( vms=[_MockVm('a'), _MockVm('b')], total_processes=10, @@ -148,12 +148,15 @@ def _CreateMpiRequest(self, include_zero_byte=False, compile_from_source=True, record_latencies=record_latencies, - multi=True) + multi=True, + ) def testRunMpiStats(self) -> None: vm = _MockVm('a') - vm.RobustRemoteCommand.return_value = ReadMpiOutput( - 'mpi_pingpong_output.txt'), '' + vm.RobustRemoteCommand.return_value = ( + ReadMpiOutput('mpi_pingpong_output.txt'), + '', + ) request = self._CreateMpiRequest(False) response = mpi.RunMpiStats(vm, request) self.assertEqual(RUN_PREFIX + ' mpirun -n 10 -hosts a,b', response.mpi_run) @@ -161,31 +164,42 @@ def testRunMpiStats(self) -> None: self.assertEqual('2019.2-057', response.version) # fully tested in testParseFiles self.assertLen(response.results, 1) - expected_args = ('mpi-benchmarks/IMB-MPI1 -msglog 10:11 -multi 0 -time 20 ' - '-off_cache -1 -iter 100000 -iter_policy off ' - '-zero_size off -show_tail yes -map 5x2 PingPong') + expected_args = ( + 'mpi-benchmarks/IMB-MPI1 -msglog 10:11 -multi 0 -time 20 ' + '-off_cache -1 -iter 100000 -iter_policy off ' + '-zero_size off -show_tail yes -map 5x2 PingPong' + ) self.assertEqual(expected_args, response.args) @mock.patch.object(mpi, '_GroupLatencyLines') @mock.patch.object(uuid, 'uuid4', side_effect=[mock.PropertyMock(hex='abc')]) - def testRunMpiStatsLatencyFile(self, mock_uuid: mock.Mock, - mock_create_histo: mock.Mock) -> None: + def testRunMpiStatsLatencyFile( + self, mock_uuid: mock.Mock, mock_create_histo: mock.Mock + ) -> None: mock_create_histo.return_value = [[ - '1024 10.0', '1024 11.0', '2048 11.10', '2048 11.11' + '1024 10.0', + '1024 11.0', + '2048 11.10', + '2048 11.11', ]] vm = _MockVm('a') vm.RobustRemoteCommand.return_value = ( - ReadMpiOutput('mpi_barrier_output.txt'), '') + ReadMpiOutput('mpi_barrier_output.txt'), + '', + ) request = self._CreateMpiRequest(True, 2) response = mpi.RunMpiStats(vm, request) # has the -show_tail and -dumpfile flags set - expected_args_re = (r'.*-zero_size off -show_tail yes ' - r'-dumpfile /tmp/latency\S+ -map 5x2 PingPong$') + expected_args_re = ( + r'.*-zero_size off -show_tail yes ' + r'-dumpfile /tmp/latency\S+ -map 5x2 PingPong$' + ) self.assertRegex(response.args, expected_args_re) mock_create_histo.assert_called_with(vm, '/tmp/latency-PingPong-abc.txt', 2) - @mock.patch('builtins.open', - mock.mock_open(read_data='\n'.join(LATENCY_DATA_FILE))) + @mock.patch( + 'builtins.open', mock.mock_open(read_data='\n'.join(LATENCY_DATA_FILE)) + ) def testGroupLatencyLines(self): vm = mock.Mock() vm.TryRemoteCommand.return_value = True @@ -206,9 +220,11 @@ def testCreateMpiDataForHistogram(self) -> None: FLAGS.run_uri = '12345678' grouped_lines = [['1024 10.0', '1024 11.0', '2048 11.10', '2048 11.11']] mpi_data1 = mpi.MpiData( - bytes=1024, repetitions=2, data={'p50': 10.5}, is_error=False) + bytes=1024, repetitions=2, data={'p50': 10.5}, is_error=False + ) mpi_data2 = mpi.MpiData( - bytes=2048, repetitions=2, data={'p50': 11.0}, is_error=False) + bytes=2048, repetitions=2, data={'p50': 11.0}, is_error=False + ) parsed_results = [ mpi.MpiResult(benchmark='PingPong', data=[mpi_data1, mpi_data2]) ] @@ -256,7 +272,7 @@ def testParseMpiEnv(self): 'I_MPI_HYDRA_TOPOLIB': 'hwloc', 'I_MPI_INTERNAL_MEM_POLICY': 'default', 'I_MPI_MPIRUN': 'mpirun', - 'I_MPI_ROOT': '/opt/intel/compilers_and_libraries_2020.0.166/linux/mpi' + 'I_MPI_ROOT': '/opt/intel/compilers_and_libraries_2020.0.166/linux/mpi', } self.assertEqual(expected_mpi_env, mpi.ParseMpiEnv(lines)) diff --git a/tests/linux_packages/netperf_test.py b/tests/linux_packages/netperf_test.py index 67e26c0c04..2569856897 100644 --- a/tests/linux_packages/netperf_test.py +++ b/tests/linux_packages/netperf_test.py @@ -16,7 +16,6 @@ import os import unittest - from perfkitbenchmarker.linux_packages import netperf @@ -30,9 +29,21 @@ def setUp(self): def testParsesHistogram(self): expected = { - 300: 5771, 400: 118948, 500: 7121, 600: 639, 700: 199, 800: 90, - 900: 53, 1000: 149, 2000: 31, 3000: 11, 4000: 8, 5000: 1, 6000: 1, - 7000: 1, 9000: 1 + 300: 5771, + 400: 118948, + 500: 7121, + 600: 639, + 700: 199, + 800: 90, + 900: 53, + 1000: 149, + 2000: 31, + 3000: 11, + 4000: 8, + 5000: 1, + 6000: 1, + 7000: 1, + 9000: 1, } hist = netperf.ParseHistogram(self.netperf_output) diff --git a/tests/linux_packages/numactl_test.py b/tests/linux_packages/numactl_test.py index 8c109f2ccb..743fe40689 100644 --- a/tests/linux_packages/numactl_test.py +++ b/tests/linux_packages/numactl_test.py @@ -32,6 +32,7 @@ def MockVm(run_cmd_response: dict[str, str]): vm = mock.Mock() + def FakeRemoteHostCommand(cmd, **_): if isinstance(run_cmd_response, dict): if cmd not in run_cmd_response: @@ -61,13 +62,8 @@ def testGetNuma(self, numactl_text, cpus, numa_list): self.assertEqual(cpus, numactl.GetNuma(MockVm(responses))) @parameterized.named_parameters( - ('singlenode', SINGLE_NUMA_NODE, { - 0: 32116 - }), - ('twonode', TWO_NUMA_NODES, { - 0: 120889, - 1: 120931 - }), + ('singlenode', SINGLE_NUMA_NODE, {0: 32116}), + ('twonode', TWO_NUMA_NODES, {0: 120889, 1: 120931}), ) def testGetNumaMemory(self, numactl_text, cpus): responses = {'numactl --hardware': numactl_text} diff --git a/tests/linux_packages/nvidia_driver_test.py b/tests/linux_packages/nvidia_driver_test.py index a09c5b2c62..07d8d2305a 100644 --- a/tests/linux_packages/nvidia_driver_test.py +++ b/tests/linux_packages/nvidia_driver_test.py @@ -16,7 +16,6 @@ import os import unittest import mock - from perfkitbenchmarker import test_util from perfkitbenchmarker.linux_packages import nvidia_driver @@ -29,8 +28,9 @@ class NvidiaDriverTestCase(unittest.TestCase, test_util.SamplesTestMixin): def setUp(self): super(NvidiaDriverTestCase, self).setUp() - path = os.path.join(os.path.dirname(__file__), '../data', - 'nvidia_smi_output.txt') + path = os.path.join( + os.path.dirname(__file__), '../data', 'nvidia_smi_output.txt' + ) with open(path) as fp: self.nvidia_smi_output = fp.read() @@ -42,30 +42,36 @@ def testQueryNumberOfGpus(self): def testQueryGpuClockSpeed(self): vm = mock.MagicMock() vm.RemoteCommand = mock.MagicMock( - return_value=('clocks.applications.graphics [MHz], ' - 'clocks.applications.memory [Mhz]\n' - '324 MHz, 527 MHz', None)) + return_value=( + ( + 'clocks.applications.graphics [MHz], ' + 'clocks.applications.memory [Mhz]\n' + '324 MHz, 527 MHz' + ), + None, + ) + ) self.assertEqual(('324', '527'), nvidia_driver.QueryGpuClockSpeed(vm, 3)) vm.RemoteCommand.assert_called_with( 'sudo nvidia-smi ' '--query-gpu=clocks.applications.memory,' - 'clocks.applications.graphics --format=csv --id=3') + 'clocks.applications.graphics --format=csv --id=3' + ) def testGetDriverVersion(self): vm = mock.MagicMock() - vm.RemoteCommand = mock.MagicMock( - return_value=(self.nvidia_smi_output, '')) + vm.RemoteCommand = mock.MagicMock(return_value=(self.nvidia_smi_output, '')) self.assertEqual('375.66', nvidia_driver.GetDriverVersion(vm)) def testGetPeerToPeerTopology(self): - path = os.path.join(os.path.dirname(__file__), '../data', - 'nvidia_smi_topo_output.txt') + path = os.path.join( + os.path.dirname(__file__), '../data', 'nvidia_smi_topo_output.txt' + ) with open(path) as fp: nvidia_smi_output = fp.read() vm = mock.MagicMock() - vm.RemoteCommand = mock.MagicMock( - return_value=(nvidia_smi_output, '')) + vm.RemoteCommand = mock.MagicMock(return_value=(nvidia_smi_output, '')) expected = 'Y Y N N;Y Y N N;N N Y Y;N N Y Y' actual = nvidia_driver.GetPeerToPeerTopology(vm) @@ -73,67 +79,84 @@ def testGetPeerToPeerTopology(self): vm.RemoteCommand.assert_called_with('nvidia-smi topo -p2p r') def testQueryAutoboostNull(self): - path = os.path.join(os.path.dirname(__file__), '../data', - 'nvidia_smi_describe_clocks_p100.txt') + path = os.path.join( + os.path.dirname(__file__), + '../data', + 'nvidia_smi_describe_clocks_p100.txt', + ) with open(path) as fp: nvidia_smi_output = fp.read() vm = mock.MagicMock() - vm.RemoteCommand = mock.MagicMock( - return_value=(nvidia_smi_output, '')) - self.assertEqual({'autoboost': None, 'autoboost_default': None}, - nvidia_driver.QueryAutoboostPolicy(vm, 0)) + vm.RemoteCommand = mock.MagicMock(return_value=(nvidia_smi_output, '')) + self.assertEqual( + {'autoboost': None, 'autoboost_default': None}, + nvidia_driver.QueryAutoboostPolicy(vm, 0), + ) def testQueryAutoboostOn(self): - path = os.path.join(os.path.dirname(__file__), '../data', - 'nvidia_smi_describe_clocks_k80.txt') + path = os.path.join( + os.path.dirname(__file__), + '../data', + 'nvidia_smi_describe_clocks_k80.txt', + ) with open(path) as fp: nvidia_smi_output = fp.read() vm = mock.MagicMock() - vm.RemoteCommand = mock.MagicMock( - return_value=(nvidia_smi_output, '')) - self.assertEqual({'autoboost': False, 'autoboost_default': True}, - nvidia_driver.QueryAutoboostPolicy(vm, 0)) + vm.RemoteCommand = mock.MagicMock(return_value=(nvidia_smi_output, '')) + self.assertEqual( + {'autoboost': False, 'autoboost_default': True}, + nvidia_driver.QueryAutoboostPolicy(vm, 0), + ) def testGetGpuTypeP100(self): - path = os.path.join(os.path.dirname(__file__), '../data', - 'list_gpus_output_p100.txt') + path = os.path.join( + os.path.dirname(__file__), '../data', 'list_gpus_output_p100.txt' + ) with open(path) as fp: nvidia_smi_output = fp.read() vm = mock.MagicMock() - vm.RemoteCommand = mock.MagicMock( - return_value=(nvidia_smi_output, '')) - self.assertEqual(nvidia_driver.NVIDIA_TESLA_P100, - nvidia_driver.GetGpuType(vm)) + vm.RemoteCommand = mock.MagicMock(return_value=(nvidia_smi_output, '')) + self.assertEqual( + nvidia_driver.NVIDIA_TESLA_P100, nvidia_driver.GetGpuType(vm) + ) def testGetGpuTypeK80(self): - path = os.path.join(os.path.dirname(__file__), '../data', - 'list_gpus_output_k80.txt') + path = os.path.join( + os.path.dirname(__file__), '../data', 'list_gpus_output_k80.txt' + ) with open(path) as fp: nvidia_smi_output = fp.read() vm = mock.MagicMock() - vm.RemoteCommand = mock.MagicMock( - return_value=(nvidia_smi_output, '')) - self.assertEqual(nvidia_driver.NVIDIA_TESLA_K80, - nvidia_driver.GetGpuType(vm)) + vm.RemoteCommand = mock.MagicMock(return_value=(nvidia_smi_output, '')) + self.assertEqual( + nvidia_driver.NVIDIA_TESLA_K80, nvidia_driver.GetGpuType(vm) + ) def testHetergeneousGpuTypes(self): - path = os.path.join(os.path.dirname(__file__), '../data', - 'list_gpus_output_heterogeneous.txt') + path = os.path.join( + os.path.dirname(__file__), + '../data', + 'list_gpus_output_heterogeneous.txt', + ) with open(path) as fp: nvidia_smi_output = fp.read() vm = mock.MagicMock() - vm.RemoteCommand = mock.MagicMock( - return_value=(nvidia_smi_output, '')) - self.assertRaisesRegexp(nvidia_driver.HeterogeneousGpuTypesError, # pytype: disable=wrong-arg-count - 'PKB only supports one type of gpu per VM', - nvidia_driver.GetGpuType, vm) + vm.RemoteCommand = mock.MagicMock(return_value=(nvidia_smi_output, '')) + self.assertRaisesRegexp( + nvidia_driver.HeterogeneousGpuTypesError, # pytype: disable=wrong-arg-count + 'PKB only supports one type of gpu per VM', + nvidia_driver.GetGpuType, + vm, + ) @mock.patch(nvidia_driver.__name__ + '.QueryNumberOfGpus', return_value=2) - @mock.patch(nvidia_driver.__name__ + '.QueryAutoboostPolicy', - return_value=AUTOBOOST_ENABLED_DICT) - def testSetAutoboostPolicyWhenValuesAreTheSame(self, - query_autoboost_mock, - num_gpus_mock): + @mock.patch( + nvidia_driver.__name__ + '.QueryAutoboostPolicy', + return_value=AUTOBOOST_ENABLED_DICT, + ) + def testSetAutoboostPolicyWhenValuesAreTheSame( + self, query_autoboost_mock, num_gpus_mock + ): vm = mock.MagicMock() vm.RemoteCommand = mock.MagicMock() @@ -142,11 +165,13 @@ def testSetAutoboostPolicyWhenValuesAreTheSame(self, vm.RemoteCommand.assert_not_called() @mock.patch(nvidia_driver.__name__ + '.QueryNumberOfGpus', return_value=2) - @mock.patch(nvidia_driver.__name__ + '.QueryAutoboostPolicy', - return_value=AUTOBOOST_DISABLED_DICT) - def testSetAutoboostPolicyWhenValuesAreDifferent(self, - query_autoboost_mock, - num_gpus_mock): + @mock.patch( + nvidia_driver.__name__ + '.QueryAutoboostPolicy', + return_value=AUTOBOOST_DISABLED_DICT, + ) + def testSetAutoboostPolicyWhenValuesAreDifferent( + self, query_autoboost_mock, num_gpus_mock + ): vm = mock.MagicMock() vm.RemoteCommand = mock.MagicMock() @@ -155,11 +180,12 @@ def testSetAutoboostPolicyWhenValuesAreDifferent(self, self.assertEqual(2, vm.RemoteCommand.call_count) @mock.patch(nvidia_driver.__name__ + '.QueryNumberOfGpus', return_value=2) - @mock.patch(nvidia_driver.__name__ + '.QueryGpuClockSpeed', - return_value=(2505, 875)) - def testSetClockSpeedWhenValuesAreTheSame(self, - query_clock_speed_mock, - num_gpus_mock): + @mock.patch( + nvidia_driver.__name__ + '.QueryGpuClockSpeed', return_value=(2505, 875) + ) + def testSetClockSpeedWhenValuesAreTheSame( + self, query_clock_speed_mock, num_gpus_mock + ): vm = mock.MagicMock() vm.RemoteCommand = mock.MagicMock() @@ -168,11 +194,12 @@ def testSetClockSpeedWhenValuesAreTheSame(self, vm.RemoteCommand.assert_not_called() @mock.patch(nvidia_driver.__name__ + '.QueryNumberOfGpus', return_value=2) - @mock.patch(nvidia_driver.__name__ + '.QueryGpuClockSpeed', - return_value=(2505, 875)) - def testSetClockSpeedWhenValuesAreDifferent(self, - query_clock_speed_mock, - num_gpus_mock): + @mock.patch( + nvidia_driver.__name__ + '.QueryGpuClockSpeed', return_value=(2505, 875) + ) + def testSetClockSpeedWhenValuesAreDifferent( + self, query_clock_speed_mock, num_gpus_mock + ): vm = mock.MagicMock() vm.RemoteCommand = mock.MagicMock() @@ -180,5 +207,6 @@ def testSetClockSpeedWhenValuesAreDifferent(self, query_clock_speed_mock.assetCalled() self.assertEqual(2, vm.RemoteCommand.call_count) + if __name__ == '__main__': unittest.main() diff --git a/tests/linux_packages/pip_test.py b/tests/linux_packages/pip_test.py index 706ad570b1..c32689c9d4 100644 --- a/tests/linux_packages/pip_test.py +++ b/tests/linux_packages/pip_test.py @@ -4,35 +4,41 @@ import unittest from absl.testing import parameterized import mock - from perfkitbenchmarker.linux_packages import pip from perfkitbenchmarker.linux_packages import python from tests import pkb_common_test_case - import requests # executed remote commands NEED_PIP_27 = [ - ('curl https://bootstrap.pypa.io/pip/2.7/get-pip.py -o get_pip.py && ' - 'sudo python get_pip.py'), + ( + 'curl https://bootstrap.pypa.io/pip/2.7/get-pip.py -o get_pip.py && ' + 'sudo python get_pip.py' + ), 'pip --version', - 'mkdir -p /opt/pkb && pip freeze | tee /opt/pkb/requirements.txt' + 'mkdir -p /opt/pkb && pip freeze | tee /opt/pkb/requirements.txt', ] NEED_PIP_38 = [ - ('curl https://bootstrap.pypa.io/pip/get-pip.py -o get_pip.py && ' - 'sudo python3 get_pip.py'), + ( + 'curl https://bootstrap.pypa.io/pip/get-pip.py -o get_pip.py && ' + 'sudo python3 get_pip.py' + ), 'pip3 --version', - 'mkdir -p /opt/pkb && pip3 freeze | tee /opt/pkb/requirements.txt' + 'mkdir -p /opt/pkb && pip3 freeze | tee /opt/pkb/requirements.txt', ] EXISTING_PIP_27 = [ - 'echo \'exec python -m pip "$@"\'| sudo tee /usr/bin/pip && ' - 'sudo chmod 755 /usr/bin/pip', + ( + 'echo \'exec python -m pip "$@"\'| sudo tee /usr/bin/pip && ' + 'sudo chmod 755 /usr/bin/pip' + ), 'pip --version', 'mkdir -p /opt/pkb && pip freeze | tee /opt/pkb/requirements.txt', ] EXISTING_PIP_38 = [ - 'echo \'exec python3 -m pip "$@"\'| sudo tee /usr/bin/pip3 && ' - 'sudo chmod 755 /usr/bin/pip3', + ( + 'echo \'exec python3 -m pip "$@"\'| sudo tee /usr/bin/pip3 && ' + 'sudo chmod 755 /usr/bin/pip3' + ), 'pip3 --version', 'mkdir -p /opt/pkb && pip3 freeze | tee /opt/pkb/requirements.txt', ] @@ -60,22 +66,27 @@ class PipTest(pkb_common_test_case.PkbCommonTestCase): ('existing_pip_38', True, '3.8', EXISTING_PIP_38, PYTHON_38_KWARGS), ) @mock.patch.object(requests, 'get', side_effect=VERSIONED_PYPI_RESPONSES.get) - def testInstall(self, - already_has_pip: bool, - python_version: str, - expected_commands: List[str], - install_kwargs: Dict[str, str], - requests_get: mock.Mock): + def testInstall( + self, + already_has_pip: bool, + python_version: str, + expected_commands: List[str], + install_kwargs: Dict[str, str], + requests_get: mock.Mock, + ): self.enter_context( mock.patch.object( - python, 'GetPythonVersion', return_value=python_version)) + python, 'GetPythonVersion', return_value=python_version + ) + ) vm = mock.Mock() vm.TryRemoteCommand.return_value = already_has_pip pip.Install(vm, **install_kwargs) vm.RemoteCommand.assert_has_calls( - [mock.call(cmd) for cmd in expected_commands]) + [mock.call(cmd) for cmd in expected_commands] + ) if not already_has_pip: requests_get.assert_called_once() diff --git a/tests/linux_packages/python_test.py b/tests/linux_packages/python_test.py index 11c0d78e8f..32b5befa86 100644 --- a/tests/linux_packages/python_test.py +++ b/tests/linux_packages/python_test.py @@ -43,18 +43,18 @@ def setUp(self): super(PythonTest, self).setUp() FLAGS['default_timeout'].parse(0) # due to @retry - def RunSetDefault(self, - responses, - expected_last_call=None): + def RunSetDefault(self, responses, expected_last_call=None): vm = mock.Mock() vm.PYTHON_2_PACKAGE = 'python2' vm.RemoteCommandWithReturnCode.side_effect = responses python._SetDefaultPythonIfNeeded(vm) if expected_last_call: vm.RemoteCommandWithReturnCode.assert_called_with( - expected_last_call, ignore_failure=True) - self.assertLen(vm.RemoteCommandWithReturnCode.call_args_list, - len(responses)) + expected_last_call, ignore_failure=True + ) + self.assertLen( + vm.RemoteCommandWithReturnCode.call_args_list, len(responses) + ) @mock.patch.object(python, '_SetDefaultPythonIfNeeded') def testYumCall(self, mock_set_default): @@ -76,8 +76,11 @@ def testDefaultPythonAlreadySet(self): def testBadAlternativesResponse(self): responses = [ - PYTHON_MISSING, PYTHON2_PATH_FOUND, ALTERNATIVES_CALL_BAD, SYMLINK_SET, - PYTHON_VERSION_2 + PYTHON_MISSING, + PYTHON2_PATH_FOUND, + ALTERNATIVES_CALL_BAD, + SYMLINK_SET, + PYTHON_VERSION_2, ] expected = 'python --version' self.RunSetDefault(responses, expected) @@ -89,16 +92,20 @@ def testMissingPythonPath(self): def testNoPythonVersionAfterSet(self): responses = [ - PYTHON_MISSING, PYTHON2_PATH_FOUND, ALTERNATIVES_CALL_GOOD, - PYTHON_VERSION_CALL_BAD + PYTHON_MISSING, + PYTHON2_PATH_FOUND, + ALTERNATIVES_CALL_GOOD, + PYTHON_VERSION_CALL_BAD, ] with self.assertRaises(errors.Setup.PythonPackageRequirementUnfulfilled): self.RunSetDefault(responses) def testPythonVersionAfterSet(self): responses = [ - PYTHON_MISSING, PYTHON2_PATH_FOUND, ALTERNATIVES_CALL_GOOD, - PYTHON_VERSION_2 + PYTHON_MISSING, + PYTHON2_PATH_FOUND, + ALTERNATIVES_CALL_GOOD, + PYTHON_VERSION_2, ] expected = 'python --version' self.RunSetDefault(responses, expected) diff --git a/tests/linux_packages/redis_enterprise_test.py b/tests/linux_packages/redis_enterprise_test.py index ed6d8343b5..a917d7bbf0 100644 --- a/tests/linux_packages/redis_enterprise_test.py +++ b/tests/linux_packages/redis_enterprise_test.py @@ -46,16 +46,22 @@ class ResultParserTest(pkb_common_test_case.PkbCommonTestCase): def testParseResults(self): output = GetRedisEnterpriseOutput('redis_enterprise_output.txt') actual = redis_enterprise.ParseResults(output) - self.assertCountEqual([r.latency_usec for r in actual], [ - 254.38488247733775, - 262.05762827864845, - 263.3261959721339, - ]) - self.assertCountEqual([r.throughput for r in actual], [ - 1420515, - 1413081, - 1419066, - ]) + self.assertCountEqual( + [r.latency_usec for r in actual], + [ + 254.38488247733775, + 262.05762827864845, + 263.3261959721339, + ], + ) + self.assertCountEqual( + [r.throughput for r in actual], + [ + 1420515, + 1413081, + 1419066, + ], + ) class ThroughputOptimizerTest(pkb_common_test_case.PkbCommonTestCase): @@ -71,10 +77,10 @@ def _CallList(self, calls): def testRun(self): # Arrange optimizer = self._GetOptimizer(16) + self.enter_context(mock.patch.object(optimizer, '_CreateAndLoadDatabases')) self.enter_context( - mock.patch.object(optimizer, '_CreateAndLoadDatabases')) - self.enter_context( - mock.patch.object(optimizer.client, 'GetDatabases', return_value={})) + mock.patch.object(optimizer.client, 'GetDatabases', return_value={}) + ) throughput_responses = [ (60, WRONG_RESULT), (40, WRONG_RESULT), @@ -83,17 +89,23 @@ def testRun(self): (80, WRONG_RESULT), (30, WRONG_RESULT), # This should be chosen as optimal throughput - (100, [ - sample.Sample('max_throughput_under_1ms', 100, 'ops/sec', - {'threads': 100}) - ]), + ( + 100, + [ + sample.Sample( + 'max_throughput_under_1ms', 100, 'ops/sec', {'threads': 100} + ) + ], + ), (20, WRONG_RESULT), (50, WRONG_RESULT), (40, WRONG_RESULT), ] self.enter_context( mock.patch.object( - redis_enterprise, 'Run', side_effect=throughput_responses)) + redis_enterprise, 'Run', side_effect=throughput_responses + ) + ) # Act throughput, _ = optimizer.GetOptimalThroughput() @@ -111,10 +123,14 @@ def testRunChecksCorrectNeighbors(self): (40, WRONG_RESULT), (70, WRONG_RESULT), # This should be chosen as optimal throughput - (100, [ - sample.Sample('max_throughput_under_1ms', 100, 'ops/sec', - {'threads': 100}) - ]), + ( + 100, + [ + sample.Sample( + 'max_throughput_under_1ms', 100, 'ops/sec', {'threads': 100} + ) + ], + ), (20, WRONG_RESULT), (50, WRONG_RESULT), (40, WRONG_RESULT), @@ -122,23 +138,27 @@ def testRunChecksCorrectNeighbors(self): ] mock_run = self.enter_context( mock.patch.object( - optimizer, '_FullRun', side_effect=throughput_responses)) + optimizer, '_FullRun', side_effect=throughput_responses + ) + ) # Act throughput, _ = optimizer.GetOptimalThroughput() # Assert self.assertEqual(throughput, 100) - mock_run.assert_has_calls(self._CallList([ - (3, 13), - (3, 12), - (3, 14), - (2, 13), - (4, 13), - (2, 12), - (2, 14), - (1, 13), - ])) + mock_run.assert_has_calls( + self._CallList([ + (3, 13), + (3, 12), + (3, 14), + (2, 13), + (4, 13), + (2, 12), + (2, 14), + (1, 13), + ]) + ) @flagsaver.flagsaver(enterprise_redis_proxy_threads=3) def testRunChecksCorrectNeighborsVaryingShards(self): @@ -148,27 +168,35 @@ def testRunChecksCorrectNeighborsVaryingShards(self): (70, WRONG_RESULT), (80, WRONG_RESULT), # This should be chosen as optimal throughput - (100, [ - sample.Sample('max_throughput_under_1ms', 100, 'ops/sec', - {'threads': 100}) - ]), + ( + 100, + [ + sample.Sample( + 'max_throughput_under_1ms', 100, 'ops/sec', {'threads': 100} + ) + ], + ), (20, WRONG_RESULT), ] mock_run = self.enter_context( mock.patch.object( - optimizer, '_FullRun', side_effect=throughput_responses)) + optimizer, '_FullRun', side_effect=throughput_responses + ) + ) # Act throughput, _ = optimizer.GetOptimalThroughput() # Assert self.assertEqual(throughput, 100) - mock_run.assert_has_calls(self._CallList([ - (1, 3), - (2, 3), - (3, 3), - (4, 3), - ])) + mock_run.assert_has_calls( + self._CallList([ + (1, 3), + (2, 3), + (3, 3), + (4, 3), + ]) + ) @flagsaver.flagsaver(enterprise_redis_shard_count=5) def testRunChecksCorrectNeighborsVaryingProxyThreads(self): @@ -178,27 +206,36 @@ def testRunChecksCorrectNeighborsVaryingProxyThreads(self): (70, WRONG_RESULT), (80, WRONG_RESULT), # This should be chosen as optimal throughput - (100, [ - sample.Sample('max_throughput_under_1ms', 100, 'ops/sec', - {'threads': 100}) - ]), + ( + 100, + [ + sample.Sample( + 'max_throughput_under_1ms', 100, 'ops/sec', {'threads': 100} + ) + ], + ), (20, WRONG_RESULT), ] mock_run = self.enter_context( mock.patch.object( - optimizer, '_FullRun', side_effect=throughput_responses)) + optimizer, '_FullRun', side_effect=throughput_responses + ) + ) # Act throughput, _ = optimizer.GetOptimalThroughput() # Assert self.assertEqual(throughput, 100) - mock_run.assert_has_calls(self._CallList([ - (5, 1), - (5, 2), - (5, 3), - (5, 4), - ])) + mock_run.assert_has_calls( + self._CallList([ + (5, 1), + (5, 2), + (5, 3), + (5, 4), + ]) + ) + if __name__ == '__main__': unittest.main() diff --git a/tests/linux_packages/s64da_test.py b/tests/linux_packages/s64da_test.py index 027f66839f..3b700ddf6d 100644 --- a/tests/linux_packages/s64da_test.py +++ b/tests/linux_packages/s64da_test.py @@ -36,116 +36,217 @@ def S64daOutput(self, file: str) -> str: @mock.patch('time.time', mock.MagicMock(return_value=0)) def testParseOLAPResults(self): results = s64da.ParseOLAPResults(self.S64daOutput(OLAP_RESULTS)) - self.assertEqual([ - sample.Sample( - metric='Query 2', - value=1.0979999999999999, - unit='s', - metadata={}, - timestamp=0), - sample.Sample( - metric='Query 9', value=0.238, unit='s', metadata={}, timestamp=0), - sample.Sample( - metric='Query 20', value=0.45, unit='s', metadata={}, timestamp=0), - sample.Sample( - metric='Query 6', value=0.352, unit='s', metadata={}, timestamp=0), - sample.Sample( - metric='Query 17', value=1.768, unit='s', metadata={}, timestamp=0), - sample.Sample( - metric='Query 18', value=6.936, unit='s', metadata={}, timestamp=0), - sample.Sample( - metric='Query 8', - value=0.21400000000000002, - unit='s', - metadata={}, - timestamp=0), - sample.Sample( - metric='Query 21', value=2.16, unit='s', metadata={}, timestamp=0), - sample.Sample( - metric='Query 13', value=0.78, unit='s', metadata={}, timestamp=0), - sample.Sample( - metric='Query 3', value=0.51, unit='s', metadata={}, timestamp=0), - sample.Sample( - metric='Query 22', - value=0.44800000000000006, - unit='s', - metadata={}, - timestamp=0), - sample.Sample( - metric='Query 16', - value=4.5920000000000005, - unit='s', - metadata={}, - timestamp=0), - sample.Sample( - metric='Query 4', value=0.144, unit='s', metadata={}, timestamp=0), - sample.Sample( - metric='Query 11', value=0.606, unit='s', metadata={}, timestamp=0), - sample.Sample( - metric='Query 15', value=0.7075, unit='s', metadata={}, - timestamp=0), - sample.Sample( - metric='Query 1', value=0.31, unit='s', metadata={}, timestamp=0), - sample.Sample( - metric='Query 10', value=0.91, unit='s', metadata={}, timestamp=0), - sample.Sample( - metric='Query 19', value=0.7825, unit='s', metadata={}, - timestamp=0), - sample.Sample( - metric='Query 5', - value=1.3599999999999999, - unit='s', - metadata={}, - timestamp=0), - sample.Sample( - metric='Query 7', value=0.4875, unit='s', metadata={}, timestamp=0), - sample.Sample( - metric='Query 12', value=0.5675, unit='s', metadata={}, - timestamp=0), - sample.Sample( - metric='Query 14', value=0.3325, unit='s', metadata={}, - timestamp=0), - sample.Sample( - metric='query_times_geomean', - value=0.6943741551291615, - unit='s', - metadata={}, - timestamp=0), - sample.Sample( - metric='TPCH_failure_rate', - value=0.0, - unit='%', - metadata={}, - timestamp=0) - ], results) + self.assertEqual( + [ + sample.Sample( + metric='Query 2', + value=1.0979999999999999, + unit='s', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='Query 9', + value=0.238, + unit='s', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='Query 20', + value=0.45, + unit='s', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='Query 6', + value=0.352, + unit='s', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='Query 17', + value=1.768, + unit='s', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='Query 18', + value=6.936, + unit='s', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='Query 8', + value=0.21400000000000002, + unit='s', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='Query 21', + value=2.16, + unit='s', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='Query 13', + value=0.78, + unit='s', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='Query 3', value=0.51, unit='s', metadata={}, timestamp=0 + ), + sample.Sample( + metric='Query 22', + value=0.44800000000000006, + unit='s', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='Query 16', + value=4.5920000000000005, + unit='s', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='Query 4', + value=0.144, + unit='s', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='Query 11', + value=0.606, + unit='s', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='Query 15', + value=0.7075, + unit='s', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='Query 1', value=0.31, unit='s', metadata={}, timestamp=0 + ), + sample.Sample( + metric='Query 10', + value=0.91, + unit='s', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='Query 19', + value=0.7825, + unit='s', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='Query 5', + value=1.3599999999999999, + unit='s', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='Query 7', + value=0.4875, + unit='s', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='Query 12', + value=0.5675, + unit='s', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='Query 14', + value=0.3325, + unit='s', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='query_times_geomean', + value=0.6943741551291615, + unit='s', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='TPCH_failure_rate', + value=0.0, + unit='%', + metadata={}, + timestamp=0, + ), + ], + results, + ) @mock.patch('time.time', mock.MagicMock(return_value=0)) def testParseOLTPResults(self): results = s64da.ParseOLTPResults(self.S64daOutput(OLTP_RESULTS), 0) - self.assertEqual([ - sample.Sample( - metric='TPM', value=15853.0, unit='TPM', metadata={}, timestamp=0), - sample.Sample( - metric='TPCC_failure_rate', - value=0.4461056185194116, - unit='%', - metadata={}, - timestamp=0) - ], results) + self.assertEqual( + [ + sample.Sample( + metric='TPM', + value=15853.0, + unit='TPM', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='TPCC_failure_rate', + value=0.4461056185194116, + unit='%', + metadata={}, + timestamp=0, + ), + ], + results, + ) @mock.patch('time.time', mock.MagicMock(return_value=0)) def testParseOLTPWithRampUpResults(self): results = s64da.ParseOLTPResults(self.S64daOutput(OLTP_RESULTS), 1) - self.assertEqual([ - sample.Sample( - metric='TPM', value=15886.0, unit='TPM', metadata={}, timestamp=0), - sample.Sample( - metric='TPCC_failure_rate', - value=0.4461056185194116, - unit='%', - metadata={}, - timestamp=0) - ], results) + self.assertEqual( + [ + sample.Sample( + metric='TPM', + value=15886.0, + unit='TPM', + metadata={}, + timestamp=0, + ), + sample.Sample( + metric='TPCC_failure_rate', + value=0.4461056185194116, + unit='%', + metadata={}, + timestamp=0, + ), + ], + results, + ) if __name__ == '__main__': diff --git a/tests/linux_packages/spark_test.py b/tests/linux_packages/spark_test.py index a2236226dc..d7103d049b 100644 --- a/tests/linux_packages/spark_test.py +++ b/tests/linux_packages/spark_test.py @@ -38,7 +38,8 @@ def setUp(self): @requests_mock.Mocker() def testDefaultSparkVersion(self, mock_requests): mock_requests.get( - 'https://downloads.apache.org/spark', text=SPARK_VERSION_LIST) + 'https://downloads.apache.org/spark', text=SPARK_VERSION_LIST + ) for _ in range(5): observed = spark.SparkVersion() self.assertEqual(version.Version('3.3.0'), observed) @@ -50,18 +51,21 @@ def testSparkVersionConnectionError(self, mock_requests): mock_requests.get('https://downloads.apache.org/spark', status_code=404) with self.assertRaisesRegex( errors.Setup.MissingExecutableError, - 'Could not load https://downloads.apache.org/spark'): + 'Could not load https://downloads.apache.org/spark', + ): spark.SparkVersion() @requests_mock.Mocker() def testSparkVersionParsingError(self, mock_requests): mock_requests.get( 'https://downloads.apache.org/spark', - text='bar') + text='bar', + ) with self.assertRaisesRegex( errors.Setup.MissingExecutableError, 'Could not find valid spark versions at ' - 'https://downloads.apache.org/spark'): + 'https://downloads.apache.org/spark', + ): spark.SparkVersion() @requests_mock.Mocker() diff --git a/tests/linux_packages/speccpu_test.py b/tests/linux_packages/speccpu_test.py index 9869197528..2792d9234b 100644 --- a/tests/linux_packages/speccpu_test.py +++ b/tests/linux_packages/speccpu_test.py @@ -17,7 +17,6 @@ import unittest from absl import flags import mock - from perfkitbenchmarker import errors from perfkitbenchmarker import linux_virtual_machine from perfkitbenchmarker import sample @@ -46,47 +45,65 @@ Est. SPECint(R)_base2006 22.7 """ -GOOD_METADATA = {'runspec_config': 'linux64-x64-gcc47.cfg', - 'runspec_config_md5sum': 'abcd', - 'runspec_define': '', - 'runspec_iterations': '3', - 'runspec_enable_32bit': 'False', - 'runspec_metric': 'rate', - 'spec_runmode': 'base', - 'spec17_fdo': False, - 'spec17_copies': None, - 'spec17_threads': None, - 'spec17_subset': ['intspeed', 'fpspeed', 'intrate', 'fprate'], - 'gcc_version': '7', - 'CPU2017_version': ''} +GOOD_METADATA = { + 'runspec_config': 'linux64-x64-gcc47.cfg', + 'runspec_config_md5sum': 'abcd', + 'runspec_define': '', + 'runspec_iterations': '3', + 'runspec_enable_32bit': 'False', + 'runspec_metric': 'rate', + 'spec_runmode': 'base', + 'spec17_fdo': False, + 'spec17_copies': None, + 'spec17_threads': None, + 'spec17_subset': ['intspeed', 'fpspeed', 'intrate', 'fprate'], + 'gcc_version': '7', + 'CPU2017_version': '', +} EXPECTED_RESULT_SPECINT = [ - sample.Sample(metric='400.perlbench', value=23.4, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='401.bzip2', value=17.1, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='403.gcc', value=22.1, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='429.mcf', value=25.1, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='445.gobmk', value=21.0, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='456.hmmer', value=19.0, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='458.sjeng', value=20.6, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='462.libquantum', value=44.2, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='464.h264ref', value=31.6, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='471.omnetpp', value=17.9, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='473.astar', value=14.6, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='483.xalancbmk', value=27.8, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='SPECint(R)_base2006', value=22.7, unit='', - metadata=GOOD_METADATA), + sample.Sample( + metric='400.perlbench', value=23.4, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='401.bzip2', value=17.1, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='403.gcc', value=22.1, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='429.mcf', value=25.1, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='445.gobmk', value=21.0, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='456.hmmer', value=19.0, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='458.sjeng', value=20.6, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='462.libquantum', value=44.2, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='464.h264ref', value=31.6, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='471.omnetpp', value=17.9, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='473.astar', value=14.6, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='483.xalancbmk', value=27.8, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='SPECint(R)_base2006', + value=22.7, + unit='', + metadata=GOOD_METADATA, + ), ] TEST_OUTPUT_SPECFP = """ @@ -112,43 +129,60 @@ """ EXPECTED_RESULT_SPECFP = [ - - sample.Sample(metric='410.bwaves', value=19.0, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='416.gamess', value=21.2, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='433.milc', value=19.1, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='434.zeusmp', value=15.2, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='435.gromacs', value=11.8, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='436.cactusADM', value=9.27, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='437.leslie3d', value=10.9, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='444.namd', value=15.9, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='447.dealII', value=28.0, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='450.soplex', value=30.6, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='453.povray', value=23.0, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='454.calculix', value=8.31, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='459.GemsFDTD', value=13.7, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='465.tonto', value=17.4, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='470.lbm', value=37.7, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='481.wrf', value=14.2, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='482.sphinx3', value=29.2, unit='', - metadata=GOOD_METADATA), - sample.Sample(metric='SPECfp(R)_base2006', value=17.5, unit='', - metadata=GOOD_METADATA), + sample.Sample( + metric='410.bwaves', value=19.0, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='416.gamess', value=21.2, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='433.milc', value=19.1, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='434.zeusmp', value=15.2, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='435.gromacs', value=11.8, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='436.cactusADM', value=9.27, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='437.leslie3d', value=10.9, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='444.namd', value=15.9, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='447.dealII', value=28.0, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='450.soplex', value=30.6, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='453.povray', value=23.0, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='454.calculix', value=8.31, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='459.GemsFDTD', value=13.7, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='465.tonto', value=17.4, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='470.lbm', value=37.7, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='481.wrf', value=14.2, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='482.sphinx3', value=29.2, unit='', metadata=GOOD_METADATA + ), + sample.Sample( + metric='SPECfp(R)_base2006', value=17.5, unit='', metadata=GOOD_METADATA + ), ] # Invalid result, multiple NR failures and no aggregate score. @@ -173,25 +207,44 @@ EXPECTED_BAD1_METADATA = GOOD_METADATA.copy() EXPECTED_BAD1_METADATA.update({ 'partial': 'true', - 'missing_results': ('400.perlbench,401.bzip2,403.gcc,429.mcf,' - '483.xalancbmk,SPECint(R)_rate_base2006')}) + 'missing_results': ( + '400.perlbench,401.bzip2,403.gcc,429.mcf,' + '483.xalancbmk,SPECint(R)_rate_base2006' + ), +}) # Invalid result, multiple NR failures, but aggregate score present. EXPECTED_RESULT_BAD1 = [ - sample.Sample(metric='445.gobmk', value=15.8, unit='', - metadata=EXPECTED_BAD1_METADATA), - sample.Sample(metric='456.hmmer', value=14.8, unit='', - metadata=EXPECTED_BAD1_METADATA), - sample.Sample(metric='458.sjeng', value=15.6, unit='', - metadata=EXPECTED_BAD1_METADATA), - sample.Sample(metric='462.libquantum', value=37.5, unit='', - metadata=EXPECTED_BAD1_METADATA), - sample.Sample(metric='464.h264ref', value=28.4, unit='', - metadata=EXPECTED_BAD1_METADATA), - sample.Sample(metric='471.omnetpp', value=11.7, unit='', - metadata=EXPECTED_BAD1_METADATA), - sample.Sample(metric='473.astar', value=10.7, unit='', - metadata=EXPECTED_BAD1_METADATA), + sample.Sample( + metric='445.gobmk', value=15.8, unit='', metadata=EXPECTED_BAD1_METADATA + ), + sample.Sample( + metric='456.hmmer', value=14.8, unit='', metadata=EXPECTED_BAD1_METADATA + ), + sample.Sample( + metric='458.sjeng', value=15.6, unit='', metadata=EXPECTED_BAD1_METADATA + ), + sample.Sample( + metric='462.libquantum', + value=37.5, + unit='', + metadata=EXPECTED_BAD1_METADATA, + ), + sample.Sample( + metric='464.h264ref', + value=28.4, + unit='', + metadata=EXPECTED_BAD1_METADATA, + ), + sample.Sample( + metric='471.omnetpp', + value=11.7, + unit='', + metadata=EXPECTED_BAD1_METADATA, + ), + sample.Sample( + metric='473.astar', value=10.7, unit='', metadata=EXPECTED_BAD1_METADATA + ), ] TEST_OUTPUT_BAD2 = """ @@ -215,25 +268,46 @@ EXPECTED_BAD2_METADATA = GOOD_METADATA.copy() EXPECTED_BAD2_METADATA.update({ 'partial': 'true', - 'missing_results': '400.perlbench,401.bzip2,403.gcc,429.mcf,483.xalancbmk'}) + 'missing_results': '400.perlbench,401.bzip2,403.gcc,429.mcf,483.xalancbmk', +}) EXPECTED_RESULT_BAD2 = [ - sample.Sample(metric='445.gobmk', value=15.8, unit='', - metadata=EXPECTED_BAD2_METADATA), - sample.Sample(metric='456.hmmer', value=14.8, unit='', - metadata=EXPECTED_BAD2_METADATA), - sample.Sample(metric='458.sjeng', value=15.6, unit='', - metadata=EXPECTED_BAD2_METADATA), - sample.Sample(metric='462.libquantum', value=37.5, unit='', - metadata=EXPECTED_BAD2_METADATA), - sample.Sample(metric='464.h264ref', value=28.4, unit='', - metadata=EXPECTED_BAD2_METADATA), - sample.Sample(metric='471.omnetpp', value=11.7, unit='', - metadata=EXPECTED_BAD2_METADATA), - sample.Sample(metric='473.astar', value=10.7, unit='', - metadata=EXPECTED_BAD2_METADATA), - sample.Sample(metric='SPECint(R)_rate_base2006', value=42.0, unit='', - metadata=EXPECTED_BAD2_METADATA), + sample.Sample( + metric='445.gobmk', value=15.8, unit='', metadata=EXPECTED_BAD2_METADATA + ), + sample.Sample( + metric='456.hmmer', value=14.8, unit='', metadata=EXPECTED_BAD2_METADATA + ), + sample.Sample( + metric='458.sjeng', value=15.6, unit='', metadata=EXPECTED_BAD2_METADATA + ), + sample.Sample( + metric='462.libquantum', + value=37.5, + unit='', + metadata=EXPECTED_BAD2_METADATA, + ), + sample.Sample( + metric='464.h264ref', + value=28.4, + unit='', + metadata=EXPECTED_BAD2_METADATA, + ), + sample.Sample( + metric='471.omnetpp', + value=11.7, + unit='', + metadata=EXPECTED_BAD2_METADATA, + ), + sample.Sample( + metric='473.astar', value=10.7, unit='', metadata=EXPECTED_BAD2_METADATA + ), + sample.Sample( + metric='SPECint(R)_rate_base2006', + value=42.0, + unit='', + metadata=EXPECTED_BAD2_METADATA, + ), ] TEST_OUTPUT_EST = """ @@ -254,38 +328,68 @@ """ EXPECTED_EST_METADATA = GOOD_METADATA.copy() -EXPECTED_EST_METADATA.update({ - 'partial': 'true', - 'missing_results': 'SPECint(R)_rate_base2006'}) +EXPECTED_EST_METADATA.update( + {'partial': 'true', 'missing_results': 'SPECint(R)_rate_base2006'} +) EXPECTED_RESULT_EST = [ - sample.Sample(metric='400.perlbench', value=27.3, unit='', - metadata=EXPECTED_EST_METADATA), - sample.Sample(metric='401.bzip2', value=16.2, unit='', - metadata=EXPECTED_EST_METADATA), - sample.Sample(metric='403.gcc', value=21.9, unit='', - metadata=EXPECTED_EST_METADATA), - sample.Sample(metric='429.mcf', value=24.4, unit='', - metadata=EXPECTED_EST_METADATA), - sample.Sample(metric='445.gobmk', value=19.4, unit='', - metadata=EXPECTED_EST_METADATA), - sample.Sample(metric='456.hmmer', value=22.0, unit='', - metadata=EXPECTED_EST_METADATA), - sample.Sample(metric='458.sjeng', value=21.2, unit='', - metadata=EXPECTED_EST_METADATA), - sample.Sample(metric='462.libquantum', value=40.1, unit='', - metadata=EXPECTED_EST_METADATA), - sample.Sample(metric='464.h264ref', value=36.4, unit='', - metadata=EXPECTED_EST_METADATA), - sample.Sample(metric='471.omnetpp', value=12.2, unit='', - metadata=EXPECTED_EST_METADATA), - sample.Sample(metric='473.astar', value=14.3, unit='', - metadata=EXPECTED_EST_METADATA), - sample.Sample(metric='483.xalancbmk', value=21.7, unit='', - metadata=EXPECTED_EST_METADATA), - sample.Sample(metric='estimated_SPECint(R)_rate_base2006', - value=21.846042257681507, unit='', - metadata=EXPECTED_EST_METADATA), + sample.Sample( + metric='400.perlbench', + value=27.3, + unit='', + metadata=EXPECTED_EST_METADATA, + ), + sample.Sample( + metric='401.bzip2', value=16.2, unit='', metadata=EXPECTED_EST_METADATA + ), + sample.Sample( + metric='403.gcc', value=21.9, unit='', metadata=EXPECTED_EST_METADATA + ), + sample.Sample( + metric='429.mcf', value=24.4, unit='', metadata=EXPECTED_EST_METADATA + ), + sample.Sample( + metric='445.gobmk', value=19.4, unit='', metadata=EXPECTED_EST_METADATA + ), + sample.Sample( + metric='456.hmmer', value=22.0, unit='', metadata=EXPECTED_EST_METADATA + ), + sample.Sample( + metric='458.sjeng', value=21.2, unit='', metadata=EXPECTED_EST_METADATA + ), + sample.Sample( + metric='462.libquantum', + value=40.1, + unit='', + metadata=EXPECTED_EST_METADATA, + ), + sample.Sample( + metric='464.h264ref', + value=36.4, + unit='', + metadata=EXPECTED_EST_METADATA, + ), + sample.Sample( + metric='471.omnetpp', + value=12.2, + unit='', + metadata=EXPECTED_EST_METADATA, + ), + sample.Sample( + metric='473.astar', value=14.3, unit='', metadata=EXPECTED_EST_METADATA + ), + sample.Sample( + metric='483.xalancbmk', + value=21.7, + unit='', + metadata=EXPECTED_EST_METADATA, + ), + sample.Sample( + metric='estimated_SPECint(R)_rate_base2006', + value=21.846042257681507, + unit='', + metadata=EXPECTED_EST_METADATA, + ), ] @@ -309,32 +413,57 @@ SPEED_METADATA = GOOD_METADATA.copy() SPEED_METADATA['runspec_metric'] = 'speed' EXPECTED_SPEED_RESULT_SPECINT = [ - sample.Sample(metric='400.perlbench:speed', value=29.5, unit='', - metadata=SPEED_METADATA), - sample.Sample(metric='401.bzip2:speed', value=17.3, unit='', - metadata=SPEED_METADATA), - sample.Sample(metric='403.gcc:speed', value=23.5, unit='', - metadata=SPEED_METADATA), - sample.Sample(metric='429.mcf:speed', value=19.0, unit='', - metadata=SPEED_METADATA), - sample.Sample(metric='445.gobmk:speed', value=20.3, unit='', - metadata=SPEED_METADATA), - sample.Sample(metric='456.hmmer:speed', value=23.8, unit='', - metadata=SPEED_METADATA), - sample.Sample(metric='458.sjeng:speed', value=22.5, unit='', - metadata=SPEED_METADATA), - sample.Sample(metric='462.libquantum:speed', value=39.5, unit='', - metadata=SPEED_METADATA), - sample.Sample(metric='464.h264ref:speed', value=38.6, unit='', - metadata=SPEED_METADATA), - sample.Sample(metric='471.omnetpp:speed', value=12.8, unit='', - metadata=SPEED_METADATA), - sample.Sample(metric='473.astar:speed', value=13.9, unit='', - metadata=SPEED_METADATA), - sample.Sample(metric='483.xalancbmk:speed', value=22.3, unit='', - metadata=SPEED_METADATA), - sample.Sample(metric='SPECint(R)_base2006:speed', value=22.3, unit='', - metadata=SPEED_METADATA), + sample.Sample( + metric='400.perlbench:speed', + value=29.5, + unit='', + metadata=SPEED_METADATA, + ), + sample.Sample( + metric='401.bzip2:speed', value=17.3, unit='', metadata=SPEED_METADATA + ), + sample.Sample( + metric='403.gcc:speed', value=23.5, unit='', metadata=SPEED_METADATA + ), + sample.Sample( + metric='429.mcf:speed', value=19.0, unit='', metadata=SPEED_METADATA + ), + sample.Sample( + metric='445.gobmk:speed', value=20.3, unit='', metadata=SPEED_METADATA + ), + sample.Sample( + metric='456.hmmer:speed', value=23.8, unit='', metadata=SPEED_METADATA + ), + sample.Sample( + metric='458.sjeng:speed', value=22.5, unit='', metadata=SPEED_METADATA + ), + sample.Sample( + metric='462.libquantum:speed', + value=39.5, + unit='', + metadata=SPEED_METADATA, + ), + sample.Sample( + metric='464.h264ref:speed', value=38.6, unit='', metadata=SPEED_METADATA + ), + sample.Sample( + metric='471.omnetpp:speed', value=12.8, unit='', metadata=SPEED_METADATA + ), + sample.Sample( + metric='473.astar:speed', value=13.9, unit='', metadata=SPEED_METADATA + ), + sample.Sample( + metric='483.xalancbmk:speed', + value=22.3, + unit='', + metadata=SPEED_METADATA, + ), + sample.Sample( + metric='SPECint(R)_base2006:speed', + value=22.3, + unit='', + metadata=SPEED_METADATA, + ), ] TEST_OUTPUT_ALL = """ @@ -349,22 +478,39 @@ ALL_METADATA['spec_runmode'] = 'all' ALL_METADATA['runspec_metric'] = None EXPECTED_ALL_RESULT_SPECINT = [ - sample.Sample(metric='410.bwaves_r', value=19.0, unit='', - metadata=ALL_METADATA), - sample.Sample(metric='410.bwaves_r:peak', value=19.1, unit='', - metadata=ALL_METADATA), - sample.Sample(metric='507.cactuBSSN_r', value=21.2, unit='', - metadata=ALL_METADATA), - sample.Sample(metric='507.cactuBSSN_r:peak', value=29.0, unit='', - metadata=ALL_METADATA), - sample.Sample(metric='508.namd_r', value=19.1, unit='', - metadata=ALL_METADATA), - sample.Sample(metric='508.namd_r:peak', value=39.0, unit='', - metadata=ALL_METADATA), - sample.Sample(metric='SPECrate2017_fp_base', value=4.90, unit='', - metadata=ALL_METADATA), - sample.Sample(metric='SPECrate2017_fp_peak', value=12.3, unit='', - metadata=ALL_METADATA), + sample.Sample( + metric='410.bwaves_r', value=19.0, unit='', metadata=ALL_METADATA + ), + sample.Sample( + metric='410.bwaves_r:peak', value=19.1, unit='', metadata=ALL_METADATA + ), + sample.Sample( + metric='507.cactuBSSN_r', value=21.2, unit='', metadata=ALL_METADATA + ), + sample.Sample( + metric='507.cactuBSSN_r:peak', + value=29.0, + unit='', + metadata=ALL_METADATA, + ), + sample.Sample( + metric='508.namd_r', value=19.1, unit='', metadata=ALL_METADATA + ), + sample.Sample( + metric='508.namd_r:peak', value=39.0, unit='', metadata=ALL_METADATA + ), + sample.Sample( + metric='SPECrate2017_fp_base', + value=4.90, + unit='', + metadata=ALL_METADATA, + ), + sample.Sample( + metric='SPECrate2017_fp_peak', + value=12.3, + unit='', + metadata=ALL_METADATA, + ), ] @@ -380,14 +526,24 @@ PEAK_METADATA['spec_runmode'] = 'peak' PEAK_METADATA['runspec_metric'] = None EXPECTED_PEAK_RESULT_SPECINT = [ - sample.Sample(metric='410.bwaves_r:peak', value=19.1, unit='', - metadata=PEAK_METADATA), - sample.Sample(metric='507.cactuBSSN_r:peak', value=29.0, unit='', - metadata=PEAK_METADATA), - sample.Sample(metric='508.namd_r:peak', value=39.0, unit='', - metadata=PEAK_METADATA), - sample.Sample(metric='SPECrate2017_fp_peak', value=12.3, unit='', - metadata=PEAK_METADATA), + sample.Sample( + metric='410.bwaves_r:peak', value=19.1, unit='', metadata=PEAK_METADATA + ), + sample.Sample( + metric='507.cactuBSSN_r:peak', + value=29.0, + unit='', + metadata=PEAK_METADATA, + ), + sample.Sample( + metric='508.namd_r:peak', value=39.0, unit='', metadata=PEAK_METADATA + ), + sample.Sample( + metric='SPECrate2017_fp_peak', + value=12.3, + unit='', + metadata=PEAK_METADATA, + ), ] @@ -402,19 +558,28 @@ PARTIAL_PEAK_METADATA = GOOD_METADATA.copy() PARTIAL_PEAK_METADATA['spec_runmode'] = 'peak' PARTIAL_PEAK_METADATA['runspec_metric'] = None -PARTIAL_PEAK_METADATA.update({ - 'partial': 'true', - 'missing_results': ('410.bwaves_r,508.namd_r')}) +PARTIAL_PEAK_METADATA.update( + {'partial': 'true', 'missing_results': '410.bwaves_r,508.namd_r'} +) EXPECTED_PARTIAL_PEAK_RESULT_SPECINT = [ - sample.Sample(metric='507.cactuBSSN_r:peak', value=29.0, unit='', - metadata=PARTIAL_PEAK_METADATA), - sample.Sample(metric='SPECrate2017_fp_peak', value=12.3, unit='', - metadata=PARTIAL_PEAK_METADATA), + sample.Sample( + metric='507.cactuBSSN_r:peak', + value=29.0, + unit='', + metadata=PARTIAL_PEAK_METADATA, + ), + sample.Sample( + metric='SPECrate2017_fp_peak', + value=12.3, + unit='', + metadata=PARTIAL_PEAK_METADATA, + ), ] -class Speccpu2006BenchmarkTestCase(unittest.TestCase, - test_util.SamplesTestMixin): +class Speccpu2006BenchmarkTestCase( + unittest.TestCase, test_util.SamplesTestMixin +): def setUp(self): super(Speccpu2006BenchmarkTestCase, self).setUp() @@ -431,36 +596,29 @@ def testParseResultsC(self): speccpu.FLAGS.spec_runmode = 'base' vm.speccpu_vm_state = spec_test_config - samples = speccpu._ExtractScore(TEST_OUTPUT_SPECINT, vm, - False, 'rate') + samples = speccpu._ExtractScore(TEST_OUTPUT_SPECINT, vm, False, 'rate') self.assertSampleListsEqualUpToTimestamp(samples, EXPECTED_RESULT_SPECINT) - samples = speccpu._ExtractScore(TEST_OUTPUT_SPECFP, vm, - False, 'rate') + samples = speccpu._ExtractScore(TEST_OUTPUT_SPECFP, vm, False, 'rate') self.assertSampleListsEqualUpToTimestamp(samples, EXPECTED_RESULT_SPECFP) # By default, incomplete results result in error. with self.assertRaises(errors.Benchmarks.RunError): - samples = speccpu._ExtractScore(TEST_OUTPUT_BAD1, vm, - False, 'rate') + samples = speccpu._ExtractScore(TEST_OUTPUT_BAD1, vm, False, 'rate') with self.assertRaises(errors.Benchmarks.RunError): - samples = speccpu._ExtractScore(TEST_OUTPUT_BAD2, vm, - False, 'rate') + samples = speccpu._ExtractScore(TEST_OUTPUT_BAD2, vm, False, 'rate') # Now use keep_partial_results - samples = speccpu._ExtractScore(TEST_OUTPUT_BAD1, vm, - True, 'rate') + samples = speccpu._ExtractScore(TEST_OUTPUT_BAD1, vm, True, 'rate') self.assertSampleListsEqualUpToTimestamp(samples, EXPECTED_RESULT_BAD1) - samples = speccpu._ExtractScore(TEST_OUTPUT_BAD2, vm, - True, 'rate') + samples = speccpu._ExtractScore(TEST_OUTPUT_BAD2, vm, True, 'rate') self.assertSampleListsEqualUpToTimestamp(samples, EXPECTED_RESULT_BAD2) # Estimate scores speccpu.FLAGS.runspec_estimate_spec = True - samples = speccpu._ExtractScore(TEST_OUTPUT_EST, vm, - True, 'rate') + samples = speccpu._ExtractScore(TEST_OUTPUT_EST, vm, True, 'rate') self.assertSampleListsEqualUpToTimestamp(samples, EXPECTED_RESULT_EST) def testParseSpeedResults(self): @@ -471,10 +629,10 @@ def testParseSpeedResults(self): spec_test_config.log_format = r'Est. (SPEC.*_base2006)\s*(\S*)' spec_test_config.runspec_config = r'linux64-x64-gcc47.cfg' vm.speccpu_vm_state = spec_test_config - samples = speccpu._ExtractScore(SPEED_OUTPUT_SPECINT, vm, - False, 'speed') + samples = speccpu._ExtractScore(SPEED_OUTPUT_SPECINT, vm, False, 'speed') self.assertSampleListsEqualUpToTimestamp( - samples, EXPECTED_SPEED_RESULT_SPECINT) + samples, EXPECTED_SPEED_RESULT_SPECINT + ) def testParseAllResults(self): speccpu.FLAGS.spec_runmode = 'all' @@ -484,10 +642,10 @@ def testParseAllResults(self): spec_test_config.log_format = r'Est. (SPEC.*2017_.*_base)\s*(\S*)' spec_test_config.runspec_config = r'linux64-x64-gcc47.cfg' vm.speccpu_vm_state = spec_test_config - samples = speccpu._ExtractScore(TEST_OUTPUT_ALL, vm, - False, None) + samples = speccpu._ExtractScore(TEST_OUTPUT_ALL, vm, False, None) self.assertSampleListsEqualUpToTimestamp( - samples, EXPECTED_ALL_RESULT_SPECINT) + samples, EXPECTED_ALL_RESULT_SPECINT + ) def testParsePeakResults(self): speccpu.FLAGS.spec_runmode = 'peak' @@ -497,10 +655,10 @@ def testParsePeakResults(self): spec_test_config.log_format = r'Est. (SPEC.*2017_.*_base)\s*(\S*)' spec_test_config.runspec_config = r'linux64-x64-gcc47.cfg' vm.speccpu_vm_state = spec_test_config - samples = speccpu._ExtractScore(TEST_OUTPUT_PEAK, vm, - False, None) + samples = speccpu._ExtractScore(TEST_OUTPUT_PEAK, vm, False, None) self.assertSampleListsEqualUpToTimestamp( - samples, EXPECTED_PEAK_RESULT_SPECINT) + samples, EXPECTED_PEAK_RESULT_SPECINT + ) def testParsePartialPeakResults(self): speccpu.FLAGS.spec_runmode = 'peak' @@ -510,10 +668,10 @@ def testParsePartialPeakResults(self): spec_test_config.log_format = r'Est. (SPEC.*2017_.*_base)\s*(\S*)' spec_test_config.runspec_config = r'linux64-x64-gcc47.cfg' vm.speccpu_vm_state = spec_test_config - samples = speccpu._ExtractScore(TEST_OUTPUT_PARTIAL_PEAK, vm, - True, None) + samples = speccpu._ExtractScore(TEST_OUTPUT_PARTIAL_PEAK, vm, True, None) self.assertSampleListsEqualUpToTimestamp( - samples, EXPECTED_PARTIAL_PEAK_RESULT_SPECINT) + samples, EXPECTED_PARTIAL_PEAK_RESULT_SPECINT + ) if __name__ == '__main__': diff --git a/tests/linux_packages/sysbench_test.py b/tests/linux_packages/sysbench_test.py index 342b778923..c3351c2506 100644 --- a/tests/linux_packages/sysbench_test.py +++ b/tests/linux_packages/sysbench_test.py @@ -16,5 +16,6 @@ def testInstallIgnoresConcurrentMods(self): sysbench._Install(vm) self.assertIn('P0001', vm.RemoteCommand.call_args_list[1][0][0]) + if __name__ == '__main__': unittest.main() diff --git a/tests/linux_packages/tailbench_test.py b/tests/linux_packages/tailbench_test.py index c114da7368..2bdb7ab4a7 100644 --- a/tests/linux_packages/tailbench_test.py +++ b/tests/linux_packages/tailbench_test.py @@ -26,22 +26,24 @@ class TailbenchTest(pkb_common_test_case.PkbCommonTestCase): def validate_results(self, results: List[tailbench._TestResult]): for result in results: histogram: sample._Histogram = result.histogram - if not (histogram[i] <= histogram[i + 1] - for i in range(len(histogram) - 1)): + if not ( + histogram[i] <= histogram[i + 1] for i in range(len(histogram) - 1) + ): return False return True def testParseResultsIsReadable(self): test_results = List[tailbench._TestResult] path = os.path.join( - os.path.dirname(__file__), '..', 'data', 'tailbench-img-dnnResult.txt') - test_results = tailbench._ParseResultsFile(path, - 'name') + os.path.dirname(__file__), '..', 'data', 'tailbench-img-dnnResult.txt' + ) + test_results = tailbench._ParseResultsFile(path, 'name') self.assertTrue(self.validate_results(test_results)) def testDataConforms(self): path = os.path.join( - os.path.dirname(__file__), '..', 'data', 'tailbench-img-dnnResult.txt') + os.path.dirname(__file__), '..', 'data', 'tailbench-img-dnnResult.txt' + ) samp = tailbench.BuildHistogramSamples(path) dictionary = samp[0].asdict() self.assertGreater(len(dictionary), 0) diff --git a/tests/linux_packages/wrk2_test.py b/tests/linux_packages/wrk2_test.py index b50de0ddac..9e43fed9b0 100644 --- a/tests/linux_packages/wrk2_test.py +++ b/tests/linux_packages/wrk2_test.py @@ -32,17 +32,21 @@ def testParseSuccessfulRun(self): wrk_output = _ReadOutputFile('wrk2_output.txt') result = list(wrk2._ParseOutput(wrk_output)) self.assertEqual( - [('p50 latency', .05331, 'ms'), - ('p75 latency', 56.99, 'ms'), - ('p90 latency', 62.62, 'ms'), - ('p99 latency', 223.23, 'ms'), - ('p99.9 latency', 244.22, 'ms'), - ('p99.99 latency', 244.22, 'ms'), - ('p99.999 latency', 36000., 'ms'), - ('p100 latency', 54000., 'ms'), - ('requests', 600, ''), - ('error_rate', 0, ''), - ('errors', 0, '')], result) + [ + ('p50 latency', 0.05331, 'ms'), + ('p75 latency', 56.99, 'ms'), + ('p90 latency', 62.62, 'ms'), + ('p99 latency', 223.23, 'ms'), + ('p99.9 latency', 244.22, 'ms'), + ('p99.99 latency', 244.22, 'ms'), + ('p99.999 latency', 36000.0, 'ms'), + ('p100 latency', 54000.0, 'ms'), + ('requests', 600, ''), + ('error_rate', 0, ''), + ('errors', 0, ''), + ], + result, + ) def testParseAllRequestsFailed(self): wrk_output = _ReadOutputFile('wrk2_output_all_error.txt') @@ -53,7 +57,7 @@ def testParseWithErrors(self): wrk_output = _ReadOutputFile('wrk2_output_errors.txt') res = list(wrk2._ParseOutput(wrk_output)) self.assertIn(('errors', 14, ''), res) - self.assertIn(('error_rate', 14. / 600, ''), res) + self.assertIn(('error_rate', 14.0 / 600, ''), res) if __name__ == '__main__': diff --git a/tests/linux_packages/wrk_test.py b/tests/linux_packages/wrk_test.py index d6ba2fd586..e08edd85a4 100644 --- a/tests/linux_packages/wrk_test.py +++ b/tests/linux_packages/wrk_test.py @@ -28,15 +28,17 @@ def setUp(self): self.wrk_results = result_file.read() def testParsesSample(self): - expected = [('p5 latency', 0.162, 'ms'), - ('p50 latency', 0.187, 'ms'), - ('p90 latency', 0.256, 'ms'), - ('p99 latency', 0.519, 'ms'), - ('p99.9 latency', 5.196, 'ms'), - ('bytes transferred', 150068000.0, 'bytes'), - ('errors', 0.0, 'n'), - ('requests', 577297.0, 'n'), - ('throughput', 9605.69, 'requests/sec')] + expected = [ + ('p5 latency', 0.162, 'ms'), + ('p50 latency', 0.187, 'ms'), + ('p90 latency', 0.256, 'ms'), + ('p99 latency', 0.519, 'ms'), + ('p99.9 latency', 5.196, 'ms'), + ('bytes transferred', 150068000.0, 'bytes'), + ('errors', 0.0, 'n'), + ('requests', 577297.0, 'n'), + ('throughput', 9605.69, 'requests/sec'), + ] actual = list(wrk._ParseOutput(self.wrk_results)) six.assertCountEqual(self, expected, actual) diff --git a/tests/linux_virtual_machine_test.py b/tests/linux_virtual_machine_test.py index 1cb158ec75..d0016b3b2f 100644 --- a/tests/linux_virtual_machine_test.py +++ b/tests/linux_virtual_machine_test.py @@ -20,9 +20,7 @@ from absl import flags from absl.testing import flagsaver from absl.testing import parameterized - import mock - from perfkitbenchmarker import errors from perfkitbenchmarker import linux_virtual_machine from perfkitbenchmarker import os_types @@ -49,14 +47,17 @@ def CreateCentos7Vm(): # /proc/cmdline on a GCP CentOS7 vm _CENTOS7_KERNEL_COMMAND_LINE = ( 'BOOT_IMAGE=/boot/vmlinuz-3.10.0-1127.13.1.el7.x86_64 ' - 'root=UUID=1-2-3-4-5 ro crashkernel=auto console=ttyS0,38400n8') + 'root=UUID=1-2-3-4-5 ro crashkernel=auto console=ttyS0,38400n8' +) _DISABLE_YUM_CRON = mock.call( - 'sudo systemctl disable yum-cron.service', ignore_failure=True) + 'sudo systemctl disable yum-cron.service', ignore_failure=True +) -class TestCentos7VirtualMachine(linux_virtual_machine.CentOs7Mixin, - pkb_common_test_case.TestVirtualMachine): +class TestCentos7VirtualMachine( + linux_virtual_machine.CentOs7Mixin, pkb_common_test_case.TestVirtualMachine +): user_name = 'perfkit' @@ -78,25 +79,40 @@ def runTest(self, set_files, calls): vm.SetFiles() self.assertCountEqual( # use assertCountEqual because order is undefined - remote_command.call_args_list, - calls) + remote_command.call_args_list, calls + ) def testNoFiles(self): - self.runTest([], - []) + self.runTest([], []) def testOneFile(self): - self.runTest(['/sys/kernel/mm/transparent_hugepage/enabled=always'], - [mock.call('echo "always" | sudo tee ' - '/sys/kernel/mm/transparent_hugepage/enabled')]) + self.runTest( + ['/sys/kernel/mm/transparent_hugepage/enabled=always'], + [ + mock.call( + 'echo "always" | sudo tee ' + '/sys/kernel/mm/transparent_hugepage/enabled' + ) + ], + ) def testMultipleFiles(self): - self.runTest(['/sys/kernel/mm/transparent_hugepage/enabled=always', - '/sys/kernel/mm/transparent_hugepage/defrag=never'], - [mock.call('echo "always" | sudo tee ' - '/sys/kernel/mm/transparent_hugepage/enabled'), - mock.call('echo "never" | sudo tee ' - '/sys/kernel/mm/transparent_hugepage/defrag')]) + self.runTest( + [ + '/sys/kernel/mm/transparent_hugepage/enabled=always', + '/sys/kernel/mm/transparent_hugepage/defrag=never', + ], + [ + mock.call( + 'echo "always" | sudo tee ' + '/sys/kernel/mm/transparent_hugepage/enabled' + ), + mock.call( + 'echo "never" | sudo tee ' + '/sys/kernel/mm/transparent_hugepage/defrag' + ), + ], + ) class TestSysctl(pkb_common_test_case.PkbCommonTestCase): @@ -113,15 +129,20 @@ def runTest(self, sysctl, calls): def testSysctl(self): self.runTest( ['vm.dirty_background_ratio=10', 'vm.dirty_ratio=25'], - [mock.call('sudo bash -c \'echo "vm.dirty_background_ratio=10" >> ' - '/etc/sysctl.conf\''), - mock.call('sudo bash -c \'echo "vm.dirty_ratio=25" >> ' - '/etc/sysctl.conf\''), - mock.call('sudo sysctl -p')]) + [ + mock.call( + 'sudo bash -c \'echo "vm.dirty_background_ratio=10" >> ' + "/etc/sysctl.conf'" + ), + mock.call( + 'sudo bash -c \'echo "vm.dirty_ratio=25" >> /etc/sysctl.conf\'' + ), + mock.call('sudo sysctl -p'), + ], + ) def testNoSysctl(self): - self.runTest([], - []) + self.runTest([], []) class TestDiskOperations(pkb_common_test_case.PkbCommonTestCase): @@ -129,42 +150,50 @@ class TestDiskOperations(pkb_common_test_case.PkbCommonTestCase): def setUp(self): super(TestDiskOperations, self).setUp() FLAGS['default_timeout'].parse(0) # due to @retry - patcher = mock.patch.object(pkb_common_test_case.TestLinuxVirtualMachine, - 'RemoteHostCommand') + patcher = mock.patch.object( + pkb_common_test_case.TestLinuxVirtualMachine, 'RemoteHostCommand' + ) self.remote_command = patcher.start() self.addCleanup(patcher.stop) self.remote_command.side_effect = [('', None, 0), ('', None, 0)] self.vm = CreateTestLinuxVm() def assertRemoteHostCalled(self, *calls): - self.assertEqual([mock.call(call) for call in calls], - self.remote_command.call_args_list) + self.assertEqual( + [mock.call(call) for call in calls], self.remote_command.call_args_list + ) def testMountDisk(self): - mkdir_cmd = ('sudo mkdir -p mp;' - 'sudo mount -o discard dp mp && ' - 'sudo chown $USER:$USER mp;') + mkdir_cmd = ( + 'sudo mkdir -p mp;' + 'sudo mount -o discard dp mp && ' + 'sudo chown $USER:$USER mp;' + ) fstab_cmd = 'echo "dp mp ext4 defaults" | sudo tee -a /etc/fstab' self.vm.MountDisk('dp', 'mp') self.assertRemoteHostCalled(mkdir_cmd, fstab_cmd) def testFormatDisk(self): - expected_command = ('[[ -d /mnt ]] && sudo umount /mnt; ' - 'sudo mke2fs -F -E lazy_itable_init=0,discard ' - '-O ^has_journal -t ext4 -b 4096 dp') + expected_command = ( + '[[ -d /mnt ]] && sudo umount /mnt; ' + 'sudo mke2fs -F -E lazy_itable_init=0,discard ' + '-O ^has_journal -t ext4 -b 4096 dp' + ) self.vm.FormatDisk('dp') self.assertRemoteHostCalled(expected_command) self.assertEqual('ext4', self.vm.os_metadata['disk_filesystem_type']) self.assertEqual(4096, self.vm.os_metadata['disk_filesystem_blocksize']) def testNfsMountDisk(self): - mkdir_cmd = ('sudo mkdir -p mp;' - 'sudo mount -t nfs -o hard,ro dp mp && ' - 'sudo chown $USER:$USER mp;') + mkdir_cmd = ( + 'sudo mkdir -p mp;' + 'sudo mount -t nfs -o hard,ro dp mp && ' + 'sudo chown $USER:$USER mp;' + ) fstab_cmd = 'echo "dp mp nfs ro" | sudo tee -a /etc/fstab' - self.vm.MountDisk('dp', 'mp', - disk_type='nfs', mount_options='hard,ro', - fstab_options='ro') + self.vm.MountDisk( + 'dp', 'mp', disk_type='nfs', mount_options='hard,ro', fstab_options='ro' + ) self.assertRemoteHostCalled(mkdir_cmd, fstab_cmd) def testNfsFormatDisk(self): @@ -192,7 +221,6 @@ def testPreDeleteCallsDmesg(self): class TestLsCpu(unittest.TestCase, test_util.SamplesTestMixin): - LSCPU_DATA = { 'NUMA node(s)': '1', 'Core(s) per socket': '2', @@ -252,8 +280,10 @@ def testMissingRequiredLsCpuEntries(self): linux_virtual_machine.LsCpuResults('') def testLsCpuParsing(self): - vm = self.CreateVm(os_types.DEFAULT, - self.LsCpuText(self.LSCPU_DATA) + '\nThis Line=Invalid') + vm = self.CreateVm( + os_types.DEFAULT, + self.LsCpuText(self.LSCPU_DATA) + '\nThis Line=Invalid', + ) results = vm.CheckLsCpu() self.assertEqual(1, results.numa_node_count) self.assertEqual(2, results.cores_per_socket) @@ -265,8 +295,10 @@ def testLsCpuParsing(self): 'Core(s) per socket': '2', 'Thread(s) per core': '2', 'Socket(s)': '3', - 'a': 'b' - }, results.data) + 'a': 'b', + }, + results.data, + ) def testProcCpuParsing(self): vm = self.CreateVm(os_types.DEFAULT, self.PROC_CPU_TEXT) @@ -278,7 +310,7 @@ def testProcCpuParsing(self): expected_common = { 'cpu family': '6', 'oddkey': 'v29;v30', - 'proccpu': 'cpu family,oddkey' + 'proccpu': 'cpu family,oddkey', } self.assertEqual(expected_mappings, results.mappings) self.assertEqual(expected_common, results.GetValues()) @@ -296,11 +328,11 @@ def testProcCpuSamples(self): 'node_name': 'pkb-test', 'proc_29': 'apicid=27;core id=13', 'proc_30': 'apicid=29;core id=14', - 'proc_31': 'apicid=31;core id=15' + 'proc_31': 'apicid=31;core id=15', } expected_samples = [ sample.Sample('proccpu', 0, '', proccpu_metadata), - sample.Sample('proccpu_mapping', 0, '', proccpu_mapping_metadata) + sample.Sample('proccpu_mapping', 0, '', proccpu_mapping_metadata), ] self.assertSampleListsEqualUpToTimestamp(expected_samples, samples) @@ -374,7 +406,8 @@ def testNonZeroReturnCodeRaises(self): def testNonZeroReturnCodeIgnored(self): self.issue_cmd_mock.return_value = ('output', 'err', 1) stdout, stderr, ret = self.vm.RemoteCommandWithReturnCode( - 'foo', ignore_failure=True) + 'foo', ignore_failure=True + ) self.assertEqual(stdout, 'output') self.assertEqual(stderr, 'err') self.assertEqual(ret, 1) @@ -430,8 +463,7 @@ def testFdiskParsingBootDiskOnly(self): /dev/sda1 * 2048 20971519 10484736 83 Linux """) results = vm.partition_table - self.assertEqual( - {'/dev/sda': 10737418240}, results) + self.assertEqual({'/dev/sda': 10737418240}, results) def testFdiskParsingWithRaidDisk(self): vm = self.CreateVm(""" @@ -465,10 +497,14 @@ def testFdiskParsingWithRaidDisk(self): """) results = vm.partition_table self.assertEqual( - {'/dev/sda': 10737418240, - '/dev/sdb': 402653184000, - '/dev/sdc': 402653184000, - '/dev/md0': 805037932544}, results) + { + '/dev/sda': 10737418240, + '/dev/sdb': 402653184000, + '/dev/sdc': 402653184000, + '/dev/md0': 805037932544, + }, + results, + ) class LinuxVirtualMachineTestCase(pkb_common_test_case.PkbCommonTestCase): @@ -488,14 +524,17 @@ class LinuxVirtualMachineTestCase(pkb_common_test_case.PkbCommonTestCase): 'uname -r': kernel_release, 'uname -m': cpu_arch, 'sudo fdisk -l': partition_table, - 'PATH="${PATH}":/usr/sbin ip link show up': - pkb_common_test_case.IP_LINK_TEXT, - 'cat /proc/cpuinfo | grep processor | wc -l': '16' + 'PATH="${PATH}":/usr/sbin ip link show up': ( + pkb_common_test_case.IP_LINK_TEXT + ), + 'cat /proc/cpuinfo | grep processor | wc -l': '16', } - def CreateVm(self, run_cmd_response: Union[str, Dict[str, str]], - metadata: bool = False): + def CreateVm( + self, run_cmd_response: Union[str, Dict[str, str]], metadata: bool = False + ): vm = CreateTestLinuxVm() + def FakeRemoteHostCommandWithReturnCode(cmd, **_): if isinstance(run_cmd_response, str): stdout = run_cmd_response @@ -517,9 +556,11 @@ def FakeWaitForBootCompletion(): vm.bootable_time = 10 vm.RemoteHostCommandWithReturnCode = mock.Mock( - side_effect=FakeRemoteHostCommandWithReturnCode) + side_effect=FakeRemoteHostCommandWithReturnCode + ) vm.CheckLsCpu = mock.Mock( - return_value=linux_virtual_machine.LsCpuResults(self.lscpu_output)) + return_value=linux_virtual_machine.LsCpuResults(self.lscpu_output) + ) vm.WaitForBootCompletion = mock.Mock(side_effect=FakeWaitForBootCompletion) if metadata: vm.WaitForBootCompletion() @@ -528,8 +569,12 @@ def FakeWaitForBootCompletion(): @parameterized.named_parameters( ('has_smt_centos7', _CENTOS7_KERNEL_COMMAND_LINE, True), - ('no_smt_centos7', _CENTOS7_KERNEL_COMMAND_LINE + ' noht nosmt nr_cpus=1', - False)) + ( + 'no_smt_centos7', + _CENTOS7_KERNEL_COMMAND_LINE + ' noht nosmt nr_cpus=1', + False, + ), + ) def testIsSmtEnabled(self, proc_cmdline, is_enabled): vm = self.CreateVm(proc_cmdline) self.assertEqual(is_enabled, vm.IsSmtEnabled()) @@ -538,8 +583,9 @@ def testIsSmtEnabled(self, proc_cmdline, is_enabled): ('hasSMT_want_real', 32, 'regular', 16), ('noSMT_want_real', 32, 'nosmt', 32), ) - def testNumCpusForBenchmarkNoSmt(self, vcpus, kernel_command_line, - expected_num_cpus): + def testNumCpusForBenchmarkNoSmt( + self, vcpus, kernel_command_line, expected_num_cpus + ): FLAGS['use_numcpu_multi_files'].parse(True) responses = { 'ls /sys/fs/cgroup/cpuset.cpus.effective >> /dev/null 2>&1 || echo file_not_exist': ( @@ -680,7 +726,7 @@ def testReboot(self): 'stat -c %z /proc/': '', 'sudo mkdir -p /opt/pkb; sudo chmod a+rwxt /opt/pkb': '', 'mkdir -p /tmp/pkb': '', - 'cat /proc/cpuinfo | grep processor | wc -l': '8' + 'cat /proc/cpuinfo | grep processor | wc -l': '8', }) vm.Reboot() self.assertEqual(os_info_new, vm.os_metadata['os_info']) @@ -693,8 +739,9 @@ def testGetNetworkDeviceNames(self): names = vm._get_network_device_mtus() self.assertEqual({'eth0': '1500', 'eth1': '1500'}, names) mock_cmd = vm.RemoteHostCommandWithReturnCode - mock_cmd.assert_called_with('PATH="${PATH}":/usr/sbin ip link show up', - stack_level=3) + mock_cmd.assert_called_with( + 'PATH="${PATH}":/usr/sbin ip link show up', stack_level=3 + ) def testCpuVulnerabilitiesEmpty(self): self.assertEqual({}, self.CreateVm('').cpu_vulnerabilities.asdict) @@ -725,7 +772,7 @@ def testCpuVulnerabilities(self): self.assertEqual(expected_mitigation, cpu_vuln.mitigations) expected_vulnerability = { 'itlb_multihit': 'KVM', - 'mds': 'Clear CPU buffers attempted, no microcode' + 'mds': 'Clear CPU buffers attempted, no microcode', } self.assertEqual(expected_vulnerability, cpu_vuln.vulnerabilities) expected_notaffecteds = set(['srbds', 'tsx_async_abort']) @@ -748,9 +795,11 @@ def testCpuVulnerabilities(self): } self.assertEqual(expected_asdict, cpu_vuln.asdict) - @parameterized.named_parameters(('flag_true', True, _DISABLE_YUM_CRON), - ('default_flag', None, _DISABLE_YUM_CRON), - ('flag_false', False, None)) + @parameterized.named_parameters( + ('flag_true', True, _DISABLE_YUM_CRON), + ('default_flag', None, _DISABLE_YUM_CRON), + ('flag_false', False, None), + ) def testCentos7OnStartup(self, flag_disable_yum_cron, additional_command): vm = CreateCentos7Vm() mock_remote = mock.Mock(return_value=('', '')) @@ -763,8 +812,9 @@ def testCentos7OnStartup(self, flag_disable_yum_cron, additional_command): # tests the default value of the flag vm.OnStartup() - common_call = ("echo 'Defaults:perfkit !requiretty' | " - 'sudo tee /etc/sudoers.d/pkb') + common_call = ( + "echo 'Defaults:perfkit !requiretty' | sudo tee /etc/sudoers.d/pkb" + ) calls = [mock.call(common_call, login_shell=True)] if additional_command: calls.append(additional_command) @@ -776,8 +826,7 @@ def testRebootCommandNonRebootable(self): with self.assertRaises(errors.VirtualMachine.VirtualMachineError): vm._Reboot() - @parameterized.named_parameters( - ('regular', CreateTestLinuxVm, False)) + @parameterized.named_parameters(('regular', CreateTestLinuxVm, False)) def testRebootCommand(self, vm_create_function, ignore_ssh_error): vm = vm_create_function() mock_remote = mock.Mock(return_value=('', '')) @@ -787,7 +836,8 @@ def testRebootCommand(self, vm_create_function, ignore_ssh_error): if ignore_ssh_error: mock_remote.assert_called_with( - 'sudo reboot', ignore_failure=True, ignore_ssh_error=True) + 'sudo reboot', ignore_failure=True, ignore_ssh_error=True + ) else: mock_remote.assert_called_with('sudo reboot', ignore_failure=True) diff --git a/tests/log_util_test.py b/tests/log_util_test.py index 3b7b8fdaac..d9b672f1aa 100644 --- a/tests/log_util_test.py +++ b/tests/log_util_test.py @@ -105,16 +105,20 @@ def RunThreadedContextCopyHelper(my_list): my_list.append(context.label) def testPkbLogFilter(self): - """Verify that PkbLogFilter sets the pkb_label of LogRecords it processes. - """ + """Verify that PkbLogFilter sets the pkb_label of LogRecords it processes.""" logger_name = 'log_util_test.LogUtilTestCase.testPkbLogFilter' context = log_util.ThreadLogContext() log_util.SetThreadLogContext(context) with context.ExtendLabel('LABEL-A'): log_record = logging.LogRecord( - name=logger_name, level=logging.INFO, pathname=__file__, + name=logger_name, + level=logging.INFO, + pathname=__file__, lineno=inspect.getframeinfo(inspect.currentframe()).lineno + 1, - msg="Log message.", args=None, exc_info=None) + msg='Log message.', + args=None, + exc_info=None, + ) log_util.PkbLogFilter().filter(log_record) self.assertEqual(log_record.pkb_label, 'LABEL-A ') @@ -125,11 +129,17 @@ def testPkbLogFilterNoContext(self): def childLog(): logger_name = 'log_util_test.LogUtilTestCase.testPkbLogFilterNoContext' self.log_record = logging.LogRecord( - name=logger_name, level=logging.INFO, pathname=__file__, + name=logger_name, + level=logging.INFO, + pathname=__file__, lineno=inspect.getframeinfo(inspect.currentframe()).lineno + 1, - msg="Log message.", args=None, exc_info=None) + msg='Log message.', + args=None, + exc_info=None, + ) log_util.PkbLogFilter().filter(self.log_record) self.completed = True + child = threading.Thread(target=childLog) child.start() child.join() diff --git a/tests/managed_memory_store_test.py b/tests/managed_memory_store_test.py index 5191d1db45..5b4c14f9d3 100644 --- a/tests/managed_memory_store_test.py +++ b/tests/managed_memory_store_test.py @@ -23,18 +23,17 @@ class ManagedMemoryStoreTest(pkb_common_test_case.PkbCommonTestCase): def testReadableVersion(self): """Test normal cases work as they should.""" - self.assertEqual( - managed_memory_store.ParseReadableVersion('6.x'), '6.x') - self.assertEqual( - managed_memory_store.ParseReadableVersion('4.0.10'), '4.0') + self.assertEqual(managed_memory_store.ParseReadableVersion('6.x'), '6.x') + self.assertEqual(managed_memory_store.ParseReadableVersion('4.0.10'), '4.0') def testReadableVersionExtraneous(self): """Test weird cases just return the version number as is.""" self.assertEqual( - managed_memory_store.ParseReadableVersion('redis.8'), 'redis.8') + managed_memory_store.ParseReadableVersion('redis.8'), 'redis.8' + ) self.assertEqual( - managed_memory_store.ParseReadableVersion('redis 9_7_5'), - 'redis 9_7_5') + managed_memory_store.ParseReadableVersion('redis 9_7_5'), 'redis 9_7_5' + ) if __name__ == '__main__': diff --git a/tests/matchers.py b/tests/matchers.py index 8ae4401b95..db730387a6 100644 --- a/tests/matchers.py +++ b/tests/matchers.py @@ -244,8 +244,9 @@ def __eq__(self, collection): True if self._values is not empty and all self._values are in collection, False otherwise. """ - if not self._values or (not self._hashable and - isinstance(collection, Hashable)): + if not self._values or ( + not self._hashable and isinstance(collection, Hashable) + ): return False if self._hashable: try: @@ -302,7 +303,6 @@ class HASMETHODVALUE(_NotEqualMixin): This calls the compared object's method and checks if it returned value. Short form for HASATTRVALUE(method, IS(lambda x: x() == value)). - """ def __init__(self, method, value): @@ -495,10 +495,14 @@ def __init__(self, sequence: Sequence[Any]): self._sequence = sequence def __eq__(self, other: Any) -> bool: - return (isinstance(other, Sequence) and - len(self._sequence) == len(other) and - all(own_item == other_item - for own_item, other_item in zip(self._sequence, other))) + return ( + isinstance(other, Sequence) + and len(self._sequence) == len(other) + and all( + own_item == other_item + for own_item, other_item in zip(self._sequence, other) + ) + ) def __repr__(self) -> str: return f"SequenceEqual({self._sequence!r})" @@ -523,9 +527,9 @@ def __init__(self, sequence: Sequence[float], **kwargs: Any): def __eq__(self, other: Any) -> bool: return ( - isinstance(other, Sequence) and - len(self._sequence) == len(other) and - all( + isinstance(other, Sequence) + and len(self._sequence) == len(other) + and all( math.isclose(own_item, other_item, **self._isclose_kwargs) for own_item, other_item in zip(self._sequence, other) ) diff --git a/tests/messaging_service_test.py b/tests/messaging_service_test.py index b69ebcb8de..d17b627c48 100644 --- a/tests/messaging_service_test.py +++ b/tests/messaging_service_test.py @@ -14,7 +14,6 @@ class DummyMessagingService(messaging_service.BaseMessagingService): - CLOUD = 'dummy' def _InstallCloudClients(self): @@ -35,14 +34,16 @@ class MessagingServiceTest(pkb_common_test_case.PkbCommonTestCase): def setUp(self): super().setUp() self.client = mock.create_autospec( - virtual_machine.BaseVirtualMachine, instance=True) + virtual_machine.BaseVirtualMachine, instance=True + ) self.messaging_service = DummyMessagingService() self.messaging_service.client_vm = self.client @mock.patch.object(DummyMessagingService, '_InstallCloudClients') @mock.patch.object(DummyMessagingService, '_InstallCommonClientPackages') - def testPrepareClientVm(self, install_cloud_clients_mock, - install_common_client_packages_mock): + def testPrepareClientVm( + self, install_cloud_clients_mock, install_common_client_packages_mock + ): self.messaging_service.PrepareClientVm() install_cloud_clients_mock.assert_called_once_with() install_common_client_packages_mock.assert_called_once_with() @@ -52,107 +53,122 @@ def testPrepareBasicVmClient(self): self.client.assert_has_calls([ mock.call.RemoteCommand('sudo pip3 install absl-py numpy'), mock.call.RemoteCommand( - 'mkdir -p ~/perfkitbenchmarker/scripts/messaging_service_scripts'), + 'mkdir -p ~/perfkitbenchmarker/scripts/messaging_service_scripts' + ), mock.call.RemoteCommand( "find ~/perfkitbenchmarker -type d -exec touch '{}/__init__.py' \\;" ), mock.call.RemoteCommand( - 'mkdir -p ~/perfkitbenchmarker/scripts/messaging_service_scripts/common' + 'mkdir -p' + ' ~/perfkitbenchmarker/scripts/messaging_service_scripts/common' ), mock.call.PushDataFile( 'messaging_service_scripts/common/__init__.py', - '~/perfkitbenchmarker/scripts/messaging_service_scripts/common/__init__.py' + '~/perfkitbenchmarker/scripts/messaging_service_scripts/common/__init__.py', ), mock.call.RemoteCommand( - 'mkdir -p ~/perfkitbenchmarker/scripts/messaging_service_scripts/common' + 'mkdir -p' + ' ~/perfkitbenchmarker/scripts/messaging_service_scripts/common' ), mock.call.PushDataFile( 'messaging_service_scripts/common/app.py', - '~/perfkitbenchmarker/scripts/messaging_service_scripts/common/app.py' + '~/perfkitbenchmarker/scripts/messaging_service_scripts/common/app.py', ), mock.call.RemoteCommand( - 'mkdir -p ~/perfkitbenchmarker/scripts/messaging_service_scripts/common' + 'mkdir -p' + ' ~/perfkitbenchmarker/scripts/messaging_service_scripts/common' ), mock.call.PushDataFile( 'messaging_service_scripts/common/client.py', - '~/perfkitbenchmarker/scripts/messaging_service_scripts/common/client.py' + '~/perfkitbenchmarker/scripts/messaging_service_scripts/common/client.py', ), mock.call.RemoteCommand( - 'mkdir -p ~/perfkitbenchmarker/scripts/messaging_service_scripts/common' + 'mkdir -p' + ' ~/perfkitbenchmarker/scripts/messaging_service_scripts/common' ), mock.call.PushDataFile( 'messaging_service_scripts/common/errors.py', - '~/perfkitbenchmarker/scripts/messaging_service_scripts/common/errors.py' + '~/perfkitbenchmarker/scripts/messaging_service_scripts/common/errors.py', ), mock.call.RemoteCommand( - 'mkdir -p ~/perfkitbenchmarker/scripts/messaging_service_scripts/common' + 'mkdir -p' + ' ~/perfkitbenchmarker/scripts/messaging_service_scripts/common' ), mock.call.PushDataFile( 'messaging_service_scripts/common/log_utils.py', - '~/perfkitbenchmarker/scripts/messaging_service_scripts/common/log_utils.py' + '~/perfkitbenchmarker/scripts/messaging_service_scripts/common/log_utils.py', ), mock.call.RemoteCommand( - 'mkdir -p ~/perfkitbenchmarker/scripts/messaging_service_scripts/common' + 'mkdir -p' + ' ~/perfkitbenchmarker/scripts/messaging_service_scripts/common' ), mock.call.PushDataFile( 'messaging_service_scripts/common/runners.py', - '~/perfkitbenchmarker/scripts/messaging_service_scripts/common/runners.py' + '~/perfkitbenchmarker/scripts/messaging_service_scripts/common/runners.py', ), mock.call.RemoteCommand( - 'mkdir -p ~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e' + 'mkdir -p' + ' ~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e' ), mock.call.PushDataFile( 'messaging_service_scripts/common/e2e/__init__.py', - '~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e/__init__.py' + '~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e/__init__.py', ), mock.call.RemoteCommand( - 'mkdir -p ~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e' + 'mkdir -p' + ' ~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e' ), mock.call.PushDataFile( 'messaging_service_scripts/common/e2e/latency_runner.py', - '~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e/latency_runner.py' + '~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e/latency_runner.py', ), mock.call.RemoteCommand( - 'mkdir -p ~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e' + 'mkdir -p' + ' ~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e' ), mock.call.PushDataFile( 'messaging_service_scripts/common/e2e/main_process.py', - '~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e/main_process.py' + '~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e/main_process.py', ), mock.call.RemoteCommand( - 'mkdir -p ~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e' + 'mkdir -p' + ' ~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e' ), mock.call.PushDataFile( 'messaging_service_scripts/common/e2e/protocol.py', - '~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e/protocol.py' + '~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e/protocol.py', ), mock.call.RemoteCommand( - 'mkdir -p ~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e' + 'mkdir -p' + ' ~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e' ), mock.call.PushDataFile( 'messaging_service_scripts/common/e2e/publisher.py', - '~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e/publisher.py' + '~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e/publisher.py', ), mock.call.RemoteCommand( - 'mkdir -p ~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e' + 'mkdir -p' + ' ~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e' ), mock.call.PushDataFile( 'messaging_service_scripts/common/e2e/receiver.py', - '~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e/receiver.py' + '~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e/receiver.py', ), mock.call.RemoteCommand( - 'mkdir -p ~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e' + 'mkdir -p' + ' ~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e' ), mock.call.PushDataFile( 'messaging_service_scripts/common/e2e/streaming_pull_receiver.py', - '~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e/streaming_pull_receiver.py' + '~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e/streaming_pull_receiver.py', ), mock.call.RemoteCommand( - 'mkdir -p ~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e' + 'mkdir -p' + ' ~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e' ), mock.call.PushDataFile( 'messaging_service_scripts/common/e2e/worker_utils.py', - '~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e/worker_utils.py' + '~/perfkitbenchmarker/scripts/messaging_service_scripts/common/e2e/worker_utils.py', ), ]) diff --git a/tests/network_test.py b/tests/network_test.py index 6deed3fa54..563ffb69bd 100644 --- a/tests/network_test.py +++ b/tests/network_test.py @@ -15,7 +15,6 @@ import unittest from absl import flags - from perfkitbenchmarker import network from tests import pkb_common_test_case @@ -30,7 +29,8 @@ class BaseNetworkTestCase(pkb_common_test_case.PkbCommonTestCase): def testGetKeyFromNetworkSpec(self): spec = network.BaseNetworkSpec( - zone='us-central1-a', cidr=None, machine_type=None) + zone='us-central1-a', cidr=None, machine_type=None + ) spec.subnet_name = 'test' actual_key = TestNetwork._GetKeyFromNetworkSpec(spec) expected_key = ('test_cloud', 'us-central1-a', 'test') diff --git a/tests/nfs_service_test.py b/tests/nfs_service_test.py index 5d491d2fc9..4f38ed24df 100644 --- a/tests/nfs_service_test.py +++ b/tests/nfs_service_test.py @@ -110,15 +110,17 @@ def setUp(self): super(UnmanagedNfsServiceTest, self).setUp() self._setUpDiskSpec() self._setUpMockServerVm() - self.nfs_service = nfs_service.UnmanagedNfsService(self.disk_spec, - self.mock_server_vm) + self.nfs_service = nfs_service.UnmanagedNfsService( + self.disk_spec, self.mock_server_vm + ) def testNewUnmanagedNfsService(self): self.assertIsNotNone(self.nfs_service) self.assertIsNotNone(self.nfs_service.server_vm) self.assertIsNotNone(self.nfs_service.disk_spec) - self.assertEqual(self.nfs_service.server_directory, - self.disk_spec.device_path) + self.assertEqual( + self.nfs_service.server_directory, self.disk_spec.device_path + ) def testCreateNfsDisk(self): nfs_disk = self.nfs_service.CreateNfsDisk() @@ -154,20 +156,23 @@ def testNfsExportDirectoryAlreadyExported(self): def testNfsExportAndMount(self): mock_nfs_create = self.enter_context( - mock.patch.object(nfs_service, 'UnmanagedNfsService')) + mock.patch.object(nfs_service, 'UnmanagedNfsService') + ) headnode = mock.Mock(internal_ip='10.0.1.11') vm1 = mock.Mock(user_name='perfkit') vm2 = mock.Mock(user_name='perfkit') - nfs_service.NfsExportAndMount([headnode, vm1, vm2], '/client_path', - '/server_path') + nfs_service.NfsExportAndMount( + [headnode, vm1, vm2], '/client_path', '/server_path' + ) mock_nfs_create.assert_called_with(None, headnode, False, '/server_path') mount_cmd = ( 'sudo mkdir -p /client_path; ' 'sudo chown perfkit /client_path; ' 'echo "10.0.1.11:/server_path /client_path nfs defaults 0 0\n" ' - '| sudo tee -a /etc/fstab; sudo mount -a') + '| sudo tee -a /etc/fstab; sudo mount -a' + ) for vm in (vm1, vm2): vm.Install.assert_called_with('nfs_utils') vm.RemoteCommand.assert_called_with(mount_cmd) diff --git a/tests/omb_benchmark_test.py b/tests/omb_benchmark_test.py index 27ca78ec54..857aab7fc2 100644 --- a/tests/omb_benchmark_test.py +++ b/tests/omb_benchmark_test.py @@ -3,7 +3,6 @@ import unittest from absl.testing import flagsaver import mock - from perfkitbenchmarker import benchmark_spec from perfkitbenchmarker import sample from perfkitbenchmarker import test_util @@ -15,9 +14,11 @@ def MockBenchmarkSpec(): benchmark_module = mock.Mock(BENCHMARK_NAME='omb') benchmark_config = mock.Mock( - vm_groups={}, relational_db=mock.Mock(vm_groups={})) - spec = benchmark_spec.BenchmarkSpec(benchmark_module, benchmark_config, - 'abcdefg') + vm_groups={}, relational_db=mock.Mock(vm_groups={}) + ) + spec = benchmark_spec.BenchmarkSpec( + benchmark_module, benchmark_config, 'abcdefg' + ) spec.vms = [mock.Mock(), mock.Mock()] return spec @@ -25,13 +26,7 @@ def MockBenchmarkSpec(): _RUN_RESULT = result = omb.RunResult( name='acc_latency', metadata={'a': 1}, - data=[{ - 'latency': 10, - 'foo': 100 - }, { - 'latency': 20, - 'foo': 200 - }], + data=[{'latency': 10, 'foo': 100}, {'latency': 20, 'foo': 200}], full_cmd='mpirun path/to/acc_latency', units='usec', params={'b': 2}, @@ -42,10 +37,8 @@ def MockBenchmarkSpec(): run_time=0, pinning=['0:0:0,1,15', '1:1:0,1,15', '2:0:2,16,17', '3:1:2,16,17'], perhost=1, - mpi_env={ - 'I_MPI_DEBUG': '6', - 'I_MPI_PIN_PROCESSOR_LIST': '0' - }) + mpi_env={'I_MPI_DEBUG': '6', 'I_MPI_PIN_PROCESSOR_LIST': '0'}, +) _COMMON_METADATA = { 'cmd': 'mpirun path/to/acc_latency', @@ -69,8 +62,9 @@ def MockBenchmarkSpec(): ] -class OmbBenchmarkTest(pkb_common_test_case.PkbCommonTestCase, - test_util.SamplesTestMixin): +class OmbBenchmarkTest( + pkb_common_test_case.PkbCommonTestCase, test_util.SamplesTestMixin +): def setUp(self): super().setUp() @@ -107,7 +101,8 @@ def testMessageSizeRequest(self, mock_run): continue for size in (1024, 2048): expected_calls.append( - mock.call(omb.RunRequest(name, bm_spec.vms, size))) + mock.call(omb.RunRequest(name, bm_spec.vms, size)) + ) mock_run.assert_has_calls(expected_calls) diff --git a/tests/omb_test.py b/tests/omb_test.py index b3d99954ff..5bb296deac 100644 --- a/tests/omb_test.py +++ b/tests/omb_test.py @@ -34,16 +34,49 @@ def setUp(self): self.enter_context(mock.patch.object(omb.time, 'time', return_value=0)) @parameterized.parameters( - ('acc_latency',), ('allgather',), ('allgatherv',), ('allreduce',), - ('alltoall',), ('alltoallv',), ('barrier',), ('bcast',), ('bibw',), - ('bw',), ('cas_latency',), ('fop_latency',), ('gather',), ('gatherv',), - ('get_acc_latency',), ('get_bw',), ('get_latency',), ('iallgather',), - ('iallgatherv',), ('iallreduce',), ('ialltoall',), ('ialltoallv',), - ('ialltoallw',), ('ibarrier',), ('ibcast',), ('igather',), ('igatherv',), - ('ireduce',), ('iscatter',), ('iscatterv',), ('latency',), - ('latency_mp',), ('latency_mt',), ('mbw_mr',), ('multi_lat',), - ('put_bibw',), ('put_bw',), ('put_latency',), ('reduce',), - ('reduce_scatter',), ('scatter',), ('scatterv',)) + ('acc_latency',), + ('allgather',), + ('allgatherv',), + ('allreduce',), + ('alltoall',), + ('alltoallv',), + ('barrier',), + ('bcast',), + ('bibw',), + ('bw',), + ('cas_latency',), + ('fop_latency',), + ('gather',), + ('gatherv',), + ('get_acc_latency',), + ('get_bw',), + ('get_latency',), + ('iallgather',), + ('iallgatherv',), + ('iallreduce',), + ('ialltoall',), + ('ialltoallv',), + ('ialltoallw',), + ('ibarrier',), + ('ibcast',), + ('igather',), + ('igatherv',), + ('ireduce',), + ('iscatter',), + ('iscatterv',), + ('latency',), + ('latency_mp',), + ('latency_mt',), + ('mbw_mr',), + ('multi_lat',), + ('put_bibw',), + ('put_bw',), + ('put_latency',), + ('reduce',), + ('reduce_scatter',), + ('scatter',), + ('scatterv',), + ) def testParseData(self, test_name): input_text = ReadFile(f'{test_name}.txt') @@ -52,17 +85,14 @@ def testParseData(self, test_name): self.assertEqual(json.loads(ReadFile(f'{test_name}.json')), values) @parameterized.parameters( - ('acc_latency.txt', { - 'sync': 'MPI_Win_flush', - 'window_creation': 'MPI_Win_allocate' - }), + ( + 'acc_latency.txt', + {'sync': 'MPI_Win_flush', 'window_creation': 'MPI_Win_allocate'}, + ), ('barrier.txt', {}), ('ibarrier.txt', {}), ('ibcast.txt', {}), - ('mbw_mr.txt', { - 'pairs': '4', - 'window_size': '64' - }), + ('mbw_mr.txt', {'pairs': '4', 'window_size': '64'}), ) def testParseMetadata(self, test_file, expected_metadata): input_text = ReadFile(test_file) @@ -75,12 +105,10 @@ def testParseMetadata(self, test_file, expected_metadata): { 'lines': ('a', 'b1', 'b2', 'c'), 'regex_text': 'b', - 'expected': ['b2', 'c'] - }, { - 'lines': ('a', 'b', 'c'), - 'regex_text': 'd', - 'expected': [] - }) + 'expected': ['b2', 'c'], + }, + {'lines': ('a', 'b', 'c'), 'regex_text': 'd', 'expected': []}, + ) def testLinesAfterMarker(self, lines, regex_text, expected): line_re = re.compile(regex_text) input_text = '\n'.join(lines) @@ -101,7 +129,10 @@ def testParseMpiPinningInfo(self): pinning = omb.ParseMpiPinning(txt.splitlines()) expected_pinning = [ - '0:0:0,1,15', '1:1:0,1,15', '2:0:2,16,17', '3:1:2,16,17' + '0:0:0,1,15', + '1:1:0,1,15', + '2:0:2,16,17', + '3:1:2,16,17', ] self.assertEqual(expected_pinning, pinning) @@ -113,25 +144,31 @@ def setUp(self): self.enter_context(flagsaver.flagsaver(mpi_vendor='intel')) self.enter_context( mock.patch.object( - intelmpi, 'SourceMpiVarsCommand', return_value='. mpivars.sh')) + intelmpi, 'SourceMpiVarsCommand', return_value='. mpivars.sh' + ) + ) self.enter_context( - mock.patch.object(intelmpi, 'MpirunMpiVersion', return_value='2019.6')) + mock.patch.object(intelmpi, 'MpirunMpiVersion', return_value='2019.6') + ) self.enter_context(mock.patch.object(omb.time, 'time', return_value=0)) @flagsaver.flagsaver( omb_mpi_env=['IMPI_DEBUG=5'], - omb_mpi_genv=['I_MPI_PIN_PROCESSOR_LIST=0', 'I_MPI_PIN=1']) + omb_mpi_genv=['I_MPI_PIN_PROCESSOR_LIST=0', 'I_MPI_PIN=1'], + ) def testRunBenchmarkNormal(self): # all calls done with vm.RemoteCommand benchmark = 'barrier' benchmark_path = f'path/to/osu_{benchmark}' textoutput = 'textoutput' ls_cmd = f'ls {omb._RUN_DIR}/*/osu_{benchmark}' - expected_full_cmd = ('. mpivars.sh; IMPI_DEBUG=5 mpirun ' - '-genv I_MPI_PIN=1 ' - '-genv I_MPI_PIN_PROCESSOR_LIST=0 -perhost 1 -n 2 ' - f'-hosts 10.0.0.1,10.0.0.2 {benchmark_path} ' - '-t 1 --full') + expected_full_cmd = ( + '. mpivars.sh; IMPI_DEBUG=5 mpirun ' + '-genv I_MPI_PIN=1 ' + '-genv I_MPI_PIN_PROCESSOR_LIST=0 -perhost 1 -n 2 ' + f'-hosts 10.0.0.1,10.0.0.2 {benchmark_path} ' + '-t 1 --full' + ) vm = mock.Mock(internal_ip='10.0.0.1') vm.RemoteCommand.side_effect = [(benchmark_path, '')] vm.RobustRemoteCommand.side_effect = [(textoutput, '')] @@ -142,7 +179,8 @@ def testRunBenchmarkNormal(self): number_processes=2, hosts=[vm, mock.Mock(internal_ip='10.0.0.2')], options={'-t': 1}, - perhost=1) + perhost=1, + ) self.assertEqual(textoutput, txt) self.assertEqual(expected_full_cmd, full_cmd) @@ -152,10 +190,12 @@ def testRunBenchmarkNormal(self): def testPrepareWorkers(self): # to export /opt/intel mock_nfs_opt_intel = self.enter_context( - mock.patch.object(intelmpi, 'NfsExportIntelDirectory')) + mock.patch.object(intelmpi, 'NfsExportIntelDirectory') + ) # to export /usr/.../osu-microbenchmarks mock_nfs_osu = self.enter_context( - mock.patch.object(nfs_service, 'NfsExportAndMount')) + mock.patch.object(nfs_service, 'NfsExportAndMount') + ) mpi_dir = '/usr/local/libexec/osu-micro-benchmarks/mpi' vm = mock.Mock(internal_ip='10.0.0.1') vm.RemoteCommand.side_effect = [(f'{mpi_dir}/startup/osu_hello', '')] @@ -169,12 +209,14 @@ def testPrepareWorkers(self): vm.RemoteCommand.assert_called_with(f'ls {mpi_dir}/*/osu_hello') vm.RobustRemoteCommand.assert_called_with( '. mpivars.sh; mpirun -perhost 1 -n 2 ' - f'-hosts 10.0.0.1,10.0.0.2 {mpi_dir}/startup/osu_hello') + f'-hosts 10.0.0.1,10.0.0.2 {mpi_dir}/startup/osu_hello' + ) @flagsaver.flagsaver( omb_iterations=10, omb_mpi_env=['IMPI_DEBUG=5'], - omb_mpi_genv=['I_MPI_PIN_PROCESSOR_LIST=0', 'I_MPI_PIN=1']) + omb_mpi_genv=['I_MPI_PIN_PROCESSOR_LIST=0', 'I_MPI_PIN=1'], + ) def testRunResult(self): test_output = inspect.cleandoc(""" [0] MPI startup(): Rank Pid Node name Pin cpu @@ -195,19 +237,14 @@ def testRunResult(self): expected_result = omb.RunResult( name='mbw_mr', - metadata={ - 'pairs': '15', - 'window_size': '64' - }, - data=[{ - 'size': 1, - 'bandwidth': 6.39, - 'messages_per_second': 6385003.8 - }], - full_cmd=('. mpivars.sh; IMPI_DEBUG=5 mpirun ' - '-genv I_MPI_PIN=1 ' - '-genv I_MPI_PIN_PROCESSOR_LIST=0 -perhost 1 -n 2 ' - f'-hosts 10.0.0.1,10.0.0.2 {mpitest_path} --iterations 10'), + metadata={'pairs': '15', 'window_size': '64'}, + data=[{'size': 1, 'bandwidth': 6.39, 'messages_per_second': 6385003.8}], + full_cmd=( + '. mpivars.sh; IMPI_DEBUG=5 mpirun ' + '-genv I_MPI_PIN=1 ' + '-genv I_MPI_PIN_PROCESSOR_LIST=0 -perhost 1 -n 2 ' + f'-hosts 10.0.0.1,10.0.0.2 {mpitest_path} --iterations 10' + ), units='MB/s', params={'--iterations': 10}, mpi_vendor='intel', @@ -220,8 +257,9 @@ def testRunResult(self): mpi_env={ 'I_MPI_PIN_PROCESSOR_LIST': '0', 'I_MPI_PIN': '1', - 'IMPI_DEBUG': '5' - }) + 'IMPI_DEBUG': '5', + }, + ) self.assertEqual(expected_result, results[0]) self.assertLen(results, 2) # Called twice, the second time with 4*2=8 processes @@ -244,7 +282,8 @@ def setUp(self): super().setUp() self.enter_context(flagsaver.flagsaver(mpi_vendor='openmpi')) self.enter_context( - mock.patch.object(openmpi, 'GetMpiVersion', return_value='3.1.2')) + mock.patch.object(openmpi, 'GetMpiVersion', return_value='3.1.2') + ) self.enter_context(mock.patch.object(omb.time, 'time', return_value=0)) def testRunBenchmarkNormal(self): @@ -253,10 +292,12 @@ def testRunBenchmarkNormal(self): benchmark_path = f'path/to/osu_{benchmark}' textoutput = 'textoutput' ls_cmd = f'ls {omb._RUN_DIR}/*/osu_{benchmark}' - expected_full_cmd = ('mpirun -report-bindings -display-map ' - '-n 2 -npernode 1 --use-hwthread-cpus ' - '-host 10.0.0.1:slots=2,10.0.0.2:slots=2 ' - f'{benchmark_path} -t 1 --full') + expected_full_cmd = ( + 'mpirun -report-bindings -display-map ' + '-n 2 -npernode 1 --use-hwthread-cpus ' + '-host 10.0.0.1:slots=2,10.0.0.2:slots=2 ' + f'{benchmark_path} -t 1 --full' + ) vm = mock.Mock(internal_ip='10.0.0.1') vm.RemoteCommand.side_effect = [(benchmark_path, '')] vm.RobustRemoteCommand.side_effect = [(textoutput, '')] @@ -267,7 +308,8 @@ def testRunBenchmarkNormal(self): number_processes=2, hosts=[vm, mock.Mock(internal_ip='10.0.0.2')], options={'-t': 1}, - perhost=1) + perhost=1, + ) self.assertEqual(textoutput, txt) self.assertEqual(expected_full_cmd, full_cmd) @@ -277,7 +319,8 @@ def testRunBenchmarkNormal(self): def testPrepareWorkers(self): # to export /usr/.../osu-microbenchmarks mock_nfs_osu = self.enter_context( - mock.patch.object(nfs_service, 'NfsExportAndMount')) + mock.patch.object(nfs_service, 'NfsExportAndMount') + ) mpi_dir = '/usr/local/libexec/osu-micro-benchmarks/mpi' vm = mock.Mock(internal_ip='10.0.0.1') vm.RemoteCommand.side_effect = [(f'{mpi_dir}/startup/osu_hello', '')] @@ -292,13 +335,15 @@ def testPrepareWorkers(self): vm.RobustRemoteCommand.assert_called_with( 'mpirun -report-bindings -display-map -n 2 -npernode 1 ' '--use-hwthread-cpus -host 10.0.0.1:slots=2,10.0.0.2:slots=2 ' - f'{mpi_dir}/startup/osu_hello') + f'{mpi_dir}/startup/osu_hello' + ) @flagsaver.flagsaver( omb_iterations=10, # For OpenMPI, env and genv are treated the same. omb_mpi_env=['OMPI_MCA_btl=self,tcp'], - omb_mpi_genv=['OMPI_MCA_hwloc_base_binding_policy=core']) + omb_mpi_genv=['OMPI_MCA_hwloc_base_binding_policy=core'], + ) def testRunResult(self): test_output = inspect.cleandoc(""" [0] MPI startup(): Rank Pid Node name Pin cpu @@ -319,22 +364,17 @@ def testRunResult(self): expected_result = omb.RunResult( name='mbw_mr', - metadata={ - 'pairs': '15', - 'window_size': '64' - }, - data=[{ - 'size': 1, - 'bandwidth': 6.39, - 'messages_per_second': 6385003.8 - }], - full_cmd=('OMPI_MCA_btl=self,tcp ' - 'OMPI_MCA_hwloc_base_binding_policy=core ' - 'mpirun -x OMPI_MCA_btl ' - '-x OMPI_MCA_hwloc_base_binding_policy ' - '-report-bindings -display-map -n 2 -npernode 1 ' - '--use-hwthread-cpus -host 10.0.0.1:slots=2,10.0.0.2:slots=2 ' - f'{mpitest_path} --iterations 10'), + metadata={'pairs': '15', 'window_size': '64'}, + data=[{'size': 1, 'bandwidth': 6.39, 'messages_per_second': 6385003.8}], + full_cmd=( + 'OMPI_MCA_btl=self,tcp ' + 'OMPI_MCA_hwloc_base_binding_policy=core ' + 'mpirun -x OMPI_MCA_btl ' + '-x OMPI_MCA_hwloc_base_binding_policy ' + '-report-bindings -display-map -n 2 -npernode 1 ' + '--use-hwthread-cpus -host 10.0.0.1:slots=2,10.0.0.2:slots=2 ' + f'{mpitest_path} --iterations 10' + ), units='MB/s', params={'--iterations': 10}, mpi_vendor='openmpi', @@ -347,7 +387,8 @@ def testRunResult(self): mpi_env={ 'OMPI_MCA_btl': 'self,tcp', 'OMPI_MCA_hwloc_base_binding_policy': 'core', - }) + }, + ) self.assertEqual(expected_result, results[0]) self.assertLen(results, 2) # Called twice, the second time with 4*2=8 processes diff --git a/tests/pkb_common_test_case.py b/tests/pkb_common_test_case.py index b5bf7fb9f0..6c320f21b3 100644 --- a/tests/pkb_common_test_case.py +++ b/tests/pkb_common_test_case.py @@ -78,6 +78,7 @@ def CreateTestVmSpec() -> TestVmSpec: class TestOsMixin(virtual_machine.BaseOsMixin): """Test class that provides dummy implementations of abstract functions.""" + OS_TYPE = 'test_os_type' BASE_OS_TYPE = 'debian' @@ -161,8 +162,10 @@ def _Delete(self): class TestVirtualMachine( - TestResource, TestOsMixin, virtual_machine.BaseVirtualMachine): + TestResource, TestOsMixin, virtual_machine.BaseVirtualMachine +): """Test class that has dummy methods for a base virtual machine.""" + CLOUD = 'test_vm_cloud' def _Start(self): @@ -180,8 +183,9 @@ def _Resume(self): # Need to provide implementations for all of the abstract methods in # order to instantiate linux_virtual_machine.BaseLinuxMixin. -class TestLinuxVirtualMachine(linux_virtual_machine.BaseLinuxVirtualMachine, - TestVirtualMachine): +class TestLinuxVirtualMachine( + linux_virtual_machine.BaseLinuxVirtualMachine, TestVirtualMachine +): def InstallPackages(self, packages): pass @@ -207,19 +211,23 @@ def _PreemptibleMetadataKeyValue(self): def CreateBenchmarkSpecFromYaml( - yaml_string: str = SIMPLE_CONFIG, - benchmark_name: str = 'cluster_boot') -> benchmark_spec.BenchmarkSpec: + yaml_string: str = SIMPLE_CONFIG, benchmark_name: str = 'cluster_boot' +) -> benchmark_spec.BenchmarkSpec: config = configs.LoadConfig(yaml_string, {}, benchmark_name) return CreateBenchmarkSpecFromConfigDict(config, benchmark_name) def CreateBenchmarkSpecFromConfigDict( - config_dict: Dict[str, Any], - benchmark_name: str) -> benchmark_spec.BenchmarkSpec: + config_dict: Dict[str, Any], benchmark_name: str +) -> benchmark_spec.BenchmarkSpec: config_spec = benchmark_config_spec.BenchmarkConfigSpec( - benchmark_name, flag_values=FLAGS, **config_dict) - benchmark_module = next((b for b in linux_benchmarks.BENCHMARKS - if b.BENCHMARK_NAME == benchmark_name)) + benchmark_name, flag_values=FLAGS, **config_dict + ) + benchmark_module = next(( + b + for b in linux_benchmarks.BENCHMARKS + if b.BENCHMARK_NAME == benchmark_name + )) return benchmark_spec.BenchmarkSpec(benchmark_module, config_spec, 'name0') @@ -245,8 +253,8 @@ def setUp(self): self.addCleanup(context.SetThreadBenchmarkSpec, None) p = mock.patch( - util.__name__ + '.GetDefaultProject', - return_value='test_project') + util.__name__ + '.GetDefaultProject', return_value='test_project' + ) self.enter_context(p) # TODO(user): Extend MockIssueCommand to support multiple calls to @@ -264,8 +272,7 @@ def MockIssueCommand(self, stdout: str, stderr: str, retcode: int) -> None: retcode: Int. Return code from running the command. """ - p = mock.patch( - 'subprocess.Popen', spec=subprocess.Popen) + p = mock.patch('subprocess.Popen', spec=subprocess.Popen) cmd_output = mock.patch.object(vm_util, '_ReadIssueCommandOutput') self.addCleanup(p.stop) diff --git a/tests/pkb_test.py b/tests/pkb_test.py index b56b89ddd4..31a564f114 100644 --- a/tests/pkb_test.py +++ b/tests/pkb_test.py @@ -58,12 +58,16 @@ def setUp(self): self.cleanup_mock = self.PatchPkbFunction('DoCleanupPhase') self.teardown_mock = self.PatchPkbFunction('DoTeardownPhase') self.publish_failed_run_sample_mock = self.PatchPkbFunction( - 'PublishFailedRunSample') + 'PublishFailedRunSample' + ) self.flags_mock.skip_pending_runs_file = None self.flags_mock.run_stage = [ - stages.PROVISION, stages.PREPARE, stages.RUN, stages.CLEANUP, - stages.TEARDOWN + stages.PROVISION, + stages.PREPARE, + stages.RUN, + stages.CLEANUP, + stages.TEARDOWN, ] self.spec = mock.MagicMock() @@ -76,7 +80,8 @@ def testCreateProvisionFailedSample(self): self.assertRaises(Exception, pkb.RunBenchmark, self.spec, self.collector) self.publish_failed_run_sample_mock.assert_called_once_with( - self.spec, error_msg, stages.PROVISION, self.collector) + self.spec, error_msg, stages.PROVISION, self.collector + ) def testCreatePrepareFailedSample(self): self.flags_mock.create_failed_run_samples = True @@ -85,7 +90,8 @@ def testCreatePrepareFailedSample(self): self.assertRaises(Exception, pkb.RunBenchmark, self.spec, self.collector) self.publish_failed_run_sample_mock.assert_called_once_with( - self.spec, error_msg, stages.PREPARE, self.collector) + self.spec, error_msg, stages.PREPARE, self.collector + ) def testCreateRunFailedSample(self): self.flags_mock.create_failed_run_samples = True @@ -94,7 +100,8 @@ def testCreateRunFailedSample(self): self.assertRaises(Exception, pkb.RunBenchmark, self.spec, self.collector) self.publish_failed_run_sample_mock.assert_called_once_with( - self.spec, error_msg, stages.RUN, self.collector) + self.spec, error_msg, stages.RUN, self.collector + ) def testCreateCleanupFailedSample(self): self.flags_mock.create_failed_run_samples = True @@ -103,7 +110,8 @@ def testCreateCleanupFailedSample(self): self.assertRaises(Exception, pkb.RunBenchmark, self.spec, self.collector) self.publish_failed_run_sample_mock.assert_called_once_with( - self.spec, error_msg, stages.CLEANUP, self.collector) + self.spec, error_msg, stages.CLEANUP, self.collector + ) def testCreateTeardownFailedSample(self): self.flags_mock.create_failed_run_samples = True @@ -112,7 +120,8 @@ def testCreateTeardownFailedSample(self): self.assertRaises(Exception, pkb.RunBenchmark, self.spec, self.collector) self.publish_failed_run_sample_mock.assert_called_once_with( - self.spec, error_msg, stages.TEARDOWN, self.collector) + self.spec, error_msg, stages.TEARDOWN, self.collector + ) def testDontCreateFailedRunSample(self): self.flags_mock.create_failed_run_samples = False @@ -127,16 +136,21 @@ class TestPublishStageStartedSamples(unittest.TestCase): def setUp(self): super().setUp() self.addCleanup(mock.patch.stopall) - self.collector = mock.patch.object( - publisher, 'SampleCollector', autospec=True).start().return_value + self.collector = ( + mock.patch.object(publisher, 'SampleCollector', autospec=True) + .start() + .return_value + ) self.benchmark_spec = mock.create_autospec( - 'perfkitbenchmarker.benchmark_spec.BenchmarkSpec', instance=True) + 'perfkitbenchmarker.benchmark_spec.BenchmarkSpec', instance=True + ) self.benchmark_spec.name = 'fake_benchmark' @mock.patch('time.time', return_value=123.45) def testProvisionStageDefault(self, _): pkb._PublishStageStartedSamples( - stages.PROVISION, benchmark_spec=self.benchmark_spec) + stages.PROVISION, benchmark_spec=self.benchmark_spec + ) self.collector.PublishSamples.assert_not_called() @flagsaver.flagsaver(create_started_run_sample=True) @@ -144,14 +158,19 @@ def testProvisionStageDefault(self, _): @mock.patch('time.time', return_value=123.45) def testProvisionStageFlags(self, _): pkb._PublishStageStartedSamples( - stages.PROVISION, benchmark_spec=self.benchmark_spec) + stages.PROVISION, benchmark_spec=self.benchmark_spec + ) self.collector.AddSamples.assert_has_calls([ mock.call( [sample.Sample('Run Started', 123.45, 'seconds', {'flags': '{}'})], - 'fake_benchmark', self.benchmark_spec), + 'fake_benchmark', + self.benchmark_spec, + ), mock.call( [sample.Sample('Provision Stage Started', 123.45, 'seconds', {})], - 'fake_benchmark', self.benchmark_spec) + 'fake_benchmark', + self.benchmark_spec, + ), ]) self.collector.PublishSamples.assert_called() @@ -166,13 +185,19 @@ def testPublishFailedRunSample(self, sample_mock, _): spec.vms = [] spec.failed_substatus = None pkb.PublishFailedRunSample( - spec, error_msg, stages.PROVISION, collector=mock.Mock()) - - sample_mock.assert_called_once_with('Run Failed', 123.45, 'seconds', { - 'error_message': error_msg, - 'run_stage': stages.PROVISION, - 'flags': '{}' - }) + spec, error_msg, stages.PROVISION, collector=mock.Mock() + ) + + sample_mock.assert_called_once_with( + 'Run Failed', + 123.45, + 'seconds', + { + 'error_message': error_msg, + 'run_stage': stages.PROVISION, + 'flags': '{}', + }, + ) @mock.patch('time.time', return_value=123.45) @mock.patch('perfkitbenchmarker.sample.Sample') @@ -184,19 +209,25 @@ def testPublishFailedRunSampleWithTruncation(self, sample_mock, _): pkb.FLAGS.failed_run_samples_error_length = 7 pkb.PublishFailedRunSample( - spec, error_msg, stages.PROVISION, collector=mock.Mock()) - - sample_mock.assert_called_once_with('Run Failed', 123.45, 'seconds', { - 'error_message': 'This is', - 'run_stage': stages.PROVISION, - 'flags': '{}', - 'failed_substatus': 'QuotaExceeded' - }) - - -class TestMiscFunctions(pkb_common_test_case.PkbCommonTestCase, - test_util.SamplesTestMixin): - + spec, error_msg, stages.PROVISION, collector=mock.Mock() + ) + + sample_mock.assert_called_once_with( + 'Run Failed', + 123.45, + 'seconds', + { + 'error_message': 'This is', + 'run_stage': stages.PROVISION, + 'flags': '{}', + 'failed_substatus': 'QuotaExceeded', + }, + ) + + +class TestMiscFunctions( + pkb_common_test_case.PkbCommonTestCase, test_util.SamplesTestMixin +): """Testing for various functions in pkb.py.""" def _MockVm( @@ -210,8 +241,8 @@ def _MockVm( return vm def _MockVmWithVuln( - self, name: str, - cpu_vuln: linux_virtual_machine.CpuVulnerabilities) -> mock.Mock: + self, name: str, cpu_vuln: linux_virtual_machine.CpuVulnerabilities + ) -> mock.Mock: vm = mock.Mock(OS_TYPE='debian9') vm.name = name type(vm).cpu_vulnerabilities = mock.PropertyMock(return_value=cpu_vuln) @@ -328,14 +359,14 @@ def test_IsException_subclass(self): self.assertTrue(pkb._IsException(e, errors.Resource.CreationInternalError)) def test_IsException_substring(self): - def RaiseCreationInternalError(unused): raise errors.Resource.CreationInternalError('internal error') with self.assertRaises(errors.VmUtil.ThreadException) as cm: background_tasks.RunThreaded(RaiseCreationInternalError, [None]) self.assertTrue( - pkb._IsException(cm.exception, errors.Resource.CreationInternalError)) + pkb._IsException(cm.exception, errors.Resource.CreationInternalError) + ) def test_IsException_false(self): e = errors.Resource.CreationInternalError('internal error') @@ -350,7 +381,9 @@ def _MockLoadProviderUtils(self, utils_module): providers, 'LoadProviderUtils', autospec=True, - return_value=utils_module)) + return_value=utils_module, + ) + ) def _MockGcpUtils(self, function_name, return_value=None, side_effect=None): return self.enter_context( @@ -359,7 +392,9 @@ def _MockGcpUtils(self, function_name, return_value=None, side_effect=None): function_name, autospec=True, return_value=return_value, - side_effect=side_effect)) + side_effect=side_effect, + ) + ) @flagsaver.flagsaver(retries=3) def testRunRetries(self): @@ -369,7 +404,9 @@ def testRunRetries(self): mock.patch.object( pkb, 'DoProvisionPhase', - side_effect=errors.Benchmarks.QuotaFailure())) + side_effect=errors.Benchmarks.QuotaFailure(), + ) + ) benchmark_specs, _ = pkb.RunBenchmarkTask(spec=test_spec) @@ -398,20 +435,20 @@ def testRunRetries(self): 'expected_retry': False, }, { - 'testcase_name': - 'FailedSubstatusIncluded', - 'status': - benchmark_status.FAILED, - 'failed_substatus': - benchmark_status.FailedSubstatus.INSUFFICIENT_CAPACITY, - 'retry_substatuses': - [benchmark_status.FailedSubstatus.INSUFFICIENT_CAPACITY], - 'expected_retry': - True, + 'testcase_name': 'FailedSubstatusIncluded', + 'status': benchmark_status.FAILED, + 'failed_substatus': ( + benchmark_status.FailedSubstatus.INSUFFICIENT_CAPACITY + ), + 'retry_substatuses': [ + benchmark_status.FailedSubstatus.INSUFFICIENT_CAPACITY + ], + 'expected_retry': True, }, ) - def testShouldRetry(self, status, failed_substatus, retry_substatuses, - expected_retry): + def testShouldRetry( + self, status, failed_substatus, retry_substatuses, expected_retry + ): test_spec = pkb_common_test_case.CreateBenchmarkSpecFromYaml() test_spec.status = status test_spec.failed_substatus = failed_substatus @@ -432,7 +469,8 @@ def testShouldRetry(self, status, failed_substatus, retry_substatuses, 'expected_run_count': 3, 'expected_quota_retry_calls': 1, 'expected_capacity_retry_calls': 1, - }, { + }, + { 'testcase_name': 'Default', 'quota_flag_value': False, 'capacity_flag_value': False, @@ -445,12 +483,19 @@ def testShouldRetry(self, status, failed_substatus, retry_substatuses, 'expected_run_count': 3, 'expected_quota_retry_calls': 0, 'expected_capacity_retry_calls': 0, - }) + }, + ) @flagsaver.flagsaver - def testRunBenchmarkTask(self, quota_flag_value, capacity_flag_value, - retry_count, run_results, expected_run_count, - expected_quota_retry_calls, - expected_capacity_retry_calls): + def testRunBenchmarkTask( + self, + quota_flag_value, + capacity_flag_value, + retry_count, + run_results, + expected_run_count, + expected_quota_retry_calls, + expected_capacity_retry_calls, + ): FLAGS.zones = ['test_zone'] FLAGS.cloud = 'GCP' FLAGS.retries = retry_count @@ -459,12 +504,15 @@ def testRunBenchmarkTask(self, quota_flag_value, capacity_flag_value, test_spec = pkb_common_test_case.CreateBenchmarkSpecFromYaml() # Generate some exceptions for each run. self.enter_context( - mock.patch.object(pkb, 'DoProvisionPhase', side_effect=run_results)) + mock.patch.object(pkb, 'DoProvisionPhase', side_effect=run_results) + ) # Mock the retry options. mock_quota_retry = self.enter_context( - mock.patch.object(pkb.ZoneRetryManager, '_AssignZoneToNewRegion')) + mock.patch.object(pkb.ZoneRetryManager, '_AssignZoneToNewRegion') + ) mock_capacity_retry = self.enter_context( - mock.patch.object(pkb.ZoneRetryManager, '_AssignZoneToSameRegion')) + mock.patch.object(pkb.ZoneRetryManager, '_AssignZoneToSameRegion') + ) self._MockGcpUtils('GetZonesFromMachineType', return_value=set()) benchmark_specs, _ = pkb.RunBenchmarkTask(spec=test_spec) @@ -472,8 +520,9 @@ def testRunBenchmarkTask(self, quota_flag_value, capacity_flag_value, self.assertEqual(len(benchmark_specs), expected_run_count) # Retry preparation functions should have the right calls. self.assertEqual(mock_quota_retry.call_count, expected_quota_retry_calls) - self.assertEqual(mock_capacity_retry.call_count, - expected_capacity_retry_calls) + self.assertEqual( + mock_capacity_retry.call_count, expected_capacity_retry_calls + ) @parameterized.named_parameters( ('1', False, False, ['test_zone_1'], ['test_zone_2'], 1, True), @@ -484,9 +533,15 @@ def testRunBenchmarkTask(self, quota_flag_value, capacity_flag_value, ('6', True, False, ['test_zone_1', 'test_zone_2'], [], 1, False), ('7', True, True, ['test_zone_2'], [], 0, False), ) - def testValidateSmartZoneRetryFlags(self, smart_quota_retry, - smart_capacity_retry, zone, zones, - retries, is_valid): + def testValidateSmartZoneRetryFlags( + self, + smart_quota_retry, + smart_capacity_retry, + zone, + zones, + retries, + is_valid, + ): flags_dict = { 'retries': retries, 'smart_quota_retry': smart_quota_retry, @@ -518,7 +573,8 @@ def testSmartQuotaRetry(self): self._MockGcpUtils('GetRegionsInGeo', return_value={'region_1', 'region_2'}) # Expect that region_1 is skipped when getting zones. mock_get_zones = self._MockGcpUtils( - 'GetZonesInRegion', return_value={'zone_2'}) + 'GetZonesInRegion', return_value={'zone_2'} + ) test_retry_manager = pkb.ZoneRetryManager(test_machine_type) test_retry_manager.HandleSmartRetries(test_spec) @@ -534,13 +590,15 @@ def testSmartQuotaRetry(self): def testSmartCapacityRetry(self): test_spec = pkb_common_test_case.CreateBenchmarkSpecFromYaml() test_spec.failed_substatus = ( - benchmark_status.FailedSubstatus.INSUFFICIENT_CAPACITY) + benchmark_status.FailedSubstatus.INSUFFICIENT_CAPACITY + ) self._MockLoadProviderUtils(gcp_utils) self._MockGcpUtils('GetRegionFromZone') # Expect that the correct possible zones are passed to the function below. self._MockGcpUtils('GetZonesInRegion', return_value={'zone_1', 'zone_2'}) self._MockGcpUtils( - 'GetZonesFromMachineType', return_value={'zone_1', 'zone_2'}) + 'GetZonesFromMachineType', return_value={'zone_1', 'zone_2'} + ) test_retry_manager = pkb.ZoneRetryManager(test_machine_type) test_retry_manager.HandleSmartRetries(test_spec) @@ -576,15 +634,16 @@ class FreezeRestoreTest(pkb_common_test_case.PkbCommonTestCase): def testSpecPickledEvenWithException(self): # Make provision have an exception. self.enter_context( - mock.patch.object(pkb, 'DoProvisionPhase', side_effect=Exception())) + mock.patch.object(pkb, 'DoProvisionPhase', side_effect=Exception()) + ) # Make benchmark_spec avoid pickling for test. - test_benchmark_spec = ( - pkb_common_test_case.CreateBenchmarkSpecFromYaml( - pkb_common_test_case.SIMPLE_CONFIG, 'cluster_boot')) - self.enter_context( - mock.patch.object(test_benchmark_spec, 'Pickle')) + test_benchmark_spec = pkb_common_test_case.CreateBenchmarkSpecFromYaml( + pkb_common_test_case.SIMPLE_CONFIG, 'cluster_boot' + ) + self.enter_context(mock.patch.object(test_benchmark_spec, 'Pickle')) mock_freeze = self.enter_context( - mock.patch.object(test_benchmark_spec, 'Freeze')) + mock.patch.object(test_benchmark_spec, 'Freeze') + ) collector = publisher.SampleCollector() # Run the benchmark loop. @@ -612,9 +671,11 @@ def testCompletionStatusesContainFreezeRestoreStatus(self): 'status': benchmark_status.SUCCEEDED, 'flags': { 'freeze': 'mock_freeze_path', - 'restore': 'mock_restore_path' - } - }) + '\n') + 'restore': 'mock_restore_path', + }, + }) + + '\n' + ) def testRestoreRelationalDb(self): test_bm_spec = pkb_common_test_case.CreateBenchmarkSpecFromYaml() @@ -635,5 +696,6 @@ def testFreezeRelationalDb(self): test_bm_spec.relational_db.Delete.assert_called_with(freeze=True) + if __name__ == '__main__': unittest.main() diff --git a/tests/plot_sysbench_results_test.py b/tests/plot_sysbench_results_test.py index 9b6bf33746..e4d77362fc 100644 --- a/tests/plot_sysbench_results_test.py +++ b/tests/plot_sysbench_results_test.py @@ -11,8 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Tests for perfkitbenchmarker.scripts.database_scripts.plot_sysbench_results. -""" +"""Tests for perfkitbenchmarker.scripts.database_scripts.plot_sysbench_results.""" import os import unittest from perfkitbenchmarker.scripts.database_scripts import plot_sysbench_results @@ -28,16 +27,19 @@ class PlotterTestCase(unittest.TestCase): def setUp(self): self.plotter = plot_sysbench_results.Plotter( - TEST_FILE_1_RUN_SECONDS, TEST_FILE_1_REPORT_INTERVAL, TEST_RUN_URI) + TEST_FILE_1_RUN_SECONDS, TEST_FILE_1_REPORT_INTERVAL, TEST_RUN_URI + ) def testadd_file(self): - self.assertRaises(plot_sysbench_results.STDERRFileDoesNotExistError, - self.plotter.add_file, '') + self.assertRaises( + plot_sysbench_results.STDERRFileDoesNotExistError, + self.plotter.add_file, + '', + ) def testparse_file(self): # TODO(user): Implement test that will raise PatternNotFoundError. - path1 = os.path.join( - os.path.dirname(__file__), TEST_FILE_1) + path1 = os.path.join(os.path.dirname(__file__), TEST_FILE_1) with open(path1) as file1: results = self.plotter._parse_file(file1) self.assertEqual(len(results), self.plotter.data_entries_per_file) diff --git a/tests/provider_benchmark_test.py b/tests/provider_benchmark_test.py index 02bf310ed2..ab93e856cf 100644 --- a/tests/provider_benchmark_test.py +++ b/tests/provider_benchmark_test.py @@ -18,7 +18,6 @@ import unittest import mock - from perfkitbenchmarker import provider_info from perfkitbenchmarker import providers import six @@ -36,9 +35,16 @@ def _VerifyProviderBenchmarkSupport(self, cloud, benchmark, support_expected): provider_info_class = provider_info.GetProviderInfoClass(cloud) supported = provider_info_class.IsBenchmarkSupported(benchmark) fmt_args = ('', ' not') if support_expected else (' not', '') - self.assertEqual(supported, support_expected, ( - 'Expected provider {provider} {0}to support benchmark {benchmark}, but ' - 'it did{1}.'.format(*fmt_args, provider=cloud, benchmark=benchmark))) + self.assertEqual( + supported, + support_expected, + ( + 'Expected provider {provider} {0}to support benchmark {benchmark},' + ' but it did{1}.'.format( + *fmt_args, provider=cloud, benchmark=benchmark + ) + ), + ) def testIperfSupport(self): expected = {provider_info.GCP: True, provider_info.DIGITALOCEAN: True} @@ -48,8 +54,9 @@ def testIperfSupport(self): def testMYSQLSupport(self): expected = {provider_info.GCP: True, provider_info.DIGITALOCEAN: False} for cloud, support_expected in six.iteritems(expected): - self._VerifyProviderBenchmarkSupport(cloud, 'mysql_service', - support_expected) + self._VerifyProviderBenchmarkSupport( + cloud, 'mysql_service', support_expected + ) if __name__ == '__main__': diff --git a/tests/providers/aws/aws_capacity_reservation_test.py b/tests/providers/aws/aws_capacity_reservation_test.py index 0abd653efe..3e36df06b2 100644 --- a/tests/providers/aws/aws_capacity_reservation_test.py +++ b/tests/providers/aws/aws_capacity_reservation_test.py @@ -1,13 +1,11 @@ """Tests for perfkitbenchmarker.tests.providers.aws.aws_capacity_reservation.""" - import datetime import unittest from absl import flags import freezegun import mock - from perfkitbenchmarker import vm_util from perfkitbenchmarker.providers.aws import aws_capacity_reservation from perfkitbenchmarker.providers.aws import util @@ -63,29 +61,34 @@ def setUp(self): self._create_patch( util.__name__ + '.GetZonesInRegion', - return_val=['us-west-1a', 'us-west-1b']) + return_val=['us-west-1a', 'us-west-1b'], + ) @freezegun.freeze_time(FAKE_DATETIME_NOW) def test_create(self): vm_group = [FakeAwsVirtualMachine()] capacity_reservation = aws_capacity_reservation.AwsCapacityReservation( - vm_group) + vm_group + ) with mock.patch( vm_util.__name__ + '.IssueCommand', - return_value=(CREATE_STDOUT_SUCCESSFUL, '', 0)) as issue_command: + return_value=(CREATE_STDOUT_SUCCESSFUL, '', 0), + ) as issue_command: capacity_reservation._Create() command_string = ' '.join(issue_command.call_args[0][0]) expected_end_date = FAKE_DATETIME_NOW + datetime.timedelta( - minutes=FLAGS.timeout_minutes) + minutes=FLAGS.timeout_minutes + ) expected_command = ( 'aws --output json ec2 create-capacity-reservation ' '--instance-type=fake_machine_type ' '--instance-platform=Linux/UNIX --availability-zone=us-west-1a ' '--instance-count=1 --instance-match-criteria=targeted ' - '--region=us-west-1 --end-date-type=limited --end-date=%s' % - expected_end_date.isoformat()) + '--region=us-west-1 --end-date-type=limited --end-date=%s' + % expected_end_date.isoformat() + ) self.assertEqual(issue_command.call_count, 1) self.assertIn(expected_command, command_string) @@ -93,15 +96,18 @@ def test_create(self): def test_delete(self): vm_group = [FakeAwsVirtualMachine()] capacity_reservation = aws_capacity_reservation.AwsCapacityReservation( - vm_group) + vm_group + ) capacity_reservation.capacity_reservation_id = 'foo' with mock.patch(vm_util.__name__ + '.IssueCommand') as issue_command: capacity_reservation._Delete() command_string = ' '.join(issue_command.call_args[0][0]) - expected_command = ('aws --output json ec2 cancel-capacity-reservation ' - '--capacity-reservation-id=foo --region=us-west-1') + expected_command = ( + 'aws --output json ec2 cancel-capacity-reservation ' + '--capacity-reservation-id=foo --region=us-west-1' + ) self.assertEqual(issue_command.call_count, 1) self.assertIn(expected_command, command_string) @@ -112,7 +118,8 @@ def test_update_vms_in_group(self): vm_group = [vm_1, vm_2, vm_3] capacity_reservation = aws_capacity_reservation.AwsCapacityReservation( - vm_group) + vm_group + ) capacity_reservation.capacity_reservation_id = 'foo' capacity_reservation._UpdateVmsInGroup('foo', 'us-west-1z') diff --git a/tests/providers/aws/aws_cluster_parameter_group_test.py b/tests/providers/aws/aws_cluster_parameter_group_test.py index 486136e2a3..16b3669317 100644 --- a/tests/providers/aws/aws_cluster_parameter_group_test.py +++ b/tests/providers/aws/aws_cluster_parameter_group_test.py @@ -16,7 +16,6 @@ import unittest from absl import flags import mock - from perfkitbenchmarker import vm_util from perfkitbenchmarker.providers.aws import aws_cluster_parameter_group from perfkitbenchmarker.providers.aws import util @@ -30,7 +29,8 @@ class RedshiftClusterParameterGroupTestCase( - pkb_common_test_case.PkbCommonTestCase): + pkb_common_test_case.PkbCommonTestCase +): def setUp(self): super(RedshiftClusterParameterGroupTestCase, self).setUp() @@ -39,28 +39,37 @@ def setUp(self): def testValidClusterParameterGroupCreation(self): cpg = aws_cluster_parameter_group.RedshiftClusterParameterGroup( - list(util.AWS_PREFIX)) + list(util.AWS_PREFIX) + ) self.assertEqual(cpg.name, 'pkb-%s' % TEST_RUN_URI) with mock.patch( - vm_util.__name__ + '.IssueCommand', - return_value=('out_', 'err_', 0)) as mock_issue: + vm_util.__name__ + '.IssueCommand', return_value=('out_', 'err_', 0) + ) as mock_issue: cpg._Create() self.assertEqual(mock_issue.call_count, 1) def testValidClusterParameterGroupDeletion(self): cpg = aws_cluster_parameter_group.RedshiftClusterParameterGroup( - list(util.AWS_PREFIX)) + list(util.AWS_PREFIX) + ) self.assertEqual(cpg.name, 'pkb-%s' % TEST_RUN_URI) with mock.patch( - vm_util.__name__ + '.IssueCommand', - return_value=('out_', 'err_', 0)) as mock_issue: + vm_util.__name__ + '.IssueCommand', return_value=('out_', 'err_', 0) + ) as mock_issue: cpg._Delete() mock_issue.assert_called_once() - mock_issue.assert_called_with([ - 'aws', '--output', 'json', 'redshift', - 'delete-cluster-parameter-group', '--parameter-group-name', - 'pkb-%s' % TEST_RUN_URI - ], raise_on_failure=False) + mock_issue.assert_called_with( + [ + 'aws', + '--output', + 'json', + 'redshift', + 'delete-cluster-parameter-group', + '--parameter-group-name', + 'pkb-%s' % TEST_RUN_URI, + ], + raise_on_failure=False, + ) if __name__ == '__main__': diff --git a/tests/providers/aws/aws_cluster_subnet_group_test.py b/tests/providers/aws/aws_cluster_subnet_group_test.py index 2f437b539a..043e83cd44 100644 --- a/tests/providers/aws/aws_cluster_subnet_group_test.py +++ b/tests/providers/aws/aws_cluster_subnet_group_test.py @@ -16,7 +16,6 @@ import unittest from absl import flags import mock - from perfkitbenchmarker import vm_util from perfkitbenchmarker.providers.aws import aws_cluster_subnet_group from perfkitbenchmarker.providers.aws import util @@ -30,7 +29,8 @@ class RedshiftClusterSubnetGroupTestCase( - pkb_common_test_case.PkbCommonTestCase): + pkb_common_test_case.PkbCommonTestCase +): def setUp(self): super(RedshiftClusterSubnetGroupTestCase, self).setUp() @@ -39,33 +39,51 @@ def setUp(self): def testValidClusterParameterGroupCreation(self): csg = aws_cluster_subnet_group.RedshiftClusterSubnetGroup( - list(util.AWS_PREFIX)) + list(util.AWS_PREFIX) + ) csg.subnet_id = CLUSTER_SUBNET_ID self.assertEqual(csg.name, 'pkb-%s' % TEST_RUN_URI) with mock.patch( - vm_util.__name__ + '.IssueCommand', - return_value=('out_', 'err_', 0)) as mock_issue: + vm_util.__name__ + '.IssueCommand', return_value=('out_', 'err_', 0) + ) as mock_issue: csg._Create() mock_issue.assert_called_once() mock_issue.assert_called_with([ - 'aws', '--output', 'json', 'redshift', 'create-cluster-subnet-group', - '--cluster-subnet-group-name', 'pkb-%s' % TEST_RUN_URI, - '--description', 'Cluster Subnet Group for run uri %s' % TEST_RUN_URI, - '--subnet-ids', CLUSTER_SUBNET_ID]) + 'aws', + '--output', + 'json', + 'redshift', + 'create-cluster-subnet-group', + '--cluster-subnet-group-name', + 'pkb-%s' % TEST_RUN_URI, + '--description', + 'Cluster Subnet Group for run uri %s' % TEST_RUN_URI, + '--subnet-ids', + CLUSTER_SUBNET_ID, + ]) def testValidClusterParameterGroupDeletion(self): csg = aws_cluster_subnet_group.RedshiftClusterSubnetGroup( - list(util.AWS_PREFIX)) + list(util.AWS_PREFIX) + ) self.assertEqual(csg.name, 'pkb-%s' % TEST_RUN_URI) with mock.patch( - vm_util.__name__ + '.IssueCommand', - return_value=('out_', 'err_', 0)) as mock_issue: + vm_util.__name__ + '.IssueCommand', return_value=('out_', 'err_', 0) + ) as mock_issue: csg._Delete() mock_issue.assert_called_once() mock_issue.assert_called_with( - ['aws', '--output', 'json', 'redshift', 'delete-cluster-subnet-group', - '--cluster-subnet-group-name', 'pkb-%s' % TEST_RUN_URI], - raise_on_failure=False) + [ + 'aws', + '--output', + 'json', + 'redshift', + 'delete-cluster-subnet-group', + '--cluster-subnet-group-name', + 'pkb-%s' % TEST_RUN_URI, + ], + raise_on_failure=False, + ) if __name__ == '__main__': diff --git a/tests/providers/aws/aws_disk_test.py b/tests/providers/aws/aws_disk_test.py index ba46011087..87764b2704 100644 --- a/tests/providers/aws/aws_disk_test.py +++ b/tests/providers/aws/aws_disk_test.py @@ -15,7 +15,6 @@ import unittest from absl import flags - from perfkitbenchmarker import errors from perfkitbenchmarker.providers.aws import aws_disk from tests import pkb_common_test_case @@ -39,9 +38,16 @@ def testDefaults(self): def testProvidedValid(self): spec = aws_disk.AwsDiskSpec( - _COMPONENT, device_path='test_device_path', disk_number=1, - disk_size=75, disk_type='test_disk_type', iops=1000, throughput=100, - mount_point='/mountpoint', num_striped_disks=2) + _COMPONENT, + device_path='test_device_path', + disk_number=1, + disk_size=75, + disk_type='test_disk_type', + iops=1000, + throughput=100, + mount_point='/mountpoint', + num_striped_disks=2, + ) self.assertEqual(spec.device_path, 'test_device_path') self.assertEqual(spec.disk_number, 1) self.assertEqual(spec.disk_size, 75) @@ -65,7 +71,8 @@ def testNonPresentFlagsDoNotOverrideConfigs(self): FLAGS.aws_provisioned_throughput = 200 FLAGS.data_disk_size = 100 spec = aws_disk.AwsDiskSpec( - _COMPONENT, FLAGS, disk_size=75, iops=1000, throughput=150) + _COMPONENT, FLAGS, disk_size=75, iops=1000, throughput=150 + ) self.assertEqual(spec.disk_size, 75) self.assertEqual(spec.iops, 1000) self.assertEqual(spec.throughput, 150) @@ -75,7 +82,8 @@ def testPresentFlagsOverrideConfigs(self): FLAGS['aws_provisioned_throughput'].parse(200) FLAGS['data_disk_size'].parse(100) spec = aws_disk.AwsDiskSpec( - _COMPONENT, FLAGS, disk_size=75, iops=1000, throughput=150) + _COMPONENT, FLAGS, disk_size=75, iops=1000, throughput=150 + ) self.assertEqual(spec.disk_size, 100) self.assertEqual(spec.iops, 2000) self.assertEqual(spec.throughput, 200) diff --git a/tests/providers/aws/aws_dynamodb_test.py b/tests/providers/aws/aws_dynamodb_test.py index d5f2020008..0150ccc2ca 100644 --- a/tests/providers/aws/aws_dynamodb_test.py +++ b/tests/providers/aws/aws_dynamodb_test.py @@ -78,7 +78,8 @@ def GetTestDynamoDBInstance(minimal=False): spec = _MINIMAL_TEST_BENCHMARK_SPEC if minimal else _TEST_BENCHMARK_SPEC test_benchmark_spec = pkb_common_test_case.CreateBenchmarkSpecFromYaml( - yaml_string=spec, benchmark_name='aws_dynamodb_ycsb') + yaml_string=spec, benchmark_name='aws_dynamodb_ycsb' + ) test_benchmark_spec.ConstructNonRelationalDb() return test_benchmark_spec.non_relational_db @@ -90,7 +91,9 @@ def _MockHasAutoscalingPolicies(self, test_instance, policies_exist): mock.patch.object( test_instance, '_HasAutoscalingPolicies', - return_value=policies_exist)) + return_value=policies_exist, + ) + ) def assertArgumentInCommand(self, mock_cmd, arg): """Given an AWS command, checks that the argument is present.""" @@ -135,15 +138,20 @@ def testInitKeysAndAttributes(self): self.assertEqual( test_instance._PrimaryKeyJson(), - '{"AttributeName": "test_primary_key", "KeyType": "HASH"}') - self.assertEqual(test_instance._SortKeyJson(), - '{"AttributeName": "test_sort_key", "KeyType": "RANGE"}') + '{"AttributeName": "test_primary_key", "KeyType": "HASH"}', + ) + self.assertEqual( + test_instance._SortKeyJson(), + '{"AttributeName": "test_sort_key", "KeyType": "RANGE"}', + ) self.assertEqual( test_instance._PrimaryAttrsJson(), - '{"AttributeName": "test_primary_key", "AttributeType": "S"}') + '{"AttributeName": "test_primary_key", "AttributeType": "S"}', + ) self.assertEqual( test_instance._SortAttrsJson(), - '{"AttributeName": "test_sort_key", "AttributeType": "S"}') + '{"AttributeName": "test_sort_key", "AttributeType": "S"}', + ) @flagsaver.flagsaver def testInitThroughput(self): @@ -152,8 +160,9 @@ def testInitThroughput(self): test_instance = GetTestDynamoDBInstance() - self.assertEqual(test_instance.throughput, - 'ReadCapacityUnits=1,WriteCapacityUnits=2') + self.assertEqual( + test_instance.throughput, 'ReadCapacityUnits=1,WriteCapacityUnits=2' + ) def testInitThroughputWithOnDemandRaises(self): FLAGS['aws_dynamodb_read_capacity'].parse(1) @@ -235,16 +244,15 @@ def testIsServerlessDefault(self): { 'testcase_name': 'ValidOutput', 'output': json.loads(_DESCRIBE_TABLE_OUTPUT)['Table'], - 'expected': True - }, { - 'testcase_name': 'EmptyOutput', - 'output': {}, - 'expected': False - }) + 'expected': True, + }, + {'testcase_name': 'EmptyOutput', 'output': {}, 'expected': False}, + ) def testExists(self, output, expected): test_instance = GetTestDynamoDBInstance() self.enter_context( - mock.patch.object(test_instance, '_DescribeTable', return_value=output)) + mock.patch.object(test_instance, '_DescribeTable', return_value=output) + ) actual = test_instance._Exists() @@ -256,7 +264,8 @@ def testSetThroughputNotCalled(self): cmd = self.enter_context(mock.patch.object(util, 'IssueRetryableCommand')) self.enter_context(mock.patch.object(test_instance, '_IsReady')) self.enter_context( - mock.patch.object(test_instance, '_GetThroughput', return_value=(5, 5))) + mock.patch.object(test_instance, '_GetThroughput', return_value=(5, 5)) + ) # Act test_instance.SetThroughput(5, 5) @@ -270,7 +279,8 @@ def testSetThroughput(self): cmd = self.enter_context(mock.patch.object(util, 'IssueRetryableCommand')) self.enter_context(mock.patch.object(test_instance, '_IsReady')) self.enter_context( - mock.patch.object(test_instance, '_GetThroughput', return_value=(5, 5))) + mock.patch.object(test_instance, '_GetThroughput', return_value=(5, 5)) + ) # Act test_instance.SetThroughput(10, 10) @@ -280,13 +290,15 @@ def testSetThroughput(self): self.assertArgumentInCommand(cmd, '--region us-east-1') self.assertArgumentInCommand( cmd, - '--provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=10') + '--provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=10', + ) def testGetThroughput(self): test_instance = GetTestDynamoDBInstance() output = json.loads(_DESCRIBE_TABLE_OUTPUT)['Table'] self.enter_context( - mock.patch.object(test_instance, '_DescribeTable', return_value=output)) + mock.patch.object(test_instance, '_DescribeTable', return_value=output) + ) actual_rcu, actual_wcu = test_instance._GetThroughput() @@ -297,7 +309,8 @@ def testTagResourceFailsWithNonExistentResource(self): test_instance = GetTestDynamoDBInstance() # Mark instance as non-existing. self.enter_context( - mock.patch.object(test_instance, '_Exists', return_value=False)) + mock.patch.object(test_instance, '_Exists', return_value=False) + ) with self.assertRaises(vm_util.RetriesExceededRetryError) as e: test_instance._GetTagResourceCommand(['test', 'tag']) @@ -309,7 +322,8 @@ def testUpdateWithDefaultTags(self): cmd = self.enter_context(mock.patch.object(util, 'IssueRetryableCommand')) # Mark instance as existing. self.enter_context( - mock.patch.object(test_instance, '_Exists', return_value=True)) + mock.patch.object(test_instance, '_Exists', return_value=True) + ) test_instance.UpdateWithDefaultTags() @@ -325,12 +339,15 @@ def testUpdateTimeout(self): util, 'MakeDefaultTags', autospec=True, - return_value={'timeout_utc': 60})) + return_value={'timeout_utc': 60}, + ) + ) # Mock the actual call to the CLI cmd = self.enter_context(mock.patch.object(util, 'IssueRetryableCommand')) # Mark instance as existing. self.enter_context( - mock.patch.object(test_instance, '_Exists', return_value=True)) + mock.patch.object(test_instance, '_Exists', return_value=True) + ) test_instance.UpdateTimeout(timeout_minutes=60) @@ -341,36 +358,45 @@ def testUpdateTimeout(self): 'testcase_name': 'OnlyRcu', 'rcu': 5, 'wcu': 500, - }, { + }, + { 'testcase_name': 'OnlyWcu', 'rcu': 500, 'wcu': 5, - }, { + }, + { 'testcase_name': 'Both', 'rcu': 500, 'wcu': 500, - }) + }, + ) def testFreezeLowersThroughputToFreeTier(self, rcu, wcu): test_instance = GetTestDynamoDBInstance() self._MockHasAutoscalingPolicies(test_instance, False) self.enter_context( mock.patch.object( - test_instance, '_GetThroughput', return_value=(rcu, wcu))) + test_instance, '_GetThroughput', return_value=(rcu, wcu) + ) + ) mock_set_throughput = self.enter_context( - mock.patch.object(test_instance, 'SetThroughput', autospec=True)) + mock.patch.object(test_instance, 'SetThroughput', autospec=True) + ) test_instance._Freeze() mock_set_throughput.assert_called_once_with( - rcu=aws_dynamodb._FREE_TIER_RCU, wcu=aws_dynamodb._FREE_TIER_WCU) + rcu=aws_dynamodb._FREE_TIER_RCU, wcu=aws_dynamodb._FREE_TIER_WCU + ) def testFreezeDoesNotLowerThroughputIfAlreadyAtFreeTier(self): test_instance = GetTestDynamoDBInstance() self._MockHasAutoscalingPolicies(test_instance, False) self.enter_context( - mock.patch.object(test_instance, '_GetThroughput', return_value=(5, 5))) + mock.patch.object(test_instance, '_GetThroughput', return_value=(5, 5)) + ) mock_set_throughput = self.enter_context( - mock.patch.object(test_instance, 'SetThroughput', autospec=True)) + mock.patch.object(test_instance, 'SetThroughput', autospec=True) + ) test_instance._Freeze() @@ -379,20 +405,28 @@ def testFreezeDoesNotLowerThroughputIfAlreadyAtFreeTier(self): @flagsaver.flagsaver( aws_dynamodb_autoscaling_target=50, aws_dynamodb_autoscaling_wcu_max=100, - aws_dynamodb_autoscaling_rcu_max=200) + aws_dynamodb_autoscaling_rcu_max=200, + ) def testFreezeAutoscalingUsesFreeTierAsMin(self): test_instance = GetTestDynamoDBInstance() self._MockHasAutoscalingPolicies(test_instance, True) mock_autoscale = self.enter_context( - mock.patch.object(test_instance, '_CreateScalableTarget')) + mock.patch.object(test_instance, '_CreateScalableTarget') + ) test_instance._Freeze() mock_autoscale.assert_has_calls([ - mock.call(aws_dynamodb._RCU_SCALABLE_DIMENSION, - aws_dynamodb._FREE_TIER_RCU, 200), - mock.call(aws_dynamodb._WCU_SCALABLE_DIMENSION, - aws_dynamodb._FREE_TIER_WCU, 100) + mock.call( + aws_dynamodb._RCU_SCALABLE_DIMENSION, + aws_dynamodb._FREE_TIER_RCU, + 200, + ), + mock.call( + aws_dynamodb._WCU_SCALABLE_DIMENSION, + aws_dynamodb._FREE_TIER_WCU, + 100, + ), ]) def testRestoreSetsThroughputBackToOriginalLevels(self): @@ -400,7 +434,8 @@ def testRestoreSetsThroughputBackToOriginalLevels(self): test_instance.rcu = 5000 test_instance.wcu = 1000 mock_set_throughput = self.enter_context( - mock.patch.object(test_instance, 'SetThroughput', autospec=True)) + mock.patch.object(test_instance, 'SetThroughput', autospec=True) + ) self._MockHasAutoscalingPolicies(test_instance, False) test_instance._Restore() @@ -410,32 +445,37 @@ def testRestoreSetsThroughputBackToOriginalLevels(self): @flagsaver.flagsaver( aws_dynamodb_autoscaling_target=50, aws_dynamodb_autoscaling_wcu_max=100, - aws_dynamodb_autoscaling_rcu_max=200) + aws_dynamodb_autoscaling_rcu_max=200, + ) def testRestoreAutoscalingUsesOriginalThroughputLevels(self): test_instance = GetTestDynamoDBInstance() mock_set_throughput = self.enter_context( - mock.patch.object(test_instance, 'SetThroughput', autospec=True)) + mock.patch.object(test_instance, 'SetThroughput', autospec=True) + ) self._MockHasAutoscalingPolicies(test_instance, True) mock_autoscale = self.enter_context( - mock.patch.object(test_instance, '_CreateScalableTarget')) + mock.patch.object(test_instance, '_CreateScalableTarget') + ) test_instance._Restore() mock_autoscale.assert_has_calls([ mock.call(aws_dynamodb._RCU_SCALABLE_DIMENSION, 5, 200), - mock.call(aws_dynamodb._WCU_SCALABLE_DIMENSION, 25, 100) + mock.call(aws_dynamodb._WCU_SCALABLE_DIMENSION, 25, 100), ]) mock_set_throughput.assert_not_called() @flagsaver.flagsaver( aws_dynamodb_autoscaling_target=50, aws_dynamodb_autoscaling_wcu_max=100, - aws_dynamodb_autoscaling_rcu_max=100) + aws_dynamodb_autoscaling_rcu_max=100, + ) def testPostCreateAutoscalingPoliciesCreatedCorrectly(self): test_instance = GetTestDynamoDBInstance() mock_create_policy = self.enter_context( - mock.patch.object(test_instance, '_CreateAutoscalingPolicy')) + mock.patch.object(test_instance, '_CreateAutoscalingPolicy') + ) test_instance._PostCreate() @@ -444,7 +484,8 @@ def testPostCreateAutoscalingPoliciesCreatedCorrectly(self): def testShouldNotAutoscale(self): test_instance = GetTestDynamoDBInstance() mock_create_policy = self.enter_context( - mock.patch.object(test_instance, '_CreateAutoscalingPolicy')) + mock.patch.object(test_instance, '_CreateAutoscalingPolicy') + ) test_instance._PostCreate() diff --git a/tests/providers/aws/aws_elasticache_redis_test.py b/tests/providers/aws/aws_elasticache_redis_test.py index ffa3ff45aa..451c75bd37 100644 --- a/tests/providers/aws/aws_elasticache_redis_test.py +++ b/tests/providers/aws/aws_elasticache_redis_test.py @@ -15,7 +15,6 @@ import unittest from absl import flags import mock - from perfkitbenchmarker import vm_util from perfkitbenchmarker.providers.aws import aws_elasticache_redis from tests import pkb_common_test_case @@ -42,36 +41,63 @@ def setUp(self): def testCreate(self): self.mock_command.return_value = (None, '', None) expected_output = [ - 'aws', 'elasticache', 'create-replication-group', '--engine', 'redis', - '--engine-version', '4.0.10', '--replication-group-id', 'pkb-run12345', - '--replication-group-description', 'pkb-run12345', '--region', - 'us-east-1', '--cache-node-type', 'cache.m4.large', - '--cache-subnet-group-name', 'subnet-pkb-run12345', - '--preferred-cache-cluster-a-zs', 'us-east-1a', '--tags' + 'aws', + 'elasticache', + 'create-replication-group', + '--engine', + 'redis', + '--engine-version', + '4.0.10', + '--replication-group-id', + 'pkb-run12345', + '--replication-group-description', + 'pkb-run12345', + '--region', + 'us-east-1', + '--cache-node-type', + 'cache.m4.large', + '--cache-subnet-group-name', + 'subnet-pkb-run12345', + '--preferred-cache-cluster-a-zs', + 'us-east-1a', + '--tags', ] self.redis._Create() self.mock_command.assert_called_once_with( - expected_output, raise_on_failure=False) + expected_output, raise_on_failure=False + ) def testDelete(self): self.mock_command.return_value = (None, '', None) expected_output = [ - 'aws', 'elasticache', 'delete-replication-group', '--region', - 'us-east-1', '--replication-group-id', 'pkb-run12345' + 'aws', + 'elasticache', + 'delete-replication-group', + '--region', + 'us-east-1', + '--replication-group-id', + 'pkb-run12345', ] self.redis._Delete() self.mock_command.assert_called_once_with( - expected_output, raise_on_failure=False) + expected_output, raise_on_failure=False + ) def testExistTrue(self): self.mock_command.return_value = (None, '', None) expected_output = [ - 'aws', 'elasticache', 'describe-replication-groups', '--region', - 'us-east-1', '--replication-group-id', 'pkb-run12345' + 'aws', + 'elasticache', + 'describe-replication-groups', + '--region', + 'us-east-1', + '--replication-group-id', + 'pkb-run12345', ] self.redis._Exists() self.mock_command.assert_called_once_with( - expected_output, raise_on_failure=False) + expected_output, raise_on_failure=False + ) if __name__ == '__main__': diff --git a/tests/providers/aws/aws_glue_crawler_test.py b/tests/providers/aws/aws_glue_crawler_test.py index 54bd3a0341..cb58cfeda9 100644 --- a/tests/providers/aws/aws_glue_crawler_test.py +++ b/tests/providers/aws/aws_glue_crawler_test.py @@ -19,20 +19,25 @@ class AwsGlueCrawlerTest(pkb_common_test_case.PkbCommonTestCase): def setUp(self): super().setUp() self.path_flag_mock = self.enter_context( - mock.patch.object(data_discovery_service, - '_DATA_DISCOVERY_OBJECT_STORE_PATH')) + mock.patch.object( + data_discovery_service, '_DATA_DISCOVERY_OBJECT_STORE_PATH' + ) + ) self.path_flag_mock.value = 's3://foo/bar' self.region_flag_mock = self.enter_context( - mock.patch.object(data_discovery_service, '_DATA_DISCOVERY_REGION')) + mock.patch.object(data_discovery_service, '_DATA_DISCOVERY_REGION') + ) self.region_flag_mock.value = 'us-east-1' self.flags_mock = self.enter_context( - mock.patch.object(aws_glue_crawler, 'FLAGS')) + mock.patch.object(aws_glue_crawler, 'FLAGS') + ) self.flags_mock.run_uri = 'deadbeef' self.flags_mock.aws_glue_crawler_role = CRAWLER_ROLE self.flags_mock.aws_glue_crawler_sample_size = None self.service = aws_glue_crawler.AwsGlueCrawler() self.issue_command_mock = self.enter_context( - mock.patch.object(vm_util, 'IssueCommand')) + mock.patch.object(vm_util, 'IssueCommand') + ) def testInitialization(self): self.assertFalse(self.service.user_managed) @@ -45,36 +50,51 @@ def testInitialization(self): def testGetMetadata(self): self.assertEqual( - self.service.GetMetadata(), { + self.service.GetMetadata(), + { 'cloud': 'AWS', 'data_discovery_region': 'us-east-1', 'aws_glue_crawler_sample_size': None, 'aws_glue_crawler_name': 'pkb-crawler-deadbeef', 'aws_glue_db_name': 'pkb-db-deadbeef', - }) + }, + ) def testGetCrawler(self): result = self.service._GetCrawler() self.assertEqual(result, self.issue_command_mock.return_value) self.issue_command_mock.assert_called_once_with( - util.AWS_PREFIX + [ - 'glue', 'get-crawler', '--name', 'pkb-crawler-deadbeef', '--region', - 'us-east-1' + util.AWS_PREFIX + + [ + 'glue', + 'get-crawler', + '--name', + 'pkb-crawler-deadbeef', + '--region', + 'us-east-1', ], - raise_on_failure=True) + raise_on_failure=True, + ) def testGetCrawlerRaiseOnFailureFalse(self): result = self.service._GetCrawler(raise_on_failure=False) self.assertEqual(result, self.issue_command_mock.return_value) self.issue_command_mock.assert_called_once_with( - util.AWS_PREFIX + [ - 'glue', 'get-crawler', '--name', 'pkb-crawler-deadbeef', '--region', - 'us-east-1' + util.AWS_PREFIX + + [ + 'glue', + 'get-crawler', + '--name', + 'pkb-crawler-deadbeef', + '--region', + 'us-east-1', ], - raise_on_failure=False) + raise_on_failure=False, + ) @mock.patch.object( - aws_glue_crawler.AwsGlueCrawler, '_GetCrawler', return_value=('', '', 0)) + aws_glue_crawler.AwsGlueCrawler, '_GetCrawler', return_value=('', '', 0) + ) def testCrawlerExists(self, get_crawler_mock): self.assertTrue(self.service._CrawlerExists()) get_crawler_mock.assert_called_once_with(raise_on_failure=False) @@ -82,7 +102,8 @@ def testCrawlerExists(self, get_crawler_mock): @mock.patch.object( aws_glue_crawler.AwsGlueCrawler, '_GetCrawler', - return_value=('', 'EntityNotFoundException', 1)) + return_value=('', 'EntityNotFoundException', 1), + ) def testCrawlerDoesntExist(self, get_crawler_mock): self.assertFalse(self.service._CrawlerExists()) get_crawler_mock.assert_called_once_with(raise_on_failure=False) @@ -91,7 +112,8 @@ def testDbExists(self): self.issue_command_mock.return_value = ('', '', 0) self.assertTrue(self.service._DbExists()) self.issue_command_mock.assert_called_once_with( - util.AWS_PREFIX + [ + util.AWS_PREFIX + + [ 'glue', 'get-database', '--name', @@ -99,13 +121,15 @@ def testDbExists(self): '--region', 'us-east-1', ], - raise_on_failure=False) + raise_on_failure=False, + ) def testDbDoesntExist(self): self.issue_command_mock.return_value = ('', 'EntityNotFoundException', 1) self.assertFalse(self.service._DbExists()) self.issue_command_mock.assert_called_once_with( - util.AWS_PREFIX + [ + util.AWS_PREFIX + + [ 'glue', 'get-database', '--name', @@ -113,48 +137,74 @@ def testDbDoesntExist(self): '--region', 'us-east-1', ], - raise_on_failure=False) + raise_on_failure=False, + ) def testCreate(self): self.service._Create() self.assertEqual(self.issue_command_mock.call_count, 2) self.issue_command_mock.assert_has_calls([ mock.call([ - 'aws', '--output', 'json', 'glue', 'create-database', + 'aws', + '--output', + 'json', + 'glue', + 'create-database', '--database-input', '{"Name": "pkb-db-deadbeef", "Description": ""}', - '--region=us-east-1' + '--region=us-east-1', ]), mock.call([ - 'aws', '--output', 'json', 'glue', 'create-crawler', '--name', - 'pkb-crawler-deadbeef', '--role', CRAWLER_ROLE, '--database-name', - 'pkb-db-deadbeef', '--targets', - '{"S3Targets": [{"Path": "s3://foo/bar"}]}', '--region', - 'us-east-1', '--tags', '' - ]) + 'aws', + '--output', + 'json', + 'glue', + 'create-crawler', + '--name', + 'pkb-crawler-deadbeef', + '--role', + CRAWLER_ROLE, + '--database-name', + 'pkb-db-deadbeef', + '--targets', + '{"S3Targets": [{"Path": "s3://foo/bar"}]}', + '--region', + 'us-east-1', + '--tags', + '', + ]), ]) - @parameterized.named_parameters(('WithCrawlerAndDb', True, True, True), - ('WithCrawlerOnly', True, False, False), - ('WithDbOnly', False, True, False), - ('WithoutCrawlerAndDb', False, False, False)) + @parameterized.named_parameters( + ('WithCrawlerAndDb', True, True, True), + ('WithCrawlerOnly', True, False, False), + ('WithDbOnly', False, True, False), + ('WithoutCrawlerAndDb', False, False, False), + ) def testExists(self, crawler_exists, db_exists, result): crawler_exists_patch = mock.patch.object( aws_glue_crawler.AwsGlueCrawler, '_CrawlerExists', - return_value=crawler_exists) + return_value=crawler_exists, + ) db_exists_patch = mock.patch.object( - aws_glue_crawler.AwsGlueCrawler, '_DbExists', return_value=db_exists) + aws_glue_crawler.AwsGlueCrawler, '_DbExists', return_value=db_exists + ) with db_exists_patch, crawler_exists_patch: self.assertEqual(self.service._Exists(), result) - @parameterized.named_parameters(('True', 'READY', True), - ('False', 'RUNNING', False)) + @parameterized.named_parameters( + ('True', 'READY', True), ('False', 'RUNNING', False) + ) def testIsReady(self, state_str, expected_result): - with mock.patch.object(aws_glue_crawler.AwsGlueCrawler, - '_GetCrawler') as get_crawler_mock: + with mock.patch.object( + aws_glue_crawler.AwsGlueCrawler, '_GetCrawler' + ) as get_crawler_mock: get_crawler_mock.return_value = ( - f'{{"Crawler":{{"State":"{state_str}"}}}}', '', 0) + f'{{"Crawler":{{"State":"{state_str}"}}}}', + '', + 0, + ) self.assertEqual(self.service._IsReady(), expected_result) def testIsReadyRaiseOnCrawlFailure(self): @@ -166,9 +216,9 @@ def testIsReadyRaiseOnCrawlFailure(self): 'LogStream': 'pkb-crawler-deadbeef', 'MessagePrefix': '12345678-1234-1234-1234-1234567890ab', 'StartTime': 1640736374.0, - 'Status': 'FAILED' + 'Status': 'FAILED', }, - 'State': 'READY' + 'State': 'READY', } } self.issue_command_mock.return_value = (json.dumps(crawler_data), '', 0) @@ -181,7 +231,8 @@ def testDelete(self): self.assertEqual(self.issue_command_mock.call_count, 2) self.issue_command_mock.assert_has_calls([ mock.call( - util.AWS_PREFIX + [ + util.AWS_PREFIX + + [ 'glue', 'delete-database', '--name', @@ -189,9 +240,11 @@ def testDelete(self): '--region', 'us-east-1', ], - raise_on_failure=False), + raise_on_failure=False, + ), mock.call( - util.AWS_PREFIX + [ + util.AWS_PREFIX + + [ 'glue', 'delete-crawler', '--name', @@ -199,28 +252,37 @@ def testDelete(self): '--region', 'us-east-1', ], - raise_on_failure=False) + raise_on_failure=False, + ), ]) - @parameterized.named_parameters(('WithCrawlerAndDb', True, True, True), - ('WithCrawlerOnly', True, False, True), - ('WithDbOnly', False, True, True), - ('WithoutCrawlerAndDb', False, False, False)) + @parameterized.named_parameters( + ('WithCrawlerAndDb', True, True, True), + ('WithCrawlerOnly', True, False, True), + ('WithDbOnly', False, True, True), + ('WithoutCrawlerAndDb', False, False, False), + ) def testIsDeleting(self, crawler_exists, db_exists, result): crawler_exists_patch = mock.patch.object( aws_glue_crawler.AwsGlueCrawler, '_CrawlerExists', - return_value=crawler_exists) + return_value=crawler_exists, + ) db_exists_patch = mock.patch.object( - aws_glue_crawler.AwsGlueCrawler, '_DbExists', return_value=db_exists) + aws_glue_crawler.AwsGlueCrawler, '_DbExists', return_value=db_exists + ) with db_exists_patch, crawler_exists_patch: self.assertEqual(self.service._IsDeleting(), result) @mock.patch.object( - aws_glue_crawler.AwsGlueCrawler, '_IsReady', return_value=True) + aws_glue_crawler.AwsGlueCrawler, '_IsReady', return_value=True + ) def testDiscoverData(self, is_ready_mock): self.issue_command_mock.return_value = ( - '{"CrawlerMetricsList":[{"LastRuntimeSeconds":42.7}]}', '', 0) + '{"CrawlerMetricsList":[{"LastRuntimeSeconds":42.7}]}', + '', + 0, + ) self.assertEqual(self.service.DiscoverData(), 42.7) is_ready_mock.assert_called_once_with(raise_on_crawl_failure=True) self.assertEqual(self.issue_command_mock.call_count, 2) diff --git a/tests/providers/aws/aws_network_test.py b/tests/providers/aws/aws_network_test.py index 5603e56227..211dc9dfed 100644 --- a/tests/providers/aws/aws_network_test.py +++ b/tests/providers/aws/aws_network_test.py @@ -32,9 +32,7 @@ def RouteTable(cidr_block): return { 'RouteTables': [{ 'RouteTableId': ROUTE_ID, - 'Routes': [{ - 'DestinationCidrBlock': cidr_block - }] + 'Routes': [{'DestinationCidrBlock': cidr_block}], }] } @@ -47,26 +45,23 @@ def RouteTable(cidr_block): QUERY_ROUTE_HAS_DEFAULT = ('describe-route-tables', RouteTable('0.0.0.0/0')) CREATE_ROUTE = ('create-route', {}) GATEWAY_ID = 'igw-1234' -DESCRIBE_SECURITY_GROUPS_ONLY_DEFAULT = ('describe-security-groups', { - 'SecurityGroups': [{ - 'GroupId': SG_DEFAULT - }] -}) +DESCRIBE_SECURITY_GROUPS_ONLY_DEFAULT = ( + 'describe-security-groups', + {'SecurityGroups': [{'GroupId': SG_DEFAULT}]}, +) VPC_QUERY_NONE = ('describe-vpcs', {'Vpcs': []}) VPC_QUERY_ONE = ('describe-vpcs', {'Vpcs': [{'VpcId': VPC_ID}]}) VPC_CREATE = ('create-vpc', {'Vpc': {'VpcId': VPC_ID}}) MODIFY_VPC = ('modify-vpc-attribute', {}) -CREATE_GATEWAY = ('create-internet-gateway', { - 'InternetGateway': { - 'InternetGatewayId': GATEWAY_ID - } -}) +CREATE_GATEWAY = ( + 'create-internet-gateway', + {'InternetGateway': {'InternetGatewayId': GATEWAY_ID}}, +) GATEWAY_QUERY_NONE = ('describe-internet-gateways', {'InternetGateways': []}) -GATEWAY_QUERY_ONE = ('describe-internet-gateways', { - 'InternetGateways': [{ - 'InternetGatewayId': GATEWAY_ID - }] -}) +GATEWAY_QUERY_ONE = ( + 'describe-internet-gateways', + {'InternetGateways': [{'InternetGatewayId': GATEWAY_ID}]}, +) ATTACH_GATEWAY = ('attach-internet-gateway', {}) DETACH_GATEWAY = ('detach-internet-gateway', {}) @@ -93,15 +88,16 @@ def AwsFilter(name, value): class BaseAwsTest(pkb_common_test_case.PkbCommonTestCase): - expected_commands: List[str] def setUp(self): super(BaseAwsTest, self).setUp() self.mock_aws = self.enter_context( - mock.patch.object(vm_util, 'IssueCommand')) + mock.patch.object(vm_util, 'IssueCommand') + ) self.mock_tags = self.enter_context( - mock.patch.object(util, 'AddDefaultTags')) + mock.patch.object(util, 'AddDefaultTags') + ) def SetExpectedCommands(self, *commands): self.mock_aws.reset_mock() @@ -109,11 +105,13 @@ def SetExpectedCommands(self, *commands): self.expected_commands = [command for command, _ in commands] def assertCommandsCalled(self): - for expected_call, found_call in zip(self.expected_commands, - self.mock_aws.call_args_list): + for expected_call, found_call in zip( + self.expected_commands, self.mock_aws.call_args_list + ): self.assertEqual(expected_call, FindAwsCommand(found_call[0][0])) self.assertEqual( - len(self.expected_commands), len(self.mock_aws.call_args_list)) + len(self.expected_commands), len(self.mock_aws.call_args_list) + ) def assertLastCommandContains(self, want_phrase): self.assertIn(want_phrase, ' '.join(self.mock_aws.call_args[0][0])) @@ -144,8 +142,12 @@ def testExistsVpcsOne(self): self.assertCommandsCalled() def testCreate(self): - self.SetExpectedCommands(VPC_CREATE, MODIFY_VPC, VPC_QUERY_ONE, - DESCRIBE_SECURITY_GROUPS_ONLY_DEFAULT) + self.SetExpectedCommands( + VPC_CREATE, + MODIFY_VPC, + VPC_QUERY_ONE, + DESCRIBE_SECURITY_GROUPS_ONLY_DEFAULT, + ) vpc = aws_network.AwsVpc(REGION) vpc.Create() self.assertEqual(VPC_ID, vpc.id) @@ -181,8 +183,9 @@ def testCreate(self): def testCreateRouteNoDefault(self): # 'create-route' is executed - self.SetExpectedCommands(QUERY_ROUTE_NO_DEFAULT, QUERY_ROUTE_NO_DEFAULT, - CREATE_ROUTE) + self.SetExpectedCommands( + QUERY_ROUTE_NO_DEFAULT, QUERY_ROUTE_NO_DEFAULT, CREATE_ROUTE + ) self.route.Create() self.route.CreateRoute(GATEWAY_ID) self.assertCommandsCalled() diff --git a/tests/providers/aws/aws_nfs_service_test.py b/tests/providers/aws/aws_nfs_service_test.py index 513a7d51ee..e7bfd24055 100644 --- a/tests/providers/aws/aws_nfs_service_test.py +++ b/tests/providers/aws/aws_nfs_service_test.py @@ -56,51 +56,53 @@ AwsResponses = collections.namedtuple('Responses', 'create describe') -_FILER = AwsResponses({ - 'SizeInBytes': { - 'Value': 0 - }, - 'CreationToken': _NFS_TOKEN, - 'CreationTime': 1513322422.0, - 'PerformanceMode': 'generalPurpose', - 'FileSystemId': _FILE_ID, - 'NumberOfMountTargets': 0, - 'LifeCycleState': 'creating', - 'OwnerId': '835761027970' -}, { - 'FileSystems': [{ - 'SizeInBytes': { - 'Value': 6144 - }, +_FILER = AwsResponses( + { + 'SizeInBytes': {'Value': 0}, 'CreationToken': _NFS_TOKEN, 'CreationTime': 1513322422.0, 'PerformanceMode': 'generalPurpose', 'FileSystemId': _FILE_ID, 'NumberOfMountTargets': 0, - 'LifeCycleState': 'available', - 'OwnerId': '835761027970' - }] -}) - -_MOUNT = AwsResponses({ - 'MountTargetId': _MOUNT_ID, - 'NetworkInterfaceId': 'eni-9956273b', - 'FileSystemId': _FILE_ID, - 'LifeCycleState': 'creating', - 'SubnetId': _SUBNET_ID, - 'OwnerId': '835761027970', - 'IpAddress': '10.0.0.182' -}, { - 'MountTargets': [{ + 'LifeCycleState': 'creating', + 'OwnerId': '835761027970', + }, + { + 'FileSystems': [{ + 'SizeInBytes': {'Value': 6144}, + 'CreationToken': _NFS_TOKEN, + 'CreationTime': 1513322422.0, + 'PerformanceMode': 'generalPurpose', + 'FileSystemId': _FILE_ID, + 'NumberOfMountTargets': 0, + 'LifeCycleState': 'available', + 'OwnerId': '835761027970', + }] + }, +) + +_MOUNT = AwsResponses( + { 'MountTargetId': _MOUNT_ID, 'NetworkInterfaceId': 'eni-9956273b', 'FileSystemId': _FILE_ID, - 'LifeCycleState': 'available', + 'LifeCycleState': 'creating', 'SubnetId': _SUBNET_ID, 'OwnerId': '835761027970', - 'IpAddress': '10.0.0.182' - }] -}) + 'IpAddress': '10.0.0.182', + }, + { + 'MountTargets': [{ + 'MountTargetId': _MOUNT_ID, + 'NetworkInterfaceId': 'eni-9956273b', + 'FileSystemId': _FILE_ID, + 'LifeCycleState': 'available', + 'SubnetId': _SUBNET_ID, + 'OwnerId': '835761027970', + 'IpAddress': '10.0.0.182', + }] + }, +) class BaseTest(pkb_common_test_case.PkbCommonTestCase): @@ -203,7 +205,8 @@ def testCreateFiler(self): nfs = self._CreateFiler() self.assertEqual(_FILE_ID, nfs.filer_id) self.issue_cmd.CreateFiler.assert_called_with( - _NFS_TOKEN, _TIER, _THROUGHPUT_MODE, _PROVISIONED_THROUGHPUT) + _NFS_TOKEN, _TIER, _THROUGHPUT_MODE, _PROVISIONED_THROUGHPUT + ) def testDeleteFiler(self): nfs = self._CreateFiler() @@ -222,8 +225,9 @@ def testDeleteFilerWithoutDeletingMountFirst(self): def testCreateMount(self): nfs = self._CreateMount() self.assertEqual(_MOUNT_ID, nfs.mount_id) - self.issue_cmd.CreateMount.assert_called_with(_FILE_ID, _SUBNET_ID, - _SECURITY_GROUP_ID) + self.issue_cmd.CreateMount.assert_called_with( + _FILE_ID, _SUBNET_ID, _SECURITY_GROUP_ID + ) def testCreateMountNoFiler(self): nfs = self._CreateNfsService() @@ -246,11 +250,13 @@ def testFullLifeCycle(self): nfs.Create() nfs.Delete() self.issue_cmd.CreateFiler.assert_called_with( - _NFS_TOKEN, _TIER, _THROUGHPUT_MODE, _PROVISIONED_THROUGHPUT) + _NFS_TOKEN, _TIER, _THROUGHPUT_MODE, _PROVISIONED_THROUGHPUT + ) self.issue_cmd.AddTagsToFiler.assert_called_with(_FILE_ID) self.issue_cmd.WaitUntilFilerAvailable.assert_called_with(_FILE_ID) - self.issue_cmd.CreateMount.assert_called_with(_FILE_ID, _SUBNET_ID, - _SECURITY_GROUP_ID) + self.issue_cmd.CreateMount.assert_called_with( + _FILE_ID, _SUBNET_ID, _SECURITY_GROUP_ID + ) self.issue_cmd.DeleteMount.assert_called_with(_MOUNT_ID) self.issue_cmd.DeleteFiler.assert_called_with(_FILE_ID) @@ -261,7 +267,8 @@ def _CreateMockVm(self): self._CreatePatched(aws_network, 'AwsNetwork') self._CreatePatched(aws_network, 'AwsFirewall') vm_spec = aws_virtual_machine.AwsVmSpec( - _COMPONENT, zone=_AWS_ZONE, machine_type='m2.2xlarge') + _COMPONENT, zone=_AWS_ZONE, machine_type='m2.2xlarge' + ) aws_machine = aws_virtual_machine.Rhel7BasedAwsVirtualMachine(vm_spec) aws_machine.RemoteCommand = mock.Mock() aws_machine.RemoteHostCommand = mock.Mock() @@ -289,41 +296,48 @@ def _CallCreateScratchDisk(self, fs_type): return aws_machine def testCreateNfsDisk(self): - mount_opt = ('hard,nfsvers=4.1,retrans=2,rsize=1048576,timeo=600,' - 'wsize=1048576') + mount_opt = ( + 'hard,nfsvers=4.1,retrans=2,rsize=1048576,timeo=600,wsize=1048576' + ) host = 'FSID.efs.us-east-1.amazonaws.com' - mount_cmd = ('sudo mkdir -p /scratch;' - 'sudo mount -t nfs -o {mount_opt} {host}:/ /scratch && ' - 'sudo chown $USER:$USER /scratch;').format( - mount_opt=mount_opt, host=host) - fstab_cmd = ('echo "{host}:/ /scratch nfs {mount_opt}"' - ' | sudo tee -a /etc/fstab').format( - mount_opt=mount_opt, host=host) + mount_cmd = ( + 'sudo mkdir -p /scratch;' + 'sudo mount -t nfs -o {mount_opt} {host}:/ /scratch && ' + 'sudo chown $USER:$USER /scratch;' + ).format(mount_opt=mount_opt, host=host) + fstab_cmd = ( + 'echo "{host}:/ /scratch nfs {mount_opt}" | sudo tee -a /etc/fstab' + ).format(mount_opt=mount_opt, host=host) install_nfs = 'sudo yum install -y nfs-utils' aws_machine = self._CallCreateScratchDisk(disk.NFS) aws_machine.RemoteCommand.assert_called_with(install_nfs) self.assertEqual( [mock.call(mount_cmd), mock.call(fstab_cmd)], - aws_machine.RemoteHostCommand.call_args_list) + aws_machine.RemoteHostCommand.call_args_list, + ) def testCreateLocalDisk(self): # show that the non-NFS case formats the disk format_cmd = ( '[[ -d /mnt ]] && sudo umount /mnt; ' 'sudo mke2fs -F -E lazy_itable_init=0,discard -O ^has_journal ' - '-t ext4 -b 4096 /dev/xvdb') - mount_cmd = ('sudo mkdir -p /scratch;' - 'sudo mount -o discard /dev/xvdb /scratch && ' - 'sudo chown $USER:$USER /scratch;') - fstab_cmd = ('echo "/dev/xvdb /scratch ext4 defaults" | sudo tee -a ' - '/etc/fstab') + '-t ext4 -b 4096 /dev/xvdb' + ) + mount_cmd = ( + 'sudo mkdir -p /scratch;' + 'sudo mount -o discard /dev/xvdb /scratch && ' + 'sudo chown $USER:$USER /scratch;' + ) + fstab_cmd = ( + 'echo "/dev/xvdb /scratch ext4 defaults" | sudo tee -a /etc/fstab' + ) aws_machine = self._CallCreateScratchDisk('ext4') self.assertEqual( - [mock.call(format_cmd), - mock.call(mount_cmd), - mock.call(fstab_cmd)], aws_machine.RemoteHostCommand.call_args_list) + [mock.call(format_cmd), mock.call(mount_cmd), mock.call(fstab_cmd)], + aws_machine.RemoteHostCommand.call_args_list, + ) class AwsEfsCommandsTest(BaseTest): @@ -339,24 +353,33 @@ def _SetResponse(self, json_value=None): self.issue_cmd.return_value = (txt, '', 0) def assertCalled(self, *args): - cmd = ['aws', '--output', 'json', '--region', _AWS_REGION, - 'efs'] + list(args) + cmd = ['aws', '--output', 'json', '--region', _AWS_REGION, 'efs'] + list( + args + ) self.issue_cmd.assert_called_with(cmd, raise_on_failure=False) def testCreateFiler(self): self._SetResponse(_FILER.create) - self.aws.CreateFiler(_NFS_TOKEN, _TIER, _THROUGHPUT_MODE, - _PROVISIONED_THROUGHPUT) - self.assertCalled('create-file-system', '--creation-token', _NFS_TOKEN, - '--performance-mode', _TIER, '--throughput-mode', - _THROUGHPUT_MODE) + self.aws.CreateFiler( + _NFS_TOKEN, _TIER, _THROUGHPUT_MODE, _PROVISIONED_THROUGHPUT + ) + self.assertCalled( + 'create-file-system', + '--creation-token', + _NFS_TOKEN, + '--performance-mode', + _TIER, + '--throughput-mode', + _THROUGHPUT_MODE, + ) def testAddTags(self): self._SetResponse() self.aws.AddTagsToFiler(_FILE_ID) tags = util.MakeFormattedDefaultTags() - self.assertCalled('create-tags', '--file-system-id', _FILE_ID, '--tags', - *tags) + self.assertCalled( + 'create-tags', '--file-system-id', _FILE_ID, '--tags', *tags + ) def testFilerAvailable(self): self._SetResponse(_FILER.describe) @@ -371,9 +394,15 @@ def testMountAvailable(self): def testCreateMount(self): self._SetResponse(_MOUNT.create) self.aws.CreateMount(_FILE_ID, _SUBNET_ID, _SECURITY_GROUP_ID) - self.assertCalled('create-mount-target', '--file-system-id', _FILE_ID, - '--subnet-id', _SUBNET_ID, '--security-groups', - _SECURITY_GROUP_ID) + self.assertCalled( + 'create-mount-target', + '--file-system-id', + _FILE_ID, + '--subnet-id', + _SUBNET_ID, + '--security-groups', + _SECURITY_GROUP_ID, + ) def testDeleteFiler(self): self._SetResponse() diff --git a/tests/providers/aws/aws_placement_group_test.py b/tests/providers/aws/aws_placement_group_test.py index e13bfe1689..b8920bbba2 100644 --- a/tests/providers/aws/aws_placement_group_test.py +++ b/tests/providers/aws/aws_placement_group_test.py @@ -6,7 +6,6 @@ from absl import flags from absl.testing import parameterized import mock - from perfkitbenchmarker import placement_group from perfkitbenchmarker import provider_info from perfkitbenchmarker import vm_util @@ -28,7 +27,7 @@ 'PlacementGroups': [{ 'GroupName': GROUP_NAME, 'State': 'available', - 'Strategy': STRATEGY + 'Strategy': STRATEGY, }] } EXISTS_TWO_RESPONSE = {'PlacementGroups': ['seat', 'filler']} @@ -42,7 +41,12 @@ def AwsCommand(topic, *aws_args, **env): # used when validating an AWS command run via vm_util.IssueCommand aws_bash_cmd = [ - 'aws', '--output', 'json', 'ec2', topic, '--region={}'.format(REGION) + 'aws', + '--output', + 'json', + 'ec2', + topic, + '--region={}'.format(REGION), ] + list(aws_args) return mock.call(aws_bash_cmd, **env) @@ -52,15 +56,19 @@ def AwsCommand(topic, *aws_args, **env): '--filter=Name=group-name,Values={}'.format(GROUP_NAME), env=None, raise_on_failure=False, - suppress_failure=None) + suppress_failure=None, +) CREATE_CALL = AwsCommand( - 'create-placement-group', '--group-name={}'.format(GROUP_NAME), + 'create-placement-group', + '--group-name={}'.format(GROUP_NAME), '--strategy={}'.format(STRATEGY), - '--tag-specifications=foobar') + '--tag-specifications=foobar', +) DELETE_CALL = AwsCommand( 'delete-placement-group', '--group-name={}'.format(GROUP_NAME), - raise_on_failure=False) + raise_on_failure=False, +) def AwsResponse(data): @@ -69,9 +77,10 @@ def AwsResponse(data): def CreateAwsPlacementGroupSpec(group_style=STRATEGY): spec_class = spec.GetSpecClass( - placement_group.BasePlacementGroupSpec, CLOUD=CLOUD) + placement_group.BasePlacementGroupSpec, CLOUD=CLOUD + ) FLAGS.placement_group_style = group_style - name = '{0}.placement_group_spec.{1}'.format(spec_class.SPEC_TYPE, CLOUD), + name = ('{0}.placement_group_spec.{1}'.format(spec_class.SPEC_TYPE, CLOUD),) return spec_class(name, zone=ZONE, flag_values=FLAGS) @@ -85,7 +94,8 @@ class AwsPlacementGroupTest(pkb_common_test_case.PkbCommonTestCase): def setUp(self): super(AwsPlacementGroupTest, self).setUp() self.mock_cmd = self.enter_context( - mock.patch.object(vm_util, 'IssueCommand')) + mock.patch.object(vm_util, 'IssueCommand') + ) uuid_call = self.enter_context(mock.patch.object(uuid, 'uuid4')) uuid_call.return_value = UUID @@ -107,7 +117,8 @@ def testGetPlacementGroup(self): @parameterized.named_parameters( ('EXISTS_NONE_RESPONSE', EXISTS_NONE_RESPONSE, False), ('EXISTS_ONE_RESPONSE', EXISTS_ONE_RESPONSE, True), - ('EXISTS_TWO_RESPONSE', EXISTS_TWO_RESPONSE, None, True)) + ('EXISTS_TWO_RESPONSE', EXISTS_TWO_RESPONSE, None, True), + ) def testExists(self, response, exists_value, throws_exception=False): self.mock_cmd.side_effect = [AwsResponse(response)] pg = CreateAwsPlacementGroup() diff --git a/tests/providers/aws/aws_relational_db_test.py b/tests/providers/aws/aws_relational_db_test.py index aad4ca0a14..1ab9945abb 100644 --- a/tests/providers/aws/aws_relational_db_test.py +++ b/tests/providers/aws/aws_relational_db_test.py @@ -52,11 +52,13 @@ def _ReadTestDataFile(filename): class AwsRelationalDbSpecTestCase(pkb_common_test_case.PkbCommonTestCase): """Class that tests the creation of an AwsRelationalDbSpec.""" + pass class AwsRelationalDbFlagsTestCase(pkb_common_test_case.PkbCommonTestCase): """Class that tests the flags defined in AwsRelationalDb.""" + pass @@ -72,10 +74,10 @@ def _PatchCriticalObjects(self, stdout='', stderr='', return_code=0): """A context manager that patches a few critical objects with mocks.""" retval = (stdout, stderr, return_code) with mock.patch( - vm_util.__name__ + '.IssueCommand', - return_value=retval) as issue_command, mock.patch( - builtins.__name__ + '.open'), mock.patch(vm_util.__name__ + - '.NamedTemporaryFile'): + vm_util.__name__ + '.IssueCommand', return_value=retval + ) as issue_command, mock.patch(builtins.__name__ + '.open'), mock.patch( + vm_util.__name__ + '.NamedTemporaryFile' + ): yield issue_command def VmGroupSpec(self): @@ -84,41 +86,30 @@ def VmGroupSpec(self): 'vm_spec': { 'GCP': { 'zone': 'us-central1-c', - 'machine_type': 'n1-standard-1' + 'machine_type': 'n1-standard-1', } }, - 'disk_spec': { - 'GCP': { - 'disk_size': 500, - 'disk_type': 'pd-ssd' - } - } + 'disk_spec': {'GCP': {'disk_size': 500, 'disk_type': 'pd-ssd'}}, }, 'servers': { 'vm_spec': { 'GCP': { 'zone': 'us-central1-c', - 'machine_type': 'n1-standard-1' + 'machine_type': 'n1-standard-1', } }, - 'disk_spec': { - 'GCP': { - 'disk_size': 500, - 'disk_type': 'pd-ssd' - } - } - } + 'disk_spec': {'GCP': {'disk_size': 500, 'disk_type': 'pd-ssd'}}, + }, } def CreateMockSpec(self, additional_spec_items=None): default_server_db_disk_spec = aws_disk.AwsDiskSpec( - _COMPONENT, disk_size=5, disk_type=aws_disk.IO1, iops=1000) + _COMPONENT, disk_size=5, disk_type=aws_disk.IO1, iops=1000 + ) default_server_db_spec = virtual_machine.BaseVmSpec( - 'NAME', **{ - 'machine_type': 'db.t1.micro', - 'zone': 'us-west-2b' - }) + 'NAME', **{'machine_type': 'db.t1.micro', 'zone': 'us-west-2b'} + ) spec_dict = { 'engine': MYSQL, 'engine_version': '5.7.11', @@ -187,9 +178,11 @@ def testCreate(self): command_string = self.Create() self.assertTrue( - command_string.startswith('%s rds create-db-instance' % _AWS_PREFIX)) - self.assertIn('--db-instance-identifier=pkb-db-instance-123', - command_string) + command_string.startswith('%s rds create-db-instance' % _AWS_PREFIX) + ) + self.assertIn( + '--db-instance-identifier=pkb-db-instance-123', command_string + ) self.assertIn('--db-instance-class=db.t1.micro', command_string) self.assertIn('--engine=mysql', command_string) self.assertIn('--master-user-password=fakepassword', command_string) @@ -203,10 +196,8 @@ def testCorrectVmGroupsPresent(self): def CreateAuroraMockSpec(self, additional_spec_items=None): default_server_db_spec = virtual_machine.BaseVmSpec( - 'NAME', **{ - 'machine_type': 'db.t1.micro', - 'zone': 'us-west-2b' - }) + 'NAME', **{'machine_type': 'db.t1.micro', 'zone': 'us-west-2b'} + ) spec_dict = { 'engine': AURORA_POSTGRES, @@ -303,9 +294,9 @@ def testDiskWithIops(self): def testDiskWithoutIops(self): spec_dict = { - 'db_disk_spec': - aws_disk.AwsDiskSpec( - _COMPONENT, disk_size=5, disk_type=aws_disk.GP2) + 'db_disk_spec': aws_disk.AwsDiskSpec( + _COMPONENT, disk_size=5, disk_type=aws_disk.GP2 + ) } command_string = self.Create(spec_dict) @@ -350,7 +341,8 @@ def testSetEndpoint(self): self.assertEqual( 'pkb-db-instance-a4499926.cqxeajwjbqne.us-west-2.rds.amazonaws.com', - db.endpoint) + db.endpoint, + ) def testDelete(self): with self._PatchCriticalObjects() as issue_command: @@ -361,8 +353,9 @@ def testDelete(self): command_string = ' '.join(issue_command.call_args[0][0]) self.assertIn('aws --output json rds delete-db-instance', command_string) - self.assertIn('--db-instance-identifier=pkb-db-instance-123', - command_string) + self.assertIn( + '--db-instance-identifier=pkb-db-instance-123', command_string + ) self.assertIn('--skip-final-snapshot', command_string) def testCreateUnmanagedDb(self): diff --git a/tests/providers/aws/aws_sqs_test.py b/tests/providers/aws/aws_sqs_test.py index 9a44707df5..b4c4d8886c 100644 --- a/tests/providers/aws/aws_sqs_test.py +++ b/tests/providers/aws/aws_sqs_test.py @@ -20,7 +20,6 @@ class AwsSqsTest(pkb_common_test_case.PkbCommonTestCase): - sqs: aws_sqs.AwsSqs client: mock.Mock @@ -35,7 +34,8 @@ def setUp(self, _): def _MockIssueCommand(self, return_value): return self.enter_context( - mock.patch.object(vm_util, 'IssueCommand', return_value=return_value)) + mock.patch.object(vm_util, 'IssueCommand', return_value=return_value) + ) def testGetQueue(self): # Don't actually issue a command. @@ -45,8 +45,12 @@ def testGetQueue(self): actual_result = self.sqs._GetQueue() cmd = ' '.join(cmd.call_args[0][0]) self.assertIn( - 'sqs get-queue-url --queue-name ' + self.sqs.queue_name + ' --region ' + - self.sqs.region, cmd) + 'sqs get-queue-url --queue-name ' + + self.sqs.queue_name + + ' --region ' + + self.sqs.region, + cmd, + ) self.assertEqual(actual_result, 'mocked_queue_url') def testCreate(self): @@ -57,8 +61,12 @@ def testCreate(self): self.sqs._Create() cmd = ' '.join(cmd.call_args[0][0]) self.assertIn( - 'sqs create-queue --queue-name ' + self.sqs.queue_name + ' --region ' + - self.sqs.region, cmd) + 'sqs create-queue --queue-name ' + + self.sqs.queue_name + + ' --region ' + + self.sqs.region, + cmd, + ) def testExists(self): # Don't actually issue a command. @@ -68,8 +76,12 @@ def testExists(self): result = self.sqs._Exists() cmd = ' '.join(cmd.call_args[0][0]) self.assertIn( - 'sqs get-queue-url --queue-name ' + self.sqs.queue_name + ' --region ' + - self.sqs.region, cmd) + 'sqs get-queue-url --queue-name ' + + self.sqs.queue_name + + ' --region ' + + self.sqs.region, + cmd, + ) self.assertTrue(result) def testDoesntExist(self): @@ -80,8 +92,12 @@ def testDoesntExist(self): result = self.sqs._Exists() cmd = ' '.join(cmd.call_args[0][0]) self.assertIn( - 'sqs get-queue-url --queue-name ' + self.sqs.queue_name + ' --region ' + - self.sqs.region, cmd) + 'sqs get-queue-url --queue-name ' + + self.sqs.queue_name + + ' --region ' + + self.sqs.region, + cmd, + ) self.assertFalse(result) @mock.patch.object(aws_sqs.AwsSqs, '_GetQueue') @@ -94,8 +110,12 @@ def testDelete(self, get_queue_mock): self.sqs._Delete() cmd = ' '.join(cmd.call_args[0][0]) self.assertIn( - 'sqs delete-queue --queue-url ' + get_queue_mock.return_value + - ' --region ' + self.sqs.region, cmd) + 'sqs delete-queue --queue-url ' + + get_queue_mock.return_value + + ' --region ' + + self.sqs.region, + cmd, + ) def testPrepareClientVm(self): # Don't actually issue a command. @@ -105,20 +125,23 @@ def testPrepareClientVm(self): self.sqs.PrepareClientVm() self.client.assert_has_calls([ mock.call.RemoteCommand( - 'sudo pip3 install boto3', ignore_failure=False), + 'sudo pip3 install boto3', ignore_failure=False + ), mock.call.RemoteCommand( - 'mkdir -p ~/perfkitbenchmarker/scripts/messaging_service_scripts/aws' + 'mkdir -p' + ' ~/perfkitbenchmarker/scripts/messaging_service_scripts/aws' ), mock.call.PushDataFile( 'messaging_service_scripts/aws/__init__.py', - '~/perfkitbenchmarker/scripts/messaging_service_scripts/aws/__init__.py' + '~/perfkitbenchmarker/scripts/messaging_service_scripts/aws/__init__.py', ), mock.call.RemoteCommand( - 'mkdir -p ~/perfkitbenchmarker/scripts/messaging_service_scripts/aws' + 'mkdir -p' + ' ~/perfkitbenchmarker/scripts/messaging_service_scripts/aws' ), mock.call.PushDataFile( 'messaging_service_scripts/aws/aws_sqs_client.py', - '~/perfkitbenchmarker/scripts/messaging_service_scripts/aws/aws_sqs_client.py' + '~/perfkitbenchmarker/scripts/messaging_service_scripts/aws/aws_sqs_client.py', ), mock.call.PushDataFile('messaging_service_scripts/aws_benchmark.py'), ]) @@ -127,16 +150,22 @@ def testPrepareClientVm(self): def testRun(self): return_value = ['{"mock1": 1}', None] self.client.RemoteCommand.return_value = return_value - remote_run_cmd = (f'python3 -m aws_benchmark ' - f'--queue_name={self.sqs.queue_name} ' - f'--region={self.sqs.region} ' - f'--benchmark_scenario={_BENCHMARK_SCENARIO} ' - f'--number_of_messages={_NUMBER_OF_MESSAGES} ' - f'--message_size={_MESSAGE_SIZE} ' - f'--warmup_messages={_WARMUP_MESSAGES}') - - self.sqs.Run(_BENCHMARK_SCENARIO, _NUMBER_OF_MESSAGES, _MESSAGE_SIZE, - _WARMUP_MESSAGES) + remote_run_cmd = ( + 'python3 -m aws_benchmark ' + f'--queue_name={self.sqs.queue_name} ' + f'--region={self.sqs.region} ' + f'--benchmark_scenario={_BENCHMARK_SCENARIO} ' + f'--number_of_messages={_NUMBER_OF_MESSAGES} ' + f'--message_size={_MESSAGE_SIZE} ' + f'--warmup_messages={_WARMUP_MESSAGES}' + ) + + self.sqs.Run( + _BENCHMARK_SCENARIO, + _NUMBER_OF_MESSAGES, + _MESSAGE_SIZE, + _WARMUP_MESSAGES, + ) self.client.RemoteCommand.assert_called_with(remote_run_cmd) diff --git a/tests/providers/aws/aws_vpc_endpoint_test.py b/tests/providers/aws/aws_vpc_endpoint_test.py index d1239a239f..c920e999ea 100644 --- a/tests/providers/aws/aws_vpc_endpoint_test.py +++ b/tests/providers/aws/aws_vpc_endpoint_test.py @@ -31,22 +31,35 @@ DELETE_RES = {'Unsuccessful': []} QUERY_ENDPOINTS_CMD = [ - 'describe-vpc-endpoints', '--filters', + 'describe-vpc-endpoints', + '--filters', 'Name=service-name,Values={}'.format(FULL_SERVICE_NAME), - 'Name=vpc-id,Values={}'.format(VPC_ID), '--query', - 'VpcEndpoints[].VpcEndpointId' + 'Name=vpc-id,Values={}'.format(VPC_ID), + '--query', + 'VpcEndpoints[].VpcEndpointId', ] DESCRIBE_ROUTES_CMD = [ - 'describe-route-tables', '--filters', - 'Name=vpc-id,Values={}'.format(VPC_ID), '--query', - 'RouteTables[].RouteTableId' + 'describe-route-tables', + '--filters', + 'Name=vpc-id,Values={}'.format(VPC_ID), + '--query', + 'RouteTables[].RouteTableId', ] CREATE_ENDPOINT_CMD = [ - 'create-vpc-endpoint', '--vpc-endpoint-type', 'Gateway', '--vpc-id', VPC_ID, - '--service-name', FULL_SERVICE_NAME, '--route-table-ids', ROUTE_TABLE_ID + 'create-vpc-endpoint', + '--vpc-endpoint-type', + 'Gateway', + '--vpc-id', + VPC_ID, + '--service-name', + FULL_SERVICE_NAME, + '--route-table-ids', + ROUTE_TABLE_ID, ] DELETE_ENDPOINT_CMD = [ - 'delete-vpc-endpoints', '--vpc-endpoint-ids', ENDPOINT_ID + 'delete-vpc-endpoints', + '--vpc-endpoint-ids', + ENDPOINT_ID, ] @@ -57,7 +70,8 @@ def setUp(self): self.mock_vpc = mock.Mock() self.mock_vpc.region = REGION self.mock_run_cmd = self.enter_context( - mock.patch.object(aws_vpc_endpoint.AwsVpcS3Endpoint, '_RunCommand')) + mock.patch.object(aws_vpc_endpoint.AwsVpcS3Endpoint, '_RunCommand') + ) def _InitEndpoint(self, vpc_id): self.mock_vpc.id = vpc_id diff --git a/tests/providers/aws/redshift_test.py b/tests/providers/aws/redshift_test.py index 1964d3e841..9fc2927154 100644 --- a/tests/providers/aws/redshift_test.py +++ b/tests/providers/aws/redshift_test.py @@ -17,7 +17,6 @@ import unittest from absl import flags import mock - from perfkitbenchmarker import errors from perfkitbenchmarker import vm_util from perfkitbenchmarker.configs import benchmark_config_spec @@ -79,25 +78,46 @@ def testSingleNodeClusterCreation(self): redshift_local = redshift.Redshift(spec) self.assertIsNone(redshift_local.snapshot) with mock.patch( - vm_util.__name__ + '.IssueCommand', - return_value=('out_', 'err_', 0)) as mock_issue: - redshift_local.Initialize(redshift_local.cluster_identifier, - redshift_local.node_type, - redshift_local.node_count, redshift_local.user, - redshift_local.password, - FakeRedshiftClusterParameterGroup(), - FakeRedshiftClusterSubnetGroup()) + vm_util.__name__ + '.IssueCommand', return_value=('out_', 'err_', 0) + ) as mock_issue: + redshift_local.Initialize( + redshift_local.cluster_identifier, + redshift_local.node_type, + redshift_local.node_count, + redshift_local.user, + redshift_local.password, + FakeRedshiftClusterParameterGroup(), + FakeRedshiftClusterSubnetGroup(), + ) mock_issue.assert_called_once() - mock_issue.assert_called_with([ - 'aws', '--output', 'json', '--region', 'us-east-1', 'redshift', - 'create-cluster', '--cluster-identifier', PKB_CLUSTER, - '--cluster-type', 'single-node', '--node-type', REDSHIFT_NODE_TYPE, - '--master-username', USERNAME, '--master-user-password', PASSWORD, - '--cluster-parameter-group-name', - 'fake_redshift_cluster_parameter_group', - '--cluster-subnet-group-name', 'fake_redshift_cluster_subnet_group', - '--publicly-accessible', '--automated-snapshot-retention-period=0' - ], raise_on_failure=False) + mock_issue.assert_called_with( + [ + 'aws', + '--output', + 'json', + '--region', + 'us-east-1', + 'redshift', + 'create-cluster', + '--cluster-identifier', + PKB_CLUSTER, + '--cluster-type', + 'single-node', + '--node-type', + REDSHIFT_NODE_TYPE, + '--master-username', + USERNAME, + '--master-user-password', + PASSWORD, + '--cluster-parameter-group-name', + 'fake_redshift_cluster_parameter_group', + '--cluster-subnet-group-name', + 'fake_redshift_cluster_subnet_group', + '--publicly-accessible', + '--automated-snapshot-retention-period=0', + ], + raise_on_failure=False, + ) def testMultiNodeClusterCreation(self): kwargs = copy.copy(BASE_REDSHIFT_SPEC) @@ -105,25 +125,46 @@ def testMultiNodeClusterCreation(self): spec = benchmark_config_spec._EdwServiceSpec('NAME', **kwargs) redshift_local = redshift.Redshift(spec) with mock.patch( - vm_util.__name__ + '.IssueCommand', - return_value=('out_', 'err_', 0)) as mock_issue: - redshift_local.Initialize(redshift_local.cluster_identifier, - redshift_local.node_type, - redshift_local.node_count, redshift_local.user, - redshift_local.password, - FakeRedshiftClusterParameterGroup(), - FakeRedshiftClusterSubnetGroup()) + vm_util.__name__ + '.IssueCommand', return_value=('out_', 'err_', 0) + ) as mock_issue: + redshift_local.Initialize( + redshift_local.cluster_identifier, + redshift_local.node_type, + redshift_local.node_count, + redshift_local.user, + redshift_local.password, + FakeRedshiftClusterParameterGroup(), + FakeRedshiftClusterSubnetGroup(), + ) mock_issue.assert_called_once() - mock_issue.assert_called_with([ - 'aws', '--output', 'json', '--region', 'us-east-1', 'redshift', - 'create-cluster', '--cluster-identifier', PKB_CLUSTER, - '--number-of-nodes', '2', '--node-type', REDSHIFT_NODE_TYPE, - '--master-username', USERNAME, '--master-user-password', PASSWORD, - '--cluster-parameter-group-name', - 'fake_redshift_cluster_parameter_group', - '--cluster-subnet-group-name', 'fake_redshift_cluster_subnet_group', - '--publicly-accessible', '--automated-snapshot-retention-period=0' - ], raise_on_failure=False) + mock_issue.assert_called_with( + [ + 'aws', + '--output', + 'json', + '--region', + 'us-east-1', + 'redshift', + 'create-cluster', + '--cluster-identifier', + PKB_CLUSTER, + '--number-of-nodes', + '2', + '--node-type', + REDSHIFT_NODE_TYPE, + '--master-username', + USERNAME, + '--master-user-password', + PASSWORD, + '--cluster-parameter-group-name', + 'fake_redshift_cluster_parameter_group', + '--cluster-subnet-group-name', + 'fake_redshift_cluster_subnet_group', + '--publicly-accessible', + '--automated-snapshot-retention-period=0', + ], + raise_on_failure=False, + ) if __name__ == '__main__': diff --git a/tests/providers/aws/s3_test.py b/tests/providers/aws/s3_test.py index 7aae7a84dd..f612060c39 100644 --- a/tests/providers/aws/s3_test.py +++ b/tests/providers/aws/s3_test.py @@ -1,9 +1,7 @@ """Tests for the AWS S3 service.""" - import unittest import mock - from perfkitbenchmarker import vm_util from perfkitbenchmarker.providers.aws import s3 from tests import pkb_common_test_case @@ -13,15 +11,14 @@ class S3Test(pkb_common_test_case.PkbCommonTestCase): def setUp(self): super(S3Test, self).setUp() - flag_values = { - 'timeout_minutes': 0, - 'persistent_timeout_minutes': 0} + flag_values = {'timeout_minutes': 0, 'persistent_timeout_minutes': 0} p = mock.patch.object(s3, 'FLAGS') flags_mock = p.start() flags_mock.configure_mock(**flag_values) self.mock_command = mock.patch.object(vm_util, 'IssueCommand').start() self.mock_retryable_command = mock.patch.object( - vm_util, 'IssueRetryableCommand').start() + vm_util, 'IssueRetryableCommand' + ).start() self.s3_service = s3.S3Service() self.s3_service.PrepareService(None) # will use s3.DEFAULT_AWS_REGION @@ -32,12 +29,27 @@ def tearDown(self): def test_make_bucket(self): self.mock_command.return_value = (None, None, None) self.s3_service.MakeBucket(bucket_name='test_bucket') - self.mock_command.assert_called_once_with([ - 'aws', 's3', 'mb', 's3://test_bucket', - '--region={}'.format(s3.DEFAULT_AWS_REGION)], raise_on_failure=False) + self.mock_command.assert_called_once_with( + [ + 'aws', + 's3', + 'mb', + 's3://test_bucket', + '--region={}'.format(s3.DEFAULT_AWS_REGION), + ], + raise_on_failure=False, + ) self.mock_retryable_command.assert_called_once_with([ - 'aws', 's3api', 'put-bucket-tagging', '--bucket', 'test_bucket', - '--tagging', 'TagSet=[]', '--region={}'.format(s3.DEFAULT_AWS_REGION)]) + 'aws', + 's3api', + 'put-bucket-tagging', + '--bucket', + 'test_bucket', + '--tagging', + 'TagSet=[]', + '--region={}'.format(s3.DEFAULT_AWS_REGION), + ]) + if __name__ == '__main__': unittest.main() diff --git a/tests/providers/aws/snowflake_aws_test.py b/tests/providers/aws/snowflake_aws_test.py index f66c318922..d788d09207 100644 --- a/tests/providers/aws/snowflake_aws_test.py +++ b/tests/providers/aws/snowflake_aws_test.py @@ -47,7 +47,9 @@ def setUp(self): FLAGS.cloud = 'AWS' FLAGS.run_uri = _TEST_RUN_URI FLAGS.zones = [_AWS_ZONE_US_EAST_1A] - FLAGS.snowflake_snowsql_config_override_file = 'snowflake_snowsql_config_override_file' + FLAGS.snowflake_snowsql_config_override_file = ( + 'snowflake_snowsql_config_override_file' + ) FLAGS.snowflake_connection = 'fake_connection' def testCreateRequestError(self): diff --git a/tests/providers/aws/util_test.py b/tests/providers/aws/util_test.py index e5b8f2342b..ba159f3d6e 100644 --- a/tests/providers/aws/util_test.py +++ b/tests/providers/aws/util_test.py @@ -13,7 +13,8 @@ def _MockIssueCommand(file_name_text_response): with open(path) as f: output = f.read() return mock.patch.object( - vm_util, 'IssueCommand', autospec=True, return_value=[output, None, None]) + vm_util, 'IssueCommand', autospec=True, return_value=[output, None, None] + ) class AwsUtilTest(pkb_common_test_case.PkbCommonTestCase): @@ -21,19 +22,25 @@ class AwsUtilTest(pkb_common_test_case.PkbCommonTestCase): def testGetZonesInRegion(self): test_region = 'us-east-1' self.enter_context( - _MockIssueCommand('aws-ec2-describe-availability-zones-output.json')) + _MockIssueCommand('aws-ec2-describe-availability-zones-output.json') + ) actual_zones = util.GetZonesInRegion(test_region) expected_zones = { - 'us-east-1a', 'us-east-1b', 'us-east-1c', 'us-east-1d', 'us-east-1e', - 'us-east-1f' + 'us-east-1a', + 'us-east-1b', + 'us-east-1c', + 'us-east-1d', + 'us-east-1e', + 'us-east-1f', } self.assertEqual(expected_zones, actual_zones) def testGetAllRegions(self): self.enter_context( - _MockIssueCommand('aws-ec2-describe-regions-output.json')) + _MockIssueCommand('aws-ec2-describe-regions-output.json') + ) actual_regions = util.GetAllRegions() @@ -50,7 +57,8 @@ def testGetGeoFromRegion(self): def testGetRegionsInGeo(self): self.enter_context( - _MockIssueCommand('aws-ec2-describe-regions-output.json')) + _MockIssueCommand('aws-ec2-describe-regions-output.json') + ) actual_regions = util.GetRegionsInGeo('us') diff --git a/tests/providers/azure/azure_disk_test.py b/tests/providers/azure/azure_disk_test.py index 5581c127bf..1b6b2b0c5a 100644 --- a/tests/providers/azure/azure_disk_test.py +++ b/tests/providers/azure/azure_disk_test.py @@ -1,10 +1,8 @@ """Tests for perfkitbenchmarker.tests.providers.azure_disk.""" - import unittest from absl import flags import mock - from perfkitbenchmarker.providers.azure import azure_disk from tests import pkb_common_test_case diff --git a/tests/providers/azure/azure_redis_cache_test.py b/tests/providers/azure/azure_redis_cache_test.py index 6b85e0443d..1809f2b7e7 100644 --- a/tests/providers/azure/azure_redis_cache_test.py +++ b/tests/providers/azure/azure_redis_cache_test.py @@ -15,7 +15,6 @@ import unittest from absl import flags import mock - from perfkitbenchmarker import vm_util from perfkitbenchmarker.providers.azure import azure_network from perfkitbenchmarker.providers.azure import azure_redis_cache @@ -32,8 +31,9 @@ def setUp(self): FLAGS.zones = ['eastus'] mock_spec = mock.Mock() mock_resource_group = mock.Mock() - self.resource_group_patch = mock.patch.object(azure_network, - 'GetResourceGroup').start() + self.resource_group_patch = mock.patch.object( + azure_network, 'GetResourceGroup' + ).start() self.resource_group_patch.return_value = mock_resource_group mock_resource_group.name = 'az_resource' self.redis = azure_redis_cache.AzureRedisCache(mock_spec) @@ -42,9 +42,20 @@ def setUp(self): def testCreate(self): self.mock_command.return_value = (None, '', None) expected_output = [ - 'az', 'redis', 'create', '--resource-group', 'az_resource', - '--location', 'us-central1', '--name', 'pkb-None', '--sku', 'Basic', - '--vm-size', 'C3', '--enable-non-ssl-port' + 'az', + 'redis', + 'create', + '--resource-group', + 'az_resource', + '--location', + 'us-central1', + '--name', + 'pkb-None', + '--sku', + 'Basic', + '--vm-size', + 'C3', + '--enable-non-ssl-port', ] self.redis._Create() self.mock_command.assert_called_once_with(expected_output, timeout=900) @@ -52,8 +63,14 @@ def testCreate(self): def testDelete(self): self.mock_command.return_value = (None, '', None) expected_output = [ - 'az', 'redis', 'delete', '--resource-group', 'az_resource', '--name', - 'pkb-None', '--yes' + 'az', + 'redis', + 'delete', + '--resource-group', + 'az_resource', + '--name', + 'pkb-None', + '--yes', ] self.redis._Delete() self.mock_command.assert_called_once_with(expected_output, timeout=900) @@ -61,12 +78,18 @@ def testDelete(self): def testExistTrue(self): self.mock_command.return_value = (None, '', 0) expected_output = [ - 'az', 'redis', 'show', '--resource-group', 'az_resource', '--name', - 'pkb-None' + 'az', + 'redis', + 'show', + '--resource-group', + 'az_resource', + '--name', + 'pkb-None', ] self.redis._Exists() self.mock_command.assert_called_once_with( - expected_output, raise_on_failure=False) + expected_output, raise_on_failure=False + ) if __name__ == '__main__': diff --git a/tests/providers/azure/azure_service_bus_test.py b/tests/providers/azure/azure_service_bus_test.py index 1cb12c7501..57ffb762cc 100644 --- a/tests/providers/azure/azure_service_bus_test.py +++ b/tests/providers/azure/azure_service_bus_test.py @@ -1,4 +1,5 @@ """Tests for azure_service_bus.""" + import unittest from absl import flags @@ -19,7 +20,6 @@ class AzureServiceBusTest(pkb_common_test_case.PkbCommonTestCase): - client: mock.Mock servicebus: asb.AzureServiceBus @@ -35,7 +35,8 @@ def setUp(self, resource_group_mock): def _MockIssueCommand(self, return_value): return self.enter_context( - mock.patch.object(vm_util, 'IssueCommand', return_value=return_value)) + mock.patch.object(vm_util, 'IssueCommand', return_value=return_value) + ) def testCreateTopic(self): # Don't actually issue a command. @@ -45,8 +46,12 @@ def testCreateTopic(self): self.servicebus._CreateTopic() cmd = ' '.join(cmd.call_args[0][0]) self.assertIn( - 'servicebus topic create --name ' + self.servicebus.topic_name + - ' --namespace-name ' + self.servicebus.namespace_name, cmd) + 'servicebus topic create --name ' + + self.servicebus.topic_name + + ' --namespace-name ' + + self.servicebus.namespace_name, + cmd, + ) def testTopicExists(self): # Don't actually issue a command. @@ -72,8 +77,12 @@ def testDeleteTopic(self): self.servicebus._DeleteTopic() cmd = ' '.join(cmd.call_args[0][0]) self.assertIn( - 'servicebus topic delete --name ' + self.servicebus.topic_name + - ' --namespace-name ' + self.servicebus.namespace_name, cmd) + 'servicebus topic delete --name ' + + self.servicebus.topic_name + + ' --namespace-name ' + + self.servicebus.namespace_name, + cmd, + ) def testCreateSubscription(self): # Don't actually issue a command. @@ -83,10 +92,14 @@ def testCreateSubscription(self): self.servicebus._CreateSubscription() cmd = ' '.join(cmd.call_args[0][0]) self.assertIn( - 'servicebus topic subscription create --name ' + - self.servicebus.subscription_name + ' --topic-name ' + - self.servicebus.topic_name + ' --namespace-name ' + - self.servicebus.namespace_name, cmd) + 'servicebus topic subscription create --name ' + + self.servicebus.subscription_name + + ' --topic-name ' + + self.servicebus.topic_name + + ' --namespace-name ' + + self.servicebus.namespace_name, + cmd, + ) def testSubscriptionExists(self): # Don't actually issue a command. @@ -112,10 +125,14 @@ def testDeleteSubscription(self): self.servicebus._DeleteSubscription() cmd = ' '.join(cmd.call_args[0][0]) self.assertIn( - 'servicebus topic subscription delete --name ' + - self.servicebus.subscription_name + ' --topic-name ' + - self.servicebus.topic_name + ' --namespace-name ' + - self.servicebus.namespace_name, cmd) + 'servicebus topic subscription delete --name ' + + self.servicebus.subscription_name + + ' --topic-name ' + + self.servicebus.topic_name + + ' --namespace-name ' + + self.servicebus.namespace_name, + cmd, + ) def testCreateNamespace(self): # Don't actually issue a command. @@ -125,8 +142,12 @@ def testCreateNamespace(self): self.servicebus._CreateNamespace() cmd = ' '.join(cmd.call_args[0][0]) self.assertIn( - 'servicebus namespace create --name ' + self.servicebus.namespace_name + - ' --location ' + self.servicebus.location, cmd) + 'servicebus namespace create --name ' + + self.servicebus.namespace_name + + ' --location ' + + self.servicebus.location, + cmd, + ) def testNamespaceExists(self): # Don't actually issue a command. @@ -153,7 +174,8 @@ def testDeleteNamespace(self): cmd = ' '.join(cmd.call_args[0][0]) self.assertIn( 'servicebus namespace delete --name ' + self.servicebus.namespace_name, - cmd) + cmd, + ) def testGetConnectionString(self): # Don't actually issue a command. @@ -163,16 +185,19 @@ def testGetConnectionString(self): self.servicebus._GetPrimaryConnectionString() cmd = ' '.join(cmd.call_args[0][0]) self.assertIn( - 'servicebus namespace authorization-rule keys list ' + - '--name=RootManageSharedAccessKey --namespace-name ' + - self.servicebus.namespace_name + - ' --query=primaryConnectionString -o=tsv', cmd) + 'servicebus namespace authorization-rule keys list ' + + '--name=RootManageSharedAccessKey --namespace-name ' + + self.servicebus.namespace_name + + ' --query=primaryConnectionString -o=tsv', + cmd, + ) @mock.patch.object(asb.AzureServiceBus, '_CreateNamespace') @mock.patch.object(asb.AzureServiceBus, '_CreateSubscription') @mock.patch.object(asb.AzureServiceBus, '_CreateTopic') - def testCreate(self, create_topic_mock, create_subscription_mock, - create_namespace_mock): + def testCreate( + self, create_topic_mock, create_subscription_mock, create_namespace_mock + ): self.servicebus._Create() self.assertEqual(create_namespace_mock.call_count, 1) self.assertEqual(create_subscription_mock.call_count, 1) @@ -185,20 +210,23 @@ def testPrepareClientVm(self): self.servicebus.PrepareClientVm() self.client.assert_has_calls([ mock.call.RemoteCommand( - 'sudo pip3 install azure-servicebus', ignore_failure=False), + 'sudo pip3 install azure-servicebus', ignore_failure=False + ), mock.call.RemoteCommand( - 'mkdir -p ~/perfkitbenchmarker/scripts/messaging_service_scripts/azure' + 'mkdir -p' + ' ~/perfkitbenchmarker/scripts/messaging_service_scripts/azure' ), mock.call.PushDataFile( 'messaging_service_scripts/azure/__init__.py', - '~/perfkitbenchmarker/scripts/messaging_service_scripts/azure/__init__.py' + '~/perfkitbenchmarker/scripts/messaging_service_scripts/azure/__init__.py', ), mock.call.RemoteCommand( - 'mkdir -p ~/perfkitbenchmarker/scripts/messaging_service_scripts/azure' + 'mkdir -p' + ' ~/perfkitbenchmarker/scripts/messaging_service_scripts/azure' ), mock.call.PushDataFile( 'messaging_service_scripts/azure/azure_service_bus_client.py', - '~/perfkitbenchmarker/scripts/messaging_service_scripts/azure/azure_service_bus_client.py' + '~/perfkitbenchmarker/scripts/messaging_service_scripts/azure/azure_service_bus_client.py', ), mock.call.PushDataFile('messaging_service_scripts/azure_benchmark.py'), ]) @@ -206,30 +234,33 @@ def testPrepareClientVm(self): @mock.patch.object( asb.AzureServiceBus, '_GetPrimaryConnectionString', - return_value='mocked_string') + return_value='mocked_string', + ) def testRun(self, get_connection_string_mock): - return_value = ['{"mock1": 1}', None] self.client.RemoteCommand.return_value = return_value remote_run_cmd = ( - f'python3 -m azure_benchmark ' + 'python3 -m azure_benchmark ' f'--topic_name={self.servicebus.topic_name} ' f'--subscription_name={self.servicebus.subscription_name} ' f'--benchmark_scenario={BENCHMARK_SCENARIO} ' f'--number_of_messages={NUMBER_OF_MESSAGES} ' f'--message_size={MESSAGE_SIZE} ' f'--warmup_messages={WARMUP_MESSAGES} ' - f'--connection_str="mocked_string"') + '--connection_str="mocked_string"' + ) - self.servicebus.Run(BENCHMARK_SCENARIO, NUMBER_OF_MESSAGES, MESSAGE_SIZE, - WARMUP_MESSAGES) + self.servicebus.Run( + BENCHMARK_SCENARIO, NUMBER_OF_MESSAGES, MESSAGE_SIZE, WARMUP_MESSAGES + ) self.client.RemoteCommand.assert_called_with(remote_run_cmd) @mock.patch.object(asb.AzureServiceBus, '_DeleteNamespace') @mock.patch.object(asb.AzureServiceBus, '_DeleteSubscription') @mock.patch.object(asb.AzureServiceBus, '_DeleteTopic') - def testDelete(self, delete_topic_mock, delete_subscription_mock, - delete_namespace_mock): + def testDelete( + self, delete_topic_mock, delete_subscription_mock, delete_namespace_mock + ): self.servicebus._Delete() self.assertEqual(delete_namespace_mock.call_count, 1) self.assertEqual(delete_subscription_mock.call_count, 1) diff --git a/tests/providers/azure/azure_util_test.py b/tests/providers/azure/azure_util_test.py index 0e76e81443..1e02b9ce01 100644 --- a/tests/providers/azure/azure_util_test.py +++ b/tests/providers/azure/azure_util_test.py @@ -1,7 +1,7 @@ """Tests for perfkitbenchmarker.providers.azure.util.""" + import unittest import mock - from perfkitbenchmarker import vm_util from perfkitbenchmarker.providers.azure import util from tests import pkb_common_test_case @@ -12,7 +12,8 @@ def _MockIssueCommand(file_name_text_response): with open(path) as f: output = f.read() return mock.patch.object( - vm_util, 'IssueCommand', autospec=True, return_value=[output, None, None]) + vm_util, 'IssueCommand', autospec=True, return_value=[output, None, None] + ) class AzureUtilTest(pkb_common_test_case.PkbCommonTestCase): @@ -24,13 +25,11 @@ def setUp(self): def test_get_region_from_zone_valid_region(self): valid_region = 'eastus2' - self.assertEqual(self.expected_region, - util.GetRegionFromZone(valid_region)) + self.assertEqual(self.expected_region, util.GetRegionFromZone(valid_region)) def test_get_region_from_zone_valid_zone(self): valid_zone = 'eastus2-1' - self.assertEqual(self.expected_region, - util.GetRegionFromZone(valid_zone)) + self.assertEqual(self.expected_region, util.GetRegionFromZone(valid_zone)) def test_get_region_from_zone_invalid_region(self): valid_region = 'us-east2' @@ -48,8 +47,10 @@ def test_get_availability_zone_from_zone_valid_region(self): def test_get_availability_zone_from_zone_valid_zone(self): valid_zone = 'eastus2-1' - self.assertEqual(self.expected_availability_zone, - util.GetAvailabilityZoneFromZone(valid_zone)) + self.assertEqual( + self.expected_availability_zone, + util.GetAvailabilityZoneFromZone(valid_zone), + ) def test_get_availability_zone_from_zone_invalid_zone(self): valid_region = 'eastus2-1a' @@ -64,7 +65,8 @@ def test_get_zones_in_region(self): def test_get_all_regions(self): self.enter_context( - _MockIssueCommand('az-account-list-locations-output.json')) + _MockIssueCommand('az-account-list-locations-output.json') + ) found_regions = util.GetAllRegions() @@ -76,7 +78,9 @@ def test_get_all_zones(self): util, 'GetAllRegions', autospec=True, - return_value={'eastus', 'eastus2'})) + return_value={'eastus', 'eastus2'}, + ) + ) found_regions = util.GetAllZones() @@ -95,9 +99,14 @@ def test_get_geo_from_region(self): "US" ] """ - self.enter_context(mock.patch.object(vm_util, 'IssueRetryableCommand', - autospec=True, - return_value=[test_output, None])) + self.enter_context( + mock.patch.object( + vm_util, + 'IssueRetryableCommand', + autospec=True, + return_value=[test_output, None], + ) + ) found_geo = util.GetGeoFromRegion('test_region') @@ -106,11 +115,13 @@ def test_get_geo_from_region(self): def test_get_region_in_geo(self): self.enter_context( - _MockIssueCommand('az-account-list-locations-output.json')) + _MockIssueCommand('az-account-list-locations-output.json') + ) found_regions = util.GetRegionsInGeo('test_geo') self.assertEqual({'eastus', 'eastus2'}, found_regions) + if __name__ == '__main__': unittest.main() diff --git a/tests/providers/azure/azure_virtual_machine_test.py b/tests/providers/azure/azure_virtual_machine_test.py index dfdfecab8e..94bf1db8fb 100644 --- a/tests/providers/azure/azure_virtual_machine_test.py +++ b/tests/providers/azure/azure_virtual_machine_test.py @@ -1,11 +1,10 @@ """Tests for perfkitbenchmarker.tests.providers.azure.azure_virtual_machine.""" + import unittest from absl.testing import parameterized import mock - from perfkitbenchmarker import errors from perfkitbenchmarker import vm_util - from perfkitbenchmarker.providers.azure import azure_virtual_machine from perfkitbenchmarker.providers.azure import util from tests import pkb_common_test_case @@ -13,8 +12,9 @@ _COMPONENT = 'test_component' -class TestAzureVirtualMachine(pkb_common_test_case.TestOsMixin, - azure_virtual_machine.AzureVirtualMachine): +class TestAzureVirtualMachine( + pkb_common_test_case.TestOsMixin, azure_virtual_machine.AzureVirtualMachine +): IMAGE_URN = 'test_image_urn' @@ -23,93 +23,112 @@ class AzureVirtualMachineTest(pkb_common_test_case.PkbCommonTestCase): def setUp(self): super(AzureVirtualMachineTest, self).setUp() self.enter_context( - mock.patch(azure_virtual_machine.__name__ + - '.azure_network.AzureNetwork.GetNetwork')) + mock.patch( + azure_virtual_machine.__name__ + + '.azure_network.AzureNetwork.GetNetwork' + ) + ) self.enter_context( - mock.patch(azure_virtual_machine.__name__ + - '.azure_network.AzureFirewall.GetFirewall')) + mock.patch( + azure_virtual_machine.__name__ + + '.azure_network.AzureFirewall.GetFirewall' + ) + ) self.enter_context( - mock.patch(azure_virtual_machine.__name__ + - '.azure_network.GetResourceGroup')) + mock.patch( + azure_virtual_machine.__name__ + '.azure_network.GetResourceGroup' + ) + ) self.mock_cmd = self.enter_context( - mock.patch.object(vm_util, 'IssueCommand')) + mock.patch.object(vm_util, 'IssueCommand') + ) self.enter_context(mock.patch.object(util, 'GetResourceTags')) @parameterized.named_parameters( { 'testcase_name': 'QuotaExceeded', 'stderror': 'Error Code: QuotaExceeded', - 'expected_error': errors.Benchmarks.QuotaFailure + 'expected_error': errors.Benchmarks.QuotaFailure, }, { 'testcase_name': 'CoreQuotaExceeded', - 'stderror': + 'stderror': ( 'Operation could not be completed as it results in exceeding ' - 'approved standardEv3Family Cores quota', - 'expected_error': errors.Benchmarks.QuotaFailure + 'approved standardEv3Family Cores quota' + ), + 'expected_error': errors.Benchmarks.QuotaFailure, }, { 'testcase_name': 'CoreQuotaExceededDifferentWording', - 'stderror': + 'stderror': ( 'The operation could not be completed as it results in exceeding ' - 'quota limit of standardEv3Family Cores', + 'quota limit of standardEv3Family Cores' + ), 'expected_error': errors.Benchmarks.QuotaFailure, }, { 'testcase_name': 'FamilyQuotaExceededWording', - 'stderror': + 'stderror': ( 'Operation could not be completed as it results in exceeding ' - 'approved Standard NDASv4_A100 Family Cores quota', - 'expected_error': errors.Benchmarks.QuotaFailure + 'approved Standard NDASv4_A100 Family Cores quota' + ), + 'expected_error': errors.Benchmarks.QuotaFailure, }, { 'testcase_name': 'UnavailableInRegion', - 'stderror': + 'stderror': ( 'The requested VM size Standard_D2s_v5 is not available in the ' - 'current region. The sizes available in the current region are', - 'expected_error': errors.Benchmarks.UnsupportedConfigError + 'current region. The sizes available in the current region are' + ), + 'expected_error': errors.Benchmarks.UnsupportedConfigError, }, { 'testcase_name': 'UnavailableInZone', - 'stderror': + 'stderror': ( "The requested VM size 'Standard_D2s_v5' is not available in the " - "current availability zone. The sizes available in the current " - "availability zone are", - 'expected_error': errors.Benchmarks.UnsupportedConfigError + 'current availability zone. The sizes available in the current ' + 'availability zone are' + ), + 'expected_error': errors.Benchmarks.UnsupportedConfigError, }, { 'testcase_name': 'UnsupportedSku', - 'stderror': - "The requested resource is currently not available in location " - "'eastus' zones '1' for subscription", - 'expected_error': errors.Benchmarks.UnsupportedConfigError + 'stderror': ( + 'The requested resource is currently not available in location ' + "'eastus' zones '1' for subscription" + ), + 'expected_error': errors.Benchmarks.UnsupportedConfigError, }, { - 'testcase_name': - 'ProvisioningTimedOut', - 'stderror': - '\"code\": \"OSProvisioningTimedOut\",\r\n \"message\": ' - '\"OS Provisioning for VM \'pkb-f1e4bae672b8-0\' did not finish ' + 'testcase_name': 'ProvisioningTimedOut', + 'stderror': ( + '"code": "OSProvisioningTimedOut",\r\n "message": ' + "\"OS Provisioning for VM 'pkb-f1e4bae672b8-0' did not finish " 'in the allotted time. The VM may still finish provisioning ' - 'successfully. Please check provisioning state later.', - 'expected_error': - errors.Resource.ProvisionTimeoutError - }, { + 'successfully. Please check provisioning state later.' + ), + 'expected_error': errors.Resource.ProvisionTimeoutError, + }, + { 'testcase_name': 'SkuNotAvailable', 'stderror': """{"error":{"code":"InvalidTemplateDeployment","message":"The template deployment 'vm_deploy_gwUdt7Sseaortu3nIvgZPN8rzgVucSOL' is not valid according to the validation procedure. The tracking id is '8b0552ca-7af5-4155-b653-8eb5f8713629'. See inner errors for details.","details":[{"code":"SkuNotAvailable","message":"The requested VM size for resource 'Following SKUs have failed for Capacity Restrictions: Standard_D2s_v4' is currently not available in location 'westus2'. Please try another size or deploy to a different location or different zone. See https://aka.ms/azureskunotavailable for details."}]}}""", # pylint: disable=line-too-long - 'expected_error': errors.Benchmarks.UnsupportedConfigError}, + 'expected_error': errors.Benchmarks.UnsupportedConfigError, + }, { 'testcase_name': 'ZonalAllocationFailed', 'stderror': """{"status":"Failed","error":{"code":"DeploymentFailed","message":"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/DeployOperations for usage details.","details":[{"code":"Conflict","message":"{\r\n \"status\": \"Failed\",\r\n \"error\": {\r\n \"code\": \"ResourceDeploymentFailure\",\r\n \"message\": \"The resource operation completed with terminal provisioning state 'Failed'.\",\r\n \"details\": [\r\n {\r\n \"code\": \"ZonalAllocationFailed\",\r\n \"message\": \"Allocation failed. We do not have sufficient capacity for the requested VM size in this zone. Read more about improving likelihood of allocation success at http://aka.ms/allocation-guidance\"\r\n }\r\n ]\r\n }\r\n}"}]}}""", # pylint: disable=line-too-long - 'expected_error': errors.Benchmarks.InsufficientCapacityCloudFailure - }, {'testcase_name': 'OverconstrainedZonalAllocationRequest', + 'expected_error': errors.Benchmarks.InsufficientCapacityCloudFailure, + }, + { + 'testcase_name': 'OverconstrainedZonalAllocationRequest', 'stderror': """{"status":"Failed","error":{"code":"DeploymentFailed","message":"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/DeployOperations for usage details.","details":[{"code":"Conflict","message":"{\r\n \"status\": \"Failed\",\r\n \"error\": {\r\n \"code\": \"ResourceDeploymentFailure\",\r\n \"message\": \"The resource operation completed with terminal provisioning state 'Failed'.\",\r\n \"details\": [\r\n {\r\n \"code\": \"OverconstrainedZonalAllocationRequest\",\r\n \"message\": \"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\n - Availability Zone\\n - VM Size\\n\"\r\n }\r\n ]\r\n }\r\n}"}]}}""", # pylint: disable=line-too-long - 'expected_error': errors.Benchmarks.UnsupportedConfigError - } + 'expected_error': errors.Benchmarks.UnsupportedConfigError, + }, ) def testVmCreationError(self, stderror, expected_error): spec = azure_virtual_machine.AzureVmSpec( - _COMPONENT, machine_type='test_machine_type', zone='testing') + _COMPONENT, machine_type='test_machine_type', zone='testing' + ) vm = TestAzureVirtualMachine(spec) self.mock_cmd.side_effect = [('', stderror, 1)] @@ -118,8 +137,11 @@ def testVmCreationError(self, stderror, expected_error): def testInsufficientSpotCapacity(self): spec = azure_virtual_machine.AzureVmSpec( - _COMPONENT, machine_type='test_machine_type', zone='testing', - low_priority=True) + _COMPONENT, + machine_type='test_machine_type', + zone='testing', + low_priority=True, + ) vm = TestAzureVirtualMachine(spec) self.mock_cmd.side_effect = [('', 'OverconstrainedAllocationRequest', 1)] @@ -132,19 +154,26 @@ class AzurePublicIPAddressTest(pkb_common_test_case.PkbCommonTestCase): def setUp(self): super(AzurePublicIPAddressTest, self).setUp() self.enter_context( - mock.patch(azure_virtual_machine.__name__ + - '.azure_network.GetResourceGroup')) + mock.patch( + azure_virtual_machine.__name__ + '.azure_network.GetResourceGroup' + ) + ) self.mock_cmd = self.enter_context( - mock.patch.object(vm_util, 'IssueCommand')) + mock.patch.object(vm_util, 'IssueCommand') + ) self.ip_address = azure_virtual_machine.AzurePublicIPAddress( - 'westus2', None, 'test_ip') + 'westus2', None, 'test_ip' + ) def testQuotaExceeded(self): - quota_error = ('ERROR: Cannot create more than 20 public IP addresses for ' - 'this subscription in this region.') + quota_error = ( + 'ERROR: Cannot create more than 20 public IP addresses for ' + 'this subscription in this region.' + ) self.mock_cmd.side_effect = [('', quota_error, 1)] with self.assertRaises(errors.Benchmarks.QuotaFailure): self.ip_address._Create() + if __name__ == '__main__': unittest.main() diff --git a/tests/providers/azure/snowflake_azure_test.py b/tests/providers/azure/snowflake_azure_test.py index 314a297e6f..57299050fa 100644 --- a/tests/providers/azure/snowflake_azure_test.py +++ b/tests/providers/azure/snowflake_azure_test.py @@ -47,7 +47,9 @@ def setUp(self): FLAGS.cloud = 'AZURE' FLAGS.run_uri = _TEST_RUN_URI FLAGS.zones = [_AZURE_ZONE_EAST_US_2] - FLAGS.snowflake_snowsql_config_override_file = 'snowflake_snowsql_config_override_file' + FLAGS.snowflake_snowsql_config_override_file = ( + 'snowflake_snowsql_config_override_file' + ) FLAGS.snowflake_connection = 'fake_connection' def testCreateRequestError(self): diff --git a/tests/providers/digitalocean/do_test.py b/tests/providers/digitalocean/do_test.py index 1dea63b430..d89daafb98 100644 --- a/tests/providers/digitalocean/do_test.py +++ b/tests/providers/digitalocean/do_test.py @@ -15,40 +15,46 @@ import unittest import mock - from perfkitbenchmarker import vm_util from perfkitbenchmarker.providers.digitalocean import util class TestDoctlAndParse(unittest.TestCase): + def testCommandSucceeds(self): - with mock.patch(vm_util.__name__ + '.IssueCommand', - return_value=('{"a": 1, "b": 2}', '', 0)): + with mock.patch( + vm_util.__name__ + '.IssueCommand', + return_value=('{"a": 1, "b": 2}', '', 0), + ): response, retval = util.DoctlAndParse(['foo', 'bar', 'baz']) self.assertEqual(response, {'a': 1, 'b': 2}) self.assertEqual(retval, 0) def testCommandFailsWithNull(self): - with mock.patch(vm_util.__name__ + '.IssueCommand', - return_value=( - 'null{"errors": [{"detail": "foo"}]}', '', 1)): + with mock.patch( + vm_util.__name__ + '.IssueCommand', + return_value=('null{"errors": [{"detail": "foo"}]}', '', 1), + ): response, retval = util.DoctlAndParse(['foo', 'bar', 'baz']) self.assertEqual(response, {'errors': [{'detail': 'foo'}]}) self.assertEqual(retval, 1) def testCommandFailsWithoutNull(self): - with mock.patch(vm_util.__name__ + '.IssueCommand', - return_value=('{"errors": [{"detail": "foo"}]}', '', 1)): + with mock.patch( + vm_util.__name__ + '.IssueCommand', + return_value=('{"errors": [{"detail": "foo"}]}', '', 1), + ): response, retval = util.DoctlAndParse(['foo', 'bar', 'baz']) self.assertEqual(response, {'errors': [{'detail': 'foo'}]}) self.assertEqual(retval, 1) def testCommandSucceedsNoOutput(self): - with mock.patch(vm_util.__name__ + '.IssueCommand', - return_value=('', '', 0)): + with mock.patch( + vm_util.__name__ + '.IssueCommand', return_value=('', '', 0) + ): response, retval = util.DoctlAndParse(['foo', 'bar', 'baz']) self.assertEqual(response, None) diff --git a/tests/providers/gcp/bigquery_slot_resource_test.py b/tests/providers/gcp/bigquery_slot_resource_test.py index 8eea3e8894..7e0f933c91 100644 --- a/tests/providers/gcp/bigquery_slot_resource_test.py +++ b/tests/providers/gcp/bigquery_slot_resource_test.py @@ -21,7 +21,6 @@ def GetBigquerySlotsTestInstance(spec=TEST_SPEC): class BigquerySlotsTestCase(pkb_common_test_case.PkbCommonTestCase): - SLOT_NUM = 500 TEST_PROJECT = 'test_project' TEST_REGION = 'aws-fake-region' @@ -110,5 +109,6 @@ def testExistsNoCommitments(self, mock_issue): self.assertIn(f'--project_id={self.TEST_PROJECT}', command_string) self.assertIn(f'--location={self.TEST_REGION}', command_string) + if __name__ == '__main__': unittest.main() diff --git a/tests/providers/gcp/bigquery_test.py b/tests/providers/gcp/bigquery_test.py index eade1d261c..93510096de 100644 --- a/tests/providers/gcp/bigquery_test.py +++ b/tests/providers/gcp/bigquery_test.py @@ -28,7 +28,7 @@ _BASE_BIGQUERY_SPEC = { 'type': 'bigquery', - 'cluster_identifier': 'bigquerypkb.tpcds_100G' + 'cluster_identifier': 'bigquerypkb.tpcds_100G', } FLAGS = flags.FLAGS @@ -48,10 +48,12 @@ def __init__(self): self.valid_install_package_list = ['pip', 'google_cloud_sdk'] self.valid_remote_command_list = [ 'sudo pip install absl-py', - '/tmp/pkb/google-cloud-sdk/bin/gcloud auth activate-service-account ' - 'SERVICE_ACCOUNT --key-file=SERVICE_ACCOUNT_KEY_FILE', + ( + '/tmp/pkb/google-cloud-sdk/bin/gcloud auth activate-service-account' + ' SERVICE_ACCOUNT --key-file=SERVICE_ACCOUNT_KEY_FILE' + ), 'chmod 755 script_runner.sh', - 'echo "\nMaxSessions 100" | sudo tee -a /etc/ssh/sshd_config' + 'echo "\nMaxSessions 100" | sudo tee -a /etc/ssh/sshd_config', ] def Install(self, package_name): @@ -62,8 +64,9 @@ def RemoteCommand(self, command): if command not in self.valid_remote_command_list: raise RuntimeError - def InstallPreprovisionedPackageData(self, package_name, filenames, - install_path): + def InstallPreprovisionedPackageData( + self, package_name, filenames, install_path + ): if package_name != 'PACKAGE_NAME': raise RuntimeError @@ -78,9 +81,10 @@ def RemoteCommand(self, command): if command == 'echo "\nMaxSessions 100" | sudo tee -a /etc/ssh/sshd_config': return None, None - expected_command = ('python script_driver.py --script={} --bq_project_id={}' - ' --bq_dataset_id={}').format(QUERY_NAME, PROJECT_ID, - DATASET_ID) + expected_command = ( + 'python script_driver.py --script={} --bq_project_id={}' + ' --bq_dataset_id={}' + ).format(QUERY_NAME, PROJECT_ID, DATASET_ID) if command != expected_command: raise RuntimeError response_object = {QUERY_NAME: {'job_id': 'JOB_ID', 'execution_time': 1.0}} @@ -104,8 +108,9 @@ def RemoteCommand(self, command): else: raise RuntimeError - def InstallPreprovisionedPackageData(self, package_name, filenames, - install_path): + def InstallPreprovisionedPackageData( + self, package_name, filenames, install_path + ): if package_name != 'PACKAGE_NAME': raise RuntimeError @@ -117,15 +122,18 @@ def RemoteCommand(self, command): if command == 'echo "\nMaxSessions 100" | sudo tee -a /etc/ssh/sshd_config': return None, None - expected_command = ('java -cp bq-java-client-2.7.jar ' - 'com.google.cloud.performance.edw.Single --project {} ' - '--credentials_file {} --dataset {} --query_file ' - '{}').format(PROJECT_ID, 'SERVICE_ACCOUNT_KEY_FILE', - DATASET_ID, QUERY_NAME) + expected_command = ( + 'java -cp bq-java-client-2.7.jar ' + 'com.google.cloud.performance.edw.Single --project {} ' + '--credentials_file {} --dataset {} --query_file ' + '{}' + ).format(PROJECT_ID, 'SERVICE_ACCOUNT_KEY_FILE', DATASET_ID, QUERY_NAME) if command != expected_command: raise RuntimeError - response_object = {'query_wall_time_in_secs': 1.0, - 'details': {'job_id': 'JOB_ID'}} + response_object = { + 'query_wall_time_in_secs': 1.0, + 'details': {'job_id': 'JOB_ID'}, + } response = json.dumps(response_object) return response, None @@ -206,7 +214,8 @@ def testJavaClientInterfaceExecuteQuery(self): self.assertIsInstance(interface, bigquery.JavaClientInterface) bm_spec = FakeBenchmarkSpec( - FakeRemoteVMForJavaClientInterfaceExecuteQuery()) + FakeRemoteVMForJavaClientInterfaceExecuteQuery() + ) interface.SetProvisionedAttributes(bm_spec) performance, details = interface.ExecuteQuery(QUERY_NAME) self.assertEqual(performance, 1.0) diff --git a/tests/providers/gcp/gce_placement_group_test.py b/tests/providers/gcp/gce_placement_group_test.py index f36fde0849..ba0e0b6374 100644 --- a/tests/providers/gcp/gce_placement_group_test.py +++ b/tests/providers/gcp/gce_placement_group_test.py @@ -4,7 +4,6 @@ import unittest from absl import flags import mock - from perfkitbenchmarker import benchmark_spec from perfkitbenchmarker import errors from perfkitbenchmarker import placement_group @@ -21,26 +20,31 @@ _STRATEGY = placement_group.PLACEMENT_GROUP_CLOSEST_SUPPORTED # The GcePlacementGroup._Create() -_CREATE_RESPONSE = (json.dumps({ - 'creationTimestamp': '2020-03-16T15:31:23.802-07:00', - 'description': 'PKB: test', - 'groupPlacementPolicy': { - 'collocation': 'COLLOCATED', - 'vmCount': 2 - }, - 'id': '123', - 'kind': 'compute#resourcePolicy', - 'name': 'perfkit-test', - 'region': 'https://www.googleapis.com/test', - 'selfLink': 'https://www.googleapis.com/test', - 'status': 'READY' -}), '', 0) +_CREATE_RESPONSE = ( + json.dumps({ + 'creationTimestamp': '2020-03-16T15:31:23.802-07:00', + 'description': 'PKB: test', + 'groupPlacementPolicy': {'collocation': 'COLLOCATED', 'vmCount': 2}, + 'id': '123', + 'kind': 'compute#resourcePolicy', + 'name': 'perfkit-test', + 'region': 'https://www.googleapis.com/test', + 'selfLink': 'https://www.googleapis.com/test', + 'status': 'READY', + }), + '', + 0, +) _QUOTA_FAILURE_RESPONSE = ( '', - 'ERROR: (gcloud.alpha.compute.resource-policies.create.group-placement) ' - 'Could not fetch resource: - Quota \'RESOURCE_POLICIES\' exceeded. Limit: ' - '10.0 in region europe-west4.', 1) + ( + 'ERROR: (gcloud.alpha.compute.resource-policies.create.group-placement)' + " Could not fetch resource: - Quota 'RESOURCE_POLICIES' exceeded. " + ' Limit: 10.0 in region europe-west4.' + ), + 1, +) # The GcePlacementGroup._Exists() response done after a Create() call. _EXISTS_RESPONSE = json.dumps({'status': 'ACTIVE'}), '', 0 # The GcePlacementGroup._Exists() response done after a Delete() call. @@ -53,7 +57,8 @@ def _BenchmarkSpec(num_vms=2, benchmark='cluster_boot'): # Creates a fake BenchmarkSpec() that will be the response for calls to # perfkitbenchmarker.context.GetThreadBenchmarkSpec(). config_spec = benchmark_config_spec.BenchmarkConfigSpec( - benchmark, flag_values=FLAGS) + benchmark, flag_values=FLAGS + ) config_spec.vm_groups = {'x': mock.Mock(vm_count=num_vms, static_vms=[])} bm_module = mock.MagicMock(BENCHMARK_NAME=benchmark) bm_spec = benchmark_spec.BenchmarkSpec(bm_module, config_spec, 'uid') @@ -67,12 +72,14 @@ def _CreateGcePlacementGroupSpec(group_style=_STRATEGY): zone=_ZONE, project=_PROJECT, num_vms=2, - flag_values=FLAGS) + flag_values=FLAGS, + ) def _CreateGcePlacementGroup(group_style=_STRATEGY): return gce_placement_group.GcePlacementGroup( - _CreateGcePlacementGroupSpec(group_style)) + _CreateGcePlacementGroupSpec(group_style) + ) class GcePlacementGroupTest(pkb_common_test_case.PkbCommonTestCase): @@ -80,31 +87,59 @@ class GcePlacementGroupTest(pkb_common_test_case.PkbCommonTestCase): def setUp(self): super(GcePlacementGroupTest, self).setUp() self.mock_cmd = self.enter_context( - mock.patch.object(vm_util, 'IssueCommand')) + mock.patch.object(vm_util, 'IssueCommand') + ) # Register a fake benchmark _BenchmarkSpec(2) def testPlacementGroupCreate(self): self.mock_cmd.return_value = _CREATE_RESPONSE _CreateGcePlacementGroup()._Create() - self.mock_cmd.assert_called_with([ - 'gcloud', 'compute', 'resource-policies', 'create', - 'group-placement', _PLACEMENT_GROUP_NAME, '--collocation', 'COLLOCATED', - '--format', 'json', - '--project', 'myproject', '--quiet', '--region', 'us-east1', - '--vm-count', '2' - ], raise_on_failure=False) + self.mock_cmd.assert_called_with( + [ + 'gcloud', + 'compute', + 'resource-policies', + 'create', + 'group-placement', + _PLACEMENT_GROUP_NAME, + '--collocation', + 'COLLOCATED', + '--format', + 'json', + '--project', + 'myproject', + '--quiet', + '--region', + 'us-east1', + '--vm-count', + '2', + ], + raise_on_failure=False, + ) def testPlacementGroupDelete(self): self.mock_cmd.side_effect = [_EXISTS_RESPONSE, _DELETE_RESPONSE] pg = _CreateGcePlacementGroup() pg.created = True pg.Delete() - self.mock_cmd.assert_called_with([ - 'gcloud', 'compute', 'resource-policies', 'describe', - _PLACEMENT_GROUP_NAME, '--format', 'json', '--project', 'myproject', - '--quiet', '--region', 'us-east1' - ], raise_on_failure=False) + self.mock_cmd.assert_called_with( + [ + 'gcloud', + 'compute', + 'resource-policies', + 'describe', + _PLACEMENT_GROUP_NAME, + '--format', + 'json', + '--project', + 'myproject', + '--quiet', + '--region', + 'us-east1', + ], + raise_on_failure=False, + ) def testPlacementGroupQuotaFailure(self): self.mock_cmd.return_value = _QUOTA_FAILURE_RESPONSE diff --git a/tests/providers/gcp/gcp_bigtable_test.py b/tests/providers/gcp/gcp_bigtable_test.py index 182947dbf9..240201de05 100644 --- a/tests/providers/gcp/gcp_bigtable_test.py +++ b/tests/providers/gcp/gcp_bigtable_test.py @@ -104,7 +104,8 @@ def GetTestBigtableInstance(spec=_TEST_BENCHMARK_SPEC): test_benchmark_spec = pkb_common_test_case.CreateBenchmarkSpecFromYaml( - yaml_string=spec, benchmark_name='cloud_bigtable_ycsb') + yaml_string=spec, benchmark_name='cloud_bigtable_ycsb' + ) test_benchmark_spec.ConstructNonRelationalDb() return test_benchmark_spec.non_relational_db @@ -116,14 +117,16 @@ def setUp(self): self.bigtable = GetTestBigtableInstance() def testNotFoundTable(self): - with mock.patch.object(util.GcloudCommand, 'Issue', - return_value=('', '', 1)): + with mock.patch.object( + util.GcloudCommand, 'Issue', return_value=('', '', 1) + ): self.assertFalse(self.bigtable._Exists()) def testFoundTable(self): stdout = VALID_JSON_BASE.format(project=PROJECT, name=NAME) - with mock.patch.object(util.GcloudCommand, 'Issue', - return_value=(stdout, '', 0)): + with mock.patch.object( + util.GcloudCommand, 'Issue', return_value=(stdout, '', 0) + ): self.assertTrue(self.bigtable._Exists()) def testQuotaError(self): @@ -131,7 +134,9 @@ def testQuotaError(self): mock.patch.object( util.GcloudCommand, 'Issue', - return_value=[None, OUT_OF_QUOTA_STDERR, None])) + return_value=[None, OUT_OF_QUOTA_STDERR, None], + ) + ) with self.assertRaises(errors.Benchmarks.QuotaFailure): self.bigtable._Create() @@ -181,7 +186,9 @@ def testGetResourceMetadataUserManaged(self): mock.patch.object( gcp_bigtable, 'GetClustersDescription', - return_value=mock_get_cluster_output)) + return_value=mock_get_cluster_output, + ) + ) # Act actual_metadata = instance.GetResourceMetadata() @@ -208,7 +215,7 @@ def testGetResourceMetadata(self): 'bigtable_replication_zone': 'parsed_rep_zone', 'bigtable_storage_type': 'ssd', 'bigtable_node_count': 3, - 'bigtable_multicluster_routing': True + 'bigtable_multicluster_routing': True, } self.assertEqual(actual_metadata, expected_metadata) @@ -235,12 +242,18 @@ def testBuildClusterConfigsWithFlags(self): actual_flag_values = bigtable._BuildClusterConfigs() # Assert - expected_flag_values = [('id=test_name-0,zone=test_zone,' - 'autoscaling-min-nodes=1,autoscaling-max-nodes=5,' - 'autoscaling-cpu-target=50'), - ('id=test_name-1,zone=test_replication_zone,' - 'autoscaling-min-nodes=1,autoscaling-max-nodes=5,' - 'autoscaling-cpu-target=50')] + expected_flag_values = [ + ( + 'id=test_name-0,zone=test_zone,' + 'autoscaling-min-nodes=1,autoscaling-max-nodes=5,' + 'autoscaling-cpu-target=50' + ), + ( + 'id=test_name-1,zone=test_replication_zone,' + 'autoscaling-min-nodes=1,autoscaling-max-nodes=5,' + 'autoscaling-cpu-target=50' + ), + ] self.assertEqual(actual_flag_values, expected_flag_values) @flagsaver.flagsaver(run_uri='test_uri') @@ -261,12 +274,17 @@ def testUpdateLabels(self): mock.patch.object( util.GcloudCommand, 'Issue', - return_value=(mock_json_response, '', 0))) + return_value=(mock_json_response, '', 0), + ) + ) self.enter_context( - mock.patch.object(util, 'GetAccessToken', return_value='test_token')) + mock.patch.object(util, 'GetAccessToken', return_value='test_token') + ) mock_request = self.enter_context( mock.patch.object( - requests, 'patch', return_value=mock.Mock(status_code=200))) + requests, 'patch', return_value=mock.Mock(status_code=200) + ) + ) # Act new_labels = { @@ -287,9 +305,10 @@ def testUpdateLabels(self): 'labels': { 'benchmark': 'test_benchmark_2', 'timeout_minutes': '10', - 'metadata': 'test_metadata' + 'metadata': 'test_metadata', } - }) + }, + ) def testBigtableGcloudCommand(self): bigtable = GetTestBigtableInstance() diff --git a/tests/providers/gcp/gcp_cloud_redis_test.py b/tests/providers/gcp/gcp_cloud_redis_test.py index 6310caeee7..d41b42c7b2 100644 --- a/tests/providers/gcp/gcp_cloud_redis_test.py +++ b/tests/providers/gcp/gcp_cloud_redis_test.py @@ -27,7 +27,8 @@ class GcpCloudRedisTestCase(pkb_common_test_case.PkbCommonTestCase): def setUp(self): super(GcpCloudRedisTestCase, self).setUp() with mock.patch.object( - util.GcloudCommand, 'Issue', return_value=('{}', '', 0)): + util.GcloudCommand, 'Issue', return_value=('{}', '', 0) + ): FLAGS.project = 'project' FLAGS.zone = ['us-central1-a'] mock_spec = mock.Mock() @@ -35,25 +36,29 @@ def setUp(self): def testCreate(self): with mock.patch.object( - util.GcloudCommand, 'Issue', return_value=('{}', '', 0)) as gcloud: + util.GcloudCommand, 'Issue', return_value=('{}', '', 0) + ) as gcloud: self.redis._Create() gcloud.assert_called_once_with(timeout=1200) self.assertTrue(self.redis._Exists()) def testDelete(self): with mock.patch.object( - util.GcloudCommand, 'Issue', return_value=('{}', '', 0)) as gcloud: + util.GcloudCommand, 'Issue', return_value=('{}', '', 0) + ) as gcloud: self.redis._Delete() gcloud.assert_called_with(raise_on_failure=False, timeout=1200) def testExistTrue(self): with mock.patch.object( - util.GcloudCommand, 'Issue', return_value=('{}', '', 0)): + util.GcloudCommand, 'Issue', return_value=('{}', '', 0) + ): self.assertTrue(self.redis._Exists()) def testExistFalse(self): with mock.patch.object( - util.GcloudCommand, 'Issue', return_value=('{}', '', 1)): + util.GcloudCommand, 'Issue', return_value=('{}', '', 1) + ): self.assertFalse(self.redis._Exists()) def testReadableVersion(self): @@ -63,19 +68,20 @@ def testReadableVersion(self): def testReadableVersionExtraneous(self): self.assertEqual(self.redis.ParseReadableVersion('redis_8'), 'redis_8') self.assertEqual( - self.redis.ParseReadableVersion('redis 9.7.5'), 'redis 9.7.5') + self.redis.ParseReadableVersion('redis 9.7.5'), 'redis 9.7.5' + ) - class TimeSeries(): + class TimeSeries: def __init__(self, points): self.points = [self.TimeSeriesValue(value) for value in points] - class TimeSeriesValue(): + class TimeSeriesValue: def __init__(self, value): self.value = self.TimeSeriesDoubleValue(value) - class TimeSeriesDoubleValue(): + class TimeSeriesDoubleValue: def __init__(self, value): self.double_value = value @@ -85,7 +91,7 @@ def testParseMonitoringTimeSeriesShort(self): self.TimeSeries([1, 2]), self.TimeSeries([2, 1]), self.TimeSeries([0, 0]), - self.TimeSeries([3, 3]) + self.TimeSeries([3, 3]), ]) self.assertEqual(avg_cpu, 0.1) @@ -94,7 +100,7 @@ def testParseMonitoringTimeSeriesMedium(self): self.TimeSeries([4.05, 2.12, 3.21, 1.58]), self.TimeSeries([2.83, 2.27, 4.71, 5.11]), self.TimeSeries([0, 0, 0, 0]), - self.TimeSeries([3.91, 3.11, 4.00, 1.65]) + self.TimeSeries([3.91, 3.11, 4.00, 1.65]), ]) self.assertEqual(avg_cpu, 0.160625) @@ -103,7 +109,7 @@ def testParseMonitoringTimeSeriesLong(self): self.TimeSeries([12, 32, 62, 51, 12, 103, 54, 85]), self.TimeSeries([81, 32, 84, 91, 25, 62, 31, 1]), self.TimeSeries([12, 93, 101, 70, 32, 58, 18, 10]), - self.TimeSeries([77, 34, 29, 83, 11, 8, 38, 68]) + self.TimeSeries([77, 34, 29, 83, 11, 8, 38, 68]), ]) self.assertEqual(avg_cpu, 3.25) diff --git a/tests/providers/gcp/gcp_key_test.py b/tests/providers/gcp/gcp_key_test.py index 4bf1f3b807..710e158e9f 100644 --- a/tests/providers/gcp/gcp_key_test.py +++ b/tests/providers/gcp/gcp_key_test.py @@ -22,8 +22,7 @@ class ConstructKeyTestCase(pkb_common_test_case.PkbCommonTestCase): @flagsaver.flagsaver(run_uri='test_uri') def testInitialization(self): - test_spec = inspect.cleandoc( - """ + test_spec = inspect.cleandoc(""" provision_key: description: Sample key benchmark key: @@ -32,8 +31,7 @@ def testInitialization(self): algorithm: test_algorithm protection_level: software location: test_location - """ - ) + """) FLAGS['cloud_kms_keyring_name'].parse('test_keyring') key_instance = _ConstructKeyFromSpec(test_spec) with self.subTest(name='ClassIsCorrect'): @@ -51,14 +49,12 @@ def testInitialization(self): @flagsaver.flagsaver(run_uri='test_uri') def testInitializationDefaults(self): - test_spec = inspect.cleandoc( - """ + test_spec = inspect.cleandoc(""" provision_key: description: Sample key benchmark key: cloud: GCP - """ - ) + """) key_instance = _ConstructKeyFromSpec(test_spec) with self.subTest(name='ClassIsCorrect'): self.assertIsInstance(key_instance, gcp_key.GcpKey) diff --git a/tests/providers/gcp/gcp_network_test.py b/tests/providers/gcp/gcp_network_test.py index 96f6ccda0c..d9995b82a0 100644 --- a/tests/providers/gcp/gcp_network_test.py +++ b/tests/providers/gcp/gcp_network_test.py @@ -19,7 +19,6 @@ from absl import flags from absl.testing import flagsaver import mock - from perfkitbenchmarker import benchmark_spec from perfkitbenchmarker import configs from perfkitbenchmarker import errors @@ -136,18 +135,21 @@ class BaseGceNetworkTest(pkb_common_test_case.PkbCommonTestCase): - def _CreateBenchmarkSpecFromYaml(self, yaml_string, - benchmark_name=_BENCHMARK_NAME): + def _CreateBenchmarkSpecFromYaml( + self, yaml_string, benchmark_name=_BENCHMARK_NAME + ): config = configs.LoadConfig(yaml_string, {}, benchmark_name) return self._CreateBenchmarkSpecFromConfigDict(config, benchmark_name) def _CreateBenchmarkSpecFromConfigDict(self, config_dict, benchmark_name): config_spec = benchmark_config_spec.BenchmarkConfigSpec( - benchmark_name, - flag_values=FLAGS, - **config_dict) - benchmark_module = next((b for b in linux_benchmarks.BENCHMARKS - if b.BENCHMARK_NAME == benchmark_name)) + benchmark_name, flag_values=FLAGS, **config_dict + ) + benchmark_module = next(( + b + for b in linux_benchmarks.BENCHMARKS + if b.BENCHMARK_NAME == benchmark_name + )) return benchmark_spec.BenchmarkSpec(benchmark_module, config_spec, _URI) @@ -183,14 +185,17 @@ def testLoadCustomConfig(self): with PatchCriticalObjects([('', '', 0)]): spec.ConstructVirtualMachines() - self.assertDictContainsSubset({'cidr': '10.0.1.0/24'}, - spec.custom_subnets['vm_1']) - self.assertDictContainsSubset({'cidr': '192.168.1.0/24'}, - spec.custom_subnets['vm_2']) + self.assertDictContainsSubset( + {'cidr': '10.0.1.0/24'}, spec.custom_subnets['vm_1'] + ) + self.assertDictContainsSubset( + {'cidr': '192.168.1.0/24'}, spec.custom_subnets['vm_2'] + ) self.assertLen(spec.networks, 2) for k in spec.networks.keys(): - self.assertCountEqual(['192.168.1.0/24', '10.0.1.0/24'], - spec.networks[k].all_nets) + self.assertCountEqual( + ['192.168.1.0/24', '10.0.1.0/24'], spec.networks[k].all_nets + ) def testLoadCustomConfigWithFlags(self): FLAGS.gce_subnet_region = 'us-north1-b' @@ -200,14 +205,17 @@ def testLoadCustomConfigWithFlags(self): with PatchCriticalObjects([('', '', 0)]): spec.ConstructVirtualMachines() - self.assertDictContainsSubset({'cidr': '10.0.1.0/24'}, - spec.custom_subnets['vm_1']) - self.assertDictContainsSubset({'cidr': '192.168.1.0/24'}, - spec.custom_subnets['vm_2']) + self.assertDictContainsSubset( + {'cidr': '10.0.1.0/24'}, spec.custom_subnets['vm_1'] + ) + self.assertDictContainsSubset( + {'cidr': '192.168.1.0/24'}, spec.custom_subnets['vm_2'] + ) self.assertLen(spec.networks, 2) for k in spec.networks.keys(): - self.assertCountEqual(['192.168.1.0/24', '10.0.1.0/24'], - spec.networks[k].all_nets) + self.assertCountEqual( + ['192.168.1.0/24', '10.0.1.0/24'], spec.networks[k].all_nets + ) def testLoadMixedConfig(self): spec = self._CreateBenchmarkSpecFromYaml(_CFG_DEFAULT_MULTI) @@ -215,12 +223,14 @@ def testLoadMixedConfig(self): spec.ConstructVirtualMachines() self.assertDictContainsSubset({'cidr': None}, spec.custom_subnets['vm_1']) - self.assertDictContainsSubset({'cidr': '192.168.1.0/24'}, - spec.custom_subnets['vm_2']) + self.assertDictContainsSubset( + {'cidr': '192.168.1.0/24'}, spec.custom_subnets['vm_2'] + ) self.assertLen(spec.networks, 2) for k in spec.networks.keys(): - self.assertCountEqual(['10.0.0.0/8', '192.168.1.0/24'], - spec.networks[k].all_nets) + self.assertCountEqual( + ['10.0.0.0/8', '192.168.1.0/24'], spec.networks[k].all_nets + ) def testLoadMixedConfigWithFlags(self): FLAGS.gce_subnet_region = 'us-north1-b' @@ -231,22 +241,26 @@ def testLoadMixedConfigWithFlags(self): spec.ConstructVirtualMachines() self.assertDictContainsSubset({'cidr': None}, spec.custom_subnets['vm_1']) - self.assertDictContainsSubset({'cidr': '192.168.1.0/24'}, - spec.custom_subnets['vm_2']) + self.assertDictContainsSubset( + {'cidr': '192.168.1.0/24'}, spec.custom_subnets['vm_2'] + ) self.assertLen(spec.networks, 2) for k in spec.networks.keys(): - self.assertCountEqual(['1.2.3.4/33', '192.168.1.0/24'], - spec.networks[k].all_nets) + self.assertCountEqual( + ['1.2.3.4/33', '192.168.1.0/24'], spec.networks[k].all_nets + ) def testLoadSameZoneCidrConfig(self): spec = self._CreateBenchmarkSpecFromYaml(_CFG_SAME_ZONE_AND_CIDR) with PatchCriticalObjects([('', '', 0)]): spec.ConstructVirtualMachines() - self.assertDictContainsSubset({'cidr': '10.0.1.0/24'}, - spec.custom_subnets['vm_1']) - self.assertDictContainsSubset({'cidr': '10.0.1.0/24'}, - spec.custom_subnets['vm_2']) + self.assertDictContainsSubset( + {'cidr': '10.0.1.0/24'}, spec.custom_subnets['vm_1'] + ) + self.assertDictContainsSubset( + {'cidr': '10.0.1.0/24'}, spec.custom_subnets['vm_2'] + ) self.assertLen(spec.networks, 1) for k in spec.networks.keys(): self.assertCountEqual(['10.0.1.0/24'], spec.networks[k].all_nets) @@ -256,14 +270,17 @@ def testLoadSameZoneDiffCidrConfig(self): with PatchCriticalObjects([('', '', 0)]): spec.ConstructVirtualMachines() - self.assertDictContainsSubset({'cidr': '10.0.1.0/24'}, - spec.custom_subnets['vm_1']) - self.assertDictContainsSubset({'cidr': '10.0.2.0/24'}, - spec.custom_subnets['vm_2']) + self.assertDictContainsSubset( + {'cidr': '10.0.1.0/24'}, spec.custom_subnets['vm_1'] + ) + self.assertDictContainsSubset( + {'cidr': '10.0.2.0/24'}, spec.custom_subnets['vm_2'] + ) self.assertLen(spec.networks, 2) for k in spec.networks.keys(): - self.assertCountEqual(['10.0.1.0/24', '10.0.2.0/24'], - spec.networks[k].all_nets) + self.assertCountEqual( + ['10.0.1.0/24', '10.0.2.0/24'], spec.networks[k].all_nets + ) class TestGceNetworkNames(BaseGceNetworkTest): @@ -273,7 +290,8 @@ def setUp(self): # need a benchmarkspec in the context to run FLAGS.run_uri = _URI config_spec = benchmark_config_spec.BenchmarkConfigSpec( - 'cluster_boot', flag_values=FLAGS) + 'cluster_boot', flag_values=FLAGS + ) benchmark_spec.BenchmarkSpec(mock.Mock(), config_spec, 'uid') ######## @@ -292,12 +310,15 @@ def testGetDefaultNetworkName(self): cidr_string = None uri = _URI expected_netname = '-'.join( - i for i in ('pkb-network', net_type, cidr_string, uri) if - i and i not in 'default') + i + for i in ('pkb-network', net_type, cidr_string, uri) + if i and i not in 'default' + ) self.assertFalse(net.is_existing_network) - self.assertEqual(expected_netname, - net_name) # pkb-network-uri45678 (default) + self.assertEqual( + expected_netname, net_name + ) # pkb-network-uri45678 (default) self.assertRegexpMatches(net_name, _REGEX_GCE_NET_NAMES) def testGetSingleNetworkName(self): @@ -315,13 +336,15 @@ def testGetSingleNetworkName(self): cidr_string = '2-2-3-4-33' uri = _URI expected_netname = '-'.join( - i for i in ('pkb-network', net_type, cidr_string, uri) if - i and i not in 'default') + i + for i in ('pkb-network', net_type, cidr_string, uri) + if i and i not in 'default' + ) self.assertFalse(net.is_existing_network) self.assertEqual( - expected_netname, - net_name) # pkb-network-single-2-2-3-4-33-uri45678 (single) + expected_netname, net_name + ) # pkb-network-single-2-2-3-4-33-uri45678 (single) self.assertRegexpMatches(net_name, _REGEX_GCE_NET_NAMES) def testGetMultiNetworkName(self): @@ -420,9 +443,17 @@ def testGetDefaultFWName(self): dst_port = None uri = _URI expected_name = '-'.join( - i for i in ( - net_type, src_cidr_string, dst_cidr_string, src_port, - dst_port, uri) if i) + i + for i in ( + net_type, + src_cidr_string, + dst_cidr_string, + src_port, + dst_port, + uri, + ) + if i + ) self.assertEqual(expected_name, fw_name) self.assertRegexpMatches(fw_name, _REGEX_GCE_FW_NAMES) @@ -439,8 +470,13 @@ def testGetSingleFWName(self): vm = mock.Mock(zone=zone, project=project, cidr=cidr) net = gce_network.GceNetwork.GetNetwork(vm) fw_name = net._MakeGceFWRuleName( - net_type=None, src_cidr=None, dst_cidr=None, port_range_lo=lo_port, - port_range_hi=hi_port, uri=None) + net_type=None, + src_cidr=None, + dst_cidr=None, + port_range_lo=lo_port, + port_range_hi=hi_port, + uri=None, + ) net_type = 'single' src_cidr_string = 'internal' @@ -449,12 +485,21 @@ def testGetSingleFWName(self): dst_port = None uri = _URI expected_name = '-'.join( - i for i in (net_type, src_cidr_string, dst_cidr_string, src_port, - dst_port, uri) - if i) + i + for i in ( + net_type, + src_cidr_string, + dst_cidr_string, + src_port, + dst_port, + uri, + ) + if i + ) - self.assertEqual(expected_name, - fw_name) # single-internal-2-2-3-4-33-uri45678 + self.assertEqual( + expected_name, fw_name + ) # single-internal-2-2-3-4-33-uri45678 self.assertRegexpMatches(fw_name, _REGEX_GCE_FW_NAMES) def testGetMultiFWNameWithPorts(self): @@ -469,9 +514,13 @@ def testGetMultiFWNameWithPorts(self): lo_port = 49152 hi_port = 65535 fw_name = net._MakeGceFWRuleName( - net_type=None, src_cidr=None, dst_cidr=dst_cidr, + net_type=None, + src_cidr=None, + dst_cidr=dst_cidr, port_range_lo=lo_port, - port_range_hi=hi_port, uri=None) + port_range_hi=hi_port, + uri=None, + ) prefix = None net_type = 'multi' @@ -481,11 +530,22 @@ def testGetMultiFWNameWithPorts(self): dst_port = '65535' uri = _URI expected_name = '-'.join( - i for i in (prefix, net_type, src_cidr_string, dst_cidr_string, - src_port, dst_port, uri) if i) + i + for i in ( + prefix, + net_type, + src_cidr_string, + dst_cidr_string, + src_port, + dst_port, + uri, + ) + if i + ) - self.assertEqual(expected_name, - fw_name) # multi-internal-1-2-3-4-56-49152-65535-uri45678 + self.assertEqual( + expected_name, fw_name + ) # multi-internal-1-2-3-4-56-49152-65535-uri45678 self.assertRegexpMatches(fw_name, _REGEX_GCE_FW_NAMES) def testGetMultiFWNameWithPortsDst(self): @@ -499,9 +559,13 @@ def testGetMultiFWNameWithPortsDst(self): lo_port = 49152 hi_port = 65535 fw_name = net._MakeGceFWRuleName( - net_type=None, src_cidr=None, dst_cidr=dst_cidr, + net_type=None, + src_cidr=None, + dst_cidr=dst_cidr, port_range_lo=lo_port, - port_range_hi=hi_port, uri=None) + port_range_hi=hi_port, + uri=None, + ) prefix = 'perfkit-firewall' net_type = 'multi' @@ -511,8 +575,18 @@ def testGetMultiFWNameWithPortsDst(self): dst_port = '65535' uri = _URI expected_name = '-'.join( - i for i in (prefix, net_type, src_cidr_string, dst_cidr_string, - src_port, dst_port, uri) if i) + i + for i in ( + prefix, + net_type, + src_cidr_string, + dst_cidr_string, + src_port, + dst_port, + uri, + ) + if i + ) # perfkit-firewall-multi-1-2-3-4-56-123-567-901-13-49152-65535-uri45678 self.assertEqual(expected_name, fw_name) @@ -530,10 +604,10 @@ def ReturnVal(*unused_arg, **unused_kwargs): return ('', '', 0) if retvals is None else retvals.pop(0) with mock.patch( - vm_util.__name__ + '.IssueCommand', - side_effect=ReturnVal) as issue_command, mock.patch( - builtins.__name__ + '.open'), mock.patch(vm_util.__name__ + - '.NamedTemporaryFile'): + vm_util.__name__ + '.IssueCommand', side_effect=ReturnVal + ) as issue_command, mock.patch(builtins.__name__ + '.open'), mock.patch( + vm_util.__name__ + '.NamedTemporaryFile' + ): yield issue_command @@ -575,42 +649,52 @@ def testMtuSupport(self, mock_issue): self.assertEqual(1500, net.mtu) create_network_cmd = mock_issue.call_args_list[0][0][0] - self.assertRegex(' '.join(create_network_cmd), - 'compute networks create .*--mtu 1500') + self.assertRegex( + ' '.join(create_network_cmd), 'compute networks create .*--mtu 1500' + ) class GceFirewallRuleTest(pkb_common_test_case.PkbCommonTestCase): @mock.patch('time.sleep', side_effect=lambda _: None) def testGceFirewallRuleSuccessfulAfterRateLimited(self, mock_cmd): - fake_rets = [('stdout', 'Rate Limit Exceeded', 1), - ('stdout', 'some warning perhaps', 0)] + fake_rets = [ + ('stdout', 'Rate Limit Exceeded', 1), + ('stdout', 'some warning perhaps', 0), + ] with PatchCriticalObjects(fake_rets) as issue_command: - fr = gce_network.GceFirewallRule('name', 'project', 'allow', - 'network_name') + fr = gce_network.GceFirewallRule( + 'name', 'project', 'allow', 'network_name' + ) fr._Create() self.assertEqual(issue_command.call_count, 2) @mock.patch('time.sleep', side_effect=lambda _: None) def testGceFirewallRuleGenericErrorAfterRateLimited(self, mock_cmd): - fake_rets = [('stdout', 'Rate Limit Exceeded', 1), - ('stdout', 'Rate Limit Exceeded', 1), - ('stdout', 'some random firewall error', 1)] + fake_rets = [ + ('stdout', 'Rate Limit Exceeded', 1), + ('stdout', 'Rate Limit Exceeded', 1), + ('stdout', 'some random firewall error', 1), + ] with PatchCriticalObjects(fake_rets) as issue_command: with self.assertRaises(errors.VmUtil.IssueCommandError): - fr = gce_network.GceFirewallRule('name', 'project', 'allow', - 'network_name') + fr = gce_network.GceFirewallRule( + 'name', 'project', 'allow', 'network_name' + ) fr._Create() self.assertEqual(issue_command.call_count, 3) @mock.patch('time.sleep', side_effect=lambda _: None) def testGceFirewallRuleAlreadyExistsAfterRateLimited(self, mock_cmd): - fake_rets = [('stdout', 'Rate Limit Exceeded', 1), - ('stdout', 'Rate Limit Exceeded', 1), - ('stdout', 'firewall already exists', 1)] + fake_rets = [ + ('stdout', 'Rate Limit Exceeded', 1), + ('stdout', 'Rate Limit Exceeded', 1), + ('stdout', 'firewall already exists', 1), + ] with PatchCriticalObjects(fake_rets) as issue_command: - fr = gce_network.GceFirewallRule('name', 'project', 'allow', - 'network_name') + fr = gce_network.GceFirewallRule( + 'name', 'project', 'allow', 'network_name' + ) fr._Create() self.assertEqual(issue_command.call_count, 3) @@ -619,8 +703,9 @@ def testGceFirewallRuleGenericError(self, mock_cmd): fake_rets = [('stdout', 'some random firewall error', 1)] with PatchCriticalObjects(fake_rets) as issue_command: with self.assertRaises(errors.VmUtil.IssueCommandError): - fr = gce_network.GceFirewallRule('name', 'project', 'allow', - 'network_name') + fr = gce_network.GceFirewallRule( + 'name', 'project', 'allow', 'network_name' + ) fr._Create() self.assertEqual(issue_command.call_count, 1) diff --git a/tests/providers/gcp/gcp_pubsub_test.py b/tests/providers/gcp/gcp_pubsub_test.py index 31b16320dd..2cbd4b0d0a 100644 --- a/tests/providers/gcp/gcp_pubsub_test.py +++ b/tests/providers/gcp/gcp_pubsub_test.py @@ -33,7 +33,8 @@ def setUp(self): def _MockIssueCommand(self, return_value): return self.enter_context( - mock.patch.object(vm_util, 'IssueCommand', return_value=return_value)) + mock.patch.object(vm_util, 'IssueCommand', return_value=return_value) + ) def testCreateTopic(self): # Don't actually issue a command. @@ -102,8 +103,9 @@ def testCreateSubscriptionError(self): return_value = ['', '', 1] cmd = self._MockIssueCommand(return_value) - self.assertRaises(errors.Resource.CreationError, - self.pubsub._CreateSubscription) + self.assertRaises( + errors.Resource.CreationError, self.pubsub._CreateSubscription + ) cmd = ' '.join(cmd.call_args[0][0]) self.assertIn('gcloud pubsub subscriptions create ' + SUBSCRIPTION, cmd) @@ -155,40 +157,46 @@ def testPrepareClientVm(self): self.pubsub.PrepareClientVm() self.client.assert_has_calls([ mock.call.RemoteCommand( - 'sudo pip3 install --upgrade --ignore-installed google-cloud-pubsub', - ignore_failure=False), + 'sudo pip3 install --upgrade --ignore-installed' + ' google-cloud-pubsub', + ignore_failure=False, + ), mock.call.RemoteCommand( - 'mkdir -p ~/perfkitbenchmarker/scripts/messaging_service_scripts/gcp' + 'mkdir -p' + ' ~/perfkitbenchmarker/scripts/messaging_service_scripts/gcp' ), mock.call.PushDataFile( 'messaging_service_scripts/gcp/__init__.py', - '~/perfkitbenchmarker/scripts/messaging_service_scripts/gcp/__init__.py' + '~/perfkitbenchmarker/scripts/messaging_service_scripts/gcp/__init__.py', ), mock.call.RemoteCommand( - 'mkdir -p ~/perfkitbenchmarker/scripts/messaging_service_scripts/gcp' + 'mkdir -p' + ' ~/perfkitbenchmarker/scripts/messaging_service_scripts/gcp' ), mock.call.PushDataFile( 'messaging_service_scripts/gcp/gcp_pubsub_client.py', - '~/perfkitbenchmarker/scripts/messaging_service_scripts/gcp/gcp_pubsub_client.py' + '~/perfkitbenchmarker/scripts/messaging_service_scripts/gcp/gcp_pubsub_client.py', ), mock.call.PushDataFile('messaging_service_scripts/gcp_benchmark.py'), ]) def testRun(self): - return_value = ['{"mock1": 1}', None] self.client.RemoteCommand.return_value = return_value - remote_run_cmd = (f'python3 -m gcp_benchmark ' - f'--pubsub_project={PROJECT} ' - f'--pubsub_topic={TOPIC} ' - f'--pubsub_subscription={SUBSCRIPTION} ' - f'--benchmark_scenario={BENCHMARK_SCENARIO} ' - f'--number_of_messages={NUMBER_OF_MESSAGES} ' - f'--message_size={MESSAGE_SIZE} ' - f'--warmup_messages={WARMUP_MESSAGES}') - - self.pubsub.Run(BENCHMARK_SCENARIO, NUMBER_OF_MESSAGES, MESSAGE_SIZE, - WARMUP_MESSAGES) + remote_run_cmd = ( + 'python3 -m gcp_benchmark ' + f'--pubsub_project={PROJECT} ' + f'--pubsub_topic={TOPIC} ' + f'--pubsub_subscription={SUBSCRIPTION} ' + f'--benchmark_scenario={BENCHMARK_SCENARIO} ' + f'--number_of_messages={NUMBER_OF_MESSAGES} ' + f'--message_size={MESSAGE_SIZE} ' + f'--warmup_messages={WARMUP_MESSAGES}' + ) + + self.pubsub.Run( + BENCHMARK_SCENARIO, NUMBER_OF_MESSAGES, MESSAGE_SIZE, WARMUP_MESSAGES + ) self.client.RemoteCommand.assert_called_with(remote_run_cmd) @mock.patch.object(pubsub.GCPCloudPubSub, '_DeleteSubscription') diff --git a/tests/providers/gcp/gcp_relational_db_test.py b/tests/providers/gcp/gcp_relational_db_test.py index e878acfcc2..3d98ee8836 100644 --- a/tests/providers/gcp/gcp_relational_db_test.py +++ b/tests/providers/gcp/gcp_relational_db_test.py @@ -20,7 +20,6 @@ import unittest from absl import flags import mock - from perfkitbenchmarker import disk from perfkitbenchmarker import relational_db from perfkitbenchmarker import relational_db_spec @@ -78,13 +77,12 @@ def PatchCriticalObjects(stdout='', stderr='', return_code=0): """A context manager that patches a few critical objects with mocks.""" retval = (stdout, stderr, return_code) with mock.patch( - vm_util.__name__ + '.IssueCommand', - return_value=retval) as issue_command, mock.patch( - builtins.__name__ + - '.open'), mock.patch(vm_util.__name__ + - '.NamedTemporaryFile'), mock.patch( - util.__name__ + '.GetDefaultProject', - return_value='fakeproject'): + vm_util.__name__ + '.IssueCommand', return_value=retval + ) as issue_command, mock.patch(builtins.__name__ + '.open'), mock.patch( + vm_util.__name__ + '.NamedTemporaryFile' + ), mock.patch( + util.__name__ + '.GetDefaultProject', return_value='fakeproject' + ): yield issue_command @@ -92,32 +90,16 @@ def VmGroupSpec(): return { 'clients': { 'vm_spec': { - 'GCP': { - 'zone': 'us-central1-c', - 'machine_type': 'n1-standard-1' - } + 'GCP': {'zone': 'us-central1-c', 'machine_type': 'n1-standard-1'} }, - 'disk_spec': { - 'GCP': { - 'disk_size': 500, - 'disk_type': 'pd-ssd' - } - } + 'disk_spec': {'GCP': {'disk_size': 500, 'disk_type': 'pd-ssd'}}, }, 'servers': { 'vm_spec': { - 'GCP': { - 'zone': 'us-central1-c', - 'machine_type': 'n1-standard-1' - } + 'GCP': {'zone': 'us-central1-c', 'machine_type': 'n1-standard-1'} }, - 'disk_spec': { - 'GCP': { - 'disk_size': 500, - 'disk_type': 'pd-ssd' - } - } - } + 'disk_spec': {'GCP': {'disk_size': 500, 'disk_type': 'pd-ssd'}}, + }, } @@ -125,10 +107,12 @@ class GcpMysqlRelationalDbTestCase(pkb_common_test_case.PkbCommonTestCase): def createMySQLSpecDict(self): db_spec = virtual_machine.BaseVmSpec( - 'NAME', **{ + 'NAME', + **{ 'machine_type': 'db-n1-standard-1', 'zone': 'us-west1-b', - }) + } + ) db_spec.cpus = None db_spec.memory = None db_disk_spec = disk.BaseDiskSpec('NAME', **{'disk_size': 50}) @@ -180,8 +164,10 @@ def testCreate(self): self.assertTrue( command_string.startswith( - 'gcloud beta sql instances create pkb-db-instance-123'), - command_string) + 'gcloud beta sql instances create pkb-db-instance-123' + ), + command_string, + ) self.assertIn('--project fakeproject', command_string) self.assertIn('--tier=db-n1-standard-1', command_string) self.assertIn('--storage-size=50', command_string) @@ -208,8 +194,10 @@ def testCreateWithBackupDisabled(self): self.assertTrue( command_string.startswith( - 'gcloud beta sql instances create pkb-db-instance-123'), - command_string) + 'gcloud beta sql instances create pkb-db-instance-123' + ), + command_string, + ) self.assertIn('--project fakeproject', command_string) self.assertIn('--tier=db-n1-standard-1', command_string) self.assertIn('--no-backup', command_string) @@ -223,12 +211,16 @@ def testDelete(self): command_string = ' '.join(issue_command.call_args[0][0]) self.assertTrue( command_string.startswith( - 'gcloud sql instances delete pkb-db-instance-123')) + 'gcloud sql instances delete pkb-db-instance-123' + ) + ) def testIsReady(self): path = os.path.join( - os.path.dirname(__file__), '../../data', - 'gcloud-describe-db-instances-available.json') + os.path.dirname(__file__), + '../../data', + 'gcloud-describe-db-instances-available.json', + ) with open(path) as fp: test_output = fp.read() @@ -238,8 +230,10 @@ def testIsReady(self): def testExists(self): path = os.path.join( - os.path.dirname(__file__), '../../data', - 'gcloud-describe-db-instances-available.json') + os.path.dirname(__file__), + '../../data', + 'gcloud-describe-db-instances-available.json', + ) with open(path) as fp: test_output = fp.read() @@ -260,16 +254,17 @@ def testHighAvailability(self): def testParseEndpoint(self): path = os.path.join( - os.path.dirname(__file__), '../../data', - 'gcloud-describe-db-instances-available.json') + os.path.dirname(__file__), + '../../data', + 'gcloud-describe-db-instances-available.json', + ) with open(path) as fp: test_output = fp.read() with PatchCriticalObjects(): db = CreateDbFromSpec(self.createMySQLSpecDict()) self.assertEqual('', db._ParseEndpoint(None)) - self.assertIn('10.10.0.35', - db._ParseEndpoint(json.loads(test_output))) + self.assertIn('10.10.0.35', db._ParseEndpoint(json.loads(test_output))) def testCreateUnmanagedDb(self): FLAGS['use_managed_db'].parse(False) diff --git a/tests/providers/gcp/gcp_spanner_test.py b/tests/providers/gcp/gcp_spanner_test.py index 2da6f24839..dc532d2768 100644 --- a/tests/providers/gcp/gcp_spanner_test.py +++ b/tests/providers/gcp/gcp_spanner_test.py @@ -12,7 +12,6 @@ from perfkitbenchmarker.providers.gcp import gcp_spanner from perfkitbenchmarker.providers.gcp import util from tests import pkb_common_test_case - import requests FLAGS = flags.FLAGS @@ -21,10 +20,12 @@ def GetTestSpannerInstance(engine='spanner-googlesql'): spec_args = {'cloud': 'GCP', 'engine': engine} spanner_spec = gcp_spanner.SpannerSpec( - 'test_component', flag_values=FLAGS, **spec_args) + 'test_component', flag_values=FLAGS, **spec_args + ) spanner_spec.spanner_database_name = 'test_database' spanner_class = relational_db.GetRelationalDbClass( - cloud='GCP', is_managed_db=True, engine=engine) + cloud='GCP', is_managed_db=True, engine=engine + ) return spanner_class(spanner_spec) @@ -67,8 +68,8 @@ def testSetNodes(self): mock.patch.object(test_instance, '_WaitUntilInstanceReady') ) cmd = self.enter_context( - mock.patch.object( - vm_util, 'IssueCommand', return_value=[None, None, 0])) + mock.patch.object(vm_util, 'IssueCommand', return_value=[None, None, 0]) + ) test_instance._SetNodes(3) @@ -93,7 +94,8 @@ def testSetNodesSkipsIfCountAlreadyCorrect(self): def testFreezeUsesCorrectNodeCount(self): instance = GetTestSpannerInstance() mock_set_nodes = self.enter_context( - mock.patch.object(instance, '_SetNodes', autospec=True)) + mock.patch.object(instance, '_SetNodes', autospec=True) + ) instance._Freeze() @@ -103,7 +105,8 @@ def testRestoreUsesCorrectNodeCount(self): instance = GetTestSpannerInstance() instance.nodes = 5 mock_set_nodes = self.enter_context( - mock.patch.object(instance, '_SetNodes', autospec=True)) + mock.patch.object(instance, '_SetNodes', autospec=True) + ) instance._Restore() @@ -129,13 +132,20 @@ def testUpdateLabels(self): mock.patch.object( util.GcloudCommand, 'Issue', - side_effect=[(mock_endpoint_response, '', 0), - (mock_labels_response, '', 0)])) + side_effect=[ + (mock_endpoint_response, '', 0), + (mock_labels_response, '', 0), + ], + ) + ) self.enter_context( - mock.patch.object(util, 'GetAccessToken', return_value='test_token')) + mock.patch.object(util, 'GetAccessToken', return_value='test_token') + ) mock_request = self.enter_context( mock.patch.object( - requests, 'patch', return_value=mock.Mock(status_code=200))) + requests, 'patch', return_value=mock.Mock(status_code=200) + ) + ) # Act new_labels = { @@ -153,11 +163,12 @@ def testUpdateLabels(self): 'labels': { 'benchmark': 'test_benchmark_2', 'timeout_minutes': '10', - 'metadata': 'test_metadata' + 'metadata': 'test_metadata', } }, - 'fieldMask': 'labels' - }) + 'fieldMask': 'labels', + }, + ) @parameterized.named_parameters([ { @@ -179,15 +190,17 @@ def testUpdateLabels(self): 'expected_qps': 10000, }, ]) - def testCalculateStartingThroughput(self, write_proportion, read_proportion, - expected_qps): + def testCalculateStartingThroughput( + self, write_proportion, read_proportion, expected_qps + ): # Arrange test_spanner = GetTestSpannerInstance() test_spanner.nodes = 3 # Act actual_qps = test_spanner.CalculateTheoreticalMaxThroughput( - read_proportion, write_proportion) + read_proportion, write_proportion + ) # Assert self.assertEqual(expected_qps, actual_qps) diff --git a/tests/providers/gcp/gcp_tpu_test.py b/tests/providers/gcp/gcp_tpu_test.py index aea4214e0e..115c1f5d02 100644 --- a/tests/providers/gcp/gcp_tpu_test.py +++ b/tests/providers/gcp/gcp_tpu_test.py @@ -17,7 +17,6 @@ import unittest from absl import flags import mock - from perfkitbenchmarker import errors from perfkitbenchmarker import vm_util from perfkitbenchmarker.configs import benchmark_config_spec @@ -44,7 +43,7 @@ def CreateTpuSpecDict(self): 'tpu_network': 'default', 'tpu_tf_version': 'nightly', 'tpu_zone': 'us-central1-a', - 'tpu_preemptible': True + 'tpu_preemptible': True, } def setUp(self): @@ -55,8 +54,7 @@ def setUp(self): FLAGS.gcloud_path = 'gcloud' mock_tpu_spec_attrs = self.CreateTpuSpecDict() - self.mock_tpu_spec = mock.Mock( - spec=benchmark_config_spec._TpuGroupSpec) + self.mock_tpu_spec = mock.Mock(spec=benchmark_config_spec._TpuGroupSpec) self.mock_tpu_spec.configure_mock(**mock_tpu_spec_attrs) @contextlib.contextmanager @@ -64,13 +62,12 @@ def _PatchCriticalObjects(self, stdout='', stderr='', return_code=0): """A context manager that patches a few critical objects with mocks.""" retval = (stdout, stderr, return_code) with mock.patch( - vm_util.__name__ + '.IssueCommand', - return_value=retval) as issue_command, mock.patch( - builtins.__name__ + - '.open'), mock.patch(vm_util.__name__ + - '.NamedTemporaryFile'), mock.patch( - util.__name__ + '.GetDefaultProject', - return_value='fakeproject'): + vm_util.__name__ + '.IssueCommand', return_value=retval + ) as issue_command, mock.patch(builtins.__name__ + '.open'), mock.patch( + vm_util.__name__ + '.NamedTemporaryFile' + ), mock.patch( + util.__name__ + '.GetDefaultProject', return_value='fakeproject' + ): yield issue_command def testCreate(self): @@ -80,9 +77,9 @@ def testCreate(self): self.assertEqual(issue_command.call_count, 1) command_string = ' '.join(issue_command.call_args[0][0]) self.assertTrue( - command_string.startswith( - 'gcloud compute tpus create pkb-tpu-123'), - command_string) + command_string.startswith('gcloud compute tpus create pkb-tpu-123'), + command_string, + ) self.assertIn('--project fakeproject', command_string) self.assertIn('--range 192.168.0.0/29', command_string) self.assertIn('--accelerator-type tpu-v2', command_string) @@ -101,7 +98,8 @@ def testStockout(self): }""" with self._PatchCriticalObjects(stderr=stderr, return_code=1): with self.assertRaises( - errors.Benchmarks.InsufficientCapacityCloudFailure): + errors.Benchmarks.InsufficientCapacityCloudFailure + ): tpu = gcp_tpu.GcpTpu(self.mock_tpu_spec) tpu._Create() @@ -112,8 +110,8 @@ def testDelete(self): self.assertEqual(issue_command.call_count, 1) command_string = ' '.join(issue_command.call_args[0][0]) self.assertTrue( - command_string.startswith( - 'gcloud compute tpus delete pkb-tpu-123')) + command_string.startswith('gcloud compute tpus delete pkb-tpu-123') + ) self.assertIn('--project fakeproject', command_string) self.assertIn('--zone us-central1-a', command_string) @@ -124,8 +122,8 @@ def testExists(self): self.assertEqual(issue_command.call_count, 1) command_string = ' '.join(issue_command.call_args[0][0]) self.assertTrue( - command_string.startswith( - 'gcloud compute tpus describe pkb-tpu-123')) + command_string.startswith('gcloud compute tpus describe pkb-tpu-123') + ) self.assertIn('--project fakeproject', command_string) self.assertIn('--zone us-central1-a', command_string) @@ -136,8 +134,11 @@ def testGetName(self): self.assertEqual(name, 'pkb-tpu-123') def testGetNumShards(self): - with self._PatchCriticalObjects(stdout='{"networkEndpoints": [{"ipAddress":' - ' "10.199.12.2", "port": 8470}]}'): + with self._PatchCriticalObjects( + stdout=( + '{"networkEndpoints": [{"ipAddress": "10.199.12.2", "port": 8470}]}' + ) + ): tpu = gcp_tpu.GcpTpu(self.mock_tpu_spec) num_shards = tpu.GetNumShards() self.assertEqual(num_shards, 8) diff --git a/tests/providers/gcp/google_kubernetes_engine_test.py b/tests/providers/gcp/google_kubernetes_engine_test.py index 0db7fb25f9..39dd67bb5b 100644 --- a/tests/providers/gcp/google_kubernetes_engine_test.py +++ b/tests/providers/gcp/google_kubernetes_engine_test.py @@ -16,13 +16,11 @@ # pylint: disable=not-context-manager import os - import unittest from unittest import mock from absl import flags as flgs import contextlib2 - from perfkitbenchmarker import container_service from perfkitbenchmarker import data from perfkitbenchmarker import errors @@ -33,17 +31,22 @@ from perfkitbenchmarker.providers.gcp import util from tests import pkb_common_test_case from six.moves import builtins + FLAGS = flgs.FLAGS _COMPONENT = 'test_component' _RUN_URI = 'fake-urn-uri' _NVIDIA_DRIVER_SETUP_DAEMON_SET_SCRIPT = 'https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/master/nvidia-driver-installer/cos/daemonset-preloaded.yaml' -_NVIDIA_UNRESTRICTED_PERMISSIONS_DAEMON_SET = 'nvidia_unrestricted_permissions_daemonset.yml' +_NVIDIA_UNRESTRICTED_PERMISSIONS_DAEMON_SET = ( + 'nvidia_unrestricted_permissions_daemonset.yml' +) _INSTANCE_GROUPS_LIST_OUTPUT = ( - '../../../tests/data/gcloud_compute_instance_groups_list_instances.json') + '../../../tests/data/gcloud_compute_instance_groups_list_instances.json' +) _NODE_POOLS_LIST_OUTPUT = ( - '../../../tests/data/gcloud_container_node_pools_list.json') + '../../../tests/data/gcloud_container_node_pools_list.json' +) _KUBECTL_VERSION = """\ serverVersion: gitVersion: v1.2.3""" @@ -66,35 +69,45 @@ def patch_critical_objects(stdout='', stderr='', return_code=0, flags=FLAGS): stack.enter_context(mock.patch(vm_util.__name__ + '.PrependTempDir')) stack.enter_context(mock.patch(vm_util.__name__ + '.NamedTemporaryFile')) stack.enter_context( - mock.patch( - util.__name__ + '.GetDefaultUser', return_value='fakeuser')) + mock.patch(util.__name__ + '.GetDefaultUser', return_value='fakeuser') + ) stack.enter_context( mock.patch( util.__name__ + '.MakeFormattedDefaultTags', - return_value='foo=bar,timeout=yesterday')) + return_value='foo=bar,timeout=yesterday', + ) + ) stack.enter_context( mock.patch( gce_network.__name__ + '.GceFirewall.GetFirewall', - return_value='fakefirewall')) + return_value='fakefirewall', + ) + ) stack.enter_context( mock.patch( gce_network.__name__ + '.GceNetwork.GetNetwork', return_value=gce_network.GceNetwork( - gce_network.GceNetworkSpec('fakeproject')))) + gce_network.GceNetworkSpec('fakeproject') + ), + ) + ) retval = (stdout, stderr, return_code) issue_command = stack.enter_context( - mock.patch(vm_util.__name__ + '.IssueCommand', return_value=retval)) + mock.patch(vm_util.__name__ + '.IssueCommand', return_value=retval) + ) yield issue_command class GoogleKubernetesEngineCustomMachineTypeTestCase( - pkb_common_test_case.PkbCommonTestCase): + pkb_common_test_case.PkbCommonTestCase +): @staticmethod def create_kubernetes_engine_spec(): kubernetes_engine_spec = container_spec.ContainerClusterSpec( - 'NAME', **{ + 'NAME', + **{ 'cloud': 'GCP', 'vm_spec': { 'GCP': { @@ -105,7 +118,8 @@ def create_kubernetes_engine_spec(): 'zone': 'us-west1-a', }, }, - }) + }, + ) return kubernetes_engine_spec def testCreate(self): @@ -125,7 +139,8 @@ class GoogleKubernetesEngineTestCase(pkb_common_test_case.PkbCommonTestCase): @staticmethod def create_kubernetes_engine_spec(): kubernetes_engine_spec = container_spec.ContainerClusterSpec( - 'NAME', **{ + 'NAME', + **{ 'cloud': 'GCP', 'vm_spec': { 'GCP': { @@ -139,7 +154,8 @@ def create_kubernetes_engine_spec(): }, }, 'vm_count': 2, - }) + }, + ) return kubernetes_engine_spec def testCreate(self): @@ -169,10 +185,10 @@ def testCreateQuotaExceeded(self): stderr=""" message=Insufficient regional quota to satisfy request: resource "CPUS": request requires '6400.0' and is short '5820.0'""", - return_code=1) as issue_command: + return_code=1, + ) as issue_command: cluster = google_kubernetes_engine.GkeCluster(spec) - with self.assertRaises( - errors.Benchmarks.QuotaFailure): + with self.assertRaises(errors.Benchmarks.QuotaFailure): cluster._Create() self.assertEqual(issue_command.call_count, 1) @@ -184,17 +200,20 @@ def testCreateResourcesExhausted(self): Instance 'test' creation failed: The zone 'projects/artemis-prod/zones/us-central1-a' does not have enough resources available to fulfill the request.""", - return_code=1) as issue_command: + return_code=1, + ) as issue_command: cluster = google_kubernetes_engine.GkeCluster(spec) with self.assertRaises( - errors.Benchmarks.InsufficientCapacityCloudFailure): + errors.Benchmarks.InsufficientCapacityCloudFailure + ): cluster._Create() self.assertEqual(issue_command.call_count, 1) def testPostCreate(self): spec = self.create_kubernetes_engine_spec() with patch_critical_objects() as issue_command, mock.patch.object( - container_service, 'RunKubectlCommand') as mock_kubectl_command: + container_service, 'RunKubectlCommand' + ) as mock_kubectl_command: cluster = google_kubernetes_engine.GkeCluster(spec) cluster._PostCreate() command_string = ' '.join(issue_command.call_args[0][0]) @@ -202,7 +221,8 @@ def testPostCreate(self): self.assertEqual(issue_command.call_count, 1) self.assertIn( 'gcloud container clusters get-credentials pkb-{0}'.format(_RUN_URI), - command_string) + command_string, + ) self.assertIn('KUBECONFIG', issue_command.call_args[1]['env']) self.assertEqual(mock_kubectl_command.call_count, 1) @@ -215,8 +235,10 @@ def testDelete(self): command_string = ' '.join(issue_command.call_args[0][0]) self.assertEqual(issue_command.call_count, 3) - self.assertIn('gcloud container clusters delete pkb-{0}'.format(_RUN_URI), - command_string) + self.assertIn( + 'gcloud container clusters delete pkb-{0}'.format(_RUN_URI), + command_string, + ) self.assertIn('--zone us-central1-a', command_string) def testExists(self): @@ -229,7 +251,8 @@ def testExists(self): self.assertEqual(issue_command.call_count, 1) self.assertIn( 'gcloud container clusters describe pkb-{0}'.format(_RUN_URI), - command_string) + command_string, + ) def testGetResourceMetadata(self): spec = self.create_kubernetes_engine_spec() @@ -251,7 +274,9 @@ def testGetResourceMetadata(self): 'zone': 'us-central1-a', 'size': 2, 'container_cluster_version': 'v1.2.3', - }, metadata) + }, + metadata, + ) def testCidrCalculations(self): self.assertEqual(google_kubernetes_engine._CalculateCidrSize(1), 19) @@ -263,12 +288,14 @@ def testCidrCalculations(self): class GoogleKubernetesEngineAutoscalingTestCase( - pkb_common_test_case.PkbCommonTestCase): + pkb_common_test_case.PkbCommonTestCase +): @staticmethod def create_kubernetes_engine_spec(): kubernetes_engine_spec = container_spec.ContainerClusterSpec( - 'NAME', **{ + 'NAME', + **{ 'cloud': 'GCP', 'vm_spec': { 'GCP': { @@ -280,7 +307,8 @@ def create_kubernetes_engine_spec(): 'min_vm_count': 1, 'vm_count': 2, 'max_vm_count': 30, - }) + }, + ) return kubernetes_engine_spec def testCreate(self): @@ -312,8 +340,10 @@ def testGetResourceMetadata(self): 'cluster_type': 'Kubernetes', 'min_size': 1, 'size': 2, - 'max_size': 30 - }, metadata) + 'max_size': 30, + }, + metadata, + ) def testLabelDisks(self): with patch_critical_objects(stdout=_PVC_LIST) as issue_command: @@ -331,12 +361,14 @@ def testLabelDisks(self): class GoogleKubernetesEngineVersionFlagTestCase( - pkb_common_test_case.PkbCommonTestCase): + pkb_common_test_case.PkbCommonTestCase +): @staticmethod def create_kubernetes_engine_spec(): kubernetes_engine_spec = container_spec.ContainerClusterSpec( - 'NAME', **{ + 'NAME', + **{ 'cloud': 'GCP', 'vm_spec': { 'GCP': { @@ -344,7 +376,8 @@ def create_kubernetes_engine_spec(): 'zone': 'us-west1-a', }, }, - }) + }, + ) return kubernetes_engine_spec def testCreateCustomVersion(self): @@ -387,12 +420,14 @@ def testCreateRapidChannel(self): class GoogleKubernetesEngineGvnicFlagTestCase( - pkb_common_test_case.PkbCommonTestCase): + pkb_common_test_case.PkbCommonTestCase +): @staticmethod def create_kubernetes_engine_spec(): kubernetes_engine_spec = container_spec.ContainerClusterSpec( - 'NAME', **{ + 'NAME', + **{ 'cloud': 'GCP', 'vm_spec': { 'GCP': { @@ -400,7 +435,8 @@ def create_kubernetes_engine_spec(): 'zone': 'us-west1-a', }, }, - }) + }, + ) return kubernetes_engine_spec def testCreateEnableGvnic(self): @@ -427,12 +463,14 @@ def testCreateDisableGvnic(self): class GoogleKubernetesEngineWithGpusTestCase( - pkb_common_test_case.PkbCommonTestCase): + pkb_common_test_case.PkbCommonTestCase +): @staticmethod def create_kubernetes_engine_spec(): kubernetes_engine_spec = container_spec.ContainerClusterSpec( - 'NAME', **{ + 'NAME', + **{ 'cloud': 'GCP', 'vm_spec': { 'GCP': { @@ -443,7 +481,8 @@ def create_kubernetes_engine_spec(): }, }, 'vm_count': 2, - }) + }, + ) return kubernetes_engine_spec def testCreate(self): @@ -457,14 +496,16 @@ def testCreate(self): self.assertIn('gcloud container clusters create', command_string) self.assertIn('--num-nodes 2', command_string) self.assertIn('--machine-type fake-machine-type', command_string) - self.assertIn('--accelerator type=nvidia-tesla-k80,count=2', - command_string) + self.assertIn( + '--accelerator type=nvidia-tesla-k80,count=2', command_string + ) @mock.patch('perfkitbenchmarker.kubernetes_helper.CreateFromFile') def testPostCreate(self, create_from_file_patch): spec = self.create_kubernetes_engine_spec() with patch_critical_objects() as issue_command, mock.patch.object( - container_service, 'RunKubectlCommand') as mock_kubectl_command: + container_service, 'RunKubectlCommand' + ) as mock_kubectl_command: cluster = google_kubernetes_engine.GkeCluster(spec) cluster._PostCreate() command_string = ' '.join(issue_command.call_args[0][0]) @@ -472,18 +513,19 @@ def testPostCreate(self, create_from_file_patch): self.assertEqual(issue_command.call_count, 1) self.assertIn( 'gcloud container clusters get-credentials pkb-{0}'.format(_RUN_URI), - command_string) + command_string, + ) self.assertIn('KUBECONFIG', issue_command.call_args[1]['env']) self.assertEqual(mock_kubectl_command.call_count, 1) expected_args_to_create_from_file = ( _NVIDIA_DRIVER_SETUP_DAEMON_SET_SCRIPT, - data.ResourcePath( - _NVIDIA_UNRESTRICTED_PERMISSIONS_DAEMON_SET) + data.ResourcePath(_NVIDIA_UNRESTRICTED_PERMISSIONS_DAEMON_SET), ) - expected_calls = [mock.call(arg) - for arg in expected_args_to_create_from_file] + expected_calls = [ + mock.call(arg) for arg in expected_args_to_create_from_file + ] # Assert that create_from_file was called twice, # and that the args were as expected (should be the NVIDIA @@ -509,18 +551,20 @@ def testGetInstanceGroups(self): expected = set([ 'gke-pkb-0c47e6fa-default-pool-167d73ee-grp', - 'gke-pkb-0c47e6fa-test-efea7796-grp' + 'gke-pkb-0c47e6fa-test-efea7796-grp', ]) self.assertEqual(expected, set(instance_groups)) # order doesn't matter class GoogleKubernetesEngineRegionalTestCase( - pkb_common_test_case.PkbCommonTestCase): + pkb_common_test_case.PkbCommonTestCase +): @staticmethod def create_kubernetes_engine_spec(use_zonal_nodepools=False): kubernetes_engine_spec = container_spec.ContainerClusterSpec( - 'NAME', **{ + 'NAME', + **{ 'cloud': 'GCP', 'vm_spec': { 'GCP': { @@ -532,29 +576,31 @@ def create_kubernetes_engine_spec(use_zonal_nodepools=False): 'nodepool1': { 'vm_spec': { 'GCP': { - 'machine_type': - 'machine-type-1', - 'zone': + 'machine_type': 'machine-type-1', + 'zone': ( 'us-west1-a,us-west1-b' - if use_zonal_nodepools else None, + if use_zonal_nodepools + else None + ), }, } }, 'nodepool2': { 'vm_spec': { 'GCP': { - 'machine_type': - 'machine-type-2', - 'zone': - 'us-west1-c' if use_zonal_nodepools else None, + 'machine_type': 'machine-type-2', + 'zone': ( + 'us-west1-c' if use_zonal_nodepools else None + ), }, }, 'sandbox_config': { 'type': 'gvisor', }, }, - } - }) + }, + }, + ) return kubernetes_engine_spec def testCreateRegionalCluster(self): @@ -563,36 +609,47 @@ def testCreateRegionalCluster(self): cluster = google_kubernetes_engine.GkeCluster(spec) cluster._Create() create_cluster, create_nodepool1, create_nodepool2 = ( - call[0][0] for call in issue_command.call_args_list) + call[0][0] for call in issue_command.call_args_list + ) self.assertNotIn('--zone', create_cluster) self.assertContainsSubsequence(create_cluster, ['--region', 'us-west1']) - self.assertContainsSubsequence(create_cluster, - ['--machine-type', 'fake-machine-type']) self.assertContainsSubsequence( - create_cluster, ['--labels', 'foo=bar,timeout=yesterday']) + create_cluster, ['--machine-type', 'fake-machine-type'] + ) + self.assertContainsSubsequence( + create_cluster, ['--labels', 'foo=bar,timeout=yesterday'] + ) self.assertContainsSubsequence( create_nodepool1, - ['gcloud', 'container', 'node-pools', 'create', 'nodepool1']) - self.assertContainsSubsequence(create_nodepool1, - ['--cluster', 'pkb-fake-urn-uri']) - self.assertContainsSubsequence(create_nodepool1, - ['--machine-type', 'machine-type-1']) + ['gcloud', 'container', 'node-pools', 'create', 'nodepool1'], + ) self.assertContainsSubsequence( - create_nodepool1, ['--node-labels', 'pkb_nodepool=nodepool1']) + create_nodepool1, ['--cluster', 'pkb-fake-urn-uri'] + ) self.assertContainsSubsequence( - create_nodepool1, ['--labels', 'foo=bar,timeout=yesterday']) + create_nodepool1, ['--machine-type', 'machine-type-1'] + ) + self.assertContainsSubsequence( + create_nodepool1, ['--node-labels', 'pkb_nodepool=nodepool1'] + ) + self.assertContainsSubsequence( + create_nodepool1, ['--labels', 'foo=bar,timeout=yesterday'] + ) self.assertNotIn('--node-locations', create_nodepool1) self.assertNotIn('--sandbox', create_nodepool1) self.assertContainsSubsequence(create_nodepool1, ['--region', 'us-west1']) self.assertContainsSubsequence( create_nodepool2, - ['gcloud', 'container', 'node-pools', 'create', 'nodepool2']) - self.assertContainsSubsequence(create_nodepool2, - ['--machine-type', 'machine-type-2']) - self.assertContainsSubsequence(create_nodepool2, - ['--sandbox', 'type=gvisor']) + ['gcloud', 'container', 'node-pools', 'create', 'nodepool2'], + ) + self.assertContainsSubsequence( + create_nodepool2, ['--machine-type', 'machine-type-2'] + ) + self.assertContainsSubsequence( + create_nodepool2, ['--sandbox', 'type=gvisor'] + ) self.assertNotIn('--node-locations', create_nodepool2) def testCreateRegionalClusterZonalNodepool(self): @@ -601,28 +658,37 @@ def testCreateRegionalClusterZonalNodepool(self): cluster = google_kubernetes_engine.GkeCluster(spec) cluster._Create() create_cluster, create_nodepool1, create_nodepool2 = ( - call[0][0] for call in issue_command.call_args_list) + call[0][0] for call in issue_command.call_args_list + ) self.assertNotIn('--zone', create_cluster) self.assertContainsSubsequence(create_cluster, ['--region', 'us-west1']) - self.assertContainsSubsequence(create_cluster, - ['--machine-type', 'fake-machine-type']) + self.assertContainsSubsequence( + create_cluster, ['--machine-type', 'fake-machine-type'] + ) self.assertContainsSubsequence( create_nodepool1, - ['gcloud', 'container', 'node-pools', 'create', 'nodepool1']) - self.assertContainsSubsequence(create_nodepool1, - ['--cluster', 'pkb-fake-urn-uri']) + ['gcloud', 'container', 'node-pools', 'create', 'nodepool1'], + ) self.assertContainsSubsequence( - create_nodepool1, ['--node-labels', 'pkb_nodepool=nodepool1']) + create_nodepool1, ['--cluster', 'pkb-fake-urn-uri'] + ) + self.assertContainsSubsequence( + create_nodepool1, ['--node-labels', 'pkb_nodepool=nodepool1'] + ) self.assertContainsSubsequence( - create_nodepool1, ['--node-locations', 'us-west1-a,us-west1-b']) + create_nodepool1, ['--node-locations', 'us-west1-a,us-west1-b'] + ) self.assertContainsSubsequence(create_nodepool1, ['--region', 'us-west1']) self.assertContainsSubsequence( create_nodepool2, - ['gcloud', 'container', 'node-pools', 'create', 'nodepool2']) - self.assertContainsSubsequence(create_nodepool2, - ['--node-locations', 'us-west1-c']) + ['gcloud', 'container', 'node-pools', 'create', 'nodepool2'], + ) + self.assertContainsSubsequence( + create_nodepool2, ['--node-locations', 'us-west1-c'] + ) + if __name__ == '__main__': unittest.main() diff --git a/tests/providers/gcp/util_test.py b/tests/providers/gcp/util_test.py index 8c3d46bbfd..7e14806ba4 100644 --- a/tests/providers/gcp/util_test.py +++ b/tests/providers/gcp/util_test.py @@ -35,7 +35,8 @@ def _MockIssueCommand(test_response): vm_util, 'IssueCommand', autospec=True, - return_value=[test_response, None, None]) + return_value=[test_response, None, None], + ) class GceResource(resource.BaseResource): @@ -63,27 +64,55 @@ def setUp(self): def testCommonFlags(self): cmd = util.GcloudCommand(None, 'compute', 'images', 'list') - self.assertEqual(cmd.GetCommand(), [ - 'path/gcloud', 'compute', 'images', 'list', '--format', 'json', - '--quiet' - ]) + self.assertEqual( + cmd.GetCommand(), + [ + 'path/gcloud', + 'compute', + 'images', + 'list', + '--format', + 'json', + '--quiet', + ], + ) def testCommonFlagsWithGlobalProject(self): self.mock_flags.project = 'default-project' cmd = util.GcloudCommand(None, 'compute', 'images', 'list') - self.assertEqual(cmd.GetCommand(), [ - 'path/gcloud', 'compute', 'images', 'list', '--format', 'json', - '--project', 'default-project', '--quiet' - ]) + self.assertEqual( + cmd.GetCommand(), + [ + 'path/gcloud', + 'compute', + 'images', + 'list', + '--format', + 'json', + '--project', + 'default-project', + '--quiet', + ], + ) def testCommonFlagsWithResource(self): gce_resource = GceResource(project=None) cmd = util.GcloudCommand( - gce_resource, 'compute', 'images', 'describe', 'fake-image') - self.assertEqual(cmd.GetCommand(), [ - 'path/gcloud', 'compute', 'images', 'describe', 'fake-image', - '--format', 'json', '--quiet' - ]) + gce_resource, 'compute', 'images', 'describe', 'fake-image' + ) + self.assertEqual( + cmd.GetCommand(), + [ + 'path/gcloud', + 'compute', + 'images', + 'describe', + 'fake-image', + '--format', + 'json', + '--quiet', + ], + ) # This case is pathological. If --project is set every GCP resource we create # should have project set. It should honestly probably raise an error. @@ -91,84 +120,150 @@ def testCommonFlagsWithResourceAndProject(self): self.mock_flags.project = 'default-project' gce_resource = GceResource(project=None) cmd = util.GcloudCommand( - gce_resource, 'compute', 'images', 'describe', 'fake-image') - self.assertEqual(cmd.GetCommand(), [ - 'path/gcloud', 'compute', 'images', 'describe', 'fake-image', - '--format', 'json', '--project', 'default-project', '--quiet' - ]) + gce_resource, 'compute', 'images', 'describe', 'fake-image' + ) + self.assertEqual( + cmd.GetCommand(), + [ + 'path/gcloud', + 'compute', + 'images', + 'describe', + 'fake-image', + '--format', + 'json', + '--project', + 'default-project', + '--quiet', + ], + ) def testCommonFlagsWithResourceWithProject(self): gce_resource = GceResource(project='test-project') cmd = util.GcloudCommand( - gce_resource, 'compute', 'images', 'describe', 'fake-image') - self.assertEqual(cmd.GetCommand(), [ - 'path/gcloud', 'compute', 'images', 'describe', 'fake-image', - '--format', 'json', '--project', 'test-project', '--quiet', - ]) + gce_resource, 'compute', 'images', 'describe', 'fake-image' + ) + self.assertEqual( + cmd.GetCommand(), + [ + 'path/gcloud', + 'compute', + 'images', + 'describe', + 'fake-image', + '--format', + 'json', + '--project', + 'test-project', + '--quiet', + ], + ) def testCommonFlagsWithResourceWithProjectAndZone(self): gce_resource = GceResource(project='test-project', zone='test-zone') cmd = util.GcloudCommand( - gce_resource, 'compute', 'images', 'describe', 'fake-image') - self.assertEqual(cmd.GetCommand(), [ - 'path/gcloud', 'compute', 'images', 'describe', 'fake-image', - '--format', 'json', '--project', 'test-project', - '--quiet', '--zone', 'test-zone' - ]) + gce_resource, 'compute', 'images', 'describe', 'fake-image' + ) + self.assertEqual( + cmd.GetCommand(), + [ + 'path/gcloud', + 'compute', + 'images', + 'describe', + 'fake-image', + '--format', + 'json', + '--project', + 'test-project', + '--quiet', + '--zone', + 'test-zone', + ], + ) def testListValue(self): gce_resource = GceResource(project=None) cmd = util.GcloudCommand(gce_resource, 'compute', 'instances', 'create') cmd.flags['local-ssd'] = ['interface=nvme', 'interface=SCSI'] - self.assertEqual(cmd.GetCommand(), [ - 'path/gcloud', - 'compute', - 'instances', - 'create', - '--format', - 'json', - '--local-ssd', - 'interface=nvme', - '--local-ssd', - 'interface=SCSI', - '--quiet', - ]) + self.assertEqual( + cmd.GetCommand(), + [ + 'path/gcloud', + 'compute', + 'instances', + 'create', + '--format', + 'json', + '--local-ssd', + 'interface=nvme', + '--local-ssd', + 'interface=SCSI', + '--quiet', + ], + ) def testIssue(self): gce_resource = GceResource(project=None) cmd = util.GcloudCommand(gce_resource, 'compute', 'images', 'list') mock_issue_return_value = ('issue-return-value', 'stderr', 0) - p = mock.patch(util.__name__ + '.vm_util.IssueCommand', - return_value=mock_issue_return_value) + p = mock.patch( + util.__name__ + '.vm_util.IssueCommand', + return_value=mock_issue_return_value, + ) with p as mock_issue: return_value = cmd.Issue() - mock_issue.assert_called_with(['path/gcloud', 'compute', 'images', 'list', - '--format', 'json', '--quiet']) + mock_issue.assert_called_with([ + 'path/gcloud', + 'compute', + 'images', + 'list', + '--format', + 'json', + '--quiet', + ]) self.assertEqual(return_value, mock_issue_return_value) def testIssueWarningSuppressed(self): gce_resource = GceResource(project=None) cmd = util.GcloudCommand(gce_resource, 'compute', 'images', 'list') mock_issue_return_value = ('issue-return-value', 'stderr', 0) - p = mock.patch(util.__name__ + '.vm_util.IssueCommand', - return_value=mock_issue_return_value) + p = mock.patch( + util.__name__ + '.vm_util.IssueCommand', + return_value=mock_issue_return_value, + ) with p as mock_issue: return_value = cmd.Issue() - mock_issue.assert_called_with( - ['path/gcloud', 'compute', 'images', 'list', '--format', 'json', - '--quiet']) + mock_issue.assert_called_with([ + 'path/gcloud', + 'compute', + 'images', + 'list', + '--format', + 'json', + '--quiet', + ]) self.assertEqual(return_value, mock_issue_return_value) def testIssueRetryable(self): gce_resource = GceResource(project=None) cmd = util.GcloudCommand(gce_resource, 'compute', 'images', 'list') mock_issue_return_value = ('issue-return-value', 'stderr', 0) - p = mock.patch(util.__name__ + '.vm_util.IssueRetryableCommand', - return_value=mock_issue_return_value) + p = mock.patch( + util.__name__ + '.vm_util.IssueRetryableCommand', + return_value=mock_issue_return_value, + ) with p as mock_issue: return_value = cmd.IssueRetryable() - mock_issue.assert_called_with(['path/gcloud', 'compute', 'images', 'list', - '--format', 'json', '--quiet']) + mock_issue.assert_called_with([ + 'path/gcloud', + 'compute', + 'images', + 'list', + '--format', + 'json', + '--quiet', + ]) self.assertEqual(return_value, mock_issue_return_value) def testGetRegionFromZone(self): @@ -248,7 +343,8 @@ def testGetZonesFromMachineType(self, mock_issue_command): """) mock_issue_command.side_effect = [ (machine_type_zone_output, None, 0), - (usable_zone_output, None, 0)] + (usable_zone_output, None, 0), + ] found_zones = util.GetZonesFromMachineType('test-machine-type') @@ -264,7 +360,8 @@ def testGetZonesFromMachineTypeNoneAvailable(self, mock_issue_command): """) mock_issue_command.side_effect = [ (machine_type_zone_output, None, 0), - (usable_zone_output, None, 0)] + (usable_zone_output, None, 0), + ] found_zones = util.GetZonesFromMachineType('test-machine-type') @@ -272,25 +369,41 @@ def testGetZonesFromMachineTypeNoneAvailable(self, mock_issue_command): self.assertEqual(found_zones, expected_zones) @parameterized.named_parameters( - ('rate_limit_exceeded', - 'ERROR: (gcloud.compute.instances.create) Could not fetch resource:\n' - ' - Rate Limit Exceeded', True), - ('legacy_add_labels', 'ERROR: (gcloud.compute.disks.add-labels) ' - "PERMISSION_DENIED: Quota exceeded for quota group 'ReadGroup' and " - "limit 'Read requests per 100 seconds' of service " - "'compute.googleapis.com' for consumer 'project_number:012345678901'.", - True), + ( + 'rate_limit_exceeded', + ( + 'ERROR: (gcloud.compute.instances.create) Could not fetch' + ' resource:\n - Rate Limit Exceeded' + ), + True, + ), + ( + 'legacy_add_labels', + ( + 'ERROR: (gcloud.compute.disks.add-labels) PERMISSION_DENIED:' + " Quota exceeded for quota group 'ReadGroup' and limit 'Read" + " requests per 100 seconds' of service 'compute.googleapis.com'" + " for consumer 'project_number:012345678901'." + ), + True, + ), ('no match', 'not a rate limit error message', False), - ('add_labels', - "ERROR: (gcloud.compute.disks.add-labels) PERMISSION_DENIED: " - "Quota exceeded for quota group 'default' and limit " - "'Queries per 100 seconds' of service 'compute.googleapis.com' for " - "consumer 'project_number:300314462293'.", True), - ) + ( + 'add_labels', + ( + 'ERROR: (gcloud.compute.disks.add-labels) PERMISSION_DENIED:' + " Quota exceeded for quota group 'default' and limit 'Queries per" + " 100 seconds' of service 'compute.googleapis.com' for consumer" + " 'project_number:300314462293'." + ), + True, + ), + ) def testGcloudCommand(self, error_text: str, is_rate_limit_message: bool): self.assertEqual( util.GcloudCommand._IsIssueRateLimitMessage(error_text), - is_rate_limit_message) + is_rate_limit_message, + ) class GcpUtilTest(pkb_common_test_case.PkbCommonTestCase): @@ -298,8 +411,8 @@ class GcpUtilTest(pkb_common_test_case.PkbCommonTestCase): def testSplitTags(self): tags = 'a=b,c=d' self.assertEqual( - collections.OrderedDict([('a', 'b'), ('c', 'd')]), - util.SplitTags(tags)) + collections.OrderedDict([('a', 'b'), ('c', 'd')]), util.SplitTags(tags) + ) def testFormatAndSplitAreInverses(self): tags = {'a': 'b', 'c': 'd'} diff --git a/tests/providers/ibmcloud/ibmcloud_disk_test.py b/tests/providers/ibmcloud/ibmcloud_disk_test.py index 08614c5f11..b47a48f68e 100644 --- a/tests/providers/ibmcloud/ibmcloud_disk_test.py +++ b/tests/providers/ibmcloud/ibmcloud_disk_test.py @@ -17,7 +17,6 @@ import unittest from absl import flags import mock - from perfkitbenchmarker.providers.ibmcloud import ibmcloud_disk from tests import pkb_common_test_case @@ -31,15 +30,19 @@ class IbmcloudDiskGetDevicePathTest(pkb_common_test_case.PkbCommonTestCase): def setUp(self): super(IbmcloudDiskGetDevicePathTest, self).setUp() # Patch the __init__ method for simplicity. - with mock.patch.object(ibmcloud_disk.IbmCloudDisk, '__init__', - lambda self: None): + with mock.patch.object( + ibmcloud_disk.IbmCloudDisk, '__init__', lambda self: None + ): self.disk = ibmcloud_disk.IbmCloudDisk() def run_cmd(self, cmd): return ( - 'Disk /dev/vde: 8589.9 GB, 8589934592000 bytes, 16777216000 ' - 'sectors\nUnits = sectors of 1 * 512 = 512 bytes\n', - None) + ( + 'Disk /dev/vde: 8589.9 GB, 8589934592000 bytes, 16777216000 ' + 'sectors\nUnits = sectors of 1 * 512 = 512 bytes\n' + ), + None, + ) def testGetDeviceFromVDisk(self): vm = mock.Mock() diff --git a/tests/providers/ibmcloud/ibmcloud_network_test.py b/tests/providers/ibmcloud/ibmcloud_network_test.py index 145e1d1f12..62db277811 100644 --- a/tests/providers/ibmcloud/ibmcloud_network_test.py +++ b/tests/providers/ibmcloud/ibmcloud_network_test.py @@ -33,10 +33,14 @@ class IbmcloudNetworkTest(pkb_common_test_case.PkbCommonTestCase): def setUp(self): super(IbmcloudNetworkTest, self).setUp() # Patch the __init__ method for simplicity. - with mock.patch.object(ibmcloud_network.IbmCloudNetwork, '__init__', - lambda self, prefix, zone: None): - self.network = ibmcloud_network.IbmCloudNetwork('fake_prefix', - 'fake_zone') + with mock.patch.object( + ibmcloud_network.IbmCloudNetwork, + '__init__', + lambda self, prefix, zone: None, + ): + self.network = ibmcloud_network.IbmCloudNetwork( + 'fake_prefix', 'fake_zone' + ) def get_vpc(self): return VPC_ID @@ -50,8 +54,9 @@ def get_fip(self): def testGetSubnetIndex(self): subnetx = '10.103.20.0/24' expected_subet_index = 2 - self.assertEqual(expected_subet_index, - ibmcloud_network.GetSubnetIndex(subnetx)) + self.assertEqual( + expected_subet_index, ibmcloud_network.GetSubnetIndex(subnetx) + ) def testCreateVpc(self): self.network = mock.Mock() diff --git a/tests/providers/ibmcloud/ibmcloud_util_test.py b/tests/providers/ibmcloud/ibmcloud_util_test.py index 7e64c5a7b6..366cc9c0a2 100644 --- a/tests/providers/ibmcloud/ibmcloud_util_test.py +++ b/tests/providers/ibmcloud/ibmcloud_util_test.py @@ -23,16 +23,13 @@ class IbmcloudUtilTest(unittest.TestCase): def testGetBaseOs(self): data = {'name': 'debian10'} - self.assertEqual('debian', - util.GetBaseOs(data)) + self.assertEqual('debian', util.GetBaseOs(data)) data = {'name': 'myos'} - self.assertEqual('unknown', - util.GetBaseOs(data)) + self.assertEqual('unknown', util.GetBaseOs(data)) def testGetOsInfo(self): data = {'name': 'Debian 10', 'operating_system': {'name': 'debian10'}} - self.assertEqual('debian', - util.GetOsInfo(data)['base_os']) + self.assertEqual('debian', util.GetOsInfo(data)['base_os']) if __name__ == '__main__': diff --git a/tests/providers/ibmcloud/ibmcloud_virtual_machine_test.py b/tests/providers/ibmcloud/ibmcloud_virtual_machine_test.py index 3c134a6f17..6c7a5d4364 100644 --- a/tests/providers/ibmcloud/ibmcloud_virtual_machine_test.py +++ b/tests/providers/ibmcloud/ibmcloud_virtual_machine_test.py @@ -37,9 +37,10 @@ URI = 'ibmuri123' -class TestIbmCloudVirtualMachine(pkb_common_test_case.TestOsMixin, - ibmcloud_virtual_machine.IbmCloudVirtualMachine - ): +class TestIbmCloudVirtualMachine( + pkb_common_test_case.TestOsMixin, + ibmcloud_virtual_machine.IbmCloudVirtualMachine, +): pass @@ -48,24 +49,32 @@ class IbmCloudVirtualMachineTest(pkb_common_test_case.PkbCommonTestCase): def setUp(self): super(IbmCloudVirtualMachineTest, self).setUp() self.mock_create_instance = self.enter_context( - mock.patch.object(ibm_api.IbmAPICommand, 'CreateInstance')) + mock.patch.object(ibm_api.IbmAPICommand, 'CreateInstance') + ) self.mock_instance_status = self.enter_context( - mock.patch.object(ibm_api.IbmAPICommand, 'InstanceStatus')) + mock.patch.object(ibm_api.IbmAPICommand, 'InstanceStatus') + ) self.mock_get_resource = self.enter_context( - mock.patch.object(ibm_api.IbmAPICommand, 'GetResource')) + mock.patch.object(ibm_api.IbmAPICommand, 'GetResource') + ) self.mock_create_vpc = self.enter_context( - mock.patch.object(ibm_api.IbmAPICommand, 'CreateVpc')) + mock.patch.object(ibm_api.IbmAPICommand, 'CreateVpc') + ) self.mock_create_subnet = self.enter_context( - mock.patch.object(ibm_api.IbmAPICommand, 'CreateSubnet')) + mock.patch.object(ibm_api.IbmAPICommand, 'CreateSubnet') + ) self.mock_check_environment = self.enter_context( - mock.patch.object(ibm_api.IbmAPICommand, '_CheckEnvironment')) - with mock.patch.object(ibm_api.IbmAPICommand, '__init__', - lambda self: None): + mock.patch.object(ibm_api.IbmAPICommand, '_CheckEnvironment') + ) + with mock.patch.object( + ibm_api.IbmAPICommand, '__init__', lambda self: None + ): self.cmd = ibm_api.IbmAPICommand() self.vm = self._CreateTestIbmCloudVirtualMachine() - def _CreateBenchmarkSpecFromYaml(self, yaml_string, - benchmark_name=BENCHMARK_NAME): + def _CreateBenchmarkSpecFromYaml( + self, yaml_string, benchmark_name=BENCHMARK_NAME + ): config = configs.LoadConfig(yaml_string, {}, benchmark_name) spec = self._CreateBenchmarkSpecFromConfigDict(config, benchmark_name) spec.disable_interrupt_moderation = False @@ -87,11 +96,13 @@ def _CreateBenchmarkSpecFromYaml(self, yaml_string, def _CreateBenchmarkSpecFromConfigDict(self, config_dict, benchmark_name): config_spec = benchmark_config_spec.BenchmarkConfigSpec( - benchmark_name, - flag_values=FLAGS, - **config_dict) - benchmark_module = next((b for b in linux_benchmarks.BENCHMARKS - if b.BENCHMARK_NAME == benchmark_name)) + benchmark_name, flag_values=FLAGS, **config_dict + ) + benchmark_module = next(( + b + for b in linux_benchmarks.BENCHMARKS + if b.BENCHMARK_NAME == benchmark_name + )) return benchmark_spec.BenchmarkSpec(benchmark_module, config_spec, URI) def _CreateTestIbmCloudVirtualMachine(self): diff --git a/tests/providers/kubernetes/kubernetes_virtual_machine_test.py b/tests/providers/kubernetes/kubernetes_virtual_machine_test.py index 1d0f04e45d..f316addd98 100644 --- a/tests/providers/kubernetes/kubernetes_virtual_machine_test.py +++ b/tests/providers/kubernetes/kubernetes_virtual_machine_test.py @@ -217,7 +217,8 @@ def patch_critical_objects(stdout='', stderr='', return_code=0, flags=FLAGS): # that the body of the file is what we expect it to be (useful for # verifying that the pod.yml body was written correctly). temp_file = stack.enter_context( - mock.patch(vm_util.__name__ + '.NamedTemporaryFile')) + mock.patch(vm_util.__name__ + '.NamedTemporaryFile') + ) issue_command = stack.enter_context( mock.patch( @@ -232,37 +233,31 @@ def patch_critical_objects(stdout='', stderr='', return_code=0, flags=FLAGS): class TestKubernetesVirtualMachine( pkb_common_test_case.TestOsMixin, - kubernetes_virtual_machine.KubernetesVirtualMachine): + kubernetes_virtual_machine.KubernetesVirtualMachine, +): pass class BaseKubernetesVirtualMachineTestCase( - pkb_common_test_case.PkbCommonTestCase): + pkb_common_test_case.PkbCommonTestCase +): def assertJsonEqual(self, str1, str2): json1 = json.loads(str1) json2 = json.loads(str2) self.assertEqual( - json.dumps(json1, sort_keys=True), - json.dumps(json2, sort_keys=True) + json.dumps(json1, sort_keys=True), json.dumps(json2, sort_keys=True) ) -class KubernetesResourcesTestCase( - BaseKubernetesVirtualMachineTestCase): +class KubernetesResourcesTestCase(BaseKubernetesVirtualMachineTestCase): @staticmethod def create_virtual_machine_spec(): spec = kubernetes_pod_spec.KubernetesPodSpec( _COMPONENT, - resource_limits={ - 'cpus': 2, - 'memory': '5GiB' - }, - resource_requests={ - 'cpus': 1.5, - 'memory': '4GiB' - }, + resource_limits={'cpus': 2, 'memory': '5GiB'}, + resource_requests={'cpus': 1.5, 'memory': '4GiB'}, gpu_count=2, gpu_type='k80', ) @@ -279,16 +274,8 @@ def testCreatePodResourceBody(self): kub_vm = TestKubernetesVirtualMachine(spec) expected = { - 'limits': { - 'cpu': '2', - 'memory': '5120Mi', - 'nvidia.com/gpu': '2' - }, - 'requests': { - 'cpu': '1.5', - 'memory': '4096Mi', - 'nvidia.com/gpu': '2' - } + 'limits': {'cpu': '2', 'memory': '5120Mi', 'nvidia.com/gpu': '2'}, + 'requests': {'cpu': '1.5', 'memory': '4096Mi', 'nvidia.com/gpu': '2'}, } actual = kub_vm._BuildResourceBody() self.assertDictEqual(expected, actual) @@ -319,14 +306,13 @@ def testAnnotations(self): class KubernetesVirtualMachineOsTypesTestCase( - BaseKubernetesVirtualMachineTestCase): + BaseKubernetesVirtualMachineTestCase +): @staticmethod def create_kubernetes_vm(os_type): - spec = kubernetes_pod_spec.KubernetesPodSpec( - _COMPONENT) - vm_class = virtual_machine.GetVmClass(provider_info.KUBERNETES, - os_type) + spec = kubernetes_pod_spec.KubernetesPodSpec(_COMPONENT) + vm_class = virtual_machine.GetVmClass(provider_info.KUBERNETES, os_type) kub_vm = vm_class(spec) kub_vm._WaitForPodBootCompletion = lambda: None kub_vm._Create() @@ -337,12 +323,14 @@ def testCreateUbuntu1604(self): write_mock = get_write_mock_from_temp_file_mock(temp_file) create_json = json.loads(write_mock.call_args[0][0]) - self.assertEqual(create_json['spec']['containers'][0]['image'], - 'ubuntu:16.04') + self.assertEqual( + create_json['spec']['containers'][0]['image'], 'ubuntu:16.04' + ) class KubernetesVirtualMachineVmGroupAffinityTestCase( - BaseKubernetesVirtualMachineTestCase): + BaseKubernetesVirtualMachineTestCase +): @staticmethod def create_kubernetes_vm(): @@ -351,7 +339,8 @@ def create_kubernetes_vm(): image='test_image', install_packages=False, machine_type='test_machine_type', - zone='test_zone') + zone='test_zone', + ) kub_vm = TestKubernetesVirtualMachine(spec) kub_vm.name = _NAME kub_vm.vm_group = 'my_vm_group' @@ -364,13 +353,11 @@ def testCreateVmGroupAffinity(self): write_mock = get_write_mock_from_temp_file_mock(temp_file) self.assertJsonEqual( - write_mock.call_args[0][0], - _EXPECTED_CALL_BODY_WITH_VM_GROUP + write_mock.call_args[0][0], _EXPECTED_CALL_BODY_WITH_VM_GROUP ) -class KubernetesVirtualMachineTestCase( - BaseKubernetesVirtualMachineTestCase): +class KubernetesVirtualMachineTestCase(BaseKubernetesVirtualMachineTestCase): @staticmethod def create_virtual_machine_spec(): @@ -379,7 +366,8 @@ def create_virtual_machine_spec(): image='test_image', install_packages=False, machine_type='test_machine_type', - zone='test_zone') + zone='test_zone', + ) return spec def testCreate(self): @@ -392,8 +380,10 @@ def testCreate(self): command_string = ' '.join(command[:4]) self.assertEqual(issue_command.call_count, 1) - self.assertIn('{0} --kubeconfig={1} create -f'.format( - _KUBECTL, _KUBECONFIG), command_string) + self.assertIn( + '{0} --kubeconfig={1} create -f'.format(_KUBECTL, _KUBECONFIG), + command_string, + ) def testCreatePodBodyWrittenCorrectly(self): spec = self.create_virtual_machine_spec() @@ -409,8 +399,7 @@ def testCreatePodBodyWrittenCorrectly(self): write_mock = get_write_mock_from_temp_file_mock(temp_file) self.assertJsonEqual( - write_mock.call_args[0][0], - _EXPECTED_CALL_BODY_WITHOUT_GPUS + write_mock.call_args[0][0], _EXPECTED_CALL_BODY_WITHOUT_GPUS ) def testDownloadPreprovisionedDataAws(self): @@ -419,7 +408,9 @@ def testDownloadPreprovisionedDataAws(self): with patch_critical_objects(flags=FLAGS) as (issue_command, _): kub_vm = ( kubernetes_virtual_machine.Ubuntu1604BasedKubernetesVirtualMachine( - spec)) + spec + ) + ) kub_vm.DownloadPreprovisionedData('path', 'name', 'filename') command = issue_command.call_args[0][0] @@ -433,7 +424,9 @@ def testDownloadPreprovisionedDataAzure(self): with patch_critical_objects() as (issue_command, _): kub_vm = ( kubernetes_virtual_machine.Ubuntu1604BasedKubernetesVirtualMachine( - spec)) + spec + ) + ) kub_vm.DownloadPreprovisionedData('path', 'name', 'filename') command = issue_command.call_args[0][0] @@ -447,7 +440,9 @@ def testDownloadPreprovisionedDataGcp(self): with patch_critical_objects() as (issue_command, _): kub_vm = ( kubernetes_virtual_machine.Ubuntu1604BasedKubernetesVirtualMachine( - spec)) + spec + ) + ) kub_vm.DownloadPreprovisionedData('path', 'name', 'filename') command = issue_command.call_args[0][0] @@ -456,7 +451,8 @@ def testDownloadPreprovisionedDataGcp(self): class KubernetesVirtualMachineWithGpusTestCase( - BaseKubernetesVirtualMachineTestCase): + BaseKubernetesVirtualMachineTestCase +): @staticmethod def create_virtual_machine_spec(): @@ -467,7 +463,8 @@ def create_virtual_machine_spec(): gpu_type='k80', install_packages=False, machine_type='test_machine_type', - zone='test_zone') + zone='test_zone', + ) return spec def testCreate(self): @@ -480,8 +477,10 @@ def testCreate(self): command_string = ' '.join(command[:4]) self.assertEqual(issue_command.call_count, 1) - self.assertIn('{0} --kubeconfig={1} create -f'.format( - _KUBECTL, _KUBECONFIG), command_string) + self.assertIn( + '{0} --kubeconfig={1} create -f'.format(_KUBECTL, _KUBECONFIG), + command_string, + ) def testCreatePodBodyWrittenCorrectly(self): spec = self.create_virtual_machine_spec() @@ -497,13 +496,13 @@ def testCreatePodBodyWrittenCorrectly(self): write_mock = get_write_mock_from_temp_file_mock(temp_file) self.assertJsonEqual( - write_mock.call_args[0][0], - _EXPECTED_CALL_BODY_WITH_2_GPUS + write_mock.call_args[0][0], _EXPECTED_CALL_BODY_WITH_2_GPUS ) class KubernetesVirtualMachineWithNvidiaCudaImage( - BaseKubernetesVirtualMachineTestCase): + BaseKubernetesVirtualMachineTestCase +): @staticmethod def create_virtual_machine_spec(): @@ -511,13 +510,15 @@ def create_virtual_machine_spec(): _COMPONENT, install_packages=False, machine_type='test_machine_type', - zone='test_zone') + zone='test_zone', + ) return spec def testCreatePodBodyWrittenCorrectly(self): spec = self.create_virtual_machine_spec() - vm_class = virtual_machine.GetVmClass(provider_info.KUBERNETES, - os_types.UBUNTU1604_CUDA9) + vm_class = virtual_machine.GetVmClass( + provider_info.KUBERNETES, os_types.UBUNTU1604_CUDA9 + ) with patch_critical_objects() as (_, temp_file): kub_vm = vm_class(spec) # Need to set the name explicitly on the instance because the test @@ -530,8 +531,7 @@ def testCreatePodBodyWrittenCorrectly(self): write_mock = get_write_mock_from_temp_file_mock(temp_file) self.assertJsonEqual( - write_mock.call_args[0][0], - _EXPECTED_CALL_BODY_WITH_NVIDIA_CUDA_IMAGE + write_mock.call_args[0][0], _EXPECTED_CALL_BODY_WITH_NVIDIA_CUDA_IMAGE ) diff --git a/tests/providers/openstack/os_virtual_machine_test.py b/tests/providers/openstack/os_virtual_machine_test.py index 446b3517f7..0493ee7e98 100644 --- a/tests/providers/openstack/os_virtual_machine_test.py +++ b/tests/providers/openstack/os_virtual_machine_test.py @@ -31,15 +31,17 @@ _network_fail = {'router:external': 'Fail'} -class TestOpenStackVirtualMachine(pkb_common_test_case.TestOsMixin, - os_virtual_machine.OpenStackVirtualMachine): +class TestOpenStackVirtualMachine( + pkb_common_test_case.TestOsMixin, os_virtual_machine.OpenStackVirtualMachine +): pass class BaseOpenStackNetworkTest(pkb_common_test_case.PkbCommonTestCase): - def _CreateBenchmarkSpecFromYaml(self, yaml_string, - benchmark_name=_BENCHMARK_NAME): + def _CreateBenchmarkSpecFromYaml( + self, yaml_string, benchmark_name=_BENCHMARK_NAME + ): config = configs.LoadConfig(yaml_string, {}, benchmark_name) spec = self._CreateBenchmarkSpecFromConfigDict(config, benchmark_name) spec.disable_interrupt_moderation = False @@ -61,11 +63,13 @@ def _CreateBenchmarkSpecFromYaml(self, yaml_string, def _CreateBenchmarkSpecFromConfigDict(self, config_dict, benchmark_name): config_spec = benchmark_config_spec.BenchmarkConfigSpec( - benchmark_name, - flag_values=FLAGS, - **config_dict) - benchmark_module = next((b for b in linux_benchmarks.BENCHMARKS - if b.BENCHMARK_NAME == benchmark_name)) + benchmark_name, flag_values=FLAGS, **config_dict + ) + benchmark_module = next(( + b + for b in linux_benchmarks.BENCHMARKS + if b.BENCHMARK_NAME == benchmark_name + )) return benchmark_spec.BenchmarkSpec(benchmark_module, config_spec, _URI) def _CreateTestOpenStackVm(self): @@ -77,9 +81,11 @@ class OpenStackVirtualMachineTest(BaseOpenStackNetworkTest): def setUp(self): super(OpenStackVirtualMachineTest, self).setUp() - self.mock_check_network_exists = self.enter_context(mock.patch.object( - os_virtual_machine.OpenStackVirtualMachine, - '_CheckNetworkExists')) + self.mock_check_network_exists = self.enter_context( + mock.patch.object( + os_virtual_machine.OpenStackVirtualMachine, '_CheckNetworkExists' + ) + ) FLAGS.ignore_package_requirements = True self.openstack_vm = self._CreateTestOpenStackVm() diff --git a/tests/providers/openstack/swift_test.py b/tests/providers/openstack/swift_test.py index 9d7cc78984..a639b4fab9 100644 --- a/tests/providers/openstack/swift_test.py +++ b/tests/providers/openstack/swift_test.py @@ -1,6 +1,5 @@ """Tests for perfkitbenchmarker.providers.openstack.swift.""" - import os import unittest import mock @@ -17,25 +16,39 @@ def setUp(self): self.addCleanup(p.stop) self.mock_flags.openstack_swift_insecure = False - @mock.patch.dict(os.environ, {'OS_AUTH_URL': 'OS_AUTH_URL', - 'OS_TENANT_NAME': 'OS_TENANT_NAME', - 'OS_USERNAME': 'OS_USERNAME', - 'OS_PASSWORD': 'OS_PASSWORD'}) + @mock.patch.dict( + os.environ, + { + 'OS_AUTH_URL': 'OS_AUTH_URL', + 'OS_TENANT_NAME': 'OS_TENANT_NAME', + 'OS_USERNAME': 'OS_USERNAME', + 'OS_PASSWORD': 'OS_PASSWORD', + }, + ) def testMakeBucket(self): swift_storage_service = swift.SwiftStorageService() swift_storage_service.PrepareService('location') - with mock.patch(vm_util.__name__ + '.IssueCommand', - return_value=('stdout', 'stderr', 0)) as mock_util: + with mock.patch( + vm_util.__name__ + '.IssueCommand', return_value=('stdout', 'stderr', 0) + ) as mock_util: swift_storage_service.MakeBucket('new_bucket') - mock_util.assert_called_with(['swift', - '--os-auth-url', 'OS_AUTH_URL', - '--os-tenant-name', 'OS_TENANT_NAME', - '--os-username', 'OS_USERNAME', - '--os-password', 'OS_PASSWORD', - 'post', - 'new_bucket'], - raise_on_failure=False) + mock_util.assert_called_with( + [ + 'swift', + '--os-auth-url', + 'OS_AUTH_URL', + '--os-tenant-name', + 'OS_TENANT_NAME', + '--os-username', + 'OS_USERNAME', + '--os-password', + 'OS_PASSWORD', + 'post', + 'new_bucket', + ], + raise_on_failure=False, + ) if __name__ == '__main__': diff --git a/tests/providers/openstack/util_test.py b/tests/providers/openstack/util_test.py index 620e50eae2..ed028321b4 100644 --- a/tests/providers/openstack/util_test.py +++ b/tests/providers/openstack/util_test.py @@ -16,7 +16,6 @@ import unittest import mock - from perfkitbenchmarker import resource from perfkitbenchmarker import vm_util from perfkitbenchmarker.providers.openstack import utils @@ -54,39 +53,46 @@ def setUp(self): def testCommonFlagsWithoutOptionalFlags(self): rack_resource = OpenStackResource() cmd = utils.OpenStackCLICommand(rack_resource, 'image', 'list') - self.assertEqual(cmd._GetCommand(), [ - 'path/openstack', 'image', 'list', '--format', 'json']) + self.assertEqual( + cmd._GetCommand(), + ['path/openstack', 'image', 'list', '--format', 'json'], + ) def testCommonFlagsWithOptionalFlags(self): rack_resource = OpenStackResource() cmd = utils.OpenStackCLICommand(rack_resource, 'image', 'list') cmd.flags['public'] = True - self.assertEqual(cmd._GetCommand(), [ - 'path/openstack', 'image', 'list', '--format', 'json', '--public']) + self.assertEqual( + cmd._GetCommand(), + ['path/openstack', 'image', 'list', '--format', 'json', '--public'], + ) @mock.patch.object(vm_util, 'IssueCommand') def testIssueCommandRaiseOnFailureDefault(self, mock_cmd): rack_resource = OpenStackResource() cmd = utils.OpenStackCLICommand(rack_resource) cmd.Issue() - mock_cmd.assert_called_with(['path/openstack', '--format', 'json'], - raise_on_failure=False) + mock_cmd.assert_called_with( + ['path/openstack', '--format', 'json'], raise_on_failure=False + ) @mock.patch.object(vm_util, 'IssueCommand') def testIssueCommandRaiseOnFailureTrue(self, mock_cmd): rack_resource = OpenStackResource() cmd = utils.OpenStackCLICommand(rack_resource) cmd.Issue(raise_on_failure=True) - mock_cmd.assert_called_with(['path/openstack', '--format', 'json'], - raise_on_failure=True) + mock_cmd.assert_called_with( + ['path/openstack', '--format', 'json'], raise_on_failure=True + ) @mock.patch.object(vm_util, 'IssueCommand') def testIssueCommandRaiseOnFailureFalse(self, mock_cmd): rack_resource = OpenStackResource() cmd = utils.OpenStackCLICommand(rack_resource) cmd.Issue(raise_on_failure=False) - mock_cmd.assert_called_with(['path/openstack', '--format', 'json'], - raise_on_failure=False) + mock_cmd.assert_called_with( + ['path/openstack', '--format', 'json'], raise_on_failure=False + ) if __name__ == '__main__': diff --git a/tests/providers/rackspace/util_test.py b/tests/providers/rackspace/util_test.py index 0f017dd1f0..8e52485727 100644 --- a/tests/providers/rackspace/util_test.py +++ b/tests/providers/rackspace/util_test.py @@ -16,7 +16,6 @@ import unittest import mock - from perfkitbenchmarker import resource from perfkitbenchmarker.providers.rackspace import util import six @@ -50,16 +49,31 @@ def setUp(self): def testCommonFlagsWithoutOptionalFlags(self): rack_resource = RackspaceResource(profile=None) cmd = util.RackCLICommand(rack_resource, 'servers', 'image', 'list') - self.assertEqual(cmd._GetCommand(), [ - 'path/rack', 'servers', 'image', 'list', '--output', 'json']) + self.assertEqual( + cmd._GetCommand(), + ['path/rack', 'servers', 'image', 'list', '--output', 'json'], + ) def testCommonFlagsWithOptionalFlags(self): rack_resource = RackspaceResource(profile='US', region='DFW') cmd = util.RackCLICommand(rack_resource, 'servers', 'keypair', 'list') cmd.flags['all-pages'] = True - self.assertEqual(cmd._GetCommand(), [ - 'path/rack', 'servers', 'keypair', 'list', '--all-pages', - '--output', 'json', '--profile', 'US', '--region', 'DFW']) + self.assertEqual( + cmd._GetCommand(), + [ + 'path/rack', + 'servers', + 'keypair', + 'list', + '--all-pages', + '--output', + 'json', + '--profile', + 'US', + '--region', + 'DFW', + ], + ) if __name__ == '__main__': diff --git a/tests/providers_test.py b/tests/providers_test.py index 5975652c5d..547e7b91ed 100644 --- a/tests/providers_test.py +++ b/tests/providers_test.py @@ -39,9 +39,11 @@ def setUp(self): self.addCleanup(p.stop) # TODO(user): See if this can be fixed. - @unittest.skip('This fails because modules are being imported multiple ' - 'times in the instance of this process. Not sure how this ' - 'ever worked.') + @unittest.skip( + 'This fails because modules are being imported multiple ' + 'times in the instance of this process. Not sure how this ' + 'ever worked.' + ) def testImportAllProviders(self): # Test that all modules can be imported successfully, but mock out the # import of CloudStack's csapi. @@ -57,14 +59,15 @@ def testLoadProviderChecksRequirements(self): with mock.patch(requirements.__name__ + '.CheckProviderRequirements'): providers.LoadProvider('GCP', ignore_package_requirements=False) typing.cast( - mock.MagicMock, - requirements.CheckProviderRequirements).assert_called_once_with('gcp') + mock.MagicMock, requirements.CheckProviderRequirements + ).assert_called_once_with('gcp') def testLoadProviderIgnoresRequirements(self): with mock.patch(requirements.__name__ + '.CheckProviderRequirements'): providers.LoadProvider('GCP') - typing.cast(mock.MagicMock, - requirements.CheckProviderRequirements).assert_not_called() + typing.cast( + mock.MagicMock, requirements.CheckProviderRequirements + ).assert_not_called() def testBenchmarkConfigSpecLoadsProvider(self): p = mock.patch(providers.__name__ + '.LoadProvider') @@ -76,14 +79,16 @@ def testBenchmarkConfigSpecLoadsProvider(self): 'cloud': 'AWS', 'os_type': 'ubuntu2004', 'vm_count': 0, - 'vm_spec': {'AWS': {}} + 'vm_spec': {'AWS': {}}, } } } benchmark_config_spec.BenchmarkConfigSpec( - 'name', flag_values=FLAGS, **config) - typing.cast(mock.MagicMock, - providers.LoadProvider).assert_called_with('AWS', True) + 'name', flag_values=FLAGS, **config + ) + typing.cast(mock.MagicMock, providers.LoadProvider).assert_called_with( + 'AWS', True + ) def testLoadProviderUtils(self): test_cloud = provider_info.GCP @@ -93,5 +98,6 @@ def testLoadProviderUtils(self): expected_utils_module = gcp_util self.assertEqual(expected_utils_module, actual_utils_module) + if __name__ == '__main__': unittest.main() diff --git a/tests/publisher_test.py b/tests/publisher_test.py index 1206f7b5bd..e22682ed0d 100644 --- a/tests/publisher_test.py +++ b/tests/publisher_test.py @@ -46,17 +46,35 @@ def testSucceedsWithNoSamples(self): instance = publisher.PrettyPrintStreamPublisher(stream) instance.PublishSamples([]) self.assertRegexpMatches( - stream.getvalue(), r'^\s*-+PerfKitBenchmarker\sResults\sSummary-+\s*$') + stream.getvalue(), r'^\s*-+PerfKitBenchmarker\sResults\sSummary-+\s*$' + ) def testWritesToStream(self): stream = six.StringIO() instance = publisher.PrettyPrintStreamPublisher(stream) - samples = [{'test': 'testb', 'metric': '1', 'value': 1.0, 'unit': 'MB', - 'metadata': {}}, - {'test': 'testb', 'metric': '2', 'value': 14.0, 'unit': 'MB', - 'metadata': {}}, - {'test': 'testa', 'metric': '3', 'value': 47.0, 'unit': 'us', - 'metadata': {}}] + samples = [ + { + 'test': 'testb', + 'metric': '1', + 'value': 1.0, + 'unit': 'MB', + 'metadata': {}, + }, + { + 'test': 'testb', + 'metric': '2', + 'value': 14.0, + 'unit': 'MB', + 'metadata': {}, + }, + { + 'test': 'testa', + 'metric': '3', + 'value': 47.0, + 'unit': 'us', + 'metadata': {}, + }, + ] instance.PublishSamples(samples) value = stream.getvalue() @@ -78,9 +96,9 @@ def testCallsLoggerAtCorrectLevel(self): class NewlineDelimitedJSONPublisherTestCase(unittest.TestCase): def setUp(self): - self.fp = tempfile.NamedTemporaryFile(mode='w+', - prefix='perfkit-test-', - suffix='.json') + self.fp = tempfile.NamedTemporaryFile( + mode='w+', prefix='perfkit-test-', suffix='.json' + ) self.addCleanup(self.fp.close) self.instance = publisher.NewlineDelimitedJSONPublisher(self.fp.name) @@ -89,24 +107,32 @@ def testEmptyInput(self): self.assertEqual('', self.fp.read()) def testMetadataConvertedToLabels(self): - samples = [{'test': 'testa', - 'metadata': collections.OrderedDict([('key', 'value'), - ('foo', 'bar')])}] + samples = [{ + 'test': 'testa', + 'metadata': collections.OrderedDict([('key', 'value'), ('foo', 'bar')]), + }] self.instance.PublishSamples(samples) d = json.load(self.fp) - self.assertDictEqual({'test': 'testa', 'labels': '|foo:bar|,|key:value|'}, - d) + self.assertDictEqual( + {'test': 'testa', 'labels': '|foo:bar|,|key:value|'}, d + ) def testJSONRecordPerLine(self): - samples = [{'test': 'testa', 'metadata': {'key': 'val'}}, - {'test': 'testb', 'metadata': {'key2': 'val2'}}] + samples = [ + {'test': 'testa', 'metadata': {'key': 'val'}}, + {'test': 'testb', 'metadata': {'key2': 'val2'}}, + ] self.instance.PublishSamples(samples) self.assertRaises(ValueError, json.load, self.fp) self.fp.seek(0) result = [json.loads(i) for i in self.fp] - self.assertListEqual([{u'test': u'testa', u'labels': u'|key:val|'}, - {u'test': u'testb', u'labels': u'|key2:val2|'}], - result) + self.assertListEqual( + [ + {'test': 'testa', 'labels': '|key:val|'}, + {'test': 'testb', 'labels': '|key2:val2|'}, + ], + result, + ) class BigQueryPublisherTestCase(unittest.TestCase): @@ -118,8 +144,10 @@ def setUp(self): self.mock_vm_util.GetTempDir.return_value = tempfile.gettempdir() self.addCleanup(p.stop) - self.samples = [{'test': 'testa', 'metadata': {}}, - {'test': 'testb', 'metadata': {}}] + self.samples = [ + {'test': 'testa', 'metadata': {}}, + {'test': 'testb', 'metadata': {}}, + ] self.table = 'samples_mart.results' def testNoSamples(self): @@ -130,31 +158,37 @@ def testNoSamples(self): def testNoProject(self): instance = publisher.BigQueryPublisher(self.table) instance.PublishSamples(self.samples) - self.mock_vm_util.IssueRetryableCommand.assert_called_once_with( - ['bq', - 'load', - '--autodetect', - '--source_format=NEWLINE_DELIMITED_JSON', - self.table, - mock.ANY]) + self.mock_vm_util.IssueRetryableCommand.assert_called_once_with([ + 'bq', + 'load', + '--autodetect', + '--source_format=NEWLINE_DELIMITED_JSON', + self.table, + mock.ANY, + ]) def testServiceAccountFlags_MissingPrivateKey(self): - self.assertRaises(ValueError, - publisher.BigQueryPublisher, - self.table, - service_account=mock.MagicMock()) + self.assertRaises( + ValueError, + publisher.BigQueryPublisher, + self.table, + service_account=mock.MagicMock(), + ) def testServiceAccountFlags_MissingServiceAccount(self): - self.assertRaises(ValueError, - publisher.BigQueryPublisher, - self.table, - service_account_private_key_file=mock.MagicMock()) + self.assertRaises( + ValueError, + publisher.BigQueryPublisher, + self.table, + service_account_private_key_file=mock.MagicMock(), + ) def testServiceAccountFlags_BothSpecified(self): instance = publisher.BigQueryPublisher( self.table, service_account=mock.MagicMock(), - service_account_private_key_file=mock.MagicMock()) + service_account_private_key_file=mock.MagicMock(), + ) instance.PublishSamples(self.samples) # No error self.mock_vm_util.IssueRetryableCommand.assert_called_once_with(mock.ANY) @@ -165,11 +199,13 @@ def testApplicationCredentialsFlag_FlagsConflict(self): self.table, service_account=mock.MagicMock(), service_account_private_key_file=mock.MagicMock(), - application_default_credential_file=mock.MagicMock()) + application_default_credential_file=mock.MagicMock(), + ) def testApplicationCredentialsFlag_WorkingNormal(self): instance = publisher.BigQueryPublisher( - self.table, application_default_credential_file=mock.MagicMock()) + self.table, application_default_credential_file=mock.MagicMock() + ) instance.PublishSamples(self.samples) # No error self.mock_vm_util.IssueRetryableCommand.assert_called_once_with(mock.ANY) @@ -191,18 +227,21 @@ def setUp(self): self.mock_uuid = p.start() self.addCleanup(p.stop) - self.samples = [{'test': 'testa', 'metadata': {}}, - {'test': 'testb', 'metadata': {}}] + self.samples = [ + {'test': 'testa', 'metadata': {}}, + {'test': 'testb', 'metadata': {}}, + ] def testPublishSamples(self): self.mock_time.time.return_value = 1417647763.387665 self.mock_uuid.uuid4.return_value = uuid.UUID( - 'be428eb3-a54a-4615-b7ca-f962b729c7ab') + 'be428eb3-a54a-4615-b7ca-f962b729c7ab' + ) instance = publisher.CloudStoragePublisher('test-bucket') instance.PublishSamples(self.samples) self.mock_vm_util.IssueRetryableCommand.assert_called_once_with( - ['gsutil', 'cp', mock.ANY, - 'gs://test-bucket/141764776338_be428eb']) + ['gsutil', 'cp', mock.ANY, 'gs://test-bucket/141764776338_be428eb'] + ) class SampleCollectorTestCase(unittest.TestCase): @@ -232,9 +271,10 @@ def _VerifyResult(self, contains_metadata=True): 'metric': 'widgets', 'unit': 'oz', 'test': self.benchmark, - 'product_name': 'PerfKitBenchmarker' + 'product_name': 'PerfKitBenchmarker', }, - collector_sample) + collector_sample, + ) if contains_metadata: self.assertDictContainsSubset({'foo': 'bar'}, metadata) else: @@ -249,11 +289,7 @@ def testAddSamples_WithTimestamp(self): timestamp_sample = sample.Sample('widgets', 100, 'oz', {}, 1.0) samples = [timestamp_sample] self.instance.AddSamples(samples, self.benchmark, self.benchmark_spec) - self.assertDictContainsSubset( - { - 'timestamp': 1.0 - }, - self.instance.samples[0]) + self.assertDictContainsSubset({'timestamp': 1.0}, self.instance.samples[0]) def CreateMockVM(hostname='Hostname', vm_id='12345', ip_address='1.2.3.4'): @@ -282,16 +318,17 @@ class DefaultMetadataProviderTestCase(unittest.TestCase): def setUp(self): p = mock.patch(publisher.__name__ + '.FLAGS') self.mock_flags = p.start() - self.mock_flags.configure_mock(metadata=[], - num_striped_disks=1, - sysctl=[], - set_files=[], - simulate_maintenance=False) + self.mock_flags.configure_mock( + metadata=[], + num_striped_disks=1, + sysctl=[], + set_files=[], + simulate_maintenance=False, + ) self.addCleanup(p.stop) self.maxDiff = None - p = mock.patch(publisher.__name__ + '.version', - VERSION='v1') + p = mock.patch(publisher.__name__ + '.version', VERSION='v1') p.start() self.addCleanup(p.stop) @@ -299,9 +336,9 @@ def setUp(self): # mock_disk.iops returns a mock.MagicMock, which is not None, # which defeats the getattr check in # publisher.DefaultMetadataProvider. - self.mock_disk = mock.MagicMock(disk_type='disk-type', - disk_size=20, num_striped_disks=1, - iops=None) + self.mock_disk = mock.MagicMock( + disk_type='disk-type', disk_size=20, num_striped_disks=1, iops=None + ) self.disk_metadata = { 'type': self.mock_disk.disk_type, 'size': self.mock_disk.disk_size, @@ -310,31 +347,34 @@ def setUp(self): self.mock_disk.GetResourceMetadata.return_value = self.disk_metadata self.mock_vm = CreateMockVM() - self.mock_spec = mock.MagicMock(vm_groups={'default': [self.mock_vm]}, - vms=[self.mock_vm]) - - self.default_meta = {'perfkitbenchmarker_version': 'v1', - 'cloud': self.mock_vm.CLOUD, - 'zone': 'us-central1-a', - 'machine_type': self.mock_vm.machine_type, - 'image': self.mock_vm.image, - 'vm_count': 1, - 'hostnames': 'Hostname', - 'vm_ids': '12345', - 'vm_names': 'Hostname', - 'vm_ip_addresses': '1.2.3.4', - 'default_vm_ids': '12345', - 'default_vm_names': 'Hostname', - 'default_vm_ip_addresses': '1.2.3.4', - } + self.mock_spec = mock.MagicMock( + vm_groups={'default': [self.mock_vm]}, vms=[self.mock_vm] + ) + + self.default_meta = { + 'perfkitbenchmarker_version': 'v1', + 'cloud': self.mock_vm.CLOUD, + 'zone': 'us-central1-a', + 'machine_type': self.mock_vm.machine_type, + 'image': self.mock_vm.image, + 'vm_count': 1, + 'hostnames': 'Hostname', + 'vm_ids': '12345', + 'vm_names': 'Hostname', + 'vm_ip_addresses': '1.2.3.4', + 'default_vm_ids': '12345', + 'default_vm_names': 'Hostname', + 'default_vm_ip_addresses': '1.2.3.4', + } def _RunTest(self, spec, expected): input_metadata = {'some_key': 'some_value'} expected = expected | input_metadata instance = publisher.DefaultMetadataProvider() result = instance.AddMetadata(input_metadata, spec) - self.assertIsNot(input_metadata, result, - msg='Input metadata was not copied.') + self.assertIsNot( + input_metadata, result, msg='Input metadata was not copied.' + ) self.assertEqual(expected, result) def testAddMetadata_ScratchDiskUndefined(self): @@ -347,10 +387,12 @@ def testAddMetadata_NoScratchDisk(self): def testAddMetadata_WithScratchDisk(self): self.mock_vm.configure_mock(scratch_disks=[self.mock_disk]) expected = self.default_meta.copy() - expected.update(data_disk_0_size=20, - data_disk_0_type='disk-type', - data_disk_count=1, - data_disk_0_num_stripes=1) + expected.update( + data_disk_0_size=20, + data_disk_0_type='disk-type', + data_disk_count=1, + data_disk_0_num_stripes=1, + ) self._RunTest(self.mock_spec, expected) def testAddMetadata_DiskSizeNone(self): @@ -358,32 +400,38 @@ def testAddMetadata_DiskSizeNone(self): self.disk_metadata['size'] = None self.mock_vm.configure_mock(scratch_disks=[self.mock_disk]) expected = self.default_meta.copy() - expected.update(data_disk_0_size=None, - data_disk_0_type='disk-type', - data_disk_count=1, - data_disk_0_num_stripes=1) + expected.update( + data_disk_0_size=None, + data_disk_0_type='disk-type', + data_disk_count=1, + data_disk_0_num_stripes=1, + ) self._RunTest(self.mock_spec, expected) def testAddMetadata_PIOPS(self): self.disk_metadata['iops'] = 1000 self.mock_vm.configure_mock(scratch_disks=[self.mock_disk]) expected = self.default_meta.copy() - expected.update(data_disk_0_size=20, - data_disk_0_type='disk-type', - data_disk_count=1, - data_disk_0_num_stripes=1, - data_disk_0_iops=1000) + expected.update( + data_disk_0_size=20, + data_disk_0_type='disk-type', + data_disk_count=1, + data_disk_0_num_stripes=1, + data_disk_0_iops=1000, + ) self._RunTest(self.mock_spec, expected) def testDiskMetadata(self): self.disk_metadata['foo'] = 'bar' self.mock_vm.configure_mock(scratch_disks=[self.mock_disk]) expected = self.default_meta.copy() - expected.update(data_disk_0_size=20, - data_disk_0_type='disk-type', - data_disk_count=1, - data_disk_0_num_stripes=1, - data_disk_0_foo='bar') + expected.update( + data_disk_0_size=20, + data_disk_0_type='disk-type', + data_disk_count=1, + data_disk_0_num_stripes=1, + data_disk_0_foo='bar', + ) self._RunTest(self.mock_spec, expected) def testMultipleVms(self): @@ -419,19 +467,36 @@ def testMultipleVms(self): class CSVPublisherTestCase(unittest.TestCase): def setUp(self): - self.tf = tempfile.NamedTemporaryFile(mode='w+', - prefix='perfkit-csv-publisher', - suffix='.csv') + self.tf = tempfile.NamedTemporaryFile( + mode='w+', prefix='perfkit-csv-publisher', suffix='.csv' + ) self.addCleanup(self.tf.close) def testWritesToStream(self): instance = publisher.CSVPublisher(self.tf.name) - samples = [{'test': 'testb', 'metric': '1', 'value': 1.0, 'unit': 'MB', - 'metadata': {}}, - {'test': 'testb', 'metric': '2', 'value': 14.0, 'unit': 'MB', - 'metadata': {}}, - {'test': 'testa', 'metric': '3', 'value': 47.0, 'unit': 'us', - 'metadata': {}}] + samples = [ + { + 'test': 'testb', + 'metric': '1', + 'value': 1.0, + 'unit': 'MB', + 'metadata': {}, + }, + { + 'test': 'testb', + 'metric': '2', + 'value': 14.0, + 'unit': 'MB', + 'metadata': {}, + }, + { + 'test': 'testa', + 'metric': '3', + 'value': 47.0, + 'unit': 'us', + 'metadata': {}, + }, + ] instance.PublishSamples(samples) self.tf.seek(0) rows = list(csv.DictReader(self.tf)) @@ -439,12 +504,29 @@ def testWritesToStream(self): def testUsesUnionOfMetaKeys(self): instance = publisher.CSVPublisher(self.tf.name) - samples = [{'test': 'testb', 'metric': '1', 'value': 1.0, 'unit': 'MB', - 'metadata': {'key1': 'value1'}}, - {'test': 'testb', 'metric': '2', 'value': 14.0, 'unit': 'MB', - 'metadata': {'key1': 'value2'}}, - {'test': 'testa', 'metric': '3', 'value': 47.0, 'unit': 'us', - 'metadata': {'key3': 'value3'}}] + samples = [ + { + 'test': 'testb', + 'metric': '1', + 'value': 1.0, + 'unit': 'MB', + 'metadata': {'key1': 'value1'}, + }, + { + 'test': 'testb', + 'metric': '2', + 'value': 14.0, + 'unit': 'MB', + 'metadata': {'key1': 'value2'}, + }, + { + 'test': 'testa', + 'metric': '3', + 'value': 47.0, + 'unit': 'us', + 'metadata': {'key3': 'value3'}, + }, + ] instance.PublishSamples(samples) self.tf.seek(0) reader = csv.DictReader(self.tf) @@ -461,25 +543,61 @@ def setUp(self): self.test_db = publisher.InfluxDBPublisher(self.db_uri, self.db_name) def testFormatToKeyValue(self): - sample_1 = {'test': 'testa', 'metric': '3', 'official': 47.0, - 'value': 'non', 'unit': 'us', 'owner': 'Rackspace', - 'run_uri': '5rtw', 'sample_uri': '5r', 'timestamp': 123} - sample_2 = {'test': 'testb', 'metric': '2', 'official': 14.0, - 'value': 'non', 'unit': 'MB', 'owner': 'Rackspace', - 'run_uri': 'bba3', 'sample_uri': 'bb', - 'timestamp': 55} - sample_3 = {'test': 'testc', 'metric': '1', 'official': 1.0, - 'value': 'non', 'unit': 'MB', 'owner': 'Rackspace', - 'run_uri': '323', 'sample_uri': '33', - 'timestamp': 123} - sample_4 = {'test': 'testc', 'metric': 'some,metric', 'official': 1.0, - 'value': 'non', 'unit': 'Some MB', 'owner': 'Rackspace', - 'run_uri': '323', 'sample_uri': '33', - 'timestamp': 123} - sample_5 = {'test': 'testc', 'metric': 'some,metric', 'official': 1.0, - 'value': 'non', 'unit': '', 'owner': 'Rackspace', - 'run_uri': '323', 'sample_uri': '', - 'timestamp': 123} + sample_1 = { + 'test': 'testa', + 'metric': '3', + 'official': 47.0, + 'value': 'non', + 'unit': 'us', + 'owner': 'Rackspace', + 'run_uri': '5rtw', + 'sample_uri': '5r', + 'timestamp': 123, + } + sample_2 = { + 'test': 'testb', + 'metric': '2', + 'official': 14.0, + 'value': 'non', + 'unit': 'MB', + 'owner': 'Rackspace', + 'run_uri': 'bba3', + 'sample_uri': 'bb', + 'timestamp': 55, + } + sample_3 = { + 'test': 'testc', + 'metric': '1', + 'official': 1.0, + 'value': 'non', + 'unit': 'MB', + 'owner': 'Rackspace', + 'run_uri': '323', + 'sample_uri': '33', + 'timestamp': 123, + } + sample_4 = { + 'test': 'testc', + 'metric': 'some,metric', + 'official': 1.0, + 'value': 'non', + 'unit': 'Some MB', + 'owner': 'Rackspace', + 'run_uri': '323', + 'sample_uri': '33', + 'timestamp': 123, + } + sample_5 = { + 'test': 'testc', + 'metric': 'some,metric', + 'official': 1.0, + 'value': 'non', + 'unit': '', + 'owner': 'Rackspace', + 'run_uri': '323', + 'sample_uri': '', + 'timestamp': 123, + } sample_1_formatted_key_value = self.test_db._FormatToKeyValue(sample_1) sample_2_formatted_key_value = self.test_db._FormatToKeyValue(sample_2) @@ -487,21 +605,61 @@ def testFormatToKeyValue(self): sample_4_formatted_key_value = self.test_db._FormatToKeyValue(sample_4) sample_5_formatted_key_value = self.test_db._FormatToKeyValue(sample_5) - expected_sample_1 = ['owner=Rackspace', 'unit=us', 'run_uri=5rtw', - 'test=testa', 'timestamp=123', 'metric=3', - 'official=47.0', 'value=non', 'sample_uri=5r'] - expected_sample_2 = ['owner=Rackspace', 'unit=MB', 'run_uri=bba3', - 'test=testb', 'timestamp=55', 'metric=2', - 'official=14.0', 'value=non', 'sample_uri=bb'] - expected_sample_3 = ['owner=Rackspace', 'unit=MB', 'run_uri=323', - 'test=testc', 'timestamp=123', 'metric=1', - 'official=1.0', 'value=non', 'sample_uri=33'] - expected_sample_4 = ['owner=Rackspace', 'unit=Some\ MB', 'run_uri=323', - 'test=testc', 'timestamp=123', 'metric=some\,metric', - 'official=1.0', 'value=non', 'sample_uri=33'] - expected_sample_5 = ['owner=Rackspace', 'unit=\\"\\"', 'run_uri=323', - 'test=testc', 'timestamp=123', 'metric=some\,metric', - 'official=1.0', 'value=non', 'sample_uri=\\"\\"'] + expected_sample_1 = [ + 'owner=Rackspace', + 'unit=us', + 'run_uri=5rtw', + 'test=testa', + 'timestamp=123', + 'metric=3', + 'official=47.0', + 'value=non', + 'sample_uri=5r', + ] + expected_sample_2 = [ + 'owner=Rackspace', + 'unit=MB', + 'run_uri=bba3', + 'test=testb', + 'timestamp=55', + 'metric=2', + 'official=14.0', + 'value=non', + 'sample_uri=bb', + ] + expected_sample_3 = [ + 'owner=Rackspace', + 'unit=MB', + 'run_uri=323', + 'test=testc', + 'timestamp=123', + 'metric=1', + 'official=1.0', + 'value=non', + 'sample_uri=33', + ] + expected_sample_4 = [ + 'owner=Rackspace', + 'unit=Some\ MB', + 'run_uri=323', + 'test=testc', + 'timestamp=123', + 'metric=some\,metric', + 'official=1.0', + 'value=non', + 'sample_uri=33', + ] + expected_sample_5 = [ + 'owner=Rackspace', + 'unit=\\"\\"', + 'run_uri=323', + 'test=testc', + 'timestamp=123', + 'metric=some\,metric', + 'official=1.0', + 'value=non', + 'sample_uri=\\"\\"', + ] six.assertCountEqual(self, sample_1_formatted_key_value, expected_sample_1) six.assertCountEqual(self, sample_2_formatted_key_value, expected_sample_2) @@ -511,21 +669,27 @@ def testFormatToKeyValue(self): def testConstructSample(self): sample_with_metadata = { - 'test': 'testc', 'metric': '1', 'official': 1.0, - 'value': 'non', 'unit': 'MB', 'owner': 'Rackspace', + 'test': 'testc', + 'metric': '1', + 'official': 1.0, + 'value': 'non', + 'unit': 'MB', + 'owner': 'Rackspace', 'product_name': 'PerfKitBenchmarker', - 'run_uri': '323', 'sample_uri': '33', + 'run_uri': '323', + 'sample_uri': '33', 'timestamp': 123, - 'metadata': collections.OrderedDict([('info', '1'), - ('more_info', '2'), - ('bar', 'foo')])} + 'metadata': collections.OrderedDict( + [('info', '1'), ('more_info', '2'), ('bar', 'foo')] + ), + } constructed_sample = self.test_db._ConstructSample(sample_with_metadata) - sample_results = ('perfkitbenchmarker,test=testc,official=1.0,' - 'owner=Rackspace,run_uri=323,sample_uri=33,' - 'metric=1,unit=MB,product_name=PerfKitBenchmarker,info=1,more_info=2,bar=foo ' - 'value=non 123000000000') + sample_results = ( + 'perfkitbenchmarker,test=testc,official=1.0,owner=Rackspace,run_uri=323,sample_uri=33,metric=1,unit=MB,product_name=PerfKitBenchmarker,info=1,more_info=2,bar=foo' + ' value=non 123000000000' + ) self.assertEqual(constructed_sample, sample_results) @@ -533,34 +697,57 @@ def testConstructSample(self): def testPublishSamples(self, mock_publish_method): samples = [ { - 'test': 'testc', 'metric': '1', 'official': 1.0, - 'value': 'non', 'unit': 'MB', 'owner': 'Rackspace', - 'run_uri': '323', 'sample_uri': '33', 'timestamp': 123, - 'metadata': collections.OrderedDict([('info', '1'), - ('more_info', '2'), - ('bar', 'foo')]) + 'test': 'testc', + 'metric': '1', + 'official': 1.0, + 'value': 'non', + 'unit': 'MB', + 'owner': 'Rackspace', + 'run_uri': '323', + 'sample_uri': '33', + 'timestamp': 123, + 'metadata': collections.OrderedDict( + [('info', '1'), ('more_info', '2'), ('bar', 'foo')] + ), }, { - 'test': 'testb', 'metric': '2', 'official': 14.0, - 'value': 'non', 'unit': 'MB', 'owner': 'Rackspace', - 'run_uri': 'bba3', 'sample_uri': 'bb', 'timestamp': 55, - 'metadata': collections.OrderedDict() + 'test': 'testb', + 'metric': '2', + 'official': 14.0, + 'value': 'non', + 'unit': 'MB', + 'owner': 'Rackspace', + 'run_uri': 'bba3', + 'sample_uri': 'bb', + 'timestamp': 55, + 'metadata': collections.OrderedDict(), }, { - 'test': 'testa', 'metric': '3', 'official': 47.0, - 'value': 'non', 'unit': 'us', 'owner': 'Rackspace', - 'run_uri': '5rtw', 'sample_uri': '5r', 'timestamp': 123 - } + 'test': 'testa', + 'metric': '3', + 'official': 47.0, + 'value': 'non', + 'unit': 'us', + 'owner': 'Rackspace', + 'run_uri': '5rtw', + 'sample_uri': '5r', + 'timestamp': 123, + }, ] expected = [ - ('perfkitbenchmarker,test=testc,official=1.0,owner=Rackspace,' - 'run_uri=323,sample_uri=33,metric=1,unit=MB,product_name=PerfKitBenchmarker,info=1,more_info=2,' - 'bar=foo value=non 123000000000'), - ('perfkitbenchmarker,test=testb,official=14.0,owner=Rackspace,' - 'run_uri=bba3,sample_uri=bb,metric=2,unit=MB,product_name=PerfKitBenchmarker value=non 55000000000'), - ('perfkitbenchmarker,test=testa,official=47.0,owner=Rackspace,' - 'run_uri=5rtw,sample_uri=5r,metric=3,unit=us,product_name=PerfKitBenchmarker value=non 123000000000') + ( + 'perfkitbenchmarker,test=testc,official=1.0,owner=Rackspace,run_uri=323,sample_uri=33,metric=1,unit=MB,product_name=PerfKitBenchmarker,info=1,more_info=2,bar=foo' + ' value=non 123000000000' + ), + ( + 'perfkitbenchmarker,test=testb,official=14.0,owner=Rackspace,run_uri=bba3,sample_uri=bb,metric=2,unit=MB,product_name=PerfKitBenchmarker' + ' value=non 55000000000' + ), + ( + 'perfkitbenchmarker,test=testa,official=47.0,owner=Rackspace,run_uri=5rtw,sample_uri=5r,metric=3,unit=us,product_name=PerfKitBenchmarker' + ' value=non 123000000000' + ), ] mock_publish_method.return_value = None @@ -571,24 +758,32 @@ def testPublishSamples(self, mock_publish_method): @mock.patch.object(publisher.InfluxDBPublisher, '_CreateDB') def testPublish(self, mock_create_db, mock_write_data): formatted_samples = [ - ('perfkitbenchmarker,test=testc,official=1.0,owner=Rackspace,' - 'run_uri=323,sample_uri=33,metric=1,unit=MB,info=1,more_info=2,' - 'bar=foo value=non 123000000000'), - ('perfkitbenchmarker,test=testb,official=14.0,owner=Rackspace,' - 'run_uri=bba3,sample_uri=bb,metric=2,unit=MB value=non 55000000000'), - ('perfkitbenchmarker,test=testa,official=47.0,owner=Rackspace,' - 'run_uri=5rtw,sample_uri=5r,metric=3,unit=us value=non 123000000000') + ( + 'perfkitbenchmarker,test=testc,official=1.0,owner=Rackspace,' + 'run_uri=323,sample_uri=33,metric=1,unit=MB,info=1,more_info=2,' + 'bar=foo value=non 123000000000' + ), + ( + 'perfkitbenchmarker,test=testb,official=14.0,owner=Rackspace,' + 'run_uri=bba3,sample_uri=bb,metric=2,unit=MB value=non 55000000000' + ), + ( + 'perfkitbenchmarker,test=testa,official=47.0,owner=Rackspace,' + 'run_uri=5rtw,sample_uri=5r,metric=3,unit=us value=non 123000000000' + ), ] - expected_output = ('perfkitbenchmarker,test=testc,official=1.0,' - 'owner=Rackspace,run_uri=323,sample_uri=33,metric=1,' - 'unit=MB,info=1,more_info=2,bar=foo value=non ' - '123000000000\nperfkitbenchmarker,test=testb,' - 'official=14.0,owner=Rackspace,run_uri=bba3,' - 'sample_uri=bb,metric=2,unit=MB value=non 55000000000\n' - 'perfkitbenchmarker,test=testa,official=47.0,' - 'owner=Rackspace,run_uri=5rtw,sample_uri=5r,' - 'metric=3,unit=us value=non 123000000000') + expected_output = ( + 'perfkitbenchmarker,test=testc,official=1.0,' + 'owner=Rackspace,run_uri=323,sample_uri=33,metric=1,' + 'unit=MB,info=1,more_info=2,bar=foo value=non ' + '123000000000\nperfkitbenchmarker,test=testb,' + 'official=14.0,owner=Rackspace,run_uri=bba3,' + 'sample_uri=bb,metric=2,unit=MB value=non 55000000000\n' + 'perfkitbenchmarker,test=testa,official=47.0,' + 'owner=Rackspace,run_uri=5rtw,sample_uri=5r,' + 'metric=3,unit=us value=non 123000000000' + ) mock_create_db.return_value = None mock_write_data.return_value = None diff --git a/tests/regex_util_test.py b/tests/regex_util_test.py index 4be5bf8a26..9f9670a5ea 100644 --- a/tests/regex_util_test.py +++ b/tests/regex_util_test.py @@ -29,8 +29,9 @@ def testMatches(self): def testNoMatch(self): regex = r'test ([\da-f]+) text' string = 'test text' - self.assertRaises(regex_util.NoMatchError, regex_util.ExtractGroup, regex, - string, group=1) + self.assertRaises( + regex_util.NoMatchError, regex_util.ExtractGroup, regex, string, group=1 + ) def testMatches_Unanchored(self): regex = r'([\da-f]+) text' @@ -40,14 +41,21 @@ def testMatches_Unanchored(self): def testNamedGroup(self): regex = r'test (?P[\da-f]+) text' string = 'test 12a3de text' - self.assertEqual('12a3de', regex_util.ExtractGroup(regex, string, - group='hex')) + self.assertEqual( + '12a3de', regex_util.ExtractGroup(regex, string, group='hex') + ) def testNumberedGroup_Invalid(self): regex = r'test ([\da-f]+) (.*)' string = 'test 12a3de text' - self.assertRaisesRegexp(IndexError, 'No such group 3 in', # pytype: disable=wrong-arg-count - regex_util.ExtractGroup, regex, string, group=3) + self.assertRaisesRegexp( + IndexError, + 'No such group 3 in', # pytype: disable=wrong-arg-count + regex_util.ExtractGroup, + regex, + string, + group=3, + ) def testNumberedGroup_Valid(self): regex = r'test ([\da-f]+) (.*)' @@ -65,14 +73,16 @@ class ExtractFloatTestCase(unittest.TestCase): def testParsesSuccessfully(self): regex = r'test (\d+|\.\d+|\d+\.\d+) string' string = 'test 12.435 string' - self.assertAlmostEqual(12.435, regex_util.ExtractFloat(regex, string, - group=1)) + self.assertAlmostEqual( + 12.435, regex_util.ExtractFloat(regex, string, group=1) + ) def testRaisesValueErrorOnInvalidInput(self): regex = r'test (invalid_float) string' string = 'test invalid_float string' - self.assertRaises(ValueError, regex_util.ExtractFloat, regex, string, - group=1) + self.assertRaises( + ValueError, regex_util.ExtractFloat, regex, string, group=1 + ) class ExtractIntTestCase(unittest.TestCase): @@ -91,8 +101,7 @@ def testRaisesValueErrorOnInvalidInput(self): class ExtractAllFloatMetricsTestCase(unittest.TestCase): def testParseSuccessful(self): - matches = regex_util.ExtractAllFloatMetrics( - """ + matches = regex_util.ExtractAllFloatMetrics(""" metric=value a=1 b=2.0 @@ -105,18 +114,18 @@ def testParseSuccessful(self): self.assertEqual(2.0, matches['b']) self.assertEqual(0.3, matches['c']) self.assertEqual(-4.5, matches['d']) - self.assertEqual(3.2e+2, matches['ef']) + self.assertEqual(3.2e2, matches['ef']) def testInvalidMetricRegex(self): self.assertRaises( NotImplementedError, regex_util.ExtractAllFloatMetrics, 'metric=1.0', - metric_regex=r'\w(\w)') + metric_regex=r'\w(\w)', + ) def testIntegerValueRegex(self): - matches = regex_util.ExtractAllFloatMetrics( - 'a=1.2,b=3', value_regex=r'\d+') + matches = regex_util.ExtractAllFloatMetrics('a=1.2,b=3', value_regex=r'\d+') self.assertEqual(len(matches), 2) self.assertEqual(1.0, matches['a']) self.assertEqual(3, matches['b']) @@ -137,11 +146,13 @@ def testParseSuccessfully(self): def testNoMatch(self): regex = r'test (\d\w\d) no match' string = 'test no match' - self.assertRaises(regex_util.NoMatchError, regex_util.ExtractAllMatches, - regex, string) + self.assertRaises( + regex_util.NoMatchError, regex_util.ExtractAllMatches, regex, string + ) class ExtractExactlyOneMatchTestCase(unittest.TestCase): + def testNoMatch(self): with self.assertRaises(regex_util.NoMatchError): regex_util.ExtractExactlyOneMatch('foo', 'bar') @@ -151,13 +162,14 @@ def testNonUniqueMatch(self): regex_util.ExtractExactlyOneMatch('spam', 'spam spam spam') def testNoCapturingGroup(self): - self.assertEqual(regex_util.ExtractExactlyOneMatch('bar+', 'foo barrr baz'), - 'barrr') + self.assertEqual( + regex_util.ExtractExactlyOneMatch('bar+', 'foo barrr baz'), 'barrr' + ) def testCapturingGroup(self): self.assertEqual( - regex_util.ExtractExactlyOneMatch('ba(r+)', 'foo barrr baz'), - 'rrr') + regex_util.ExtractExactlyOneMatch('ba(r+)', 'foo barrr baz'), 'rrr' + ) class SubstituteTestCase(unittest.TestCase): @@ -173,8 +185,9 @@ def testNoMatch(self): pattern = r'\[(\w+)\]' repl = r'\1' text = 'foo bar' - self.assertRaises(regex_util.NoMatchError, regex_util.Substitute, - pattern, repl, text) + self.assertRaises( + regex_util.NoMatchError, regex_util.Substitute, pattern, repl, text + ) if __name__ == '__main__': diff --git a/tests/relational_db_spec_test.py b/tests/relational_db_spec_test.py index cb0315c0ab..2058092cfb 100644 --- a/tests/relational_db_spec_test.py +++ b/tests/relational_db_spec_test.py @@ -15,7 +15,6 @@ import unittest from absl import flags - from perfkitbenchmarker import errors from perfkitbenchmarker import relational_db from perfkitbenchmarker import relational_db_spec @@ -66,85 +65,89 @@ def setUp(self): self.minimal_spec = { 'cloud': 'GCP', 'engine': 'mysql', - 'db_spec': { - 'GCP': { - 'machine_type': 'n1-standard-1' - } - }, - 'db_disk_spec': { - 'GCP': { - 'disk_size': 500 - } - } + 'db_spec': {'GCP': {'machine_type': 'n1-standard-1'}}, + 'db_disk_spec': {'GCP': {'disk_size': 500}}, } def testMinimalConfig(self): result = relational_db_spec.RelationalDbSpec( - _COMPONENT, flag_values=FLAGS, **self.minimal_spec) + _COMPONENT, flag_values=FLAGS, **self.minimal_spec + ) self.assertEqual(result.engine, 'mysql') self.assertEqual(result.cloud, 'GCP') self.assertIsInstance(result.db_spec, gce_virtual_machine.GceVmSpec) def testDefaultDatabaseName(self): result = relational_db_spec.RelationalDbSpec( - _COMPONENT, flag_values=FLAGS, **self.minimal_spec) + _COMPONENT, flag_values=FLAGS, **self.minimal_spec + ) self.assertEqual(result.database_name, 'pkb-db-123') def testCustomDatabaseName(self): spec = _mergeDicts(self.minimal_spec, {'database_name': 'fakename'}) result = relational_db_spec.RelationalDbSpec( - _COMPONENT, flag_values=FLAGS, **spec) + _COMPONENT, flag_values=FLAGS, **spec + ) self.assertEqual(result.database_name, 'fakename') def testCustomDatabaseVersion(self): spec = _mergeDicts(self.minimal_spec, {'engine_version': '6.6'}) result = relational_db_spec.RelationalDbSpec( - _COMPONENT, flag_values=FLAGS, **spec) + _COMPONENT, flag_values=FLAGS, **spec + ) self.assertEqual(result.engine_version, '6.6') def testDefaultDatabasePassword(self): result = relational_db_spec.RelationalDbSpec( - _COMPONENT, flag_values=FLAGS, **self.minimal_spec) + _COMPONENT, flag_values=FLAGS, **self.minimal_spec + ) self.assertIsInstance(result.database_password, str) self.assertEqual(len(result.database_password), 13) def testRandomDatabasePassword(self): spec = _mergeDicts(self.minimal_spec, {'database_password': 'fakepassword'}) result = relational_db_spec.RelationalDbSpec( - _COMPONENT, flag_values=FLAGS, **spec) + _COMPONENT, flag_values=FLAGS, **spec + ) self.assertEqual(result.database_password, 'fakepassword') def testDefaultHighAvailability(self): result = relational_db_spec.RelationalDbSpec( - _COMPONENT, flag_values=FLAGS, **self.minimal_spec) + _COMPONENT, flag_values=FLAGS, **self.minimal_spec + ) self.assertEqual(result.high_availability, False) def testCustomHighAvailability(self): spec = _mergeDicts(self.minimal_spec, {'high_availability': True}) result = relational_db_spec.RelationalDbSpec( - _COMPONENT, flag_values=FLAGS, **spec) + _COMPONENT, flag_values=FLAGS, **spec + ) self.assertEqual(result.high_availability, True) def testDefaultBackupEnabled(self): result = relational_db_spec.RelationalDbSpec( - _COMPONENT, flag_values=FLAGS, **self.minimal_spec) + _COMPONENT, flag_values=FLAGS, **self.minimal_spec + ) self.assertEqual(result.backup_enabled, True) def testCustomBackupEnabled(self): spec = _mergeDicts(self.minimal_spec, {'backup_enabled': False}) result = relational_db_spec.RelationalDbSpec( - _COMPONENT, flag_values=FLAGS, **spec) + _COMPONENT, flag_values=FLAGS, **spec + ) self.assertEqual(result.backup_enabled, False) def testDefaultBackupTime(self): result = relational_db_spec.RelationalDbSpec( - _COMPONENT, flag_values=FLAGS, **self.minimal_spec) + _COMPONENT, flag_values=FLAGS, **self.minimal_spec + ) self.assertEqual(result.backup_start_time, '07:00') def testCustomBackupTime(self): spec = _mergeDicts(self.minimal_spec, {'backup_start_time': '08:00'}) result = relational_db_spec.RelationalDbSpec( - _COMPONENT, flag_values=FLAGS, **spec) + _COMPONENT, flag_values=FLAGS, **spec + ) self.assertEqual(result.backup_start_time, '08:00') @@ -157,29 +160,23 @@ def setUp(self): self.spec = { 'cloud': 'GCP', 'engine': 'mysql', - 'db_spec': { - 'GCP': { - 'machine_type': 'n1-standard-1' - } - }, - 'db_disk_spec': { - 'GCP': { - 'disk_size': 500 - } - } + 'db_spec': {'GCP': {'machine_type': 'n1-standard-1'}}, + 'db_disk_spec': {'GCP': {'disk_size': 500}}, } def testDiskSpecRequired(self): del self.spec['db_disk_spec'] with self.assertRaisesRegexp(errors.Config.MissingOption, 'db_disk_spec'): relational_db_spec.RelationalDbSpec( - _COMPONENT, flag_values=FLAGS, **self.spec) + _COMPONENT, flag_values=FLAGS, **self.spec + ) def testVmSpecRequired(self): del self.spec['db_spec'] with self.assertRaisesRegexp(errors.Config.MissingOption, 'db_spec'): relational_db_spec.RelationalDbSpec( - _COMPONENT, flag_values=FLAGS, **self.spec) + _COMPONENT, flag_values=FLAGS, **self.spec + ) class RelationalDbFlagsTestCase(pkb_common_test_case.PkbCommonTestCase): @@ -211,31 +208,21 @@ def setUp(self): 'vm_spec': { 'GCP': { 'zone': 'us-central1-c', - 'machine_type': 'n1-standard-1' + 'machine_type': 'n1-standard-1', } }, - 'disk_spec': { - 'GCP': { - 'disk_size': 500, - 'disk_type': 'pd-ssd' - } - } + 'disk_spec': {'GCP': {'disk_size': 500, 'disk_type': 'pd-ssd'}}, }, 'servers': { 'vm_spec': { 'GCP': { 'zone': 'us-central1-c', - 'machine_type': 'n1-standard-1' + 'machine_type': 'n1-standard-1', } }, - 'disk_spec': { - 'GCP': { - 'disk_size': 500, - 'disk_type': 'pd-ssd' - } - } - } - } + 'disk_spec': {'GCP': {'disk_size': 500, 'disk_type': 'pd-ssd'}}, + }, + }, } # Not testing this yet, because it requires the implementation @@ -247,69 +234,80 @@ def testCloudFlag(self): def testDatabaseFlag(self): FLAGS['db_engine'].parse('postgres') result = relational_db_spec.RelationalDbSpec( - _COMPONENT, flag_values=FLAGS, **self.full_spec) + _COMPONENT, flag_values=FLAGS, **self.full_spec + ) self.assertEqual(result.engine, 'postgres') def testDatabaseNameFlag(self): FLAGS['database_name'].parse('fakedbname') result = relational_db_spec.RelationalDbSpec( - _COMPONENT, flag_values=FLAGS, **self.full_spec) + _COMPONENT, flag_values=FLAGS, **self.full_spec + ) self.assertEqual(result.database_name, 'fakedbname') def testDatabasePasswordFlag(self): FLAGS['database_password'].parse('fakepassword') result = relational_db_spec.RelationalDbSpec( - _COMPONENT, flag_values=FLAGS, **self.full_spec) + _COMPONENT, flag_values=FLAGS, **self.full_spec + ) self.assertEqual(result.database_password, 'fakepassword') def testHighAvailabilityFlag(self): FLAGS['db_high_availability'].parse(True) result = relational_db_spec.RelationalDbSpec( - _COMPONENT, flag_values=FLAGS, **self.full_spec) + _COMPONENT, flag_values=FLAGS, **self.full_spec + ) self.assertEqual(result.high_availability, True) def testDatabaseVersionFlag(self): FLAGS['db_engine_version'].parse('5.6') result = relational_db_spec.RelationalDbSpec( - _COMPONENT, flag_values=FLAGS, **self.full_spec) + _COMPONENT, flag_values=FLAGS, **self.full_spec + ) self.assertEqual(result.engine_version, '5.6') def testBackupEnabledFlag(self): FLAGS['db_backup_enabled'].parse(False) result = relational_db_spec.RelationalDbSpec( - _COMPONENT, flag_values=FLAGS, **self.full_spec) + _COMPONENT, flag_values=FLAGS, **self.full_spec + ) self.assertEqual(result.backup_enabled, False) def testBackupStartTimeFlag(self): FLAGS['db_backup_start_time'].parse('12:23') result = relational_db_spec.RelationalDbSpec( - _COMPONENT, flag_values=FLAGS, **self.full_spec) + _COMPONENT, flag_values=FLAGS, **self.full_spec + ) self.assertEqual(result.backup_start_time, '12:23') def testZoneFlag(self): FLAGS['db_zone'].parse('us-east1-b') result = relational_db_spec.RelationalDbSpec( - _COMPONENT, flag_values=FLAGS, **self.full_spec) + _COMPONENT, flag_values=FLAGS, **self.full_spec + ) self.assertEqual(result.db_spec.zone, 'us-east1-b') self.assertEqual(result.vm_groups['servers'].vm_spec.zone, 'us-east1-b') def testClientVmZoneFlag(self): FLAGS['client_vm_zone'].parse('us-east1-b') result = relational_db_spec.RelationalDbSpec( - _COMPONENT, flag_values=FLAGS, **self.full_spec) + _COMPONENT, flag_values=FLAGS, **self.full_spec + ) self.assertEqual(result.vm_groups['clients'].vm_spec.zone, 'us-east1-b') def testDiskSizeFlag(self): FLAGS['db_disk_size'].parse(2000) result = relational_db_spec.RelationalDbSpec( - _COMPONENT, flag_values=FLAGS, **self.full_spec) + _COMPONENT, flag_values=FLAGS, **self.full_spec + ) self.assertEqual(result.db_disk_spec.disk_size, 2000) self.assertEqual(result.vm_groups['servers'].disk_spec.disk_size, 2000) def testClientVmDiskSizeFlag(self): FLAGS['client_vm_disk_size'].parse(2000) result = relational_db_spec.RelationalDbSpec( - _COMPONENT, flag_values=FLAGS, **self.full_spec) + _COMPONENT, flag_values=FLAGS, **self.full_spec + ) self.assertEqual(result.vm_groups['clients'].disk_spec.disk_size, 2000) diff --git a/tests/requirements_test.py b/tests/requirements_test.py index 98c5a9837e..4c6759f098 100644 --- a/tests/requirements_test.py +++ b/tests/requirements_test.py @@ -111,13 +111,28 @@ def testIncludedFiles(self): with mock.patch.object(pkg_resources, 'require') as mocked_require: requirements._CheckRequirements(_PATH) mocked_open.assert_has_calls(( - mock.call('dir/file', 'r'), mock.call('dir/subfile0', 'r'), + mock.call('dir/file', 'r'), + mock.call('dir/subfile0', 'r'), mock.call('dir/subdir/subfile2', 'r'), - mock.call('dir/../subfile3', 'r'), mock.call('dir/subfile1', 'r'))) + mock.call('dir/../subfile3', 'r'), + mock.call('dir/subfile1', 'r'), + )) mocked_require.assert_has_calls( list( - map(mock.call, ('package-0', 'package-3', 'package-4', 'package-5', - 'package-1>=2.0', 'package-6', 'package-2')))) + map( + mock.call, + ( + 'package-0', + 'package-3', + 'package-4', + 'package-5', + 'package-1>=2.0', + 'package-6', + 'package-2', + ), + ) + ) + ) class CheckBasicRequirementsTestCase(unittest.TestCase): diff --git a/tests/resource_test.py b/tests/resource_test.py index d73c87fa1d..e7b608dd8e 100644 --- a/tests/resource_test.py +++ b/tests/resource_test.py @@ -14,6 +14,7 @@ class GetResourceClassTest(pkb_common_test_case.PkbCommonTestCase): + class BaseTestClassForAttributes(resource.BaseResource): REQUIRED_ATTRS = ['SERVICE_TYPE'] RESOURCE_TYPE = 'BaseTestClassForAttributes' @@ -41,12 +42,15 @@ def _Delete(self): def test_list_of_attributes(self, service): self.assertEqual( resource.GetResourceClass( - self.BaseTestClassForAttributes, SERVICE_TYPE=service), - self.TestClassForAttributes) + self.BaseTestClassForAttributes, SERVICE_TYPE=service + ), + self.TestClassForAttributes, + ) def test_pickle(self): base = resource.GetResourceClass( - self.BaseTestClassForAttributes, SERVICE_TYPE='Base')('spec') + self.BaseTestClassForAttributes, SERVICE_TYPE='Base' + )('spec') base = pickle.loads(pickle.dumps(base)) self.assertEqual(base.SERVICE_TYPE, 'Base') @@ -95,7 +99,8 @@ class ResourceTest(pkb_common_test_case.PkbCommonTestCase): def testDeleteResourceBadCreate(self): test_resource = CreateRaisesNonFreezeRestoreResource() mock_delete = self.enter_context( - mock.patch.object(test_resource, '_Delete')) + mock.patch.object(test_resource, '_Delete') + ) with self.assertRaises(errors.Resource.CreationError): test_resource.Create() @@ -136,7 +141,9 @@ def testDeleteWithFreezeErrorFailsNoisily(self): test_resource = _CreateFreezeRestoreResource() self.enter_context( mock.patch.object( - test_resource, 'Freeze', side_effect=errors.Resource.FreezeError())) + test_resource, 'Freeze', side_effect=errors.Resource.FreezeError() + ) + ) with self.assertRaises(errors.Resource.FreezeError): test_resource.Delete(freeze=True) @@ -146,7 +153,9 @@ def testDeleteWithFreezeErrorProceedsWithDeletion(self): test_resource.delete_on_freeze_error = True self.enter_context( mock.patch.object( - test_resource, 'Freeze', side_effect=errors.Resource.FreezeError())) + test_resource, 'Freeze', side_effect=errors.Resource.FreezeError() + ) + ) # At the start of the test the resource is not deleted. self.assertFalse(test_resource.deleted) @@ -160,9 +169,9 @@ def testCreateWithRestoreErrorFailsNoisily(self): test_resource = _CreateFreezeRestoreResource() self.enter_context( mock.patch.object( - test_resource, - 'Restore', - side_effect=errors.Resource.RestoreError())) + test_resource, 'Restore', side_effect=errors.Resource.RestoreError() + ) + ) with self.assertRaises(errors.Resource.RestoreError): test_resource.Create(restore=True) @@ -172,9 +181,9 @@ def testCreateWithRestoreErrorProceedsWithCreation(self): test_resource.create_on_restore_error = True self.enter_context( mock.patch.object( - test_resource, - 'Restore', - side_effect=errors.Resource.RestoreError())) + test_resource, 'Restore', side_effect=errors.Resource.RestoreError() + ) + ) # At the start of the test the resource is not deleted. self.assertFalse(test_resource.created) @@ -187,16 +196,19 @@ def testExceptionsRaisedAsFreezeError(self): # Ensures that generic exceptions in _Freeze raised as FreezeError. test_resource = _CreateFreezeRestoreResource() self.enter_context( - mock.patch.object(test_resource, '_Freeze', side_effect=Exception())) + mock.patch.object(test_resource, '_Freeze', side_effect=Exception()) + ) with self.assertRaises(errors.Resource.FreezeError): test_resource.Freeze() def testDeleteWithSuccessfulFreeze(self): test_resource = _CreateFreezeRestoreResource() mock_freeze = self.enter_context( - mock.patch.object(test_resource, '_Freeze')) + mock.patch.object(test_resource, '_Freeze') + ) mock_update_timeout = self.enter_context( - mock.patch.object(test_resource, '_UpdateTimeout')) + mock.patch.object(test_resource, '_UpdateTimeout') + ) test_resource.Delete(freeze=True) @@ -207,9 +219,11 @@ def testDeleteWithSuccessfulFreeze(self): def testCreateWithSuccessfulRestore(self): test_resource = _CreateFreezeRestoreResource() mock_restore = self.enter_context( - mock.patch.object(test_resource, '_Restore')) + mock.patch.object(test_resource, '_Restore') + ) mock_create_resource = self.enter_context( - mock.patch.object(test_resource, '_CreateResource')) + mock.patch.object(test_resource, '_CreateResource') + ) test_resource.Create(restore=True) @@ -220,9 +234,11 @@ def testCreateWithSuccessfulRestore(self): def testCreateWithRestoreErrorRaisesInsteadOfCreating(self): test_resource = _CreateFreezeRestoreResource() self.enter_context( - mock.patch.object(test_resource, '_Restore', side_effect=Exception())) + mock.patch.object(test_resource, '_Restore', side_effect=Exception()) + ) mock_create_resource = self.enter_context( - mock.patch.object(test_resource, '_CreateResource')) + mock.patch.object(test_resource, '_CreateResource') + ) with self.assertRaises(errors.Resource.RestoreError): test_resource.Create(restore=True) @@ -232,7 +248,8 @@ def testCreateWithRestoreErrorRaisesInsteadOfCreating(self): def testRestoreNotEnabled(self): test_resource = CompleteFreezeRestoreResource(enable_freeze_restore=False) mock_restore = self.enter_context( - mock.patch.object(test_resource, 'Restore')) + mock.patch.object(test_resource, 'Restore') + ) test_resource.Create(restore=True) diff --git a/tests/sample_test.py b/tests/sample_test.py index bf80a5785c..76172cd8c6 100644 --- a/tests/sample_test.py +++ b/tests/sample_test.py @@ -27,8 +27,9 @@ def testMetadataOptional(self): def testProvidedMetadataSet(self): metadata = {'origin': 'unit test'} - instance = sample.Sample(metric='Test', value=1.0, unit='Mbps', - metadata=metadata.copy()) + instance = sample.Sample( + metric='Test', value=1.0, unit='Mbps', metadata=metadata.copy() + ) self.assertDictEqual(metadata, instance.metadata) def testNoneValueShouldBeZero(self): @@ -58,8 +59,9 @@ class TestPercentileCalculator(unittest.TestCase): def testPercentileCalculator(self): numbers = list(range(0, 1001)) - percentiles = sample.PercentileCalculator(numbers, - percentiles=[0, 1, 99.9, 100]) + percentiles = sample.PercentileCalculator( + numbers, percentiles=[0, 1, 99.9, 100] + ) self.assertEqual(percentiles['p0'], 0) self.assertEqual(percentiles['p1'], 10) diff --git a/tests/scratch_disk_test.py b/tests/scratch_disk_test.py index 818bd1fdeb..7eb26ab59e 100644 --- a/tests/scratch_disk_test.py +++ b/tests/scratch_disk_test.py @@ -67,19 +67,20 @@ def setUp(self): self.patches = [] vm_prefix = linux_virtual_machine.__name__ + '.BaseLinuxMixin' - self.patches.append( - mock.patch(vm_prefix + '.FormatDisk')) - self.patches.append( - mock.patch(vm_prefix + '.MountDisk')) + self.patches.append(mock.patch(vm_prefix + '.FormatDisk')) + self.patches.append(mock.patch(vm_prefix + '.MountDisk')) self.patches.append( mock.patch( - util.__name__ + '.GetDefaultProject', side_effect='test_project')) + util.__name__ + '.GetDefaultProject', side_effect='test_project' + ) + ) # Patch subprocess.Popen to make sure we don't issue any commands to spin up # resources. self.patches.append(mock.patch('subprocess.Popen')) self.patches.append( - mock.patch(vm_util.__name__ + '.GetTempDir', return_value='/tmp/dir')) + mock.patch(vm_util.__name__ + '.GetTempDir', return_value='/tmp/dir') + ) self._PatchCloudSpecific() @@ -89,14 +90,17 @@ def setUp(self): # We need the disk class mocks to return new mocks each time they are # called. Otherwise all "disks" instantiated will be the same object. - self._GetDiskClass().side_effect = ( - lambda *args, **kwargs: mock.MagicMock(is_striped=False)) + self._GetDiskClass().side_effect = lambda *args, **kwargs: mock.MagicMock( + is_striped=False + ) # VM Creation depends on there being a BenchmarkSpec. config_spec = benchmark_config_spec.BenchmarkConfigSpec( - _BENCHMARK_NAME, flag_values=FLAGS, vm_groups={}) - self.spec = benchmark_spec.BenchmarkSpec(mock.MagicMock(), config_spec, - _BENCHMARK_UID) + _BENCHMARK_NAME, flag_values=FLAGS, vm_groups={} + ) + self.spec = benchmark_spec.BenchmarkSpec( + mock.MagicMock(), config_spec, _BENCHMARK_UID + ) self.addCleanup(context.SetThreadBenchmarkSpec, None) self.addCleanup(flagsaver.restore_flag_values, self.saved_flag_values) @@ -168,7 +172,8 @@ def _PatchCloudSpecific(self): def _CreateVm(self): vm_spec = azure_virtual_machine.AzureVmSpec( - 'test_vm_spec.Azure', zone='eastus2', machine_type='test_machine_type') + 'test_vm_spec.Azure', zone='eastus2', machine_type='test_machine_type' + ) return azure_virtual_machine.Ubuntu2004BasedAzureVirtualMachine(vm_spec) def _GetDiskClass(self): @@ -218,30 +223,37 @@ def _PatchCloudSpecific(self): # platform.system(). It is called by RemoteCommand() in # _GetNvmeBootIndex() so we'll mock that instead. self.patches.append( - mock.patch(aws_virtual_machine.__name__ + - '.AwsVirtualMachine._GetNvmeBootIndex')) + mock.patch( + aws_virtual_machine.__name__ + + '.AwsVirtualMachine._GetNvmeBootIndex' + ) + ) self.patches.append( - mock.patch(aws_virtual_machine.__name__ + - '.AwsVirtualMachine.GetVolumeIdByDevice')) + mock.patch( + aws_virtual_machine.__name__ + + '.AwsVirtualMachine.GetVolumeIdByDevice' + ) + ) self.patches.append( - mock.patch(aws_virtual_machine.__name__ + - '.AwsVirtualMachine.GetPathByDevice')) + mock.patch( + aws_virtual_machine.__name__ + '.AwsVirtualMachine.GetPathByDevice' + ) + ) def _CreateVm(self): vm_spec = aws_virtual_machine.AwsVmSpec( - 'test_vm_spec.AWS', zone='us-east-1a', machine_type='test_machine_type') + 'test_vm_spec.AWS', zone='us-east-1a', machine_type='test_machine_type' + ) vm = aws_virtual_machine.Ubuntu2004BasedAwsVirtualMachine(vm_spec) vm.LogDeviceByDiskSpecId('0_0', 'foobar_1') vm.LogDeviceByName('foobar_1', 'vol67890', None) vm.GetNVMEDeviceInfo = mock.Mock() - vm.GetNVMEDeviceInfo.return_value = [ - { - 'DevicePath': '/dev/nvme1n2', - 'SerialNumber': 'vol67890', - 'ModelNumber': 'Amazon Elastic Block Store', - } - ] + vm.GetNVMEDeviceInfo.return_value = [{ + 'DevicePath': '/dev/nvme1n2', + 'SerialNumber': 'vol67890', + 'ModelNumber': 'Amazon Elastic Block Store', + }] return vm def _GetDiskClass(self): @@ -254,10 +266,8 @@ def testDeviceId(self): with mock.patch(disk.__name__ + '.FLAGS') as disk_flags: disk_flags.os_type = 'windows' disk_spec = gce_disk.GceDiskSpec( - _COMPONENT, - disk_number=1, - disk_size=2, - disk_type=gce_disk.PD_STANDARD) + _COMPONENT, disk_number=1, disk_size=2, disk_type=gce_disk.PD_STANDARD + ) disk_obj = gce_disk.GceDisk(disk_spec, 'name', 'zone', 'project') self.assertEqual(disk_obj.GetDeviceId(), r'\\.\PHYSICALDRIVE1') diff --git a/tests/scripts/aws_sqs_client_test.py b/tests/scripts/aws_sqs_client_test.py index 8c4e3fa92c..de83fe5c7c 100644 --- a/tests/scripts/aws_sqs_client_test.py +++ b/tests/scripts/aws_sqs_client_test.py @@ -24,14 +24,13 @@ def testPublishMessage(self, _, resource_mock): aws_client.publish_message(message) # assert publish was called - resource_mock.return_value.get_queue_by_name( - ).send_message.assert_called_with(MessageBody=message) + resource_mock.return_value.get_queue_by_name().send_message.assert_called_with( + MessageBody=message + ) def testPullMessage(self, client_mock, resource_mock): client_mock.return_value.receive_message.return_value = { - 'Messages': [{ - 'ReceiptHandle': 'MockedReceipt' - }] + 'Messages': [{'ReceiptHandle': 'MockedReceipt'}] } aws_client = aws_sqs_client.AwsSqsClient(REGION_NAME, QUEUE_NAME) aws_client.pull_message() @@ -39,9 +38,8 @@ def testPullMessage(self, client_mock, resource_mock): # assert pull was called client_mock.return_value.receive_message.assert_called_with( - QueueUrl=queue_url, - MaxNumberOfMessages=1, - WaitTimeSeconds=TIMEOUT) + QueueUrl=queue_url, MaxNumberOfMessages=1, WaitTimeSeconds=TIMEOUT + ) def testAcknowledgeReceivedMessage(self, client_mock, resource_mock): response = {'ReceiptHandle': 'MockedReceipt'} @@ -52,7 +50,9 @@ def testAcknowledgeReceivedMessage(self, client_mock, resource_mock): # assert acknowledge was called client_mock.return_value.delete_message.assert_called_with( - QueueUrl=queue_url, ReceiptHandle='MockedReceipt') + QueueUrl=queue_url, ReceiptHandle='MockedReceipt' + ) + if __name__ == '__main__': unittest.main() diff --git a/tests/scripts/azure_service_bus_client_test.py b/tests/scripts/azure_service_bus_client_test.py index 0f3f33d80e..5d28c28d65 100644 --- a/tests/scripts/azure_service_bus_client_test.py +++ b/tests/scripts/azure_service_bus_client_test.py @@ -1,4 +1,5 @@ """Tests for scripts/messaging_service_scripts/azure/azure_service_bus_client.py.""" + import sys import unittest from unittest import mock @@ -17,7 +18,8 @@ class AzureServiceBusClientTest(unittest.TestCase): def testGenerateRandomMessage(self): azure_interface = azure_service_bus_client.AzureServiceBusClient( - _CONNECTION_STRING, _TOPIC, _SUBSCRIPTION) + _CONNECTION_STRING, _TOPIC, _SUBSCRIPTION + ) azure_interface.generate_message(0, _MESSAGE_SIZE) AZURE_MOCK.servicebus.ServiceBusMessage.assert_called() @@ -25,7 +27,8 @@ def testGenerateRandomMessage(self): def testPublishMessage(self): message = 'mocked_message' azure_interface = azure_service_bus_client.AzureServiceBusClient( - _CONNECTION_STRING, _TOPIC, _SUBSCRIPTION) + _CONNECTION_STRING, _TOPIC, _SUBSCRIPTION + ) azure_interface.publish_message(message) client = AZURE_MOCK.servicebus.ServiceBusClient connection_str = client.from_connection_string.return_value @@ -36,27 +39,32 @@ def testPublishMessage(self): def testPullMessage(self): azure_interface = azure_service_bus_client.AzureServiceBusClient( - _CONNECTION_STRING, _TOPIC, _SUBSCRIPTION) + _CONNECTION_STRING, _TOPIC, _SUBSCRIPTION + ) azure_interface.pull_message() client = AZURE_MOCK.servicebus.ServiceBusClient connection_str = client.from_connection_string.return_value subscription_receiver = ( - connection_str.get_subscription_receiver.return_value) + connection_str.get_subscription_receiver.return_value + ) # assert pull was called subscription_receiver.receive_messages.assert_called_with( - max_message_count=1, max_wait_time=10) + max_message_count=1, max_wait_time=10 + ) def testAcknowledgeReceivedMessage(self): message = 'mocked_message' azure_interface = azure_service_bus_client.AzureServiceBusClient( - _CONNECTION_STRING, _TOPIC, _SUBSCRIPTION) + _CONNECTION_STRING, _TOPIC, _SUBSCRIPTION + ) azure_interface.acknowledge_received_message(message) client = AZURE_MOCK.servicebus.ServiceBusClient connection_str = client.from_connection_string.return_value subscription_receiver = ( - connection_str.get_subscription_receiver.return_value) + connection_str.get_subscription_receiver.return_value + ) # assert acknowledge was called subscription_receiver.complete_message.assert_called_with(message) diff --git a/tests/scripts/gcp_pubsub_client_test.py b/tests/scripts/gcp_pubsub_client_test.py index 0afc1609e2..fb0c9527eb 100644 --- a/tests/scripts/gcp_pubsub_client_test.py +++ b/tests/scripts/gcp_pubsub_client_test.py @@ -1,4 +1,5 @@ """Tests for data/messaging_service/gcp_pubsub_client.py.""" + import unittest from unittest import mock @@ -17,18 +18,22 @@ class GCPPubSubClientTest(unittest.TestCase): def testPublishMessage(self, _, publisher_mock): message = 'test_message'.encode('utf-8') - topic_path = publisher_mock.return_value.topic_path.return_value = 'test_topic_path' + topic_path = ( + publisher_mock.return_value.topic_path.return_value + ) = 'test_topic_path' - gcp_interface = gcp_pubsub_client.GCPPubSubClient(PROJECT, TOPIC, - SUBSCRIPTION) + gcp_interface = gcp_pubsub_client.GCPPubSubClient( + PROJECT, TOPIC, SUBSCRIPTION + ) gcp_interface.publish_message(message) # assert publish was called publisher_mock.return_value.publish.assert_called_with(topic_path, message) def testPullMessage(self, subscriber_mock, _): - gcp_interface = gcp_pubsub_client.GCPPubSubClient(PROJECT, TOPIC, - SUBSCRIPTION) + gcp_interface = gcp_pubsub_client.GCPPubSubClient( + PROJECT, TOPIC, SUBSCRIPTION + ) gcp_interface.pull_message() # assert pull was called @@ -36,11 +41,13 @@ def testPullMessage(self, subscriber_mock, _): def testAcknowledgeReceivedMessage(self, subscriber_mock, _): response_mock = mock.MagicMock() - response_mock.return_value.received_messages[ - 0].message.data = 'mocked_message' + response_mock.return_value.received_messages[0].message.data = ( + 'mocked_message' + ) - gcp_interface = gcp_pubsub_client.GCPPubSubClient(PROJECT, TOPIC, - SUBSCRIPTION) + gcp_interface = gcp_pubsub_client.GCPPubSubClient( + PROJECT, TOPIC, SUBSCRIPTION + ) gcp_interface.acknowledge_received_message(response_mock) # assert acknowledge was called diff --git a/tests/scripts/messaging_service_scripts_app_test.py b/tests/scripts/messaging_service_scripts_app_test.py index 1cf04338b1..4b2e252c4f 100644 --- a/tests/scripts/messaging_service_scripts_app_test.py +++ b/tests/scripts/messaging_service_scripts_app_test.py @@ -1,4 +1,5 @@ """Tests for scripts/messaging_service_scripts/common/factories.py.""" + import sys import unittest from unittest import mock @@ -22,15 +23,20 @@ class MessagingServiceScriptsAppTest(pkb_common_test_case.PkbCommonTestCase): def setUp(self): super().setUp() self.gcp_client_mock = self.enter_context( - mock.patch.object(gcp_pubsub_client, 'GCPPubSubClient')) + mock.patch.object(gcp_pubsub_client, 'GCPPubSubClient') + ) self.aws_client_mock = self.enter_context( - mock.patch.object(aws_sqs_client, 'AwsSqsClient')) + mock.patch.object(aws_sqs_client, 'AwsSqsClient') + ) self.azure_client_mock = self.enter_context( - mock.patch.object(azure_service_bus_client, 'AzureServiceBusClient')) + mock.patch.object(azure_service_bus_client, 'AzureServiceBusClient') + ) self.pull_runner_mock = self.enter_context( - mock.patch.object(runners, 'PullLatencyRunner')) + mock.patch.object(runners, 'PullLatencyRunner') + ) self.publish_runner_mock = self.enter_context( - mock.patch.object(runners, 'PublishLatencyRunner')) + mock.patch.object(runners, 'PublishLatencyRunner') + ) def testRegisterRunner(self, benchmark_scenario_flag, *_): foo_runner = mock.Mock() @@ -63,13 +69,20 @@ def testGetRunner(self, benchmark_scenario_flag, *_): self.gcp_client_mock.from_flags.assert_called_once() self.pull_runner_mock.run_class_startup.assert_called_once() self.pull_runner_mock.assert_called_once_with( - self.gcp_client_mock.from_flags.return_value) + self.gcp_client_mock.from_flags.return_value + ) self.assertEqual(runner, self.pull_runner_mock.return_value) @mock.patch.object(app.App, '_register_runners') @mock.patch.object(app.App, 'get_runner') - def testCall(self, get_runner_mock, register_runners_mock, _, - number_of_messages_mock, message_size_mock): + def testCall( + self, + get_runner_mock, + register_runners_mock, + _, + number_of_messages_mock, + message_size_mock, + ): my_app = app.App.get_instance() parent_mock = mock.Mock() parent_mock.attach_mock(register_runners_mock, 'register_runners') @@ -78,8 +91,9 @@ def testCall(self, get_runner_mock, register_runners_mock, _, parent_mock.assert_has_calls([ mock.call.register_runners(), mock.call.get_runner(), - mock.call.get_runner().run_phase(number_of_messages_mock.value, - message_size_mock.value), + mock.call.get_runner().run_phase( + number_of_messages_mock.value, message_size_mock.value + ), mock.call.get_runner().close(), ]) diff --git a/tests/scripts/messaging_service_scripts_e2e_main_process_test.py b/tests/scripts/messaging_service_scripts_e2e_main_process_test.py index cfe36030c3..1c752200aa 100644 --- a/tests/scripts/messaging_service_scripts_e2e_main_process_test.py +++ b/tests/scripts/messaging_service_scripts_e2e_main_process_test.py @@ -24,77 +24,57 @@ 'e2e_acknowledge_latency_cold': { 'value': 1000, 'unit': 'milliseconds', - 'metadata': {} - }, - 'e2e_latency_failure_counter': { - 'value': 0, - 'unit': '', - 'metadata': {} + 'metadata': {}, }, + 'e2e_latency_failure_counter': {'value': 0, 'unit': '', 'metadata': {}}, 'e2e_latency_mean': { 'value': 500.0, 'unit': 'milliseconds', - 'metadata': { - 'samples': [500] - } - }, - 'e2e_latency_p50': { - 'value': 500.0, - 'unit': 'milliseconds', - 'metadata': {} - }, - 'e2e_latency_p99': { - 'value': 500.0, - 'unit': 'milliseconds', - 'metadata': {} + 'metadata': {'samples': [500]}, }, + 'e2e_latency_p50': {'value': 500.0, 'unit': 'milliseconds', 'metadata': {}}, + 'e2e_latency_p99': {'value': 500.0, 'unit': 'milliseconds', 'metadata': {}}, 'e2e_latency_p99_9': { 'value': 500.0, 'unit': 'milliseconds', - 'metadata': {} + 'metadata': {}, }, 'e2e_latency_percentage_received': { 'value': 100.0, 'unit': '%', - 'metadata': {} - }, - 'e2e_latency_cold': { - 'value': 500, - 'unit': 'milliseconds', - 'metadata': {} + 'metadata': {}, }, + 'e2e_latency_cold': {'value': 500, 'unit': 'milliseconds', 'metadata': {}}, 'e2e_acknowledge_latency_failure_counter': { 'value': 0, 'unit': '', - 'metadata': {} + 'metadata': {}, }, 'e2e_acknowledge_latency_mean': { 'value': 1000.0, 'unit': 'milliseconds', - 'metadata': { - 'samples': [1000] - } + 'metadata': {'samples': [1000]}, }, 'e2e_acknowledge_latency_p50': { 'value': 1000.0, 'unit': 'milliseconds', - 'metadata': {} + 'metadata': {}, }, 'e2e_acknowledge_latency_p99': { 'value': 1000.0, 'unit': 'milliseconds', - 'metadata': {} + 'metadata': {}, }, 'e2e_acknowledge_latency_p99_9': { 'value': 1000.0, 'unit': 'milliseconds', - 'metadata': {} + 'metadata': {}, }, 'e2e_acknowledge_latency_percentage_received': { 'value': 100.0, 'unit': '%', - 'metadata': {} - } + 'metadata': {}, + }, } @@ -118,12 +98,14 @@ async def MockCoro(*args, **kwargs): # pylint: disable=unused-argument class MessagingServiceScriptsE2EMainProcessTest( - pkb_common_test_case.PkbCommonTestCase): + pkb_common_test_case.PkbCommonTestCase +): def setUp(self): super().setUp() self.pipe_mock = self.enter_context( - mock.patch.object(mp, 'Pipe', side_effect=self._GetPipeMocks)) + mock.patch.object(mp, 'Pipe', side_effect=self._GetPipeMocks) + ) self.process_mock = self.enter_context(mock.patch.object(mp, 'Process')) self.subprocess_mock = self.process_mock.return_value self.flags_mock = self.enter_context(mock.patch('absl.flags.FLAGS')) @@ -147,20 +129,23 @@ def _GetSubprocessOutReader(self, worker): @mock.patch.object(main_process.BaseWorker, '_join_subprocess') @mock.patch.object(main_process.BaseWorker, '_read_subprocess_output') @AsyncTest - async def testStartStop(self, read_subprocess_output_mock, - join_subprocess_mock): + async def testStartStop( + self, read_subprocess_output_mock, join_subprocess_mock + ): worker = main_process.PublisherWorker() self.assertEqual(self.pipe_mock.call_count, 2) self.assertEqual( - self._GetSubprocessInWriter(worker)._extract_mock_name(), 'pipe_writer') + self._GetSubprocessInWriter(worker)._extract_mock_name(), 'pipe_writer' + ) self.assertEqual( - self._GetSubprocessInReader(worker)._extract_mock_name(), 'pipe_reader') + self._GetSubprocessInReader(worker)._extract_mock_name(), 'pipe_reader' + ) self.assertEqual( - self._GetSubprocessOutWriter(worker)._extract_mock_name(), - 'pipe_writer') + self._GetSubprocessOutWriter(worker)._extract_mock_name(), 'pipe_writer' + ) self.assertEqual( - self._GetSubprocessOutReader(worker)._extract_mock_name(), - 'pipe_reader') + self._GetSubprocessOutReader(worker)._extract_mock_name(), 'pipe_reader' + ) self.assertEqual(worker.subprocess_func, publisher.main) await worker.start() self.process_mock.assert_called_once_with( @@ -171,7 +156,8 @@ async def testStartStop(self, read_subprocess_output_mock, 'serialized_flags': self.flags_mock.flags_into_string(), 'app': self.app_mock.get_instance.return_value, 'pinned_cpus': None, - }) + }, + ) read_subprocess_output_mock.assert_called_once_with(protocol.Ready, None) await worker.stop() self.subprocess_mock.terminate.assert_called_once_with() @@ -191,17 +177,20 @@ async def testStartWithPinnedCpus(self, *_): 'serialized_flags': self.flags_mock.flags_into_string(), 'app': self.app_mock.get_instance.return_value, 'pinned_cpus': {3, 1, 4}, - }) + }, + ) await worker.stop() @mock.patch.object( main_process.BaseWorker, '_join_subprocess', - side_effect=(errors.EndToEnd.SubprocessTimeoutError, None)) + side_effect=(errors.EndToEnd.SubprocessTimeoutError, None), + ) @mock.patch.object(main_process.BaseWorker, '_read_subprocess_output') @AsyncTest - async def testStopKill(self, read_subprocess_output_mock, - join_subprocess_mock): + async def testStopKill( + self, read_subprocess_output_mock, join_subprocess_mock + ): worker = main_process.PublisherWorker() await worker.start() read_subprocess_output_mock.assert_called_once_with(protocol.Ready, None) @@ -214,7 +203,8 @@ async def testStopKill(self, read_subprocess_output_mock, @mock.patch.object( main_process.BaseWorker, '_join_subprocess', - side_effect=(errors.EndToEnd.SubprocessTimeoutError, None)) + side_effect=(errors.EndToEnd.SubprocessTimeoutError, None), + ) @AsyncTest async def testReadSubprocessOutput(self, _, sleep_mock): worker = main_process.PublisherWorker() @@ -231,7 +221,8 @@ async def testReadSubprocessOutput(self, _, sleep_mock): @mock.patch.object( main_process.BaseWorker, '_join_subprocess', - side_effect=(errors.EndToEnd.SubprocessTimeoutError, None)) + side_effect=(errors.EndToEnd.SubprocessTimeoutError, None), + ) @AsyncTest async def testReadSubprocessOutputTimeout(self, _): worker = main_process.PublisherWorker() @@ -246,7 +237,8 @@ async def testReadSubprocessOutputTimeout(self, _): @mock.patch.object( main_process.BaseWorker, '_join_subprocess', - side_effect=(errors.EndToEnd.SubprocessTimeoutError, None)) + side_effect=(errors.EndToEnd.SubprocessTimeoutError, None), + ) @AsyncTest async def testReadSubprocessUnexpectedObject(self, _): worker = main_process.PublisherWorker() @@ -264,17 +256,22 @@ async def testReadSubprocessUnexpectedObject(self, _): @mock.patch.object( main_process.BaseWorker, '_read_subprocess_output', - return_value=protocol.AckPublish(seq=1, publish_timestamp=1000)) + return_value=protocol.AckPublish(seq=1, publish_timestamp=1000), + ) @AsyncTest async def testPublish(self, read_subprocess_output_mock, *_): worker = main_process.PublisherWorker() await worker.start() - self.assertEqual(await worker.publish(1), - protocol.AckPublish(seq=1, publish_timestamp=1000)) + self.assertEqual( + await worker.publish(1), + protocol.AckPublish(seq=1, publish_timestamp=1000), + ) self._GetSubprocessInWriter(worker).send.assert_called_once_with( - protocol.Publish(seq=1)) - read_subprocess_output_mock.assert_called_once_with(protocol.AckPublish, - None) + protocol.Publish(seq=1) + ) + read_subprocess_output_mock.assert_called_once_with( + protocol.AckPublish, None + ) await worker.stop() @mock.patch.object(main_process.BaseWorker, 'start') @@ -282,7 +279,8 @@ async def testPublish(self, read_subprocess_output_mock, *_): @mock.patch.object( main_process.BaseWorker, '_read_subprocess_output', - return_value=protocol.AckPublish(publish_error='blahblah')) + return_value=protocol.AckPublish(publish_error='blahblah'), + ) @AsyncTest async def testPublishError(self, *_): worker = main_process.PublisherWorker() @@ -300,9 +298,11 @@ async def testStartConsumption(self, read_subprocess_output_mock, *_): await worker.start() await worker.start_consumption(1) self._GetSubprocessInWriter(worker).send.assert_called_once_with( - protocol.Consume(1)) - read_subprocess_output_mock.assert_called_once_with(protocol.AckConsume, - None) + protocol.Consume(1) + ) + read_subprocess_output_mock.assert_called_once_with( + protocol.AckConsume, None + ) await worker.stop() @mock.patch.object(main_process.BaseWorker, 'start') @@ -310,7 +310,8 @@ async def testStartConsumption(self, read_subprocess_output_mock, *_): @mock.patch.object( main_process.BaseWorker, '_read_subprocess_output', - return_value=protocol.ReceptionReport(receive_error='blahblah')) + return_value=protocol.ReceptionReport(receive_error='blahblah'), + ) @AsyncTest async def testReceive(self, *_): worker = main_process.ReceiverWorker() @@ -321,22 +322,26 @@ async def testReceive(self, *_): class MessagingServiceScriptsEndToEndLatencyRunnerTest( - pkb_common_test_case.PkbCommonTestCase): - + pkb_common_test_case.PkbCommonTestCase +): mock_coro = GetMockCoro() mock_sleep_coro = GetMockCoro() def setUp(self): super().setUp() self.publisher_mock = self.enter_context( - mock.patch.object(main_process, 'PublisherWorker', autospec=True)) + mock.patch.object(main_process, 'PublisherWorker', autospec=True) + ) self.receiver_mock = self.enter_context( mock.patch.object( latency_runner.EndToEndLatencyRunner, 'RECEIVER_WORKER', - autospec=True)) + autospec=True, + ) + ) self.set_start_method_mock = self.enter_context( - mock.patch.object(mp, 'set_start_method')) + mock.patch.object(mp, 'set_start_method') + ) self.publisher_instance_mock = self.publisher_mock.return_value @@ -346,17 +351,21 @@ def setUp(self): self.parent_mock.attach_mock(self.publisher_instance_mock, 'publisher') self.parent_mock.attach_mock(self.publisher_instance_mock, 'receiver') - def _SetupWorkerMocks(self, publish_timestamp, receive_timestamp, - ack_timestamp): + def _SetupWorkerMocks( + self, publish_timestamp, receive_timestamp, ack_timestamp + ): self.publisher_instance_mock.publish.return_value = protocol.AckPublish( - seq=0, publish_timestamp=publish_timestamp) + seq=0, publish_timestamp=publish_timestamp + ) self.receiver_instance_mock.receive.return_value = protocol.ReceptionReport( - seq=0, receive_timestamp=receive_timestamp, ack_timestamp=ack_timestamp) + seq=0, receive_timestamp=receive_timestamp, ack_timestamp=ack_timestamp + ) @mock.patch.object(asyncio, 'run') @mock.patch.object( - latency_runner.EndToEndLatencyRunner, '_async_run_phase', new=mock_coro) + latency_runner.EndToEndLatencyRunner, '_async_run_phase', new=mock_coro + ) def testRunPhase(self, asyncio_run_mock): runner = latency_runner.EndToEndLatencyRunner(mock.Mock()) runner.run_phase(13, 14) @@ -378,8 +387,9 @@ async def testAsyncRunPhase(self, print_mock): @mock.patch.object(os, 'sched_getaffinity', return_value={1, 2, 3, 4, 5, 6}) @mock.patch.object(os, 'sched_setaffinity') @AsyncTest - async def testPinnedCpus(self, sched_setaffinity_mock, sched_getaffinity_mock, - *_): + async def testPinnedCpus( + self, sched_setaffinity_mock, sched_getaffinity_mock, *_ + ): runner_cls = latency_runner.EndToEndLatencyRunner try: self._SetupWorkerMocks(1_000_000_000, 1_500_000_000, 2_000_000_000) @@ -398,8 +408,11 @@ async def testPinnedCpus(self, sched_setaffinity_mock, sched_getaffinity_mock, self.assertLess(receiver_pinned_cpus, {1, 2, 3, 4, 5, 6}) self.assertLen( main_pinned_cpus | publisher_pinned_cpus | receiver_pinned_cpus, - len(main_pinned_cpus) + len(publisher_pinned_cpus) + - len(receiver_pinned_cpus), 'test for disjointness') + len(main_pinned_cpus) + + len(publisher_pinned_cpus) + + len(receiver_pinned_cpus), + 'test for disjointness', + ) sched_setaffinity_mock.assert_called_once_with(0, main_pinned_cpus) runner = latency_runner.EndToEndLatencyRunner(mock.Mock()) await runner._async_run_phase(1) @@ -415,8 +428,9 @@ async def testPinnedCpus(self, sched_setaffinity_mock, sched_getaffinity_mock, @mock.patch.object(os, 'sched_getaffinity', return_value={1, 2}) @mock.patch.object(os, 'sched_setaffinity') @AsyncTest - async def testPinnedCpusNotEnoughCpus(self, sched_setaffinity_mock, - sched_getaffinity_mock, *_): + async def testPinnedCpusNotEnoughCpus( + self, sched_setaffinity_mock, sched_getaffinity_mock, *_ + ): self._SetupWorkerMocks(1_000_000_000, 1_500_000_000, 2_000_000_000) self.publisher_mock.CPUS_REQUIRED = 1 self.receiver_mock.CPUS_REQUIRED = 1 diff --git a/tests/scripts/messaging_service_scripts_e2e_workers_test.py b/tests/scripts/messaging_service_scripts_e2e_workers_test.py index 0a242ba03c..00e3e1df8a 100644 --- a/tests/scripts/messaging_service_scripts_e2e_workers_test.py +++ b/tests/scripts/messaging_service_scripts_e2e_workers_test.py @@ -15,14 +15,16 @@ class MessagingServiceScriptsE2ECommunicatorTest( - pkb_common_test_case.PkbCommonTestCase): + pkb_common_test_case.PkbCommonTestCase +): def setUp(self): super().setUp() self.input_conn_mock = mock.Mock() self.output_conn_mock = mock.Mock() - self.communicator = worker_utils.Communicator(self.input_conn_mock, - self.output_conn_mock) + self.communicator = worker_utils.Communicator( + self.input_conn_mock, self.output_conn_mock + ) def testSend(self): self.communicator.send('hello') @@ -55,7 +57,6 @@ def testAwaitFromMainUnexpectedObjClass(self): class BaseSubprocessTest(pkb_common_test_case.PkbCommonTestCase): - subprocess_module = None main_function = None @@ -66,23 +67,29 @@ def Main(cls, *args, **kwargs): def setUp(self): super().setUp() self.flags_mock = self.enter_context( - mock.patch.object(self.subprocess_module, 'FLAGS')) + mock.patch.object(self.subprocess_module, 'FLAGS') + ) self.communicator_mock = self.enter_context( - mock.patch.object(worker_utils, 'Communicator')) + mock.patch.object(worker_utils, 'Communicator') + ) self.get_client_class_mock = self.enter_context( - mock.patch.object(app.App, 'get_client_class')) + mock.patch.object(app.App, 'get_client_class') + ) self.time_ns_mock = self.enter_context(mock.patch('time.time_ns')) self.time_mock = self.enter_context(mock.patch('time.time')) self.sched_setaffinity_mock = self.enter_context( - mock.patch.object(os, 'sched_setaffinity')) + mock.patch.object(os, 'sched_setaffinity') + ) self.communicator_instance_mock = self.communicator_mock.return_value - self.client_instance_mock = self.get_client_class_mock( - ).from_flags.return_value + self.client_instance_mock = ( + self.get_client_class_mock().from_flags.return_value + ) self.input_conn_mock = mock.Mock() self.output_conn_mock = mock.Mock() self.parent_mock = mock.Mock() - self.parent_mock.attach_mock(self.communicator_instance_mock, - 'communicator') + self.parent_mock.attach_mock( + self.communicator_instance_mock, 'communicator' + ) self.parent_mock.attach_mock(self.client_instance_mock, 'client') self._curr_timens = 1632957090714891010 @@ -107,11 +114,13 @@ def testInitialization(self): iterations=0, ) self.sched_setaffinity_mock.assert_not_called() - self.flags_mock.assert_called_once_with(['--foo=bar', '--bar=qux'], - known_only=True) + self.flags_mock.assert_called_once_with( + ['--foo=bar', '--bar=qux'], known_only=True + ) self.get_client_class_mock().from_flags.assert_called_once_with() - self.communicator_mock.assert_called_once_with(self.input_conn_mock, - self.output_conn_mock) + self.communicator_mock.assert_called_once_with( + self.input_conn_mock, self.output_conn_mock + ) self.communicator_instance_mock.greet.assert_called_once_with() def testSchedSetAffinity(self): @@ -127,13 +136,13 @@ def testSchedSetAffinity(self): class MessagingServiceScriptsE2EReceiverTest(BaseSubprocessTest): - subprocess_module = receiver main_function = receiver.ReceiverRunner.main def testMainLoop(self): self.communicator_instance_mock.await_from_main.return_value = ( - protocol.Consume(seq=1)) + protocol.Consume(seq=1) + ) self.client_instance_mock.decode_seq_from_message.return_value = 1 self.Main( input_conn=self.input_conn_mock, @@ -148,8 +157,9 @@ def testMainLoop(self): self.assertEqual(ack_timestamp - pull_timestamp, 1_000_000_000) self.parent_mock.assert_has_calls([ mock.call.communicator.greet(), - mock.call.communicator.await_from_main(protocol.Consume, - protocol.AckConsume()), + mock.call.communicator.await_from_main( + protocol.Consume, protocol.AckConsume() + ), mock.call.client.pull_message(client.TIMEOUT), mock.call.client.acknowledge_received_message(mock.ANY), mock.call.client.decode_seq_from_message(mock.ANY), @@ -158,7 +168,9 @@ def testMainLoop(self): protocol.ReceptionReport( seq=1, receive_timestamp=pull_timestamp, - ack_timestamp=ack_timestamp)), + ack_timestamp=ack_timestamp, + ) + ), ]) def testMainLoopError(self): @@ -173,22 +185,24 @@ def testMainLoopError(self): ) self.parent_mock.assert_has_calls([ mock.call.communicator.greet(), - mock.call.communicator.await_from_main(protocol.Consume, - protocol.AckConsume()), + mock.call.communicator.await_from_main( + protocol.Consume, protocol.AckConsume() + ), mock.call.client.pull_message(client.TIMEOUT), mock.call.communicator.send( - protocol.ReceptionReport(receive_error=repr(error))), + protocol.ReceptionReport(receive_error=repr(error)) + ), ]) class MessagingServiceScriptsE2EPublisherTest(BaseSubprocessTest): - subprocess_module = publisher main_function = publisher.main def testMainLoop(self): self.communicator_instance_mock.await_from_main.return_value = ( - protocol.Consume(seq=42)) + protocol.Consume(seq=42) + ) self.Main( input_conn=self.input_conn_mock, output_conn=self.output_conn_mock, @@ -202,12 +216,14 @@ def testMainLoop(self): mock.call.client.generate_message(42, self.flags_mock.message_size), mock.call.client.publish_message(mock.ANY), mock.call.communicator.send( - protocol.AckPublish(seq=42, publish_timestamp=self._curr_timens)), + protocol.AckPublish(seq=42, publish_timestamp=self._curr_timens) + ), ]) def testMainLoopError(self): self.communicator_instance_mock.await_from_main.return_value = ( - protocol.Consume(seq=33)) + protocol.Consume(seq=33) + ) error = Exception('Too bad') self.client_instance_mock.publish_message.side_effect = error self.Main( @@ -223,7 +239,8 @@ def testMainLoopError(self): mock.call.client.generate_message(33, self.flags_mock.message_size), mock.call.client.publish_message(mock.ANY), mock.call.communicator.send( - protocol.AckPublish(publish_error=repr(error))), + protocol.AckPublish(publish_error=repr(error)) + ), ]) diff --git a/tests/scripts/messaging_service_scripts_runners_test.py b/tests/scripts/messaging_service_scripts_runners_test.py index 01c7ec4537..f8f8eba376 100644 --- a/tests/scripts/messaging_service_scripts_runners_test.py +++ b/tests/scripts/messaging_service_scripts_runners_test.py @@ -16,44 +16,48 @@ FAILURE_COUNTER = 10 METRICS = [ - 0.20989608764648438, 0.2431643009185791, 0.14051604270935059, - 0.08317422866821289, 0.11351299285888672, 0.17781305313110352, - 0.037261247634887695, 0.030757904052734375, 0.042165279388427734, - 0.036507606506347656 + 0.20989608764648438, + 0.2431643009185791, + 0.14051604270935059, + 0.08317422866821289, + 0.11351299285888672, + 0.17781305313110352, + 0.037261247634887695, + 0.030757904052734375, + 0.042165279388427734, + 0.036507606506347656, ] AGGREGATE_PUBLISH_METRICS = { 'publish_latency_mean': { 'value': 0.11147687435150147, 'unit': UNIT_OF_TIME, - 'metadata': { - 'samples': METRICS - } + 'metadata': {'samples': METRICS}, }, 'publish_latency_p50': { 'value': 0.0983436107635498, 'unit': UNIT_OF_TIME, - 'metadata': {} + 'metadata': {}, }, 'publish_latency_p99': { 'value': 0.2401701617240906, 'unit': UNIT_OF_TIME, - 'metadata': {} + 'metadata': {}, }, 'publish_latency_p99_9': { 'value': 0.2428648869991303, 'unit': UNIT_OF_TIME, - 'metadata': {} + 'metadata': {}, }, 'publish_latency_percentage_received': { 'value': 100.0, 'unit': '%', - 'metadata': {} + 'metadata': {}, }, 'publish_latency_failure_counter': { 'value': 10, 'unit': '', - 'metadata': {} + 'metadata': {}, }, } @@ -61,35 +65,29 @@ 'pull_latency_mean': { 'value': 0.11147687435150147, 'unit': UNIT_OF_TIME, - 'metadata': { - 'samples': METRICS - } + 'metadata': {'samples': METRICS}, }, 'pull_latency_p50': { 'value': 0.0983436107635498, 'unit': UNIT_OF_TIME, - 'metadata': {} + 'metadata': {}, }, 'pull_latency_p99': { 'value': 0.2401701617240906, 'unit': UNIT_OF_TIME, - 'metadata': {} + 'metadata': {}, }, 'pull_latency_p99_9': { 'value': 0.2428648869991303, 'unit': UNIT_OF_TIME, - 'metadata': {} + 'metadata': {}, }, 'pull_latency_percentage_received': { 'value': 100.0, 'unit': '%', - 'metadata': {} - }, - 'pull_latency_failure_counter': { - 'value': 10, - 'unit': '', - 'metadata': {} + 'metadata': {}, }, + 'pull_latency_failure_counter': {'value': 10, 'unit': '', 'metadata': {}}, } @@ -102,32 +100,38 @@ def setUp(self): @parameterized.named_parameters( ('Publish', 'publish_latency', AGGREGATE_PUBLISH_METRICS), - ('Pull', 'pull_latency', AGGREGATE_PULL_METRICS)) + ('Pull', 'pull_latency', AGGREGATE_PULL_METRICS), + ) def testGetSummaryStatistics(self, scenario, expected_samples): runner = runners.PublishLatencyRunner(mock.Mock()) - actual_samples = runner._get_summary_statistics(scenario, METRICS, - NUMBER_OF_MESSAGES, - FAILURE_COUNTER) + actual_samples = runner._get_summary_statistics( + scenario, METRICS, NUMBER_OF_MESSAGES, FAILURE_COUNTER + ) for expected_sample_key in expected_samples: if expected_sample_key not in actual_samples: sample_not_found_message = ( - f'Expected sample:\n{expected_sample_key}\nnot found in actual samples:' - f'\n{actual_samples}') + f'Expected sample:\n{expected_sample_key}\nnot found in actual' + f' samples:\n{actual_samples}' + ) raise Exception(sample_not_found_message) - elif expected_samples[expected_sample_key] != actual_samples[ - expected_sample_key]: + elif ( + expected_samples[expected_sample_key] + != actual_samples[expected_sample_key] + ): sample_doesnt_match_message = ( - f"Expected sample:\n{expected_samples[expected_sample_key]}\ndoesn't match actual sample:" - f'\n{actual_samples[expected_sample_key]}') + 'Expected' + f" sample:\n{expected_samples[expected_sample_key]}\ndoesn't match" + f' actual sample:\n{actual_samples[expected_sample_key]}' + ) raise Exception(sample_doesnt_match_message) @mock.patch.object( runners.PublishLatencyRunner, '_get_summary_statistics', - return_value={'mocked_dict': 'mocked_value'}) + return_value={'mocked_dict': 'mocked_value'}, + ) def testPublishMessages(self, summary_statistics_mock): - runner = runners.PublishLatencyRunner(mock.Mock()) results = runner.run_phase(NUMBER_OF_MESSAGES, MESSAGE_SIZE) @@ -138,12 +142,14 @@ def testPublishMessages(self, summary_statistics_mock): summary_statistics_mock.assert_called() self.assertEqual( typing.cast(mock.MagicMock, runner.client.publish_message).call_count, - NUMBER_OF_MESSAGES) + NUMBER_OF_MESSAGES, + ) @mock.patch.object( runners.PublishLatencyRunner, '_get_summary_statistics', - return_value={'mocked_dict': 'mocked_value'}) + return_value={'mocked_dict': 'mocked_value'}, + ) def testMeasurePublishMessagesException(self, summary_statistics_mock): client_mock = mock.Mock() client_mock.publish_message.side_effect = Exception('MockedException') @@ -153,16 +159,16 @@ def testMeasurePublishMessagesException(self, summary_statistics_mock): # check if functions were called client_mock.publish_message.assert_called() - summary_statistics_mock.assert_called_with('publish_latency', [], - NUMBER_OF_MESSAGES, - FAILURE_COUNTER) + summary_statistics_mock.assert_called_with( + 'publish_latency', [], NUMBER_OF_MESSAGES, FAILURE_COUNTER + ) @mock.patch.object( runners.PullLatencyRunner, '_get_summary_statistics', - return_value={'mocked_dict': 'mocked_value'}) + return_value={'mocked_dict': 'mocked_value'}, + ) def testPullMessages(self, summary_statistics_mock): - runner = runners.PullLatencyRunner(mock.Mock()) results = runner.run_phase(NUMBER_OF_MESSAGES, MESSAGE_SIZE) @@ -170,21 +176,26 @@ def testPullMessages(self, summary_statistics_mock): # check if functions were called typing.cast(mock.MagicMock, runner.client.pull_message).assert_called() - typing.cast(mock.MagicMock, - runner.client.acknowledge_received_message).assert_called() + typing.cast( + mock.MagicMock, runner.client.acknowledge_received_message + ).assert_called() summary_statistics_mock.assert_called() self.assertEqual( typing.cast(mock.MagicMock, runner.client.pull_message).call_count, - NUMBER_OF_MESSAGES) + NUMBER_OF_MESSAGES, + ) self.assertEqual( - typing.cast(mock.MagicMock, - runner.client.acknowledge_received_message).call_count, - NUMBER_OF_MESSAGES) + typing.cast( + mock.MagicMock, runner.client.acknowledge_received_message + ).call_count, + NUMBER_OF_MESSAGES, + ) @mock.patch.object( runners.PullLatencyRunner, '_get_summary_statistics', - return_value={'mocked_dict': 'mocked_value'}) + return_value={'mocked_dict': 'mocked_value'}, + ) def testPullMessagesException(self, summary_statistics_mock): client_mock = mock.Mock() client_mock.pull_message.side_effect = Exception('MockedException') @@ -194,9 +205,9 @@ def testPullMessagesException(self, summary_statistics_mock): # check if functions were called client_mock.pull_message.assert_called() - summary_statistics_mock.assert_called_with('pull_and_acknowledge_latency', - [], NUMBER_OF_MESSAGES, - FAILURE_COUNTER) + summary_statistics_mock.assert_called_with( + 'pull_and_acknowledge_latency', [], NUMBER_OF_MESSAGES, FAILURE_COUNTER + ) if __name__ == '__main__': diff --git a/tests/scripts/object_storage_worker_tests.py b/tests/scripts/object_storage_worker_tests.py index 251dc4a9dd..25a6b024f3 100644 --- a/tests/scripts/object_storage_worker_tests.py +++ b/tests/scripts/object_storage_worker_tests.py @@ -20,7 +20,6 @@ import unittest import mock - from perfkitbenchmarker.scripts.object_storage_api_test_scripts import object_storage_api_tests @@ -62,13 +61,11 @@ def testPointDistribution(self): dist = {10: 100.0} dist[10] = 100.0 - self.assertEqual(object_storage_api_tests.MaxSizeInDistribution(dist), - 10) + self.assertEqual(object_storage_api_tests.MaxSizeInDistribution(dist), 10) def testTwoElementDistribution(self): dist = {1: 50.0, 10: 50.0} - self.assertEqual(object_storage_api_tests.MaxSizeInDistribution(dist), - 10) + self.assertEqual(object_storage_api_tests.MaxSizeInDistribution(dist), 10) class TestIterators(unittest.TestCase): @@ -80,20 +77,21 @@ def testPrefixCounterIterator(self): def testPrefixTimestampSuffixIterator(self): iterator = object_storage_api_tests.PrefixTimestampSuffixIterator( - 'foo', 'bar') - with mock.patch(time.__name__ + '.time', - side_effect=[0, 1, 2]): + 'foo', 'bar' + ) + with mock.patch(time.__name__ + '.time', side_effect=[0, 1, 2]): values = list(itertools.islice(iterator, 3)) - self.assertEqual(values, ['foo_0.000000_bar', - 'foo_1.000000_bar', - 'foo_2.000000_bar']) + self.assertEqual( + values, ['foo_0.000000_bar', 'foo_1.000000_bar', 'foo_2.000000_bar'] + ) def testPrefixHashCountIterator(self): iterator = object_storage_api_tests.PrefixHashCountIterator('foo') values = list(itertools.islice(iterator, 3)) self.assertEqual( - values, - ['857db8c1dd/foo_0', 'a231267a73/foo_1', '538eec728d/foo_2']) + values, ['857db8c1dd/foo_0', 'a231267a73/foo_1', '538eec728d/foo_2'] + ) + if __name__ == '__main__': unittest.main() diff --git a/tests/scripts/tensorflow_serving_client_workload_test.py b/tests/scripts/tensorflow_serving_client_workload_test.py index 0cea03abd4..0c306171ac 100644 --- a/tests/scripts/tensorflow_serving_client_workload_test.py +++ b/tests/scripts/tensorflow_serving_client_workload_test.py @@ -57,7 +57,8 @@ def setUp(self): self.addCleanup(os_patch.stop) self.client_workload = ( - tensorflow_serving_client_workload.TfServingClientWorkload()) + tensorflow_serving_client_workload.TfServingClientWorkload() + ) def testPrintOutput(self): self.client_workload.num_completed_requests = 10 diff --git a/tests/smb_service_test.py b/tests/smb_service_test.py index 8a061ccb7f..8fda88f9f0 100644 --- a/tests/smb_service_test.py +++ b/tests/smb_service_test.py @@ -15,7 +15,6 @@ import unittest from absl import flags - from perfkitbenchmarker import disk from perfkitbenchmarker import smb_service from tests import pkb_common_test_case @@ -71,8 +70,9 @@ def testCreateSmbDisk(self): smb = self._NewSmbResource() smb_disk = smb.CreateSmbDisk() self.assertEqual('//remote1', smb_disk.device_path) - self.assertEqual({'user': 'hello', 'pw': 'world'}, - smb_disk.storage_account_and_key) + self.assertEqual( + {'user': 'hello', 'pw': 'world'}, smb_disk.storage_account_and_key + ) self.assertEqual('3.0', smb_disk.smb_version) def testDefaultSmbVersion(self): diff --git a/tests/spark_service_test.py b/tests/spark_service_test.py index f3ac5b1fdb..c0352c5471 100644 --- a/tests/spark_service_test.py +++ b/tests/spark_service_test.py @@ -79,7 +79,8 @@ def _CreateBenchmarkSpecFromYaml(self, yaml_string, benchmark_name=NAME): def _CreateBenchmarkSpecFromConfigDict(self, config_dict, benchmark_name): config_spec = benchmark_config_spec.BenchmarkConfigSpec( - benchmark_name, flag_values=FLAGS, **config_dict) + benchmark_name, flag_values=FLAGS, **config_dict + ) return benchmark_spec.BenchmarkSpec(mock.MagicMock(), config_spec, UID) @@ -93,14 +94,18 @@ def testDataprocConfig(self): self.assertTrue(spec.spark_service is not None) self.assertEqual(len(spec.vms), 0) machine_type = spec.config.spark_service.worker_group.vm_spec.machine_type - self.assertEqual(spec.config.spark_service.worker_group.vm_count, 4, - str(spec.config.spark_service.__dict__)) - self.assertEqual(spec.config.spark_service.service_type, - spark_service.PROVIDER_MANAGED) - self.assertEqual(machine_type, - 'n1-standard-4', str(spec.config.spark_service.__dict__)) - self.assertTrue(isinstance(spec.spark_service, - gcp_dataproc.GcpDataproc)) + self.assertEqual( + spec.config.spark_service.worker_group.vm_count, + 4, + str(spec.config.spark_service.__dict__), + ) + self.assertEqual( + spec.config.spark_service.service_type, spark_service.PROVIDER_MANAGED + ) + self.assertEqual( + machine_type, 'n1-standard-4', str(spec.config.spark_service.__dict__) + ) + self.assertTrue(isinstance(spec.spark_service, gcp_dataproc.GcpDataproc)) def testEMRConfig(self): FLAGS.cloud = provider_info.AWS @@ -111,21 +116,30 @@ def testEMRConfig(self): self.assertTrue(hasattr(spec, 'spark_service')) self.assertTrue(spec.spark_service is not None) self.assertEqual(len(spec.vms), 0) - self.assertEqual(spec.config.spark_service.worker_group.vm_count, 4, - str(spec.config.spark_service.__dict__)) + self.assertEqual( + spec.config.spark_service.worker_group.vm_count, + 4, + str(spec.config.spark_service.__dict__), + ) machine_type = spec.config.spark_service.worker_group.vm_spec.machine_type - self.assertEqual(spec.config.spark_service.service_type, - spark_service.PROVIDER_MANAGED) - self.assertEqual(machine_type, 'm4.xlarge', - str(spec.config.spark_service.__dict__)) + self.assertEqual( + spec.config.spark_service.service_type, spark_service.PROVIDER_MANAGED + ) + self.assertEqual( + machine_type, 'm4.xlarge', str(spec.config.spark_service.__dict__) + ) self.assertTrue(isinstance(spec.spark_service, aws_emr.AwsEMR)) def testPkbManaged(self): spec = self._CreateBenchmarkSpecFromYaml(PKB_MANAGED_CONFIG) - self.assertEqual(spec.config.spark_service.worker_group.vm_count, 2, - str(spec.config.spark_service.__dict__)) - self.assertEqual(spec.config.spark_service.service_type, - spark_service.PKB_MANAGED) + self.assertEqual( + spec.config.spark_service.worker_group.vm_count, + 2, + str(spec.config.spark_service.__dict__), + ) + self.assertEqual( + spec.config.spark_service.service_type, spark_service.PKB_MANAGED + ) spec.ConstructSparkService() spec.ConstructVirtualMachines() self.assertEqual(len(spec.vms), 3) @@ -133,8 +147,9 @@ def testPkbManaged(self): self.assertEqual(len(spec.vm_groups['worker_group']), 2) self.assertEqual(len(spec.spark_service.vms['worker_group']), 2) self.assertEqual(len(spec.spark_service.vms['master_group']), 1) - self.assertTrue(isinstance(spec.spark_service, - spark_service.PkbSparkService)) + self.assertTrue( + isinstance(spec.spark_service, spark_service.PkbSparkService) + ) if __name__ == '__main__': diff --git a/tests/sql_engine_utils_test.py b/tests/sql_engine_utils_test.py index 4a0d60a57e..c6e28aa534 100644 --- a/tests/sql_engine_utils_test.py +++ b/tests/sql_engine_utils_test.py @@ -10,13 +10,13 @@ class SqlEngineUtilTest(pkb_common_test_case.PkbCommonTestCase): def testGetDbEngineType(self): self.assertEqual( - sql_engine_utils.GetDbEngineType('aurora-postgresql'), 'postgres') + sql_engine_utils.GetDbEngineType('aurora-postgresql'), 'postgres' + ) + self.assertEqual(sql_engine_utils.GetDbEngineType('aurora-mysql'), 'mysql') self.assertEqual( - sql_engine_utils.GetDbEngineType('aurora-mysql'), 'mysql') - self.assertEqual( - sql_engine_utils.GetDbEngineType('sqlserver-ex'), 'sqlserver') - self.assertEqual( - sql_engine_utils.GetDbEngineType('mysql'), 'mysql') + sql_engine_utils.GetDbEngineType('sqlserver-ex'), 'sqlserver' + ) + self.assertEqual(sql_engine_utils.GetDbEngineType('mysql'), 'mysql') with self.assertRaises(TypeError): sql_engine_utils.GetDbEngineType('abc') diff --git a/tests/stages_test.py b/tests/stages_test.py index feddc3ba8a..090e32e829 100644 --- a/tests/stages_test.py +++ b/tests/stages_test.py @@ -42,8 +42,10 @@ def testIncorrectOrder(self): self._parser.parse('teardown,provision') def testAll(self): - self.assertEqual(self._parser.parse('all'), - ['provision', 'prepare', 'run', 'cleanup', 'teardown']) + self.assertEqual( + self._parser.parse('all'), + ['provision', 'prepare', 'run', 'cleanup', 'teardown'], + ) def testIndividual(self): self.assertEqual(self._parser.parse('prepare'), ['prepare']) diff --git a/tests/static_virtual_machine_test.py b/tests/static_virtual_machine_test.py index 0cf09c72c6..f29640dc80 100644 --- a/tests/static_virtual_machine_test.py +++ b/tests/static_virtual_machine_test.py @@ -25,15 +25,15 @@ FLAGS = flags.FLAGS _COMPONENT = 'test_static_vm_spec' -_DISK_SPEC_DICTS = [{ - 'device_path': '/test_device_path' -}, { - 'mount_point': '/test_mount_point' -}] +_DISK_SPEC_DICTS = [ + {'device_path': '/test_device_path'}, + {'mount_point': '/test_mount_point'}, +] -class TestStaticVirtualMachine(pkb_common_test_case.TestOsMixin, - svm.StaticVirtualMachine): +class TestStaticVirtualMachine( + pkb_common_test_case.TestOsMixin, svm.StaticVirtualMachine +): pass @@ -91,13 +91,15 @@ def _AssertStaticVMsEqual(self, vm1, vm2): def testReadFromFile_WrongFormat(self): fp = io.StringIO('{}') - self.assertRaises(ValueError, - svm.StaticVirtualMachine.ReadStaticVirtualMachineFile, fp) + self.assertRaises( + ValueError, svm.StaticVirtualMachine.ReadStaticVirtualMachineFile, fp + ) def testReadFromFile_MissingKey(self): fp = io.StringIO('[{"ip_address": "10.10.10.3"}]') - self.assertRaises(ValueError, - svm.StaticVirtualMachine.ReadStaticVirtualMachineFile, fp) + self.assertRaises( + ValueError, svm.StaticVirtualMachine.ReadStaticVirtualMachineFile, fp + ) def testReadFromFile_Empty(self): fp = io.StringIO('[]') @@ -105,18 +107,20 @@ def testReadFromFile_Empty(self): self.assertEqual([], list(svm.StaticVirtualMachine.vm_pool)) def testReadFromFile_NoErr(self): - s = ('[{' - ' "ip_address": "174.12.14.1", ' - ' "user_name": "perfkitbenchmarker", ' - ' "keyfile_path": "perfkitbenchmarker.pem" ' - '}, ' - '{ ' - ' "ip_address": "174.12.14.121", ' - ' "user_name": "ubuntu", ' - ' "keyfile_path": "rackspace.pem", ' - ' "internal_ip": "10.10.10.2", ' - ' "zone": "rackspace_dallas" ' - '}] ') + s = ( + '[{' + ' "ip_address": "174.12.14.1", ' + ' "user_name": "perfkitbenchmarker", ' + ' "keyfile_path": "perfkitbenchmarker.pem" ' + '}, ' + '{ ' + ' "ip_address": "174.12.14.121", ' + ' "user_name": "ubuntu", ' + ' "keyfile_path": "rackspace.pem", ' + ' "internal_ip": "10.10.10.2", ' + ' "zone": "rackspace_dallas" ' + '}] ' + ) fp = io.StringIO(s) svm.StaticVirtualMachine.ReadStaticVirtualMachineFile(fp) @@ -128,7 +132,11 @@ def testReadFromFile_NoErr(self): _COMPONENT, ip_address='174.12.14.1', user_name='perfkitbenchmarker', - ssh_private_key='perfkitbenchmarker.pem')), vm_pool[0]) + ssh_private_key='perfkitbenchmarker.pem', + ) + ), + vm_pool[0], + ) self._AssertStaticVMsEqual( TestStaticVirtualMachine( svm.StaticVmSpec( @@ -137,26 +145,35 @@ def testReadFromFile_NoErr(self): user_name='ubuntu', ssh_private_key='rackspace.pem', internal_ip='10.10.10.2', - zone='rackspace_dallas')), vm_pool[1]) + zone='rackspace_dallas', + ) + ), + vm_pool[1], + ) def testReadFromFile_InvalidScratchDisksType(self): - s = ('[{' - ' "ip_address": "174.12.14.1", ' - ' "user_name": "perfkitbenchmarker", ' - ' "keyfile_path": "perfkitbenchmarker.pem", ' - ' "scratch_disk_mountpoints": "/tmp/google-pkb" ' - '}]') + s = ( + '[{' + ' "ip_address": "174.12.14.1", ' + ' "user_name": "perfkitbenchmarker", ' + ' "keyfile_path": "perfkitbenchmarker.pem", ' + ' "scratch_disk_mountpoints": "/tmp/google-pkb" ' + '}]' + ) fp = io.StringIO(s) - self.assertRaises(ValueError, - svm.StaticVirtualMachine.ReadStaticVirtualMachineFile, fp) + self.assertRaises( + ValueError, svm.StaticVirtualMachine.ReadStaticVirtualMachineFile, fp + ) def testReadFromFile_UnknownOsTypeDefaultsToLinuxRequiredKeys(self): FLAGS.os_type = 'unknown_os_type' - s = ('[{' - ' "ip_address": "174.12.14.1", ' - ' "user_name": "perfkitbenchmarker", ' - ' "keyfile_path": "perfkitbenchmarker.pem"' - '}]') + s = ( + '[{' + ' "ip_address": "174.12.14.1", ' + ' "user_name": "perfkitbenchmarker", ' + ' "keyfile_path": "perfkitbenchmarker.pem"' + '}]' + ) fp = io.StringIO(s) svm.StaticVirtualMachine.ReadStaticVirtualMachineFile(fp) @@ -168,21 +185,27 @@ def testReadFromFile_UnknownOsTypeDefaultsToLinuxRequiredKeys(self): _COMPONENT, ip_address='174.12.14.1', user_name='perfkitbenchmarker', - ssh_private_key='perfkitbenchmarker.pem')), vm_pool[0]) + ssh_private_key='perfkitbenchmarker.pem', + ) + ), + vm_pool[0], + ) def testCreateReturn(self): - s = ('[{' - ' "ip_address": "174.12.14.1", ' - ' "user_name": "perfkitbenchmarker", ' - ' "keyfile_path": "perfkitbenchmarker.pem" ' - '}, ' - '{ ' - ' "ip_address": "174.12.14.121", ' - ' "user_name": "ubuntu", ' - ' "keyfile_path": "rackspace.pem", ' - ' "internal_ip": "10.10.10.2", ' - ' "zone": "rackspace_dallas" ' - '}] ') + s = ( + '[{' + ' "ip_address": "174.12.14.1", ' + ' "user_name": "perfkitbenchmarker", ' + ' "keyfile_path": "perfkitbenchmarker.pem" ' + '}, ' + '{ ' + ' "ip_address": "174.12.14.121", ' + ' "user_name": "ubuntu", ' + ' "keyfile_path": "rackspace.pem", ' + ' "internal_ip": "10.10.10.2", ' + ' "zone": "rackspace_dallas" ' + '}] ' + ) fp = io.StringIO(s) svm.StaticVirtualMachine.ReadStaticVirtualMachineFile(fp) self.assertEqual(2, len(svm.StaticVirtualMachine.vm_pool)) @@ -206,18 +229,21 @@ def testDiskSpecs(self): "/test_scratch_disk_1"] }] """ - expected_paths_and_mount_points = ((None, '/test_scratch_disk_0'), - (None, '/test_scratch_disk_1'), - ('/test_local_disk_0', - None), ('/test_local_disk_1', None)) + expected_paths_and_mount_points = ( + (None, '/test_scratch_disk_0'), + (None, '/test_scratch_disk_1'), + ('/test_local_disk_0', None), + ('/test_local_disk_1', None), + ) fp = io.StringIO(s) svm.StaticVirtualMachine.ReadStaticVirtualMachineFile(fp) self.assertEqual(1, len(svm.StaticVirtualMachine.vm_pool)) vm = svm.StaticVirtualMachine.GetStaticVirtualMachine() self.assertTrue(vm.from_pool) self.assertEqual(len(vm.disk_specs), 4) - for disk_spec, expected_paths in zip(vm.disk_specs, - expected_paths_and_mount_points): + for disk_spec, expected_paths in zip( + vm.disk_specs, expected_paths_and_mount_points + ): expected_device_path, expected_mount_point = expected_paths self.assertEqual(disk_spec.device_path, expected_device_path) self.assertEqual(disk_spec.mount_point, expected_mount_point) diff --git a/tests/time_triggers/maintenance_simulation_trigger_test.py b/tests/time_triggers/maintenance_simulation_trigger_test.py index 0825055e06..2520689215 100644 --- a/tests/time_triggers/maintenance_simulation_trigger_test.py +++ b/tests/time_triggers/maintenance_simulation_trigger_test.py @@ -17,7 +17,6 @@ import unittest from unittest import mock from absl import flags - from perfkitbenchmarker import benchmark_spec from perfkitbenchmarker import sample from perfkitbenchmarker.sample import Sample @@ -69,7 +68,8 @@ def testAppendSamples(self): trigger.vms = [vm] trigger.AppendSamples(None, vm_spec, s) self.assertEqual( - s, [Sample('LM Total Time', 10, 'seconds', time_dic, timestamp=0)]) + s, [Sample('LM Total Time', 10, 'seconds', time_dic, timestamp=0)] + ) @mock.patch('time.time', mock.MagicMock(return_value=0)) def testAppendLossFunctionWithDegradationPercent(self): @@ -431,96 +431,125 @@ def testAppendLossFunctionSamples(self): vm_spec = mock.MagicMock(spec=benchmark_spec.BenchmarkSpec) trigger = maintenance_simulation_trigger.MaintenanceEventTrigger() trigger.capture_live_migration_timestamps = False - s = sample.CreateTimeSeriesSample([1, 1, 1, 1, 0, 0.1, 0.2, 0.3], - [1000 * i for i in range(1, 9)], - sample.TPM_TIME_SERIES, 'TPM', 1) + s = sample.CreateTimeSeriesSample( + [1, 1, 1, 1, 0, 0.1, 0.2, 0.3], + [1000 * i for i in range(1, 9)], + sample.TPM_TIME_SERIES, + 'TPM', + 1, + ) samples = [s] trigger.trigger_time = datetime.datetime.fromtimestamp(4) trigger.AppendSamples(None, vm_spec, samples) - self.assertEqual(samples, [ - Sample( - metric='TPM_time_series', - value=0.0, - unit='TPM', - metadata={ - 'values': [1, 1, 1, 1, 0, 0.1, 0.2, 0.3], - 'timestamps': [1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000], - 'interval': 1 - }, - timestamp=0), - Sample( - metric='seconds_dropped_below_0_percent', - value=1.0, - unit='s', - metadata={}, - timestamp=0), - Sample( - metric='seconds_dropped_below_10_percent', - value=2.0, - unit='s', - metadata={}, - timestamp=0), - Sample( - metric='seconds_dropped_below_20_percent', - value=3.0, - unit='s', - metadata={}, - timestamp=0), - Sample( - metric='seconds_dropped_below_30_percent', - value=4.0, - unit='s', - metadata={}, - timestamp=0), - Sample( - metric='seconds_dropped_below_40_percent', - value=4.0, - unit='s', - metadata={}, - timestamp=0), - Sample( - metric='seconds_dropped_below_50_percent', - value=4.0, - unit='s', - metadata={}, - timestamp=0), - Sample( - metric='seconds_dropped_below_60_percent', - value=4.0, - unit='s', - metadata={}, - timestamp=0), - Sample( - metric='seconds_dropped_below_70_percent', - value=4.0, - unit='s', - metadata={}, - timestamp=0), - Sample( - metric='seconds_dropped_below_80_percent', - value=4.0, - unit='s', - metadata={}, - timestamp=0), - Sample( - metric='seconds_dropped_below_90_percent', - value=4.0, - unit='s', - metadata={}, - timestamp=0), - Sample( - metric='unresponsive_metric', - value=2.584, - unit='metric', - metadata={}, - timestamp=0), - Sample( - metric='total_loss_seconds', - value=3.4, - unit='seconds', - metadata={}, - timestamp=0) - ]) + self.assertEqual( + samples, + [ + Sample( + metric='TPM_time_series', + value=0.0, + unit='TPM', + metadata={ + 'values': [1, 1, 1, 1, 0, 0.1, 0.2, 0.3], + 'timestamps': [ + 1000, + 2000, + 3000, + 4000, + 5000, + 6000, + 7000, + 8000, + ], + 'interval': 1, + }, + timestamp=0, + ), + Sample( + metric='seconds_dropped_below_0_percent', + value=1.0, + unit='s', + metadata={}, + timestamp=0, + ), + Sample( + metric='seconds_dropped_below_10_percent', + value=2.0, + unit='s', + metadata={}, + timestamp=0, + ), + Sample( + metric='seconds_dropped_below_20_percent', + value=3.0, + unit='s', + metadata={}, + timestamp=0, + ), + Sample( + metric='seconds_dropped_below_30_percent', + value=4.0, + unit='s', + metadata={}, + timestamp=0, + ), + Sample( + metric='seconds_dropped_below_40_percent', + value=4.0, + unit='s', + metadata={}, + timestamp=0, + ), + Sample( + metric='seconds_dropped_below_50_percent', + value=4.0, + unit='s', + metadata={}, + timestamp=0, + ), + Sample( + metric='seconds_dropped_below_60_percent', + value=4.0, + unit='s', + metadata={}, + timestamp=0, + ), + Sample( + metric='seconds_dropped_below_70_percent', + value=4.0, + unit='s', + metadata={}, + timestamp=0, + ), + Sample( + metric='seconds_dropped_below_80_percent', + value=4.0, + unit='s', + metadata={}, + timestamp=0, + ), + Sample( + metric='seconds_dropped_below_90_percent', + value=4.0, + unit='s', + metadata={}, + timestamp=0, + ), + Sample( + metric='unresponsive_metric', + value=2.584, + unit='metric', + metadata={}, + timestamp=0, + ), + Sample( + metric='total_loss_seconds', + value=3.4, + unit='seconds', + metadata={}, + timestamp=0, + ), + ], + ) @mock.patch('time.time', mock.MagicMock(return_value=0)) def testAppendLossFunctionSamplesWithNotification(self): @@ -529,7 +558,11 @@ def testAppendLossFunctionSamplesWithNotification(self): trigger.capture_live_migration_timestamps = True s = sample.CreateTimeSeriesSample( [1, 1, 1, 1, 0, 0.1, 0.2, 0.3, 0.95, 0.95, 0.95, 0.95], - [1000 * i for i in range(1, 13)], sample.TPM_TIME_SERIES, 'TPM', 1) + [1000 * i for i in range(1, 13)], + sample.TPM_TIME_SERIES, + 'TPM', + 1, + ) samples = [s] trigger.trigger_time = datetime.datetime.fromtimestamp(4) vm = mock.MagicMock() @@ -690,7 +723,8 @@ def testAppendLossFunctionSamplesContainsMetadata(self): sample.TPM_TIME_SERIES, 'TPM', 1, - additional_metadata={'random': 'random'}) + additional_metadata={'random': 'random'}, + ) samples = [s] trigger.trigger_time = datetime.datetime.fromtimestamp(4) vm = mock.MagicMock() @@ -855,7 +889,8 @@ def testAppendLossFunctionSamplesHandleTimeDrift(self): sample.TPM_TIME_SERIES, 'TPM', 1, - additional_metadata={'random': 'random'}) + additional_metadata={'random': 'random'}, + ) samples = [s] trigger.trigger_time = datetime.datetime.fromtimestamp(4) vm = mock.MagicMock() diff --git a/tests/timing_util_test.py b/tests/timing_util_test.py index 80ad5e1b4b..867e8e248d 100644 --- a/tests/timing_util_test.py +++ b/tests/timing_util_test.py @@ -16,7 +16,6 @@ import unittest from absl import flags import mock - from perfkitbenchmarker import sample from perfkitbenchmarker import test_util from perfkitbenchmarker import timing_util @@ -53,7 +52,8 @@ def testValid(self): self.assertIs(validate(['end_to_end_runtime', 'timestamps']), True) self.assertIs(validate(['runtimes', 'timestamps']), True) self.assertIs( - validate(['end_to_end_runtime', 'runtimes', 'timestamps']), True) + validate(['end_to_end_runtime', 'runtimes', 'timestamps']), True + ) class IntervalTimerTestCase(unittest.TestCase, test_util.SamplesTestMixin): @@ -130,7 +130,8 @@ def testGenerateSamplesRuntimeNoTimestamps(self): samples = timer.GenerateSamples() exp_samples = [ sample.Sample('First Runtime', stop0 - start0, 'seconds'), - sample.Sample('Second Runtime', stop1 - start1, 'seconds')] + sample.Sample('Second Runtime', stop1 - start1, 'seconds'), + ] self.assertSampleListsEqualUpToTimestamp(samples, exp_samples) def testGenerateSamplesRuntimeAndTimestamps(self): @@ -146,7 +147,8 @@ def testGenerateSamplesRuntimeAndTimestamps(self): stop1 = timer.intervals[1][2] with mock.patch( 'perfkitbenchmarker.timing_util.TimestampMeasurementsEnabled', - return_value=True): + return_value=True, + ): samples = timer.GenerateSamples() exp_samples = [ sample.Sample('First Runtime', stop0 - start0, 'seconds'), @@ -154,7 +156,8 @@ def testGenerateSamplesRuntimeAndTimestamps(self): sample.Sample('First Stop Timestamp', stop0, 'seconds'), sample.Sample('Second Runtime', stop1 - start1, 'seconds'), sample.Sample('Second Start Timestamp', start1, 'seconds'), - sample.Sample('Second Stop Timestamp', stop1, 'seconds')] + sample.Sample('Second Stop Timestamp', stop1, 'seconds'), + ] self.assertSampleListsEqualUpToTimestamp(samples, exp_samples) diff --git a/tests/traces/asadm_test.py b/tests/traces/asadm_test.py index 5873907bd0..ded621f0fc 100644 --- a/tests/traces/asadm_test.py +++ b/tests/traces/asadm_test.py @@ -59,9 +59,9 @@ def testAsadmSummaryAnalyze(self): metadata={ 'values': [53.644, 88.0], 'interval': self.interval, - 'timestamps': expected_timestamps + 'timestamps': expected_timestamps, }, - timestamp=0 + timestamp=0, ), pkb_sample.Sample( metric=asadm.MEMORY_USED_PERCENTAGES_METRIC, @@ -70,9 +70,9 @@ def testAsadmSummaryAnalyze(self): metadata={ 'values': [4.82, 14.82], 'interval': self.interval, - 'timestamps': expected_timestamps + 'timestamps': expected_timestamps, }, - timestamp=0 + timestamp=0, ), pkb_sample.Sample( metric=asadm.DISK_USED_METRIC, @@ -81,9 +81,9 @@ def testAsadmSummaryAnalyze(self): metadata={ 'values': [898.540, 1200.0], 'interval': self.interval, - 'timestamps': expected_timestamps + 'timestamps': expected_timestamps, }, - timestamp=0 + timestamp=0, ), pkb_sample.Sample( metric=asadm.DISK_USED_PERCENTAGES_METRIC, @@ -92,9 +92,9 @@ def testAsadmSummaryAnalyze(self): metadata={ 'values': [3.66, 13.66], 'interval': self.interval, - 'timestamps': expected_timestamps + 'timestamps': expected_timestamps, }, - timestamp=0 + timestamp=0, ), ] self.assertSampleListsEqualUpToTimestamp(samples, expected_samples) @@ -114,9 +114,9 @@ def testAsadmLatencyAnalyze(self): metadata={ 'values': [12.0, 21.0], 'interval': self.interval, - 'timestamps': expected_timestamps + 'timestamps': expected_timestamps, }, - timestamp=0 + timestamp=0, ), pkb_sample.Sample( metric=asadm.READ_LATENCY_OVER_1_MS_METRIC, @@ -125,9 +125,9 @@ def testAsadmLatencyAnalyze(self): metadata={ 'values': [1.1, 9.1], 'interval': self.interval, - 'timestamps': expected_timestamps + 'timestamps': expected_timestamps, }, - timestamp=0 + timestamp=0, ), pkb_sample.Sample( metric=asadm.READ_LATENCY_OVER_8_MS_METRIC, @@ -136,9 +136,9 @@ def testAsadmLatencyAnalyze(self): metadata={ 'values': [2.2, 8.2], 'interval': self.interval, - 'timestamps': expected_timestamps + 'timestamps': expected_timestamps, }, - timestamp=0 + timestamp=0, ), pkb_sample.Sample( metric=asadm.READ_LATENCY_OVER_64_MS_METRIC, @@ -147,9 +147,9 @@ def testAsadmLatencyAnalyze(self): metadata={ 'values': [3.3, 7.3], 'interval': self.interval, - 'timestamps': expected_timestamps + 'timestamps': expected_timestamps, }, - timestamp=0 + timestamp=0, ), pkb_sample.Sample( metric=asadm.WRITE_IOPS_METRIC, @@ -158,9 +158,9 @@ def testAsadmLatencyAnalyze(self): metadata={ 'values': [10.0, 22.0], 'interval': self.interval, - 'timestamps': expected_timestamps + 'timestamps': expected_timestamps, }, - timestamp=0 + timestamp=0, ), pkb_sample.Sample( metric=asadm.WRITE_LATENCY_OVER_1_MS_METRIC, @@ -169,9 +169,9 @@ def testAsadmLatencyAnalyze(self): metadata={ 'values': [7.7, 3.7], 'interval': self.interval, - 'timestamps': expected_timestamps + 'timestamps': expected_timestamps, }, - timestamp=0 + timestamp=0, ), pkb_sample.Sample( metric=asadm.WRITE_LATENCY_OVER_8_MS_METRIC, @@ -180,9 +180,9 @@ def testAsadmLatencyAnalyze(self): metadata={ 'values': [8.8, 2.8], 'interval': self.interval, - 'timestamps': expected_timestamps + 'timestamps': expected_timestamps, }, - timestamp=0 + timestamp=0, ), pkb_sample.Sample( metric=asadm.WRITE_LATENCY_OVER_64_MS_METRIC, @@ -191,9 +191,9 @@ def testAsadmLatencyAnalyze(self): metadata={ 'values': [9.9, 1.9], 'interval': self.interval, - 'timestamps': expected_timestamps + 'timestamps': expected_timestamps, }, - timestamp=0 + timestamp=0, ), ] self.assertSampleListsEqualUpToTimestamp(samples, expected_samples) diff --git a/tests/traces/dstat_test.py b/tests/traces/dstat_test.py index c242625e7d..b97e4f521d 100644 --- a/tests/traces/dstat_test.py +++ b/tests/traces/dstat_test.py @@ -16,7 +16,6 @@ import os import unittest from absl import flags - from perfkitbenchmarker import events from perfkitbenchmarker.sample import Sample from tests import pkb_common_test_case @@ -46,61 +45,52 @@ def testAnalyzeInvalidEventTimestamps(self): self.assertEqual(self.samples, []) def testAnalyzeValidEventSingleRow(self): - events.AddEvent('sender', 'event', 1475708693, 1475708694, - {'label1': 123}) + events.AddEvent('sender', 'event', 1475708693, 1475708694, {'label1': 123}) self.collector.Analyze('testSender', None, self.samples) # 61 metrics self.assertTrue(len(self.samples), 61) - expected = Sample(metric='usr__total cpu usage', - value=6.4000000000000004, - unit='', - metadata={'vm_role': 'test_vm0', - 'label1': 123, 'event': 'event'}, - timestamp=0.0) - self.assertEqual( - expected.metric, self.samples[0].metric) - self.assertEqual( - expected.value, self.samples[0].value) - self.assertEqual( - expected.metadata, self.samples[0].metadata) + expected = Sample( + metric='usr__total cpu usage', + value=6.4000000000000004, + unit='', + metadata={'vm_role': 'test_vm0', 'label1': 123, 'event': 'event'}, + timestamp=0.0, + ) + self.assertEqual(expected.metric, self.samples[0].metric) + self.assertEqual(expected.value, self.samples[0].value) + self.assertEqual(expected.metadata, self.samples[0].metadata) def testAnalyzeValidEventTwoRows(self): - events.AddEvent('sender', 'event', 1475708693, 1475708695, - {'label1': 123}) + events.AddEvent('sender', 'event', 1475708693, 1475708695, {'label1': 123}) self.collector.Analyze('testSender', None, self.samples) # 61 metrics self.assertTrue(len(self.samples), 61) - expected = Sample(metric='usr__total cpu usage', - value=3.200000000000000, - unit='', - metadata={'vm_role': 'test_vm0', - 'label1': 123, 'event': 'event'}, - timestamp=0.0) - self.assertEqual( - expected.metric, self.samples[0].metric) - self.assertEqual( - expected.value, self.samples[0].value) - self.assertEqual( - expected.metadata, self.samples[0].metadata) + expected = Sample( + metric='usr__total cpu usage', + value=3.200000000000000, + unit='', + metadata={'vm_role': 'test_vm0', 'label1': 123, 'event': 'event'}, + timestamp=0.0, + ) + self.assertEqual(expected.metric, self.samples[0].metric) + self.assertEqual(expected.value, self.samples[0].value) + self.assertEqual(expected.metadata, self.samples[0].metadata) def testAnalyzeValidEventEntireFile(self): - events.AddEvent('sender', 'event', 1475708693, 1475709076, - {'label1': 123}) + events.AddEvent('sender', 'event', 1475708693, 1475709076, {'label1': 123}) self.collector.Analyze('testSender', None, self.samples) # 61 metrics self.assertTrue(len(self.samples), 61) - expected = Sample(metric='usr__total cpu usage', - value=10.063689295039159, - unit='', - metadata={'vm_role': 'test_vm0', - 'label1': 123, 'event': 'event'}, - timestamp=0.0) - self.assertEqual( - expected.metric, self.samples[0].metric) - self.assertEqual( - expected.value, self.samples[0].value) - self.assertEqual( - expected.metadata, self.samples[0].metadata) + expected = Sample( + metric='usr__total cpu usage', + value=10.063689295039159, + unit='', + metadata={'vm_role': 'test_vm0', 'label1': 123, 'event': 'event'}, + timestamp=0.0, + ) + self.assertEqual(expected.metric, self.samples[0].metric) + self.assertEqual(expected.value, self.samples[0].value) + self.assertEqual(expected.metadata, self.samples[0].metadata) if __name__ == '__main__': diff --git a/tests/traces/mpstat_test.py b/tests/traces/mpstat_test.py index c5922a196e..b03c3d3fb0 100644 --- a/tests/traces/mpstat_test.py +++ b/tests/traces/mpstat_test.py @@ -38,9 +38,10 @@ 'event': 'mpstat', 'sender': 'run', 'mpstat_cpu_id': -1, - 'nodename': 'instance-3' + 'nodename': 'instance-3', }, - timestamp=1621447265.0), + timestamp=1621447265.0, + ), sample.Sample( metric='mpstat_avg_irq', value=0.0, @@ -49,9 +50,10 @@ 'event': 'mpstat', 'sender': 'run', 'mpstat_cpu_id': -1, - 'nodename': 'instance-3' + 'nodename': 'instance-3', }, - timestamp=1621447265.0), + timestamp=1621447265.0, + ), sample.Sample( metric='mpstat_avg_soft', value=0.01, @@ -60,9 +62,10 @@ 'event': 'mpstat', 'sender': 'run', 'mpstat_cpu_id': -1, - 'nodename': 'instance-3' + 'nodename': 'instance-3', }, - timestamp=1621447265.0), + timestamp=1621447265.0, + ), sample.Sample( metric='mpstat_avg_intr', value=264.5, @@ -71,9 +74,10 @@ 'event': 'mpstat', 'sender': 'run', 'mpstat_cpu_id': 0, - 'nodename': 'instance-3' + 'nodename': 'instance-3', }, - timestamp=1621447265.0), + timestamp=1621447265.0, + ), sample.Sample( metric='mpstat_avg_intr', value=14.375, @@ -82,9 +86,10 @@ 'event': 'mpstat', 'sender': 'run', 'mpstat_cpu_id': 1, - 'nodename': 'instance-3' + 'nodename': 'instance-3', }, - timestamp=1621447265.0) + timestamp=1621447265.0, + ), ] # Besides verifying that per-interval samples are produced, the @@ -101,9 +106,10 @@ 'sender': 'run', 'mpstat_cpu_id': -1, 'ordinal': 0, - 'nodename': 'instance-3' + 'nodename': 'instance-3', }, - timestamp=1621447265.0), + timestamp=1621447265.0, + ), sample.Sample( metric='mpstat_avg_idle', value=49.98, @@ -113,9 +119,10 @@ 'sender': 'run', 'mpstat_cpu_id': -1, 'ordinal': 1, - 'nodename': 'instance-3' + 'nodename': 'instance-3', }, - timestamp=1621447325.0) + timestamp=1621447325.0, + ), ] @@ -130,7 +137,8 @@ def setUp(self): # export S_TIME_FORMAT=ISO # mpstat -I ALL -u -P ALL 60 2 -o JSON path = os.path.join( - os.path.dirname(__file__), '../data', 'mpstat_output.json') + os.path.dirname(__file__), '../data', 'mpstat_output.json' + ) with open(path) as fp: self.contents = json.loads(fp.read()) @@ -142,14 +150,21 @@ def setUp(self): 60, 93, _AGGREGATE_SAMPLES + _PER_INTERVAL_SAMPLES, - )) - def testMpstatParse(self, per_interval_samples, interval, - expected_number_of_samples, expected_samples): + ), + ) + def testMpstatParse( + self, + per_interval_samples, + interval, + expected_number_of_samples, + expected_samples, + ): actual_samples = mpstat._MpstatResults( MPSTAT_METADATA, self.contents, per_interval_samples=per_interval_samples, - interval=interval) + interval=interval, + ) self.assertLen(actual_samples, expected_number_of_samples) @@ -157,7 +172,8 @@ def testMpstatParse(self, per_interval_samples, interval, if expected_sample not in actual_samples: sample_not_found_message = ( f'Expected sample:\n{expected_sample}\nnot found in actual samples:' - f'\n{actual_samples}') + f'\n{actual_samples}' + ) raise Exception(sample_not_found_message) diff --git a/tests/traces/nvidia_power_test.py b/tests/traces/nvidia_power_test.py index 49d2ec3b95..c045271f62 100644 --- a/tests/traces/nvidia_power_test.py +++ b/tests/traces/nvidia_power_test.py @@ -25,7 +25,8 @@ class NvidiaPowerTestCase(unittest.TestCase): def setUp(self): super(NvidiaPowerTestCase, self).setUp() path = os.path.join( - os.path.dirname(__file__), '../data', 'nvidia_power_output.txt') + os.path.dirname(__file__), '../data', 'nvidia_power_output.txt' + ) with open(path, 'r') as fp: self.contents = fp.read() @@ -34,14 +35,16 @@ def testParseNvidiaPowerResult(self): 'event': 'nvidia_power', 'sender': 'run', 'nvidia_power_interval': '1', - 'role': 'default_0' + 'role': 'default_0', } samples = [] - nvidia_power._NvidiaPowerResults(metadata, - csv.DictReader(io.StringIO(self.contents)), - samples, - ['index', 'timestamp', 'power.draw']) + nvidia_power._NvidiaPowerResults( + metadata, + csv.DictReader(io.StringIO(self.contents)), + samples, + ['index', 'timestamp', 'power.draw'], + ) expected_values = [ 16.48, diff --git a/tests/traces/ops_agent_test.py b/tests/traces/ops_agent_test.py index 954254212d..9481253c98 100644 --- a/tests/traces/ops_agent_test.py +++ b/tests/traces/ops_agent_test.py @@ -18,14 +18,16 @@ from absl.testing import flagsaver import mock from perfkitbenchmarker import data - from tests import pkb_common_test_case from perfkitbenchmarker.traces import ops_agent FLAGS = flags.FLAGS _COLLECTOR_FILE = 'collector.stdout' -_RUN_COMMAND = f'sudo service google-cloud-ops-agent restart > {_COLLECTOR_FILE} 2>&1 & echo $!' +_RUN_COMMAND = ( + f'sudo service google-cloud-ops-agent restart > {_COLLECTOR_FILE} 2>&1 &' + ' echo $!' +) OPS_AGENT_BASH_SCRIPT_PATH = './ops_agent/add-google-cloud-ops-agent-repo.sh' @@ -48,7 +50,8 @@ def testInstallCommand(self): vm = mock.Mock() self.collector._InstallCollector(vm) vm.RemoteCommand.assert_called_with( - 'sudo rm -rf ops_agent && mkdir ops_agent') + 'sudo rm -rf ops_agent && mkdir ops_agent' + ) vm.RenderTemplate.assert_called_with( data.ResourcePath(OPS_AGENT_BASH_SCRIPT_PATH), OPS_AGENT_BASH_SCRIPT_PATH, @@ -56,9 +59,11 @@ def testInstallCommand(self): 'DEBIAN_REPO_NAME': str(FLAGS.ops_agent_debian_rapture_repo), 'RPM_REPO_NAME': str(FLAGS.ops_agent_rpm_rapture_repo), 'SUSE_REPO_NAME': str(FLAGS.ops_agent_suse_rapture_repo), - }) + }, + ) vm.RobustRemoteCommand.assert_called_with( - f'sudo bash {OPS_AGENT_BASH_SCRIPT_PATH} --also-install') + f'sudo bash {OPS_AGENT_BASH_SCRIPT_PATH} --also-install' + ) vm.PushFile.assert_not_called() @@ -69,7 +74,8 @@ def testInstallCommandWithCustomRepoFlag(self): vm = mock.Mock() self.collector._InstallCollector(vm) vm.RemoteCommand.assert_called_with( - 'sudo rm -rf ops_agent && mkdir ops_agent') + 'sudo rm -rf ops_agent && mkdir ops_agent' + ) vm.RenderTemplate.assert_called_with( data.ResourcePath(OPS_AGENT_BASH_SCRIPT_PATH), OPS_AGENT_BASH_SCRIPT_PATH, @@ -77,9 +83,11 @@ def testInstallCommandWithCustomRepoFlag(self): 'DEBIAN_REPO_NAME': 'debian_repo', 'RPM_REPO_NAME': 'rpm_repo', 'SUSE_REPO_NAME': 'suse_repo', - }) + }, + ) vm.RobustRemoteCommand.assert_called_with( - f'sudo bash {OPS_AGENT_BASH_SCRIPT_PATH} --also-install') + f'sudo bash {OPS_AGENT_BASH_SCRIPT_PATH} --also-install' + ) vm.PushFile.assert_not_called() @@ -89,7 +97,8 @@ def testInstallCommandWithConfig(self): self.collector._InstallCollector(vm) vm.PushFile.assert_called_with('config.yaml', '/tmp/config.yaml') vm.RemoteCommand.assert_called_with( - 'sudo cp /tmp/config.yaml /etc/google-cloud-ops-agent/config.yaml') + 'sudo cp /tmp/config.yaml /etc/google-cloud-ops-agent/config.yaml' + ) if __name__ == '__main__': diff --git a/tests/traces/otel_test.py b/tests/traces/otel_test.py index 44df7fd247..8fe8f5c6c9 100644 --- a/tests/traces/otel_test.py +++ b/tests/traces/otel_test.py @@ -46,14 +46,19 @@ def testOtelAnalyze(self): metric_names = self.contents.keys() for sample in samples: self.assertIn(sample.metric, metric_names) - self.assertEqual(sample.metadata['values'], - self.contents[sample.metric]['values']) - self.assertEqual(sample.metadata['values'], - self.contents[sample.metric]['values']) - self.assertEqual(sample.metadata['timestamps'], - self.contents[sample.metric]['timestamps']) - self.assertEqual(sample.metadata['unit'], - self.contents[sample.metric]['unit']) + self.assertEqual( + sample.metadata['values'], self.contents[sample.metric]['values'] + ) + self.assertEqual( + sample.metadata['values'], self.contents[sample.metric]['values'] + ) + self.assertEqual( + sample.metadata['timestamps'], + self.contents[sample.metric]['timestamps'], + ) + self.assertEqual( + sample.metadata['unit'], self.contents[sample.metric]['unit'] + ) self.assertEqual(sample.metadata['vm_role'], TEST_VM) diff --git a/tests/traces/sar_test.py b/tests/traces/sar_test.py index 9c66fa569d..d5c8aed4f7 100644 --- a/tests/traces/sar_test.py +++ b/tests/traces/sar_test.py @@ -15,7 +15,6 @@ import os import unittest - from perfkitbenchmarker.traces import sar @@ -23,8 +22,7 @@ class SarTestCase(unittest.TestCase): def setUp(self): super(SarTestCase, self).setUp() - path = os.path.join( - os.path.dirname(__file__), '../data', 'sar_output.txt') + path = os.path.join(os.path.dirname(__file__), '../data', 'sar_output.txt') with open(path) as fp: self.contents = fp.read() @@ -42,8 +40,17 @@ def testParseSarResult(self): metadata = samples[0].metadata expected_steal_values = [ - 0.150000, 0.220000, 0.300000, 0.190000, 0.370000, 0.300000, 0.250000, - 0.350000, 0.210000, 0.170000, 17.990000 + 0.150000, + 0.220000, + 0.300000, + 0.190000, + 0.370000, + 0.300000, + 0.250000, + 0.350000, + 0.210000, + 0.170000, + 17.990000, ] for i in range(0, 11): @@ -56,5 +63,6 @@ def testParseSarResult(self): last_sample = samples[-1] self.assertEqual('average_steal', last_sample.metric) + if __name__ == '__main__': unittest.main() diff --git a/tests/traces/tcpdump_test.py b/tests/traces/tcpdump_test.py index 051b3f238b..2d7e502a0a 100644 --- a/tests/traces/tcpdump_test.py +++ b/tests/traces/tcpdump_test.py @@ -17,7 +17,6 @@ from absl import flags from absl.testing import flagsaver import mock - from tests import pkb_common_test_case from perfkitbenchmarker.traces import tcpdump @@ -25,8 +24,9 @@ _OUTPUT_FILE = '/tmp/x.pcap' # all vm.RemoteCommands to launch tcpdump look like this -_CMD_FORMAT = ('sudo tcpdump -n -w {output_file} {{command}} ' - '> /dev/null 2>&1 & echo $!').format(output_file=_OUTPUT_FILE) +_CMD_FORMAT = ( + 'sudo tcpdump -n -w {output_file} {{command}} > /dev/null 2>&1 & echo $!' +).format(output_file=_OUTPUT_FILE) class TcpdumpTestCase(pkb_common_test_case.PkbCommonTestCase): @@ -76,7 +76,8 @@ def testKillCommand(self): vm.RemoteCommand.reset_mock() collector._StopOnVm(vm, 'roleA') vm.RemoteCommand.assert_called_with( - 'sudo kill -s INT pid1234; sleep 3', ignore_failure=True) + 'sudo kill -s INT pid1234; sleep 3', ignore_failure=True + ) if __name__ == '__main__': diff --git a/tests/units_test.py b/tests/units_test.py index 53d9826f7a..3b0338c0f8 100644 --- a/tests/units_test.py +++ b/tests/units_test.py @@ -27,8 +27,9 @@ def testUnitNotEqual(self): self.assertFalse(units.byte != units.Unit('byte')) def testKB(self): - self.assertEqual(units.ParseExpression('12KB'), - units.ParseExpression('12000 bytes')) + self.assertEqual( + units.ParseExpression('12KB'), units.ParseExpression('12000 bytes') + ) def testIntPercent(self): q = units.ParseExpression('10%') @@ -41,8 +42,7 @@ def testFloatPercent(self): self.assertEqual(q.units, units.percent) def testKubernetes(self): - self.assertEqual(units.ParseExpression('2048Ki'), - 2 * units.mebibyte) + self.assertEqual(units.ParseExpression('2048Ki'), 2 * units.mebibyte) class TestPintPickling(unittest.TestCase): diff --git a/tests/virtual_machine_test.py b/tests/virtual_machine_test.py index 8f41a33fb9..19d15edaf3 100644 --- a/tests/virtual_machine_test.py +++ b/tests/virtual_machine_test.py @@ -47,8 +47,12 @@ def testDefaults(self): def testProvidedValid(self): spec = virtual_machine.BaseVmSpec( - _COMPONENT, image='test_image', install_packages=False, - machine_type='test_machine_type', zone='test_zone') + _COMPONENT, + image='test_image', + install_packages=False, + machine_type='test_machine_type', + zone='test_zone', + ) self.assertEqual(spec.image, 'test_image') self.assertEqual(spec.install_packages, False) self.assertEqual(spec.machine_type, 'test_machine_type') @@ -56,18 +60,27 @@ def testProvidedValid(self): def testUnrecognizedOptions(self): with self.assertRaises(errors.Config.UnrecognizedOption) as cm: - virtual_machine.BaseVmSpec(_COMPONENT, color='red', flavor='cherry', - texture=None) - self.assertEqual(str(cm.exception), ( - 'Unrecognized options were found in test_component: color, flavor, ' - 'texture.')) + virtual_machine.BaseVmSpec( + _COMPONENT, color='red', flavor='cherry', texture=None + ) + self.assertEqual( + str(cm.exception), + ( + 'Unrecognized options were found in test_component: color, flavor, ' + 'texture.' + ), + ) def testMissingOptions(self): with self.assertRaises(errors.Config.MissingOption) as cm: TestVmSpec(_COMPONENT) - self.assertEqual(str(cm.exception), ( - 'Required options were missing from test_component: required_int, ' - 'required_string.')) + self.assertEqual( + str(cm.exception), + ( + 'Required options were missing from test_component: required_int, ' + 'required_string.' + ), + ) def testInvalidImage(self): with self.assertRaises(errors.Config.InvalidValue): @@ -88,33 +101,40 @@ def testInvalidZone(self): def testGpus(self): gpu_count = 2 gpu_type = 'k80' - result = virtual_machine.BaseVmSpec(_COMPONENT, - gpu_count=gpu_count, - gpu_type=gpu_type) + result = virtual_machine.BaseVmSpec( + _COMPONENT, gpu_count=gpu_count, gpu_type=gpu_type + ) self.assertEqual(result.gpu_type, 'k80') self.assertEqual(result.gpu_count, 2) def testMissingGpuCount(self): with self.assertRaises(errors.Config.MissingOption) as cm: virtual_machine.BaseVmSpec(_COMPONENT, flag_values=FLAGS, gpu_type='k80') - self.assertEqual(str(cm.exception), ( - 'gpu_count must be specified if gpu_type is set')) + self.assertEqual( + str(cm.exception), 'gpu_count must be specified if gpu_type is set' + ) def testMissingGpuType(self): with self.assertRaises(errors.Config.MissingOption) as cm: virtual_machine.BaseVmSpec(_COMPONENT, flag_values=FLAGS, gpu_count=1) - self.assertEqual(str(cm.exception), ( - 'gpu_type must be specified if gpu_count is set')) + self.assertEqual( + str(cm.exception), 'gpu_type must be specified if gpu_count is set' + ) def testInvalidGpuType(self): with self.assertRaises(errors.Config.InvalidValue) as cm: virtual_machine.BaseVmSpec( - _COMPONENT, flag_values=FLAGS, gpu_count=1, gpu_type='bad_type') + _COMPONENT, flag_values=FLAGS, gpu_count=1, gpu_type='bad_type' + ) - self.assertIn(( - 'Invalid test_component.gpu_type value: "bad_type". ' - 'Value must be one of the following:'), str(cm.exception)) + self.assertIn( + ( + 'Invalid test_component.gpu_type value: "bad_type". ' + 'Value must be one of the following:' + ), + str(cm.exception), + ) self.assertIn('k80', str(cm.exception)) self.assertIn('p100', str(cm.exception)) @@ -122,11 +142,16 @@ def testInvalidGpuType(self): def testInvalidGpuCount(self): with self.assertRaises(errors.Config.InvalidValue) as cm: virtual_machine.BaseVmSpec( - _COMPONENT, flag_values=FLAGS, gpu_count=0, gpu_type='k80') + _COMPONENT, flag_values=FLAGS, gpu_count=0, gpu_type='k80' + ) - self.assertEqual(str(cm.exception), ( - 'Invalid test_component.gpu_count value: "0". ' - 'Value must be at least 1.')) + self.assertEqual( + str(cm.exception), + ( + 'Invalid test_component.gpu_count value: "0". ' + 'Value must be at least 1.' + ), + ) def CreateTestVm(): @@ -139,63 +164,63 @@ class TestInstallData(pkb_common_test_case.PkbCommonTestCase): def setUp(self): super(TestInstallData, self).setUp() self.vm = CreateTestVm() - self.preprovisioned_data = { - 'fake_pkg': 'fake_checksum' - } + self.preprovisioned_data = {'fake_pkg': 'fake_checksum'} self.module_name = 'fake_module' self.filenames = ['fake_pkg'] self.install_path = '/fake_path' - self.fallback_url = { - 'fake_pkg': 'https://fake_url/fake_pkg.tar.gz'} + self.fallback_url = {'fake_pkg': 'https://fake_url/fake_pkg.tar.gz'} def testPreprovisionNotAvailableFallBackInstallation(self): - with mock.patch.object( - self.vm, 'ShouldDownloadPreprovisionedData') as show: + with mock.patch.object(self.vm, 'ShouldDownloadPreprovisionedData') as show: with mock.patch.object(self.vm, 'RemoteCommand') as remote_command: with mock.patch.object(self.vm, 'CheckPreprovisionedData') as check: show.side_effect = [False] remote_command.side_effect = None - self.vm._InstallData(self.preprovisioned_data, - self.module_name, - self.filenames, - self.install_path, - self.fallback_url) + self.vm._InstallData( + self.preprovisioned_data, + self.module_name, + self.filenames, + self.install_path, + self.fallback_url, + ) show.assert_called_once_with(self.module_name, 'fake_pkg') remote_command.assert_called_once_with( - 'wget -O /fake_path/fake_pkg.tar.gz https://fake_url/fake_pkg.tar.gz') + 'wget -O /fake_path/fake_pkg.tar.gz https://fake_url/fake_pkg.tar.gz' + ) check.assert_called_once_with( - self.install_path, self.module_name, 'fake_pkg', 'fake_checksum') + self.install_path, self.module_name, 'fake_pkg', 'fake_checksum' + ) def testPreprovisionSucceed(self): - with mock.patch.object( - self.vm, 'ShouldDownloadPreprovisionedData') as show: + with mock.patch.object(self.vm, 'ShouldDownloadPreprovisionedData') as show: with mock.patch.object(self.vm, 'DownloadPreprovisionedData') as download: with mock.patch.object(self.vm, 'RemoteCommand') as remote_command: with mock.patch.object(self.vm, 'CheckPreprovisionedData') as check: show.side_effect = [True] - self.vm._InstallData(self.preprovisioned_data, - self.module_name, - self.filenames, - self.install_path, - self.fallback_url) + self.vm._InstallData( + self.preprovisioned_data, + self.module_name, + self.filenames, + self.install_path, + self.fallback_url, + ) download.assert_called_once_with( - self.install_path, self.module_name, 'fake_pkg') + self.install_path, self.module_name, 'fake_pkg' + ) remote_command.assert_not_called() check.assert_called_once_with( - self.install_path, self.module_name, 'fake_pkg', 'fake_checksum') + self.install_path, self.module_name, 'fake_pkg', 'fake_checksum' + ) def testPreprovisionNotAvailableFallBackNotAvailable(self): - with mock.patch.object( - self.vm, 'ShouldDownloadPreprovisionedData') as show: + with mock.patch.object(self.vm, 'ShouldDownloadPreprovisionedData') as show: with mock.patch.object(self.vm, 'RemoteCommand') as remote_command: with mock.patch.object(self.vm, 'CheckPreprovisionedData') as check: show.side_effect = [False] with self.assertRaises(errors.Setup.BadPreprovisionedDataError): - self.vm._InstallData({}, - self.module_name, - self.filenames, - self.install_path, - {}) + self.vm._InstallData( + {}, self.module_name, self.filenames, self.install_path, {} + ) remote_command.assert_not_called() check.assert_not_called() diff --git a/tests/vm_util_test.py b/tests/vm_util_test.py index 8297a62285..1fd466c043 100644 --- a/tests/vm_util_test.py +++ b/tests/vm_util_test.py @@ -21,7 +21,6 @@ import unittest from absl import flags import mock - from perfkitbenchmarker import errors from perfkitbenchmarker import vm_util from tests import pkb_common_test_case @@ -31,7 +30,8 @@ class ShouldRunOnInternalIpAddressTestCase( - pkb_common_test_case.PkbCommonTestCase): + pkb_common_test_case.PkbCommonTestCase +): def setUp(self): super(ShouldRunOnInternalIpAddressTestCase, self).setUp() @@ -44,7 +44,9 @@ def _RunTest(self, expectation, ip_addresses, is_reachable=True): self.assertEqual( expectation, vm_util.ShouldRunOnInternalIpAddress( - self.sending_vm, self.receiving_vm)) + self.sending_vm, self.receiving_vm + ), + ) def testExternal_Reachable(self): self._RunTest(False, vm_util.IpAddressSubset.EXTERNAL, True) @@ -68,8 +70,7 @@ def testReachable_Reachable(self): self._RunTest(True, vm_util.IpAddressSubset.REACHABLE, True) def testReachable_Unreachable(self): - self._RunTest( - False, vm_util.IpAddressSubset.REACHABLE, False) + self._RunTest(False, vm_util.IpAddressSubset.REACHABLE, False) def HaveSleepSubprocess(): @@ -134,15 +135,19 @@ def testTimeoutNotReached(self): @mock.patch('threading.Timer', new=WaitUntilSleepTimer) def testTimeoutReachedThrows(self): with self.assertRaises(errors.VmUtil.IssueCommandTimeoutError): - _, _, _ = vm_util.IssueCommand(['sleep', '2s'], timeout=1, - raise_on_failure=False) + _, _, _ = vm_util.IssueCommand( + ['sleep', '2s'], timeout=1, raise_on_failure=False + ) self.assertFalse(HaveSleepSubprocess()) @mock.patch('threading.Timer', new=WaitUntilSleepTimer) def testTimeoutReached(self): - _, _, retcode = vm_util.IssueCommand(['sleep', '2s'], timeout=1, - raise_on_failure=False, - raise_on_timeout=False) + _, _, retcode = vm_util.IssueCommand( + ['sleep', '2s'], + timeout=1, + raise_on_failure=False, + raise_on_timeout=False, + ) self.assertEqual(retcode, -9) self.assertFalse(HaveSleepSubprocess()) @@ -177,13 +182,15 @@ def _SuppressFailure(stdout, stderr, retcode): del stdout # unused del stderr # unused self.assertNotEqual( - retcode, 0, - '_SuppressFailure should not have been called for retcode=0.') + retcode, + 0, + '_SuppressFailure should not have been called for retcode=0.', + ) return True stdout, stderr, retcode = vm_util.IssueCommand( - ['cat', 'non_existent_file'], - suppress_failure=_SuppressFailure) + ['cat', 'non_existent_file'], suppress_failure=_SuppressFailure + ) # Ideally our command would produce stdout that we could verify is preserved # but that's hard with the way IssueCommand creates local files for getting @@ -198,29 +205,36 @@ def _SuppressFailure(stdout, stderr, retcode): self.assertEqual(retcode, 0) def testRaiseOnFailureUnsuppressed_ExceptionRaised(self): - def _DoNotSuppressFailure(stdout, stderr, retcode): del stdout # unused del stderr # unused self.assertNotEqual( - retcode, 0, - '_DoNotSuppressFailure should not have been called for retcode=0.') + retcode, + 0, + '_DoNotSuppressFailure should not have been called for retcode=0.', + ) return False with self.assertRaises(errors.VmUtil.IssueCommandError) as cm: - vm_util.IssueCommand(['cat', 'non_existent_file'], - raise_on_failure=True, - suppress_failure=_DoNotSuppressFailure) - self.assertIn('cat: non_existent_file: No such file or directory', - str(cm.exception)) + vm_util.IssueCommand( + ['cat', 'non_existent_file'], + raise_on_failure=True, + suppress_failure=_DoNotSuppressFailure, + ) + self.assertIn( + 'cat: non_existent_file: No such file or directory', str(cm.exception) + ) def testRaiseOnFailureWithNoSuppression_ExceptionRaised(self): with self.assertRaises(errors.VmUtil.IssueCommandError) as cm: - vm_util.IssueCommand(['cat', 'non_existent_file'], - raise_on_failure=True, - suppress_failure=None) - self.assertIn('cat: non_existent_file: No such file or directory', - str(cm.exception)) + vm_util.IssueCommand( + ['cat', 'non_existent_file'], + raise_on_failure=True, + suppress_failure=None, + ) + self.assertIn( + 'cat: non_existent_file: No such file or directory', str(cm.exception) + ) class VmUtilTest(pkb_common_test_case.PkbCommonTestCase): @@ -232,9 +246,11 @@ def setUp(self): def testReplaceTextUsesCorrectCommand(self): """Test of vm_util.ReplaceText().""" vm_util.ReplaceText( - self.mock_vm, 'current', 'new', 'test_file', regex_char='|') + self.mock_vm, 'current', 'new', 'test_file', regex_char='|' + ) self.mock_vm.RemoteCommand.assert_called_with( - 'sed -i -r "s|current|new|" test_file') + 'sed -i -r "s|current|new|" test_file' + ) def testDictionaryToEnvString(self): self.assertEqual('', vm_util.DictionaryToEnvString({})) @@ -242,5 +258,6 @@ def testDictionaryToEnvString(self): self.assertEqual('a=b c=d', vm_util.DictionaryToEnvString(test_dict)) self.assertEqual('a=b;c=d', vm_util.DictionaryToEnvString(test_dict, ';')) + if __name__ == '__main__': unittest.main() diff --git a/tests/vpn_service_test.py b/tests/vpn_service_test.py index 69c55f9431..70d5475ae9 100644 --- a/tests/vpn_service_test.py +++ b/tests/vpn_service_test.py @@ -68,19 +68,24 @@ def setUp(self): super(BaseVPNServiceTest, self).setUp() if not sys.warnoptions: # https://bugs.python.org/issue33154 import warnings + warnings.simplefilter('ignore', ResourceWarning) - def _CreateBenchmarkSpecFromYaml(self, yaml_string, - benchmark_name=BENCHMARK_NAME): + def _CreateBenchmarkSpecFromYaml( + self, yaml_string, benchmark_name=BENCHMARK_NAME + ): config = configs.LoadConfig(yaml_string, {}, benchmark_name) return self._CreateBenchmarkSpecFromConfigDict(config, benchmark_name) def _CreateBenchmarkSpecFromConfigDict(self, config_dict, benchmark_name): - config_spec = benchmark_config_spec.BenchmarkConfigSpec(benchmark_name, - flag_values=FLAGS, - **config_dict) - benchmark_module = next((b for b in linux_benchmarks.BENCHMARKS if - b.BENCHMARK_NAME == benchmark_name)) + config_spec = benchmark_config_spec.BenchmarkConfigSpec( + benchmark_name, flag_values=FLAGS, **config_dict + ) + benchmark_module = next(( + b + for b in linux_benchmarks.BENCHMARKS + if b.BENCHMARK_NAME == benchmark_name + )) return benchmark_spec.BenchmarkSpec(benchmark_module, config_spec, URI) def extractDictAFromB(self, A, B): # assertDictContainsSubset deprecated @@ -161,7 +166,7 @@ def testTunnelConfigHash(self): 'ike_version': '3', 'routing': 'static', 'psk': 'private', - 'endpoints': endpoints + 'endpoints': endpoints, } tunnel_config = TunnelConfig() diff --git a/tests/windows_packages/diskspd_test.py b/tests/windows_packages/diskspd_test.py index 81cb779cba..2fbf9daa7f 100644 --- a/tests/windows_packages/diskspd_test.py +++ b/tests/windows_packages/diskspd_test.py @@ -34,62 +34,62 @@ def setUp(self): self.result_xml = self.getDataContents('diskspd_result.xml') def testDiskSpdParsing(self): - single_sample = diskspd.ParseDiskSpdResults(self.result_xml, {}, - 'ReadSpeed') - expected_metadata = {'DisableAffinity': 'false', - 'MaxFileSize': '0', - 'BlockSize': '65536', - 'IdlePercent': '96.80', - 'CompletionRoutines': 'false', - 'StrideSize': '65536', - 'RandomAccess': 'false', - 'Weight': '1', - 'UserPercent': '0.25', - 'WriteBytes': 0, - 'Warmup': '5', - 'Pattern': 'sequential', - 'IOPriority': '3', - 'ThreadsPerFile': '4', - 'KernelPercent': '2.95', - 'ReadIops': 3030, - 'BytesCount': 5961809920, - 'InterlockedSequential': 'false', - 'MeasureLatency': 'false', - 'WriteRatio': '0', - 'FileSize': '838860800', - 'BaseFileOffset': '0', - 'Cooldown': '0', - 'IOCount': 90970, - 'UseLargePages': 'false', - 'UsagePercent': '3.20', - 'SequentialScan': 'true', - 'TotalIops': 3030, - 'WriteSpeed': 0, - 'ProcCount': '4', - 'WriteCount': 0, - 'ReadSpeed': 189, - 'TestTimeSeconds': '30.02', - 'ThreadCount': '4', - 'ReadBytes': 5961809920, - 'TemporaryFile': 'false', - 'ReadCount': 90970, - 'Duration': '30', - 'ThreadStride': '0', - 'TotalSpeed': 189, - 'RandSeed': '0', - 'RequestCount': '0', - 'Path': 'C:\\scratch\\testfile.dat', - 'WriteThrough': 'true', - 'CalculateIopsStdDev': 'false', - 'IoBucketDuration': '1000', - 'ParallelAsyncIO': 'false', - 'Throughput': '0', - 'DisableOSCache': 'true', - 'WriteIops': 0} - sample_list = [ - sample.Sample('ReadSpeed', 189, 'MB/s', - expected_metadata) - ] + single_sample = diskspd.ParseDiskSpdResults( + self.result_xml, {}, 'ReadSpeed' + ) + expected_metadata = { + 'DisableAffinity': 'false', + 'MaxFileSize': '0', + 'BlockSize': '65536', + 'IdlePercent': '96.80', + 'CompletionRoutines': 'false', + 'StrideSize': '65536', + 'RandomAccess': 'false', + 'Weight': '1', + 'UserPercent': '0.25', + 'WriteBytes': 0, + 'Warmup': '5', + 'Pattern': 'sequential', + 'IOPriority': '3', + 'ThreadsPerFile': '4', + 'KernelPercent': '2.95', + 'ReadIops': 3030, + 'BytesCount': 5961809920, + 'InterlockedSequential': 'false', + 'MeasureLatency': 'false', + 'WriteRatio': '0', + 'FileSize': '838860800', + 'BaseFileOffset': '0', + 'Cooldown': '0', + 'IOCount': 90970, + 'UseLargePages': 'false', + 'UsagePercent': '3.20', + 'SequentialScan': 'true', + 'TotalIops': 3030, + 'WriteSpeed': 0, + 'ProcCount': '4', + 'WriteCount': 0, + 'ReadSpeed': 189, + 'TestTimeSeconds': '30.02', + 'ThreadCount': '4', + 'ReadBytes': 5961809920, + 'TemporaryFile': 'false', + 'ReadCount': 90970, + 'Duration': '30', + 'ThreadStride': '0', + 'TotalSpeed': 189, + 'RandSeed': '0', + 'RequestCount': '0', + 'Path': 'C:\\scratch\\testfile.dat', + 'WriteThrough': 'true', + 'CalculateIopsStdDev': 'false', + 'IoBucketDuration': '1000', + 'ParallelAsyncIO': 'false', + 'Throughput': '0', + 'DisableOSCache': 'true', + 'WriteIops': 0, + } + sample_list = [sample.Sample('ReadSpeed', 189, 'MB/s', expected_metadata)] self.assertSampleListsEqualUpToTimestamp([single_sample], sample_list) diff --git a/tests/windows_packages/iperf3_test.py b/tests/windows_packages/iperf3_test.py index dddae9405f..b26e05cbd1 100644 --- a/tests/windows_packages/iperf3_test.py +++ b/tests/windows_packages/iperf3_test.py @@ -15,7 +15,6 @@ from collections import namedtuple import unittest from absl import flags - from perfkitbenchmarker import sample from perfkitbenchmarker import test_util from perfkitbenchmarker.windows_packages import iperf3 @@ -102,7 +101,8 @@ def testIperfUDPStreamSamples(self): receiving_vm = fake_vm(machine_type='A', zone='B') samples = iperf3.GetUDPStreamSamples( - sending_vm, receiving_vm, iperf3_results, bandwidth, internal_ip_used) + sending_vm, receiving_vm, iperf3_results, bandwidth, internal_ip_used + ) expected_metadata = { 'protocol': 'UDP', @@ -117,18 +117,16 @@ def testIperfUDPStreamSamples(self): } expected_samples = [ - sample.Sample('Loss Rate', 55.836, 'Percent', - expected_metadata), - sample.Sample('Bandwidth Achieved', 1380, 'Mbits/sec', - expected_metadata), - sample.Sample('Jitter', 0.072, 'ms', - expected_metadata), + sample.Sample('Loss Rate', 55.836, 'Percent', expected_metadata), + sample.Sample( + 'Bandwidth Achieved', 1380, 'Mbits/sec', expected_metadata + ), + sample.Sample('Jitter', 0.072, 'ms', expected_metadata), ] self.assertSampleListsEqualUpToTimestamp(samples, expected_samples) def testIperfTCPMultiStream(self): - tcp_number_of_streams = 10 fake_vm = namedtuple('fake_vm', 'machine_type zone') @@ -158,9 +156,13 @@ def _Metadata(thread_id): sample.Sample('Bandwidth', 4670.0, 'Mbits/sec', _Metadata('SUM')), ] - samples = iperf3.ParseTCPMultiStreamOutput(iperf3_tcp_results, sending_vm, - receiving_vm, - tcp_number_of_streams, True) + samples = iperf3.ParseTCPMultiStreamOutput( + iperf3_tcp_results, + sending_vm, + receiving_vm, + tcp_number_of_streams, + True, + ) self.assertSampleListsEqualUpToTimestamp(samples, expected_samples) diff --git a/tests/windows_packages/ntttcp_test.py b/tests/windows_packages/ntttcp_test.py index 4a031332e6..64dd87e7cc 100644 --- a/tests/windows_packages/ntttcp_test.py +++ b/tests/windows_packages/ntttcp_test.py @@ -18,7 +18,6 @@ import unittest from absl import flags from absl.testing import parameterized - from perfkitbenchmarker import sample from perfkitbenchmarker import test_util from perfkitbenchmarker.windows_packages import ntttcp @@ -29,8 +28,9 @@ NtttcpConf = ntttcp.NtttcpConf -class NtttcpBenchmarkTestCase(parameterized.TestCase, unittest.TestCase, - test_util.SamplesTestMixin): +class NtttcpBenchmarkTestCase( + parameterized.TestCase, unittest.TestCase, test_util.SamplesTestMixin +): def getDataContents(self, file_name): path = os.path.join(os.path.dirname(__file__), '..', 'data', file_name) @@ -46,8 +46,9 @@ def setUp(self): self.xml_udp_rec_results = self.getDataContents('ntttcp_udp_receiver.xml') def testNtttcpTcpParsing(self): - samples = ntttcp.ParseNtttcpResults(self.xml_tcp_send_results, - self.xml_tcp_rec_results, {}) + samples = ntttcp.ParseNtttcpResults( + self.xml_tcp_send_results, self.xml_tcp_rec_results, {} + ) expected_metadata = { 'async': 'False', @@ -123,17 +124,20 @@ def testNtttcpTcpParsing(self): expected_samples = [ sample.Sample('Total Throughput', 291.485, 'Mbps', expected_metadata), - sample.Sample('Thread Throughput', 147.105, 'Mbps', - expected_thread_0_metadata), - sample.Sample('Thread Throughput', 144.379, 'Mbps', - expected_thread_1_metadata) + sample.Sample( + 'Thread Throughput', 147.105, 'Mbps', expected_thread_0_metadata + ), + sample.Sample( + 'Thread Throughput', 144.379, 'Mbps', expected_thread_1_metadata + ), ] self.assertSampleListsEqualUpToTimestamp(expected_samples, samples) def testNtttcpUdpParsing(self): - samples = ntttcp.ParseNtttcpResults(self.xml_udp_send_results, - self.xml_udp_rec_results, {}) + samples = ntttcp.ParseNtttcpResults( + self.xml_udp_send_results, self.xml_udp_rec_results, {} + ) expected_metadata = { 'async': 'False', @@ -209,10 +213,12 @@ def testNtttcpUdpParsing(self): expected_samples = [ sample.Sample('Total Throughput', 245.153, 'Mbps', expected_metadata), - sample.Sample('Thread Throughput', 121.160, 'Mbps', - expected_thread_0_metadata), - sample.Sample('Thread Throughput', 123.993, 'Mbps', - expected_thread_1_metadata) + sample.Sample( + 'Thread Throughput', 121.160, 'Mbps', expected_thread_0_metadata + ), + sample.Sample( + 'Thread Throughput', 123.993, 'Mbps', expected_thread_1_metadata + ), ] self.assertSampleListsEqualUpToTimestamp(expected_samples, samples) @@ -221,7 +227,8 @@ def testSingleConfigParse(self): ntttcp.FLAGS.ntttcp_config_list = ['True:7:800:INTERNAL:1'] expected_list = [ NtttcpConf( - udp=True, threads=7, time_s=800, ip_type='INTERNAL', packet_size=1) + udp=True, threads=7, time_s=800, ip_type='INTERNAL', packet_size=1 + ) ] conf_list = ntttcp.ParseConfigList() self.assertListEqual(conf_list, expected_list) @@ -234,27 +241,28 @@ def testEmptyConfig(self): threads=FLAGS.ntttcp_threads, time_s=FLAGS.ntttcp_time, ip_type=FLAGS.ip_addresses, - packet_size=FLAGS.ntttcp_packet_size) + packet_size=FLAGS.ntttcp_packet_size, + ) ] conf_list = ntttcp.ParseConfigList() self.assertListEqual(conf_list, expected_list) def testMultiConfigParse(self): ntttcp.FLAGS.ntttcp_config_list = [ - 'True:7:800:INTERNAL:1', 'False:1:2:EXTERNAL:2', - 'True:44:1001:INTERNAL:3' + 'True:7:800:INTERNAL:1', + 'False:1:2:EXTERNAL:2', + 'True:44:1001:INTERNAL:3', ] expected_list = [ NtttcpConf( - udp=True, threads=7, time_s=800, ip_type='INTERNAL', packet_size=1), + udp=True, threads=7, time_s=800, ip_type='INTERNAL', packet_size=1 + ), NtttcpConf( - udp=False, threads=1, time_s=2, ip_type='EXTERNAL', packet_size=2), + udp=False, threads=1, time_s=2, ip_type='EXTERNAL', packet_size=2 + ), NtttcpConf( - udp=True, - threads=44, - time_s=1001, - ip_type='INTERNAL', - packet_size=3), + udp=True, threads=44, time_s=1001, ip_type='INTERNAL', packet_size=3 + ), ] conf_list = ntttcp.ParseConfigList() self.assertListEqual(conf_list, expected_list) @@ -262,7 +270,8 @@ def testMultiConfigParse(self): @parameterized.named_parameters( ('MissingVal', ['True:7:800:INTERNAL:1', 'False::2:EXTERNAL:2']), ('Misspell', ['rue:7:800:INTERNAL:3', 'True:44:1001:EXTERNAL:4']), - ('WrongOrder', ['True:7:INTERNAL:800:1', '44:True:1001:EXTERNAL:6'])) + ('WrongOrder', ['True:7:INTERNAL:800:1', '44:True:1001:EXTERNAL:6']), + ) def testMalformedConfig(self, conf): with self.assertRaises(flags.IllegalFlagValueError): ntttcp.FLAGS.ntttcp_config_list = conf diff --git a/tests/windows_packages/nuttcp_test.py b/tests/windows_packages/nuttcp_test.py index 2ebe4c3d55..0cb0954398 100644 --- a/tests/windows_packages/nuttcp_test.py +++ b/tests/windows_packages/nuttcp_test.py @@ -28,11 +28,13 @@ %RX 104429 / 1554763 drop/pkt 6.72 %loss """ -cpu_results = ('\r\nInstanceName CookedValue\r\n------------ ' - ' -----------\r\n0 22.7976893740141\r\n1 ' - ' 32.6422793196096\r\n2 18.6525988706' - '054\r\n3 44.5594145169094\r\n_total 2' - '9.6629938622484\r\n\r\n\r\n') +cpu_results = ( + '\r\nInstanceName CookedValue\r\n------------ ' + ' -----------\r\n0 22.7976893740141\r\n1 ' + ' 32.6422793196096\r\n2 18.6525988706' + '054\r\n3 44.5594145169094\r\n_total 2' + '9.6629938622484\r\n\r\n\r\n' +) class NuttcpTestCase(unittest.TestCase, test_util.SamplesTestMixin): @@ -47,8 +49,16 @@ def testNuttcpUDPStreamSamples(self): server = machine(machine_type='sB', zone='sZ') result_sample = [ - nuttcp.GetUDPStreamSample(nuttcp_results, cpu_results, cpu_results, - client, server, bandwidth, 'external', 1) + nuttcp.GetUDPStreamSample( + nuttcp_results, + cpu_results, + cpu_results, + client, + server, + bandwidth, + 'external', + 1, + ) ] expected_metadata = { diff --git a/tests/windows_packages/psping_test.py b/tests/windows_packages/psping_test.py index 82f685c6d8..945439adcb 100644 --- a/tests/windows_packages/psping_test.py +++ b/tests/windows_packages/psping_test.py @@ -49,7 +49,6 @@ class PspingBenchmarkTestCase(unittest.TestCase, test_util.SamplesTestMixin): def testPspingParsing(self): - minimum = 0.19 maximum = 0.58 average = 0.27 @@ -60,8 +59,9 @@ def testPspingParsing(self): client = machine(machine_type='cA', zone='cZ') server = machine(machine_type='sB', zone='sZ') - samples = psping.ParsePspingResults(psping_results, client, server, - use_internal_ip) + samples = psping.ParsePspingResults( + psping_results, client, server, use_internal_ip + ) expected_metadata = { 'internal_ip_used': use_internal_ip, @@ -88,8 +88,9 @@ def testPspingParsing(self): expected_histogram_metadata = expected_metadata.copy() expected_histogram_metadata['histogram'] = histogram - expected_samples.append(sample.Sample('latency:histogram', 0, 'ms', - expected_histogram_metadata)) + expected_samples.append( + sample.Sample('latency:histogram', 0, 'ms', expected_histogram_metadata) + ) self.assertSampleListsEqualUpToTimestamp(expected_samples, samples) diff --git a/tools/demos/process_histogram_samples.py b/tools/demos/process_histogram_samples.py index 4adb6d445b..04d910f0e0 100644 --- a/tools/demos/process_histogram_samples.py +++ b/tools/demos/process_histogram_samples.py @@ -15,7 +15,8 @@ def main(): sys.exit(1) latency_histogram_by_label = collections.defaultdict( - lambda: collections.defaultdict(int)) + lambda: collections.defaultdict(int) + ) total_samples = collections.defaultdict(int) with open(sys.argv[1]) as samples_file: for line in samples_file: @@ -25,7 +26,8 @@ def main(): regex = r'\|%s:(.*?)\|' % sys.argv[3] label = re.search(regex, labels).group(1) histogram = json.loads( - re.search(r'\|histogram:(.*?)\|', labels).group(1)) + re.search(r'\|histogram:(.*?)\|', labels).group(1) + ) for bucket, count in histogram.iteritems(): latency_histogram_by_label[label][float(bucket)] += int(count) total_samples[label] += int(count) diff --git a/tools/object_storage_timeline.py b/tools/object_storage_timeline.py index cd770f690c..dc5c790969 100755 --- a/tools/object_storage_timeline.py +++ b/tools/object_storage_timeline.py @@ -23,6 +23,7 @@ class DraggableXRange: + def __init__(self, figure, updater): self.figure = figure self.span = None @@ -34,11 +35,14 @@ def __init__(self, figure, updater): def connect(self): 'connect to all the events we need' self.cidpress = self.figure.canvas.mpl_connect( - 'button_press_event', self.on_press) + 'button_press_event', self.on_press + ) self.cidrelease = self.figure.canvas.mpl_connect( - 'button_release_event', self.on_release) + 'button_release_event', self.on_release + ) self.cidmotion = self.figure.canvas.mpl_connect( - 'motion_notify_event', self.on_motion) + 'motion_notify_event', self.on_motion + ) def on_press(self, event): 'on button press we will see if the mouse is over us and store some data' @@ -117,6 +121,7 @@ def disconnect(self): class SelectionUpdate: + def __init__(self, figure, ax, start_times, latencies): self.text = None self.figure = figure @@ -152,8 +157,9 @@ def update(self, start, end): active_stop_indexes.append(i + 1) break active_latencies = [ - self.latencies[i][active_start_indexes[i]:active_stop_indexes[i]] - for i in range(len(self.latencies))] + self.latencies[i][active_start_indexes[i] : active_stop_indexes[i]] + for i in range(len(self.latencies)) + ] all_active_latencies = np.concatenate(active_latencies) qps = len(all_active_latencies) / (end - start) @@ -162,16 +168,30 @@ def update(self, start, end): latency_avg = sum(all_active_latencies) / len(all_active_latencies) latency_stddev = np.std(all_active_latencies) - text_str = ('Duration: %s\nQPS: %s\nlatency min: %s\nlatency max: %s\n' - 'latency avg: %s\nlatency stddev: %s' - % (end - start, qps, latency_min, latency_max, - latency_avg, latency_stddev)) + text_str = ( + 'Duration: %s\nQPS: %s\nlatency min: %s\nlatency max: %s\n' + 'latency avg: %s\nlatency stddev: %s' + % ( + end - start, + qps, + latency_min, + latency_max, + latency_avg, + latency_stddev, + ) + ) # place a text box in upper left in axes coords props = dict(boxstyle='round', facecolor='wheat', alpha=0.5) - self.text = self.ax.text(0.05, 0.95, text_str, - transform=self.ax.transAxes, fontsize=14, - verticalalignment='top', bbox=props) + self.text = self.ax.text( + 0.05, + 0.95, + text_str, + transform=self.ax.transAxes, + fontsize=14, + verticalalignment='top', + bbox=props, + ) # redraw just the text self.text.axes.draw_artist(self.text) # blit just the redrawn area @@ -179,26 +199,31 @@ def update(self, start, end): def GenerateObjectTimeline(file_name, start_times, latencies): - print("Generating object timeline") + print('Generating object timeline') assert len(start_times) == len(latencies) rects = [] for i, worker_times in enumerate(zip(start_times, latencies)): for j, (start_time, latency) in enumerate(np.vstack(worker_times).T): - rect = mpl_patches.Rectangle((start_time, i + 0.5), latency, 1.0, - color=(0.5 * (j % 2) + 0.5, 0, 0), - linewidth=0) + rect = mpl_patches.Rectangle( + (start_time, i + 0.5), + latency, + 1.0, + color=(0.5 * (j % 2) + 0.5, 0, 0), + linewidth=0, + ) rects.append(rect) pc = mplc.PatchCollection(rects, match_original=True) fig, ax = plt.subplots(figsize=(30, 5)) ax.add_collection(pc) ax.autoscale() ax.margins(0.1) - print("Saving figure as %s" % file_name) + print('Saving figure as %s' % file_name) plt.savefig(file_name, bbox_inches='tight', dpi=1200) - print("Figured saved. Rendering figure...") + print('Figured saved. Rendering figure...') - selection = DraggableXRange(fig, SelectionUpdate(fig, ax, start_times, - latencies)) + selection = DraggableXRange( + fig, SelectionUpdate(fig, ax, start_times, latencies) + ) selection.connect() plt.show() selection.disconnect() @@ -253,13 +278,14 @@ def LoadWorkerOutput(output): def main(): worker_output = None - print("Reading worker output") + print('Reading worker output') with open(sys.argv[1], 'r') as worker_out_file: worker_output = json.loads(worker_out_file.read()) - print("Parsing worker output") + print('Parsing worker output') start_times, latencies, _ = LoadWorkerOutput(worker_output) GenerateObjectTimeline(sys.argv[2], start_times, latencies) + ######################################## if __name__ == '__main__': diff --git a/tools/side-by-side/side_by_side.py b/tools/side-by-side/side_by_side.py index 154fb738fe..337300d16e 100755 --- a/tools/side-by-side/side_by_side.py +++ b/tools/side-by-side/side_by_side.py @@ -38,8 +38,11 @@ import jinja2 -DEFAULT_FLAGS = ('--cloud=GCP', '--machine_type=n1-standard-4', - '--benchmarks=netperf') +DEFAULT_FLAGS = ( + '--cloud=GCP', + '--machine_type=n1-standard-4', + '--benchmarks=netperf', +) # Keys in the sample JSON we expect to vary between runs. # These will be removed prior to diffing samples. VARYING_KEYS = 'run_uri', 'sample_uri', 'timestamp', 'value' @@ -54,7 +57,8 @@ PerfKitBenchmarkerResult = collections.namedtuple( 'PerfKitBenchmarkerResult', - ['name', 'description', 'sha1', 'samples', 'flags']) + ['name', 'description', 'sha1', 'samples', 'flags'], +) @contextlib.contextmanager @@ -98,15 +102,17 @@ def _GitCommandPrefix(): def _GitRevParse(revision): """Returns the output of 'git rev-parse' for 'revision'.""" - output = subprocess.check_output(_GitCommandPrefix() + - ['rev-parse', revision]) + output = subprocess.check_output( + _GitCommandPrefix() + ['rev-parse', revision] + ) return output.rstrip() def _GitDescribe(revision): """Returns the output of 'git describe' for 'revision'.""" - output = subprocess.check_output(_GitCommandPrefix() + - ['describe', '--always', revision]) + output = subprocess.check_output( + _GitCommandPrefix() + ['describe', '--always', revision] + ) return output.rstrip() @@ -151,8 +157,13 @@ def RunPerfKitBenchmarker(revision, flags): logging.info('Running %s in %s', cmd, td) subprocess.check_call(cmd, cwd=td) samples = [json.loads(line) for line in tf] - return PerfKitBenchmarkerResult(name=revision, sha1=sha1, flags=flags, - samples=samples, description=description) + return PerfKitBenchmarkerResult( + name=revision, + sha1=sha1, + flags=flags, + samples=samples, + description=description, + ) def _SplitLabels(labels): @@ -183,6 +194,7 @@ def _CompareSamples(a, b, context=True, numlines=1): context: boolean. Show context in diff? If False, all lines are output, even those which are equal. numlines: int. Passed to difflib.Htmldiff.make_table. + Returns: string or None. An HTML table, or None if there are no differences. """ @@ -218,6 +230,7 @@ def _MatchSamples(base_samples, head_samples): Returns: List of pairs, each item of the pair containing either a dict or None. """ + def ExtractKeys(samples): return [(i['test'], i['metric'], i['unit']) for i in samples] @@ -230,26 +243,45 @@ def ExtractKeys(samples): for opcode, base_begin, base_end, head_begin, head_end in sm.get_opcodes(): if opcode == 'equal': - result.extend(zip(base_samples[base_begin:base_end], - head_samples[head_begin:head_end])) + result.extend( + zip( + base_samples[base_begin:base_end], + head_samples[head_begin:head_end], + ) + ) elif opcode == 'replace': - result.extend(zip(base_samples[base_begin:base_end], - [None] * (base_end - base_begin))) - result.extend(zip([None] * (head_end - head_begin), - head_samples[head_begin:head_end])) + result.extend( + zip( + base_samples[base_begin:base_end], + [None] * (base_end - base_begin), + ) + ) + result.extend( + zip( + [None] * (head_end - head_begin), + head_samples[head_begin:head_end], + ) + ) elif opcode == 'delete': - result.extend(zip(base_samples[base_begin:base_end], - [None] * (base_end - base_begin))) + result.extend( + zip( + base_samples[base_begin:base_end], + [None] * (base_end - base_begin), + ) + ) elif opcode == 'insert': - result.extend(zip([None] * (head_end - head_begin), - head_samples[head_begin:head_end])) + result.extend( + zip( + [None] * (head_end - head_begin), + head_samples[head_begin:head_end], + ) + ) else: raise AssertionError('Unknown op: ' + opcode) return result -def RenderResults(base_result, head_result, template_name=TEMPLATE, - **kwargs): +def RenderResults(base_result, head_result, template_name=TEMPLATE, **kwargs): """Render the results of a comparison as an HTML page. Args: @@ -263,6 +295,7 @@ def RenderResults(base_result, head_result, template_name=TEMPLATE, Returns: String. The HTML template. """ + def _ClassForPercentDifference(percent_diff): """Crude highlighting of differences between runs. @@ -292,14 +325,14 @@ def _ClassForPercentDifference(percent_diff): env = jinja2.Environment( loader=jinja2.FileSystemLoader(os.path.dirname(__file__)), - undefined=jinja2.StrictUndefined) + undefined=jinja2.StrictUndefined, + ) env.globals['class_for_percent_diff'] = _ClassForPercentDifference env.globals['izip_longest'] = itertools.izip_longest template = env.get_template('side_by_side.html.j2') - matched = _MatchSamples(base_result.samples, - head_result.samples) + matched = _MatchSamples(base_result.samples, head_result.samples) # Generate sample diffs sample_context_diffs = [] @@ -308,63 +341,98 @@ def _ClassForPercentDifference(percent_diff): if not base_sample or not head_sample: # Sample inserted or deleted. continue - sample_context_diffs.append( - _CompareSamples(base_sample, head_sample)) + sample_context_diffs.append(_CompareSamples(base_sample, head_sample)) sample_diffs.append( - _CompareSamples(base_sample, head_sample, context=False)) + _CompareSamples(base_sample, head_sample, context=False) + ) # Generate flag diffs flag_diffs = difflib.HtmlDiff().make_table( - base_result.flags, head_result.flags, context=False) + base_result.flags, head_result.flags, context=False + ) # Used for generating a chart with differences. - matched_json = json.dumps(matched)\ - .replace(u'<', u'\\u003c') \ - .replace(u'>', u'\\u003e') \ - .replace(u'&', u'\\u0026') \ - .replace(u"'", u'\\u0027') - - return template.render(base=base_result, - head=head_result, - matched_samples=matched, - matched_samples_json=matched_json, - sample_diffs=sample_diffs, - sample_context_diffs=sample_context_diffs, - flag_diffs=flag_diffs, - infinity=float('inf'), - **kwargs) + matched_json = ( + json.dumps(matched) + .replace('<', '\\u003c') + .replace('>', '\\u003e') + .replace('&', '\\u0026') + .replace("'", '\\u0027') + ) + + return template.render( + base=base_result, + head=head_result, + matched_samples=matched, + matched_samples_json=matched_json, + sample_diffs=sample_diffs, + sample_context_diffs=sample_context_diffs, + flag_diffs=flag_diffs, + infinity=float('inf'), + **kwargs + ) def main(): p = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description=__doc__) - p.add_argument('-t', '--title', default='PerfKitBenchmarker Comparison', - help="""HTML report title""") + description=__doc__, + ) + p.add_argument( + '-t', + '--title', + default='PerfKitBenchmarker Comparison', + help="""HTML report title""", + ) p.add_argument('--base', default='master', help="""Base revision.""") p.add_argument('--head', default='dev', help="""Head revision.""") - p.add_argument('--base-flags', default=None, help="""Flags for run against + p.add_argument( + '--base-flags', + default=None, + help="""Flags for run against '--base' revision. Will be combined with --flags.""", - type=shlex.split) - p.add_argument('--head-flags', default=None, help="""Flags for run against + type=shlex.split, + ) + p.add_argument( + '--head-flags', + default=None, + help="""Flags for run against '--head' revision. Will be combined with --flags.""", - type=shlex.split) - p.add_argument('-f', '--flags', type=shlex.split, - help="""Command line flags (Default: {0})""".format( - ' '.join(DEFAULT_FLAGS))) - p.add_argument('-p', '--parallel', default=False, action='store_true', - help="""Run concurrently""") - p.add_argument('--rerender', help="""Re-render the HTML report from a JSON - file [for developers].""", action='store_true') + type=shlex.split, + ) + p.add_argument( + '-f', + '--flags', + type=shlex.split, + help="""Command line flags (Default: {0})""".format( + ' '.join(DEFAULT_FLAGS) + ), + ) + p.add_argument( + '-p', + '--parallel', + default=False, + action='store_true', + help="""Run concurrently""", + ) + p.add_argument( + '--rerender', + help="""Re-render the HTML report from a JSON + file [for developers].""", + action='store_true', + ) p.add_argument('json_output', help="""JSON output path.""") p.add_argument('html_output', help="""HTML output path.""") a = p.parse_args() - if (a.base_flags or a.head_flags): + if a.base_flags or a.head_flags: if not (a.base_flags and a.head_flags): - p.error('--base-flags and --head-flags must be specified together.\n' - '\tbase flags={0}\n\thead flags={1}'.format( - a.base_flags, a.head_flags)) + p.error( + '--base-flags and --head-flags must be specified together.\n' + '\tbase flags={0}\n\thead flags={1}'.format( + a.base_flags, a.head_flags + ) + ) a.base_flags = a.base_flags + (a.flags or []) a.head_flags = a.head_flags + (a.flags or []) else: @@ -377,11 +445,14 @@ def main(): if not a.rerender: if a.parallel: from concurrent import futures + with futures.ThreadPoolExecutor(max_workers=2) as executor: - base_res_fut = executor.submit(RunPerfKitBenchmarker, a.base, - a.base_flags) - head_res_fut = executor.submit(RunPerfKitBenchmarker, a.head, - a.head_flags) + base_res_fut = executor.submit( + RunPerfKitBenchmarker, a.base, a.base_flags + ) + head_res_fut = executor.submit( + RunPerfKitBenchmarker, a.head, a.head_flags + ) base_res = base_res_fut.result() head_res = head_res_fut.result() else: @@ -393,10 +464,11 @@ def main(): with argparse.FileType('w')(a.json_output) as json_fp: logging.info('Writing JSON to %s', a.json_output) - json.dump({'head': head_res._asdict(), - 'base': base_res._asdict()}, - json_fp, - indent=2) + json.dump( + {'head': head_res._asdict(), 'base': base_res._asdict()}, + json_fp, + indent=2, + ) json_fp.write('\n') else: logging.info('Loading results from %s', a.json_output) @@ -407,10 +479,14 @@ def main(): with argparse.FileType('w')(a.html_output) as html_fp: logging.info('Writing HTML to %s', a.html_output) - html_fp.write(RenderResults(base_result=base_res, - head_result=head_res, - varying_keys=VARYING_KEYS, - title=a.title)) + html_fp.write( + RenderResults( + base_result=base_res, + head_result=head_res, + varying_keys=VARYING_KEYS, + title=a.title, + ) + ) if __name__ == '__main__': diff --git a/tools/zip/zip.py b/tools/zip/zip.py index 0e03f81feb..43a01a9498 100755 --- a/tools/zip/zip.py +++ b/tools/zip/zip.py @@ -27,15 +27,22 @@ def main(argv): zip_file_path = argv[1] version = subprocess.check_output( - (os.path.join(os.getcwd(), 'pkb.py'), '--version')).rstrip() + (os.path.join(os.getcwd(), 'pkb.py'), '--version') + ).rstrip() with zipfile.ZipFile(zip_file_path, 'w') as zip_file: for dir_path, _, file_names in os.walk('perfkitbenchmarker'): for file_name in file_names: if not file_name.endswith('.pyc'): zip_file.write(os.path.join(dir_path, file_name)) - for file_name in ('AUTHORS', 'CHANGES.md', 'CONTRIBUTING.md', 'LICENSE', - 'README.md', 'requirements.txt'): + for file_name in ( + 'AUTHORS', + 'CHANGES.md', + 'CONTRIBUTING.md', + 'LICENSE', + 'README.md', + 'requirements.txt', + ): zip_file.write(file_name) zip_file.write('pkb.py', '__main__.py') zip_file.writestr('perfkitbenchmarker/version.txt', version)