Merge pull request #1545 from GoogleCloudPlatform/moe

Rollup changes with MOE

CHANGES.next.md

@@ -12,6 +12,7 @@
 - Support for ProfitBricks API v4:
   - Add `profitbricks_image_alias` flag and support for image aliases
   - Add new location, `us/ewr`
+- Add aws_image_name_filter flag to ease specifying images.
 
 ###Bug fixes and maintenance updates:
 - Moved GPU-related specs from GceVmSpec to BaseVmSpec

perfkitbenchmarker/configs/benchmark_config_spec.py

@@ -551,6 +551,13 @@ def _GetOptionDecoderConstructions(cls):
                 managed_relational_db.AURORA_POSTGRES,
             ]
         }),
+        'zones': (option_decoders.ListDecoder, {
+            'item_decoder': option_decoders.StringDecoder(),
+            'default': None
+        }),
+        'machine_type': (option_decoders.StringDecoder, {
+            'default': None
+        }),
         'engine_version': (option_decoders.StringDecoder, {
             'default': None
         }),

perfkitbenchmarker/container_service.py

@@ -90,6 +90,11 @@ def GetResourceMetadata(self):
         'zone': self.zone,
         'size': self.num_nodes,
     }
+    if self.gpu_count:
+      metadata.update({
+          'gpu_type': self.gpu_type,
+          'num_gpus': self.gpu_count,
+      })
     return metadata
 
 

perfkitbenchmarker/data/edw/redshift_driver.sh

@@ -30,7 +30,7 @@ START_TIME=$SECONDS
 
 for REDSHIFT_SCRIPT in "${REDSHIFT_SCRIPT_LIST[@]}"
 do
-  PGPASSWORD=$REDSHIFT_PASSWORD psql -h $REDSHIFT_HOST -p 5439 -d $REDSHIFT_DB -U $REDSHIFT_USER -f redshift_sql/$REDSHIFT_SCRIPT > /dev/null &
+  PGPASSWORD=$REDSHIFT_PASSWORD psql -h $REDSHIFT_HOST -p 5439 -d $REDSHIFT_DB -U $REDSHIFT_USER -f $REDSHIFT_SCRIPT > /dev/null &
   pid=$!
   pids="$pids $pid"
 done

perfkitbenchmarker/data/nvidia_unrestricted_permissions_daemonset.yml

@@ -0,0 +1,44 @@
+# This file defines a daemonset which runs automatically on all
+# kubernetes nodes. It is used like so: kubectl create -f <this file_path>.
+# The daemonset does the following:
+#   - waits until nvidia-smi is mounted and available on PATH
+#   - enables persistence mode on the nvidia driver
+#   - allows all users to set the GPU clock speed
+# In effect, this allows pods created without a privileged security context to
+# set the GPU clock speeds
+# This daemonset config does not define GPU resources, because otherwise it
+# would consume them, leaving them unavailable to pods. Instead, it runs in
+# privileged mode (so it can see all GPUs), and manually mounts the CUDA
+# lib and bin directories.
+
+apiVersion: apps/v1beta2
+kind: DaemonSet
+metadata:
+  name: nvidia-add-unrestricted-permissions-dameon-set
+spec:
+  selector:
+    matchLabels:
+      name: nvidia-add-unrestricted-permissions
+  template:
+    metadata:
+      labels:
+        name: nvidia-add-unrestricted-permissions
+    spec:
+      containers:
+      - name: nvidia-add-unrestricted-permissions
+        image: nvidia/cuda:8.0-devel-ubuntu16.04
+        securityContext:
+          privileged: true
+        command: [ "/bin/bash", "-c", "export PATH=$PATH:/usr/local/bin/nvidia/ && while [ ! $(type -p nvidia-smi) ]; do echo waiting for nvidia-smi to mount...; sleep 2; done && nvidia-smi -pm 1 && nvidia-smi --applications-clocks-permission=UNRESTRICTED && nvidia-smi --auto-boost-permission=UNRESTRICTED && tail -f /dev/null" ]
+        volumeMounts:
+          - name: nvidia-debug-tools
+            mountPath: /usr/local/bin/nvidia
+          - name: nvidia-libraries
+            mountPath: /usr/local/nvidia/lib64
+      volumes:
+        - name: nvidia-debug-tools
+          hostPath:
+            path: /home/kubernetes/bin/nvidia/bin
+        - name: nvidia-libraries
+          hostPath:
+            path: /home/kubernetes/bin/nvidia/lib

perfkitbenchmarker/flag_util.py

@@ -109,8 +109,8 @@ def __str__(self):
 
   def _CreateXrangeFromTuple(self, input_tuple):
     start = input_tuple[0]
-    stop_inclusive = input_tuple[1] + 1
     step = 1 if len(input_tuple) == 2 else input_tuple[2]
+    stop_inclusive = input_tuple[1] + (1 if step > 0 else -1)
     return xrange(start, stop_inclusive, step)
 
 
@@ -187,6 +187,7 @@ def HandleNonIncreasing():
         low = int(match.group(1))
         high = int(match.group(3))
         step = int(match.group(5)) if match.group(5) is not None else 1
+        step = step if low <= high else -step
 
         if high <= low or (len(result) > 0 and low <= result[-1]):
           HandleNonIncreasing()

perfkitbenchmarker/linux_benchmarks/dacapo_benchmark.py

@@ -0,0 +1,97 @@
+# Copyright 2016 PerfKitBenchmarker Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Runs DaCapo benchmarks.
+
+This benchmark runs the various DaCapo benchmarks. More information can be found
+at: http://dacapobench.org/
+"""
+
+import os
+import re
+
+from perfkitbenchmarker import configs
+from perfkitbenchmarker import errors
+from perfkitbenchmarker import flags
+from perfkitbenchmarker import linux_packages
+from perfkitbenchmarker import sample
+
+flags.DEFINE_string('dacapo_jar_filename', 'dacapo-9.12-bach.jar',
+                    'Filename of DaCapo jar file.')
+flags.DEFINE_enum('dacapo_benchmark', 'luindex', ['luindex', 'lusearch'],
+                  'Name of specific DaCapo benchmark to execute.')
+flags.DEFINE_integer('dacapo_num_iters', 1, 'Number of iterations to execute.')
+
+FLAGS = flags.FLAGS
+
+BENCHMARK_NAME = 'dacapo'
+BENCHMARK_CONFIG = """
+dacapo:
+  description: Runs DaCapo benchmarks
+  vm_groups:
+    default:
+      vm_spec: *default_single_core
+"""
+_PASS_PATTERN = re.compile(r'^=====.*PASSED in (\d+) msec =====$')
+
+
+def GetConfig(user_config):
+  return configs.LoadConfig(BENCHMARK_CONFIG, user_config, BENCHMARK_NAME)
+
+
+def Prepare(benchmark_spec):
+  """Install the DaCapo benchmark suite on the vms.
+
+  Args:
+    benchmark_spec: The benchmark specification. Contains all data that is
+        required to run the benchmark.
+  """
+  benchmark_spec.vms[0].Install('dacapo')
+
+
+def Run(benchmark_spec):
+  """Run the DaCapo benchmark on the vms.
+
+  Args:
+    benchmark_spec: The benchmark specification. Contains all data that is
+        required to run the benchmark.
+
+  Returns:
+    A singleton list of sample.Sample objects containing the DaCapo benchmark
+    run time (in msec).
+
+  Raises:
+    errors.Benchmarks.RunError if the DaCapo benchmark didn't succeed.
+  """
+  _, stderr = benchmark_spec.vms[0].RemoteCommand(
+      'java -jar %s %s -n %i --scratch-directory=%s' %
+      (os.path.join(linux_packages.INSTALL_DIR, FLAGS.dacapo_jar_filename),
+       FLAGS.dacapo_benchmark, FLAGS.dacapo_num_iters,
+       os.path.join(linux_packages.INSTALL_DIR, 'dacapo_scratch')))
+  for line in stderr.splitlines():
+    m = _PASS_PATTERN.match(line)
+    if m:
+      return [sample.Sample('run_time', float(m.group(1)), 'ms')]
+  raise errors.Benchmarks.RunError(
+      'DaCapo benchmark %s failed.' % FLAGS.dacapo_benchmark)
+
+
+def Cleanup(benchmark_spec):
+  """Cleanup the DaCapo benchmark on the target vm (by uninstalling).
+
+  Args:
+    benchmark_spec: The benchmark specification. Contains all data that is
+        required to run the benchmark.
+  """
+  benchmark_spec.vms[0].RemoteCommand(
+      'rm -rf %s' % os.path.join(linux_packages.INSTALL_DIR, 'dacapo_scratch'))

perfkitbenchmarker/linux_benchmarks/edw_benchmark.py

@@ -20,6 +20,7 @@
 
 
 import copy
+import os
 
 from perfkitbenchmarker import configs
 from perfkitbenchmarker import data
@@ -59,15 +60,16 @@ def Prepare(benchmark_spec):
 
 def Run(benchmark_spec):
   """Run phase executes the sql scripts on edw cluster and collects duration."""
-  driver_name = '{}_driver.sh'.format(benchmark_spec.edw_service.SERVICE_TYPE)
-  driver_path = data.ResourcePath(driver_name)
-
-  scripts_name = '{}_sql'.format(benchmark_spec.edw_service.SERVICE_TYPE)
-  scripts_path = data.ResourcePath(scripts_name)
-
   vm = benchmark_spec.vms[0]
+  driver_name = '{}_driver.sh'.format(benchmark_spec.edw_service.SERVICE_TYPE)
+  driver_path = data.ResourcePath(os.path.join('edw', driver_name))
   vm.PushFile(driver_path)
-  vm.PushFile(scripts_path)
+
+  scripts_dir = '{}_sql'.format(benchmark_spec.edw_service.SERVICE_TYPE)
+  scripts_list = FLAGS.edw_benchmark_scripts
+  for script in scripts_list:
+    script_path = data.ResourcePath(os.path.join('edw', scripts_dir, script))
+    vm.PushFile(script_path)
 
   driver_perms_update_cmd = 'chmod 755 {}'.format(driver_name)
   vm.RemoteCommand(driver_perms_update_cmd)

perfkitbenchmarker/linux_benchmarks/multichase_benchmark.py

@@ -285,7 +285,7 @@ def Run(benchmark_spec):
 
   for thread_count in FLAGS.multichase_thread_count:
     if thread_count > vm.num_cpus:
-      break
+      continue
     memory_size_iterator = _IterMemorySizes(
         lambda: vm.total_memory_kb * 1024, FLAGS.multichase_memory_size_min,
         FLAGS.multichase_memory_size_max)

perfkitbenchmarker/linux_benchmarks/speccpu2006_benchmark.py

@@ -24,18 +24,19 @@
 
 import itertools
 import logging
+from operator import mul
 import os
 import posixpath
 import re
 import tarfile
 
-from operator import mul
 from perfkitbenchmarker import configs
 from perfkitbenchmarker import data
 from perfkitbenchmarker import errors
 from perfkitbenchmarker import flags
 from perfkitbenchmarker import sample
 from perfkitbenchmarker import stages
+from perfkitbenchmarker.linux_packages import build_tools
 
 
 FLAGS = flags.FLAGS
@@ -62,6 +63,14 @@
     'cfg file must be placed in the local PKB data directory and will be '
     'copied to the remote machine prior to executing runspec. See README.md '
     'for instructions if running with a repackaged cpu2006v1.2.tgz file.')
+flags.DEFINE_string(
+    'runspec_build_tool_version', None,
+    'Version of gcc/g++/gfortran. This should match runspec_config. Note, if '
+    'neither runspec_config and runspec_build_tool_version is set, the test '
+    'install gcc/g++/gfortran-4.7, since that matches default config version. '
+    'If runspec_config is set, but not runspec_build_tool_version, default '
+    'version of build tools will be installed. Also this flag only works with '
+    'debian.')
 flags.DEFINE_integer(
     'runspec_iterations', 3,
     'Used by the PKB speccpu2006 benchmark. The number of benchmark iterations '
@@ -119,7 +128,7 @@ def GetConfig(user_config):
   return configs.LoadConfig(BENCHMARK_CONFIG, user_config, BENCHMARK_NAME)
 
 
-def CheckPrerequisites(benchmark_config):
+def CheckPrerequisites(unused_benchmark_config):
   """Verifies that the required input files are present."""
   try:
     # Peeking into the tar file is slow. If running in stages, it's
@@ -227,6 +236,7 @@ class _SpecCpu2006SpecificState(object):
         where the SPEC files are stored.
     tar_file_path: Optional string. Path of the tar file on the remote machine.
   """
+
   def __init__(self):
     self.cfg_file_path = None
     self.iso_file_path = None
@@ -246,8 +256,15 @@ def Prepare(benchmark_spec):
   speccpu_vm_state = _SpecCpu2006SpecificState()
   setattr(vm, _BENCHMARK_SPECIFIC_VM_STATE_ATTR, speccpu_vm_state)
   vm.Install('wget')
-  vm.Install('build_tools')
   vm.Install('fortran')
+  vm.Install('build_tools')
+
+  # If using default config files and runspec_build_tool_version is not set,
+  # install 4.7 gcc/g++/gfortan. If either one of the flag is set, we assume
+  # user is smart
+  if not FLAGS['runspec_config'].present or FLAGS.runspec_build_tool_version:
+    build_tool_version = FLAGS.runspec_build_tool_version or '4.7'
+    build_tools.Reinstall(vm, version=build_tool_version)
   if FLAGS.runspec_enable_32bit:
     vm.Install('multilib')
   vm.Install('numactl')
@@ -329,6 +346,7 @@ def _ExtractScore(stdout, vm, keep_partial_results, estimate_spec):
     keep_partial_results: A boolean indicating whether partial results should
         be extracted in the event that not all benchmarks were successfully
         run. See the "runspec_keep_partial_results" flag for more info.
+    estimate_spec: A boolean indicating whether should we estimate spec score.
 
   Sample input for SPECint:
       ...
@@ -449,7 +467,7 @@ def _ExtractScore(stdout, vm, keep_partial_results, estimate_spec):
 
 
 def _GeometricMean(arr):
-  "Calculates the geometric mean of the array."
+  """Calculates the geometric mean of the array."""
   return reduce(mul, arr) ** (1.0 / len(arr))