Merge branch 'master' of https://github.com/AppScale/appscale-tools i…

…nto java-1.8.1

debian/appscale_build.sh

@@ -2,7 +2,7 @@
 
 export DIST=`lsb_release -c -s`
 
-if [ "$DIST" == "n/a" ]
+if [ "$DIST" = "n/a" ]
 then
   DIST="na"
 fi

debian/appscale_install_functions.sh

@@ -48,7 +48,6 @@ installpylibs()
   pip_wrapper oauth2client
   pip_wrapper google-api-python-client
   pip_wrapper argparse
-  pip_wrapper python-gflags
 }
 
 installappscaletools()

lib/agents/gce_agent.py

@@ -6,6 +6,7 @@
 """
 
 # General-purpose Python library imports
+import argparse
 import datetime
 import json
 import os.path
@@ -1087,7 +1088,8 @@ def open_connection(self, parameters):
     credentials = storage.get()
 
     if credentials is None or credentials.invalid:
-      credentials = oauth2client.tools.run(flow, storage)
+      flags = oauth2client.tools.argparser.parse_args(args=[])
+      credentials = oauth2client.tools.run_flow(flow, storage, flags)
 
     # Build the service
     return discovery.build('compute', self.API_VERSION), credentials

lib/remote_helper.py

@@ -8,7 +8,6 @@
 import re
 import socket
 import subprocess
-import tempfile
 import threading
 import time
 import uuid
@@ -291,6 +290,39 @@ def is_port_open(cls, host, port, is_verbose):
       AppScaleLogger.verbose(str(exception), is_verbose)
       return False
 
+  @classmethod
+  def merge_authorized_keys(cls, host, keyname, user, is_verbose):
+    """ Adds the contents of the user's authorized_keys file to the root's
+    authorized_keys file.
+
+    Args:
+      host: A str representing the host to enable root logins on.
+      keyname: A str representing the name of the SSH keypair to login with.
+      user: A str representing the name of the user to login as.
+      is_verbose: A bool indicating if we should print the command we execute to
+        enable root login to stdout.
+    """
+    AppScaleLogger.log('Root login not enabled - enabling it now.')
+
+    create_root_keys = 'sudo touch /root/.ssh/authorized_keys'
+    cls.ssh(host, keyname, create_root_keys, is_verbose, user=user)
+
+    set_permissions = 'sudo chmod 600 /root/.ssh/authorized_keys'
+    cls.ssh(host, keyname, set_permissions, is_verbose, user=user)
+
+    temp_file = cls.ssh(host, keyname, 'mktemp', is_verbose, user=user)
+
+    merge_to_tempfile = 'sudo sort -u ~/.ssh/authorized_keys '\
+      '/root/.ssh/authorized_keys -o {}'.format(temp_file)
+    cls.ssh(host, keyname, merge_to_tempfile, is_verbose, user=user)
+
+    overwrite_root_keys = "sudo sed -n '/.*Please login/d; "\
+      "w/root/.ssh/authorized_keys' {}".format(temp_file)
+    cls.ssh(host, keyname, overwrite_root_keys, is_verbose, user=user)
+
+    remove_tempfile = 'rm -f {0}'.format(temp_file)
+    cls.ssh(host, keyname, remove_tempfile, is_verbose, user=user)
+    return
 
   @classmethod
   def enable_root_login(cls, host, keyname, infrastructure, is_verbose):
@@ -313,35 +345,16 @@ def enable_root_login(cls, host, keyname, infrastructure, is_verbose):
       # Google Compute Engine creates a user with the same name as the currently
       # logged-in user, so log in as that user to enable root login.
       if infrastructure == "gce":
-        AppScaleLogger.log("Root login not enabled - enabling it now.")
-        temp_file = tempfile.NamedTemporaryFile()
-        cls.ssh(host, keyname, "sudo sort -u ~/.ssh/authorized_keys " \
-          "/root/.ssh/authorized_keys -o {0}".format(temp_file.name),
-          is_verbose, user=getpass.getuser())
-        cls.ssh(host, keyname, "sudo sed -n '/.*Please login/d; " \
-          "w/root/.ssh/authorized_keys' {0}".format(temp_file.name),
-          is_verbose, user=getpass.getuser())
-        cls.ssh(host, keyname, "sudo rm -f {0}".format(temp_file.name),
-          is_verbose, user=getpass.getuser())
-        temp_file.close()
+        cls.merge_authorized_keys(host, keyname, getpass.getuser(),
+          is_verbose)
         return
       else:
         raise exception
 
     # Amazon EC2 rejects a root login request and tells the user to log in as
     # the ubuntu user, so do that to enable root login.
     if re.search(cls.LOGIN_AS_UBUNTU_USER, output):
-      AppScaleLogger.log("Root login not enabled - enabling it now.")
-      temp_file = tempfile.NamedTemporaryFile()
-      cls.ssh(host, keyname, "sudo sort -u ~/.ssh/authorized_keys " \
-        "/root/.ssh/authorized_keys -o {0}".format(temp_file.name),
-        is_verbose, user='ubuntu')
-      cls.ssh(host, keyname, "sudo sed -n '/.*Please login/d; " \
-        "w/root/.ssh/authorized_keys' {0}".format(temp_file.name),
-        is_verbose, user='ubuntu')
-      cls.ssh(host, keyname, "sudo rm -f {0}".format(temp_file.name),
-        is_verbose, user='ubuntu')
-      temp_file.close()
+      cls.merge_authorized_keys(host, keyname, 'ubuntu', is_verbose)
     else:
       AppScaleLogger.log("Root login already enabled - not re-enabling it.")
 
@@ -363,8 +376,9 @@ def ssh(cls, host, keyname, command, is_verbose, user='root', \
         representing the standard error of the remote command.
     """
     ssh_key = LocalState.get_key_path_from_name(keyname)
-    return LocalState.shell("ssh -i {0} {1} {2}@{3} ".format(ssh_key,
-      cls.SSH_OPTIONS, user, host), is_verbose, num_retries, stdin=command)
+    return LocalState.shell("ssh -F /dev/null -i {0} {1} {2}@{3} bash".format(
+      ssh_key, cls.SSH_OPTIONS, user, host),
+      is_verbose, num_retries, stdin=command)
 
 
   @classmethod

osx/appscale_install.sh

@@ -8,7 +8,7 @@ mkdir -p $TARGETDIR
 cp -r bin lib templates LICENSE README.md $TARGETDIR || exit 1
 
 # from installpylibs
-easy_install termcolor SOAPpy pyyaml boto argparse oauth2client google-api-python-client httplib2 python-gflags
+easy_install termcolor SOAPpy pyyaml boto argparse oauth2client google-api-python-client httplib2
 
 # create ssh private key if it does not exist
 test -e ~/.ssh/id_rsa || ssh-keygen -q -t rsa -f ~/.ssh/id_rsa -N ""

setup.py

@@ -27,8 +27,7 @@ def readme():
     'PyYAML',
     'boto',
     'google-api-python-client',
-    'argparse',
-    'python-gflags'
+    'argparse'
   ],
   classifiers=[
     'Development Status :: 5 - Production/Stable',

test/test_appscale_run_instances.py

@@ -462,6 +462,17 @@ def test_appscale_in_one_node_cloud_deployment_auto_spot_price(self):
       False, 5, stdin='ls').and_return(RemoteHelper.LOGIN_AS_UBUNTU_USER)
 
     # assume that we can enable root login
+    self.local_state.should_receive('shell').with_args(
+      re.compile('ssh'), False, 5,
+      stdin='sudo touch /root/.ssh/authorized_keys').and_return()
+
+    self.local_state.should_receive('shell').with_args(
+      re.compile('ssh'), False, 5,
+      stdin='sudo chmod 600 /root/.ssh/authorized_keys').and_return()
+
+    self.local_state.should_receive('shell').with_args(
+      re.compile('ssh'), False, 5, stdin='mktemp').and_return()
+
     self.local_state.should_receive('shell').with_args(
       re.compile('ssh'), False, 5,
       stdin=re.compile(
@@ -478,7 +489,7 @@ def test_appscale_in_one_node_cloud_deployment_auto_spot_price(self):
     ).and_return()
 
     self.local_state.should_receive('shell').with_args(
-      re.compile('ssh'), False, 5, stdin=re.compile('sudo rm -f ')
+      re.compile('ssh'), False, 5, stdin=re.compile('rm -f ')
     ).and_return()
 
     # and assume that we can copy over our ssh keys fine
@@ -589,6 +600,17 @@ def test_appscale_in_one_node_cloud_deployment_manual_spot_price(self):
       False, 5, stdin='ls').and_return(RemoteHelper.LOGIN_AS_UBUNTU_USER)
 
     # assume that we can enable root login
+    self.local_state.should_receive('shell').with_args(
+      re.compile('ssh'), False, 5,
+      stdin='sudo touch /root/.ssh/authorized_keys').and_return()
+
+    self.local_state.should_receive('shell').with_args(
+      re.compile('ssh'), False, 5,
+      stdin='sudo chmod 600 /root/.ssh/authorized_keys').and_return()
+
+    self.local_state.should_receive('shell').with_args(
+      re.compile('ssh'), False, 5, stdin='mktemp').and_return()
+
     self.local_state.should_receive('shell').with_args(
       re.compile('ssh'), False, 5,
       stdin=re.compile(
@@ -605,7 +627,7 @@ def test_appscale_in_one_node_cloud_deployment_manual_spot_price(self):
     ).and_return()
 
     self.local_state.should_receive('shell').with_args(
-      re.compile('ssh'), False, 5, stdin=re.compile('sudo rm -f ')
+      re.compile('ssh'), False, 5, stdin=re.compile('rm -f ')
     ).and_return()
 
     # and assume that we can copy over our ssh keys fine

test/test_appscale_terminate_instances.py

@@ -372,14 +372,16 @@ def test_terminate_in_gce_and_succeeds(self):
     fake_storage = flexmock(name='fake_storage')
     fake_storage.should_receive('get').and_return(None)
 
+    fake_flags = oauth2client.tools.argparser.parse_args(args=[])
+
     flexmock(oauth2client.file)
     oauth2client.file.should_receive('Storage').with_args(str).and_return(
       fake_storage)
 
     fake_credentials = flexmock(name='fake_credentials')
     flexmock(oauth2client.tools)
-    oauth2client.tools.should_receive('run').with_args(fake_flow,
-      fake_storage).and_return(fake_credentials)
+    oauth2client.tools.should_receive('run_flow').with_args(fake_flow,
+      fake_storage, fake_flags).and_return(fake_credentials)
 
     # next, mock out http calls to GCE
     fake_http = flexmock(name='fake_http')

test/test_remote_helper.py

@@ -182,9 +182,21 @@ def setUp(self):
 
 
   def test_start_head_node_in_cloud_but_ami_not_appscale(self):
-    # mock out our attempts to find /etc/appscale and presume it doesn't exist
+    # Mock out our attempts to find /etc/appscale and presume it doesn't exist.
     local_state = flexmock(LocalState)
-    # mock out our attempts to enable the root login. 
+
+    # Mock out our attempts to enable the root login.
+    local_state.should_receive('shell').with_args(
+      re.compile('ssh'), False, 5,
+      stdin='sudo touch /root/.ssh/authorized_keys').and_return()
+
+    local_state.should_receive('shell').with_args(
+      re.compile('ssh'), False, 5,
+      stdin='sudo chmod 600 /root/.ssh/authorized_keys').and_return()
+
+    local_state.should_receive('shell').with_args(
+      re.compile('ssh'), False, 5, stdin='mktemp').and_return()
+
     local_state.should_receive('shell') \
       .with_args(re.compile('^ssh'), False, 5,
         stdin='ls') \
@@ -206,15 +218,15 @@ def test_start_head_node_in_cloud_but_ami_not_appscale(self):
     ).and_return()
 
     local_state.should_receive('shell').with_args(
-      re.compile('ssh'), False, 5, stdin=re.compile('sudo rm -f ')
+      re.compile('ssh'), False, 5, stdin=re.compile('rm -f ')
     ).and_return()
 
     local_state.should_receive('shell')\
       .with_args(re.compile('^ssh'),False,5,\
         stdin=re.compile('ls /etc/appscale'))\
       .and_raise(ShellException).ordered()
 
-    # check that the cleanup routine is called on error
+    # Check that the cleanup routine is called on error.
     flexmock(AppScaleTools).should_receive('terminate_instances')\
       .and_return().ordered()
 
@@ -224,7 +236,18 @@ def test_start_head_node_in_cloud_but_ami_not_appscale(self):
 
   def test_start_head_node_in_cloud_but_ami_wrong_version(self):
     local_state = flexmock(LocalState)
-    # mock out our attempts to enable the root login. 
+    # mock out our attempts to enable the root login.
+    local_state.should_receive('shell').with_args(
+      re.compile('ssh'), False, 5,
+      stdin='sudo touch /root/.ssh/authorized_keys').and_return()
+
+    local_state.should_receive('shell').with_args(
+      re.compile('ssh'), False, 5,
+      stdin='sudo chmod 600 /root/.ssh/authorized_keys').and_return()
+
+    local_state.should_receive('shell').with_args(
+      re.compile('ssh'), False, 5, stdin='mktemp').and_return()
+
     local_state.should_receive('shell') \
       .with_args(re.compile('^ssh'), False, 5,
         stdin='ls') \
@@ -246,7 +269,7 @@ def test_start_head_node_in_cloud_but_ami_wrong_version(self):
     ).and_return()
 
     local_state.should_receive('shell').with_args(
-      re.compile('ssh'), False, 5, stdin=re.compile('sudo rm -f ')
+      re.compile('ssh'), False, 5, stdin=re.compile('rm -f ')
     ).and_return()
 
     # mock out our attempts to find /etc/appscale and presume it does exist
@@ -273,7 +296,18 @@ def test_start_head_node_in_cloud_but_ami_wrong_version(self):
   def test_start_head_node_in_cloud_but_using_unsupported_database(self):
     local_state = flexmock(LocalState)
 
-    # mock out our attempts to enable the root login. 
+    # Mock out our attempts to enable the root login.
+    local_state.should_receive('shell').with_args(
+      re.compile('ssh'), False, 5,
+      stdin='sudo touch /root/.ssh/authorized_keys').and_return()
+
+    local_state.should_receive('shell').with_args(
+      re.compile('ssh'), False, 5,
+      stdin='sudo chmod 600 /root/.ssh/authorized_keys').and_return()
+
+    local_state.should_receive('shell').with_args(
+      re.compile('ssh'), False, 5, stdin='mktemp').and_return()
+
     local_state.should_receive('shell') \
       .with_args(re.compile('^ssh'), False, 5,
         stdin='ls') \
@@ -295,7 +329,7 @@ def test_start_head_node_in_cloud_but_using_unsupported_database(self):
     ).and_return()
 
     local_state.should_receive('shell').with_args(
-      re.compile('ssh'), False, 5, stdin=re.compile('sudo rm -f ')
+      re.compile('ssh'), False, 5, stdin=re.compile('rm -f ')
     ).and_return()
 
     # mock out our attempts to find /etc/appscale and presume it does exist