[FLOC-3017] Allow run-acceptance-tests to boot devstack guests

NEWS

@@ -2,6 +2,7 @@ Flocker 1.4.0.dev1 (2015-09-08)
 ===============================
 
 - Improved the error handling when Cinder volumes enter an unexpected state. (FLOC-2970)
+- ``run-acceptance-tests`` now supports all OpenStack variants, not just RackSpace. (FLOC-3017)
 
 Flocker 1.3.0 (2015-09-04)
 =====================================

admin/acceptance.py

@@ -791,6 +791,34 @@ def _runner_RACKSPACE(self, package_source, dataset_backend,
             package_source, dataset_backend, "rackspace", provider_config
         )
 
+    def _runner_OPENSTACK(self, package_source, dataset_backend,
+                          provider_config):
+        """
+        :param PackageSource package_source: The source of omnibus packages.
+        :param DatasetBackend dataset_backend: A ``DatasetBackend`` constant.
+        :param provider_config: The ``openstack`` section of the acceptance
+            testing configuration file.  The section of the configuration
+            file should look something like:
+
+               openstack:
+                 auth_plugin: <auth_plugin, one of "password" or "apikey">
+                 auth_url: <keystone_url, e.g. "https://192.168.1.101:5000/v2.0/tokens/">
+                 region: <region, e.g. "RegionOne">
+                 tenant: <tenant_name e.g. "demo">
+                 username: <username>
+                 secret: <password or apikey>
+                 keyname: <ssh-key-name>
+                 images:
+                   ubuntu-14.04: <image_name, e.g. "ubuntu-14_04">
+                   centos-7: <image_id, e.g. "327637d0-b744-4567-837e-100f01017d56">  # noqa
+                 flavor: <flavor, e.g. "m1.medium">
+
+        :see: :ref:`acceptance-testing-openstack-config`
+        """
+        return self._libcloud_runner(
+            package_source, dataset_backend, "openstack", provider_config
+        )
+
     def _runner_AWS(self, package_source, dataset_backend,
                     provider_config):
         """

docs/gettinginvolved/acceptance-testing.rst

@@ -162,6 +162,47 @@ Rackspace can use these dataset backends:
 
   admin/run-acceptance-tests --distribution centos-7 --provider rackspace --config-file config.yml
 
+.. _acceptance-testing-openstack-config:
+
+OpenStack
+~~~~~~~~~
+
+To run the acceptance tests on OpenStack, you need:
+
+- An OpenStack account and the associated password or API key.
+- The URL of the OpenStack keystone service.
+- An ssh-key registered with the OpenStack account.
+- The OpenStack image name or image ID for images containing supported Flocker node operating systems.
+- The OpenStack flavor to use for acceptance test nodes.
+
+To use the OpenStack provider, the configuration file should include an item like:
+
+.. code-block:: yaml
+
+   openstack:
+     auth_plugin: <auth_plugin, one of "password" or "apikey">
+     auth_url: <keystone_url, e.g. "https://192.168.1.101:5000/v2.0/tokens/">
+     region: <region, e.g. "RegionOne">
+     tenant: <tenant, e.g. "demo">
+     username: <username>
+     secret: <password or apikey>
+     keyname: <ssh-key-name>
+     images:
+       ubuntu-14.04: <image_name, e.g. "ubuntu-14_04">
+       centos-7: <image_id, e.g. "327637d0-b744-4567-837e-100f01017d56">  # noqa
+     flavor: <flavour, e.g. "m1.medium">
+
+You will need a ssh agent running with access to the corresponding private key.
+
+Openstack can use these dataset backends:
+
+  * :ref:`OpenStack<openstack-dataset-backend>`.
+  * :ref:`ZFS<zfs-dataset-backend>`.
+  * :ref:`Loopback<loopback-dataset-backend>`.
+
+.. prompt:: bash $
+
+  admin/run-acceptance-tests --distribution centos-7 --provider openstack --config-file config.yml
 
 .. _acceptance-testing-aws-config:
 

docs/gettinginvolved/example-acceptance.yml

@@ -14,6 +14,19 @@ rackspace:
   key: "33333333333333333333333333333333"
   keyname: "your.rackspace.ssh.key.identifier"
 
+openstack:
+  auth_plugin: <auth_plugin, one of "password" or "apikey">
+  auth_url: <keystone_url, e.g. "https://192.168.1.101:5000/v2.0/tokens/">
+  region: <region, e.g. "RegionOne">
+  tenant: <tenant, e.g. "demo">
+  username: <username>
+  secret: <password or apikey>
+  keyname: <ssh-key-name>
+  images:
+    ubuntu-14.04: <image_name, e.g. "ubuntu-14_04">
+    centos-7: <image_id, e.g. "327637d0-b744-4567-837e-100f01017d56">  # noqa
+  flavor: <flavor, e.g. "m1.medium">
+
 managed:
   addresses:
     - "192.0.2.15"

flocker/provision/__init__.py

@@ -7,11 +7,13 @@
 from ._common import PackageSource, Variants
 from ._install import provision, configure_cluster
 from ._rackspace import rackspace_provisioner
+from ._openstack import openstack_provisioner
 from ._aws import aws_provisioner
 from ._ca import Certificates
 
 CLOUD_PROVIDERS = {
     'rackspace': rackspace_provisioner,
+    'openstack': openstack_provisioner,
     'aws': aws_provisioner,
 }
 

flocker/provision/_libcloud.py

@@ -12,16 +12,16 @@
 from flocker.provision._ssh import run_remotely, run_from_args
 
 
-def get_size(driver, size_id):
+def get_size(driver, size_name_or_id):
     """
     Return a ``NodeSize`` corresponding to a given id.
 
     :param driver: The libcloud driver to query for sizes.
     """
     try:
-        return [s for s in driver.list_sizes() if s.id == size_id][0]
+        return [s for s in driver.list_sizes() if size_name_or_id in (s.id, s.name)][0]
     except IndexError:
-        raise ValueError("Unknown size.", size_id)
+        raise ValueError("Unknown size.", size_name_or_id)
 
 
 def get_image(driver, image_name):
@@ -224,7 +224,13 @@ def create_node(self, name, distribution,
         )
 
         node, addresses = self._driver.wait_until_running(
-            [node], wait_period=15)[0]
+            [node], 
+            wait_period=15,
+            # XXX: devstack guests don't have public_ips by default.
+            # Need to figure out how to handle this generally. Look at
+            # self.use_private_addresses
+            ssh_interface="private_ips",
+        )[0]
 
         public_address = addresses[0]
 

flocker/provision/_openstack.py

@@ -0,0 +1,154 @@
+# Copyright ClusterHQ Inc.  See LICENSE file for details.
+
+"""
+OpenStack provisioner.
+"""
+
+from textwrap import dedent
+from time import time
+
+from effect.retry import retry
+from effect import Effect, Constant
+
+from ._libcloud import LibcloudProvisioner
+from ._install import (
+    provision,
+    task_install_ssh_key,
+    task_open_control_firewall,
+)
+from ._ssh import run_remotely
+
+from ._effect import sequence
+
+# XXX: Copied from _aws. Needs refactoring
+_usernames = {
+    'centos-7': 'centos',
+    'ubuntu-14.04': 'ubuntu',
+    'ubuntu-15.04': 'ubuntu',
+}
+
+
+def get_default_username(distribution):
+    """
+    Return the username available by default on a system.
+
+    :param str distribution: Name of the operating system distribution
+    :return str: The username made available by AWS for this distribution.
+    """
+    return _usernames[distribution]
+
+
+def provision_openstack(node, package_source, distribution, variants):
+    """
+    Provision flocker on this node.
+
+    :param LibcloudNode node: Node to provision.
+    :param PackageSource package_source: See func:`task_install_flocker`
+    :param bytes distribution: See func:`task_install_flocker`
+    :param set variants: The set of variant configurations to use when
+        provisioning
+    """
+    username = get_default_username(distribution)
+    commands = []
+    # cloud-init may not have allowed sudo without tty yet, so try SSH key
+    # installation for a few more seconds:
+    start = []
+
+    # XXX Copied from _aws.py -- refactor.
+    def for_ten_seconds(*args, **kwargs):
+        if not start:
+            start.append(time())
+        return Effect(Constant((time() - start[0]) < 30))
+
+    commands.append(run_remotely(
+        username=username,
+        address=node.address,
+        commands=retry(task_install_ssh_key(), for_ten_seconds),
+    ))
+
+    commands.append(run_remotely(
+        username='root',
+        address=node.address,
+        commands=provision(
+            package_source=package_source,
+            distribution=node.distribution,
+            variants=variants,
+        ),
+    ))
+
+    commands.append(run_remotely(
+        username='root',
+        address=node.address,
+        commands=sequence([
+            provision(
+                package_source=package_source,
+                distribution=node.distribution,
+                variants=variants,
+            ),
+            # https://clusterhq.atlassian.net/browse/FLOC-1550
+            # This should be part of ._install.configure_cluster
+            # task_open_control_firewall(node.distribution),
+        ]),
+    ))
+
+    return sequence(commands)
+
+
+def openstack_provisioner(auth_url, auth_plugin, username, secret, region,
+                          keyname, images, flavor, tenant):
+    """
+    Create a LibCloudProvisioner for provisioning nodes on openstack.
+
+    :param bytes auth_url: The keystone URL.
+    :param bytes auth_plugin: The OpenStack authentication mechanism. One of
+        password or apikey.
+    :param bytes username: The user to connect to with.
+    :param bytes secret: The password or API key associated with the user.
+    :param bytes region: The region in which to launch the instance.
+    :param bytes keyname: The name of an existing ssh public key configured in
+       openstack. The provision step assumes the corresponding private key is
+       available from an agent.
+    :param dict images: A mapping of supported operating systems to a
+        corresponding OpenStack image name or image ID.
+    :param bytes flavor: A flavor name or flavor ID available in the target
+        OpenStack installation.
+    :param bytes tenant: The name of an OpenStack tenant or project.
+    """
+    # Import these here, so that this can be imported without
+    # installng libcloud.
+    from libcloud.compute.providers import get_driver, Provider
+
+    # LibCloud chooses OpenStack auth plugins using a weird naming scheme.
+    # See https://libcloud.readthedocs.org/en/latest/compute/drivers/openstack.html#connecting-to-the-openstack-installation  # noqa
+    auth_versions = {
+        "apikey": "2.0_apikey",
+        "password": "2.0_password",
+    }
+    # See https://libcloud.readthedocs.org/en/latest/compute/drivers/openstack.html  # noqa
+    driver = get_driver(Provider.OPENSTACK)(
+        key=username,
+        secret=secret,
+        region=region,
+        ex_force_auth_url=auth_url,
+        ex_force_auth_version=auth_versions[auth_plugin],
+        ex_force_service_region=region,
+        ex_tenant_name=tenant,
+    )
+
+    provisioner = LibcloudProvisioner(
+        driver=driver,
+        keyname=keyname,
+        image_names=images,
+        create_node_arguments=lambda **kwargs: {
+            "ex_config_drive": "true",
+            "ex_userdata": dedent("""\
+                #!/bin/sh
+                sed -i '/Defaults *requiretty/d' /etc/sudoers
+                """),
+        },
+        provision=provision_openstack,
+        default_size=flavor,
+        get_default_user=get_default_username,
+    )
+
+    return provisioner