Merge pull request #24 from cf-platform-eng/acceptance

Major update

.gitignore

@@ -1,2 +1 @@
 *.pyc
-bin

README.md

@@ -18,34 +18,38 @@ For a 7-minute introduction into what it is and does, see [this screencast]
 (https://www.youtube.com/watch?v=_WeJbqNJWzQ).
 
 ## How to Use
-1. check out the tile-generator repo:
+
+1. Check out the tile-generator repo:
 
   ```bash
   git clone https://github.com/cf-platform-eng/tile-generator.git
   ```
 
-1. change to the root directory of the tile generator, and pull down the generator's dependencies:
+2. Change to the root directory of the tile generator, and pull down the generator's dependencies:
 
   ```bash
   cd tile-generator
   pip install -r requirements.txt
   ```
 
-1. Add root directory of tile-generator to your path:
+3. Add the `bin` directory of tile-generator to your path:
+
   ```bash
-  export PATH=<path to root dir of tile-generator>:$PATH
+  export PATH=`pwd`/bin:$PATH
   ```
 
-1. then, from within the root directory of the project for which you wish to create a tile, initialize it as a tile repo (we recommend that this be a git repo, but this is not required):
+*If you expect to frequently use the tile generator, you may want to add this to your shell's startup script, i.e. `.profile`*
+
+4. Then, from within the root directory of the project for which you wish to create a tile, initialize it as a tile repo (we recommend that this be a git repo, but this is not required):
 
    ```bash
    cd <your project dir>
    tile init
    ```
 
-1. Edit the generated `tile.yml` file to define your tile (more details below)
+5. Edit the generated `tile.yml` file to define your tile (more details below)
 
-1. Build your tile
+6. Build your tile
    ```bash
    tile build
    ```

bin/pcf

@@ -0,0 +1,200 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import yaml
+import json
+import time
+import click
+import subprocess
+
+PATH = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(PATH, os.path.join('..', 'lib')))
+import opsmgr
+import erb
+
+@click.group()
+def cli():
+	pass
+
+@cli.command('products')
+def products_cmd():
+	products = opsmgr.get_products()
+	for product in products:
+		print "-", product["name"], product["product_version"], "(installed)" if product["installed"] else ""
+
+@cli.command('is-available')
+@click.argument('product')
+@click.argument('version', None, required=False)
+def is_available_cmd(product, version):
+	products = opsmgr.get_products()
+	matches = [ p for p in products if p['name'] == product and (version is None or p['product_version'] == version) ]
+	if len(matches) < 1:
+		print >> sys.stderr, 'No match found for product', product, 'version', version
+		sys.exit(1)
+
+@cli.command('is-installed')
+@click.argument('product')
+@click.argument('version', None, required=False)
+def is_installed_cmd(product, version):
+	products = opsmgr.get_products()
+	matches = [ p for p in products if p['name'] == product and (version is None or p['product_version'] == version) and p['installed'] ]
+	if len(matches) < 1:
+		print >> sys.stderr, 'Product', product, 'version', version, 'is not installed'
+		sys.exit(1)
+
+@cli.command('configure')
+@click.argument('product')
+@click.argument('properties_file')
+def configure_cmd(product, properties_file):
+	with open(properties_file) as f:
+		properties = yaml.safe_load(f)
+	settings = opsmgr.get('/api/installation_settings').json()
+	opsmgr.configure(settings, product, properties)
+	opsmgr.post_yaml('/api/installation_settings', 'installation[file]', settings)
+
+@cli.command('settings')
+@click.argument('product', None, required=False)
+def settings_cmd(product):
+	settings = opsmgr.get('/api/installation_settings').json()
+	if product is not None:
+		settings = [ p for p in settings['products'] if p['identifier'] == product ]
+		if len(settings) < 1:
+			print >> sys.stderr, 'No settings found for product', product
+			sys.exit(1)
+		settings = settings[0]
+	print json.dumps(settings, indent=4)
+
+@cli.command('cf-info')
+def cf_info_cmd():
+	cfinfo = opsmgr.get_cfinfo()
+	for key, value in cfinfo.items():
+		print '-', key + ':', value
+
+@cli.command('import')
+@click.argument('zipfile')
+def import_cmd(zipfile):
+	opsmgr.upload('/api/products', zipfile)
+
+@cli.command('install')
+@click.argument('product')
+@click.argument('version')
+def install_cmd(product, version):
+	payload = {
+		'name': product,
+		'product_version': version,
+	}
+	opsmgr.post('/api/installation_settings/products', payload)
+
+@cli.command('uninstall')
+@click.argument('product')
+def install_cmd(product):
+	products = opsmgr.get('/api/installation_settings/products').json()
+	matches = [ p for p in products if p['type'] == product ]
+	for match in matches:
+		opsmgr.delete('/api/installation_settings/products/' + match['guid'])
+
+@cli.command('delete-unused-products')
+def delete_unused_products_cmd():
+	opsmgr.delete('/api/products')
+
+@cli.command('backup')
+@click.argument('backup_file')
+def backup_cmd(backup_file):
+	response = opsmgr.get('/api/installation_asset_collection', stream=True)
+	with open(backup_file, 'wb') as f:
+		for chunk in response.iter_content(1024):
+			f.write(chunk)
+
+@cli.command('restore')
+@click.argument('backup_file')
+def restore_cmd(backup_file):
+	creds = get_credentials()
+	with open(backup_file, 'rb') as f:
+		payload = { 'installation[file]': f, 'password': creds['opsmgr']['password'] }
+		opsmgr.post('/api/installation_asset_collection', f)
+
+@cli.command('cleanup')
+@click.argument('product')
+def cleanup_cmd(product):
+	#
+	# Attempt 1 - Delete any uninstalled versions
+	#
+	products = opsmgr.get('/api/installation_settings/products').json()
+	matches = [ p for p in products if p['type'] == product ]
+	for match in matches:
+		print >> sys.stderr, '- attempting to delete', match['name']
+		opsmgr.delete('/api/installation_settings/products/' + match['guid'])
+	products = opsmgr.get('/api/installation_settings/products').json()
+	matches = [ p for p in products if p['type'] == product ]
+	if len(matches) < 1:
+		sys.exit(0)
+	if len(matches) > 1:
+		print >> sys.stderr, '- more than one match remains installed'
+		sys.exit(1)
+	#
+	# Attempt 2 - Uninstall deployed version
+	#
+	match = matches[0]
+	print >> sys.stderr, '- product was deployed, applying changes to uninstall it'
+	apply_changes_cmd()
+	opsmgr.delete('/api/products')
+	products = opsmgr.get('/api/installation_settings/products').json()
+	matches = [ p for p in products if p['type'] == product ]
+	if len(matches) < 1:
+		sys.exit(0)
+	#
+	# Attempt 3 - Re-deploy with errands disabled, then uninstall
+	#
+	match = matches[0]
+	print >> sys.stderr, '- uninstall appears to have failed'
+	print >> sys.stderr, '- re-deploying with disabled errands'
+	opsmgr.disable_errands(product)
+	apply_changes_cmd()
+	print >> sys.stderr, '- uninstalling with disabled errands'
+	opsmgr.delete('/api/installation_settings/products/' + match['guid'])
+	apply_changes_cmd()
+	opsmgr.delete('/api/products')
+	products = opsmgr.get('/api/installation_settings/products').json()
+	matches = [ p for p in products if p['type'] == product ]
+	if len(matches) > 0:
+		print >> sys.stderr, '- failed to uninstall'
+		sys.exit(1)
+
+@cli.command('apply-changes')
+def apply_changes_cmd():
+	install = opsmgr.post('/api/installation', { 'ignore_warnings': 'true' }).json()['install']
+	lines_shown = 0
+	while True:
+		log_lines = opsmgr.get('/api/installation/' + str(install['id']) + '/logs').json()['logs'].splitlines()
+		for line in log_lines[lines_shown:]:
+			if not line.startswith('{'):
+				print ' ', line
+		lines_shown = len(log_lines)
+		install_status = opsmgr.get('/api/installation/' + str(install['id'])).json()['status']
+		if not install_status == 'running':
+			break
+		time.sleep(1)
+	if not install_status == 'success':
+		print >> sys.stderr, '- install finished with status:', install_status
+		sys.exit(1)
+
+@cli.command('test-errand')
+@click.argument('tile_repo')
+@click.argument('errand_name')
+def test_errand_cmd(tile_repo, errand_name):
+	errand_file = os.path.join(tile_repo, 'release/jobs', errand_name, 'templates', errand_name + '.sh.erb')
+	rendered_errand = errand_name + '.sh'
+	erb.render(rendered_errand, errand_file, tile_repo)
+	env = os.environ
+	env['PACKAGE_PATH'] = os.path.join(tile_repo, 'release/blobs')
+	os.execlpe('bash', 'bash', rendered_errand, env)
+
+@cli.command('target')
+def target_cmd():
+	cf = opsmgr.get_cfinfo()
+	subprocess.call(['cf', 'api', 'api.' + cf['system_domain'], '--skip-ssl-validation'])
+	subprocess.call(['cf', 'login', '-u', cf['admin_username'], '-p', cf['admin_password']])
+
+if __name__ == '__main__':
+	cli()

tile → bin/tile

@@ -6,7 +6,7 @@ import os
 import yaml
 
 PATH = os.path.dirname(os.path.realpath(__file__))
-sys.path.append(os.path.join(PATH, 'lib'))
+sys.path.append(os.path.join(PATH, os.path.join('..', 'lib')))
 import build
 import template
 

ci/Dockerfile

@@ -1,6 +1,7 @@
 FROM alpine
 
-RUN apk add --no-cache python && \
+RUN apk add --no-cache zip &&\
+    apk add --no-cache python && \
     apk add --no-cache --virtual=build-dependencies wget ca-certificates && \
     wget "https://bootstrap.pypa.io/get-pip.py" -O /dev/stdout | python && \
     apk del build-dependencies
@@ -14,4 +15,8 @@ RUN gem install bosh_cli --no-ri --no-rdoc
 
 RUN apk add --update bash
 
+RUN apk add openrc
+RUN apk add docker
+RUN rc-update add docker boot
+
 RUN rm -rf /var/cache/apk/*

ci/acceptance-tests/bosh_acceptancetest.py

@@ -0,0 +1,62 @@
+import unittest
+import sys
+import os
+import glob
+import yaml
+
+class VerifyBoshRelease(unittest.TestCase):
+
+	def test_has_manifest(self):
+		self.assertTrue(os.path.exists('release/release.MF'))
+
+	def test_all_jobs_have_monit(self):
+		self.assertEqual(len(glob.glob('release/jobs/*/monit')), len(glob.glob('release/jobs/*')))
+
+	def test_errands_have_empty_monit(self):
+		for monit in glob.glob('release/jobs/*/monit'):
+			if not monit.startswith('release/jobs/docker-bosh-'):
+				self.assertTrue(os.path.exists(monit), monit)
+				self.assertEqual(os.path.getsize(monit), 0, monit)
+
+	def test_non_errands_have_nonempty_monit(self):
+		for monit in glob.glob('release/jobs/*'):
+			if monit.startswith('release/jobs/docker-bosh-'):
+				self.assertTrue(os.path.exists(monit), monit)
+				self.assertNotEqual(os.path.getsize(monit), 0, monit)
+
+	def test_all_jobs_have_manifest(self):
+		self.assertEqual(len(glob.glob('release/jobs/*/job.MF')), len(glob.glob('release/jobs/*')))
+
+	def test_cf_errand_manifest_has_cf_cli_package(self):
+		for manifest in glob.glob('release/jobs/*/job.MF'):
+			if not manifest.startswith('release/jobs/docker-bosh-'):
+				self.assertTrue('cf_cli' in read_yaml(manifest).get('packages', []), manifest)
+
+	def test_bosh_job_spec_has_no_cf_cli_package(self):
+		for manifest in glob.glob('release/jobs/*/job.MF'):
+			if manifest.startswith('release/jobs/docker-bosh-'):
+				self.assertFalse('cf_cli' in read_yaml(manifest).get('packages', []), manifest)
+
+	def test_all_jobs_have_template(self):
+		self.assertEqual(len(glob.glob('release/jobs/*/templates/*.sh.erb')), len(glob.glob('release/jobs/*')))
+
+	def test_has_complete_cf_cli_package(self):
+		self.assertEqual(len(glob.glob('release/packages/cf_cli')), 1)
+		self.assertEqual(len(glob.glob('release/packages/cf_cli/cf_cli/cf-linux-amd64.tgz')), 1)
+		self.assertEqual(len(glob.glob('release/packages/cf_cli/packaging')), 1)
+		self.assertTrue('cf-linux-amd64.tgz' in read_file('release/packages/cf_cli/packaging'))
+
+	def test_has_complete_common_package(self):
+		self.assertEqual(len(glob.glob('release/packages/common')), 1)
+		self.assertEqual(len(glob.glob('release/packages/common/common/utils.sh')), 1)
+
+def read_yaml(filename):
+	with open(filename, 'rb') as file:
+		return yaml.safe_load(file)
+
+def read_file(filename):
+	with open(filename, 'rb') as file:
+		return file.read()
+
+if __name__ == '__main__':
+	unittest.main()

ci/acceptance-tests/tile_acceptancetest.py

@@ -0,0 +1,30 @@
+import unittest
+import sys
+import os
+import glob
+import yaml
+
+class VerifyTile(unittest.TestCase):
+
+	def test_has_valid_content_migrations(self):
+		self.assertTrue(os.path.exists('product/content_migrations'))
+		files = glob.glob('product/content_migrations/*.yml')
+		self.assertEqual(len(files), 1)
+		read_yaml(files[0]) # Ensure corrent yaml syntax
+
+	def test_has_valid_metadata(self):
+		self.assertTrue(os.path.exists('product/metadata'))
+		files = glob.glob('product/metadata/*.yml')
+		self.assertEqual(len(files), 1)
+		read_yaml(files[0]) # Ensure corrent yaml syntax
+
+def read_yaml(filename):
+	with open(filename, 'rb') as file:
+		return yaml.safe_load(file)
+
+def read_file(filename):
+	with open(filename, 'rb') as file:
+		return file.read()
+
+if __name__ == '__main__':
+	unittest.main()