Merge pull request #36 from Mirantis/engines

Provisioners reworked to Engines

devenv.py

@@ -17,13 +17,14 @@
     provisioner = Provisioner(
         id=uuid_provisioner_jenkins,
         name='My AWS',
-        engine='kqueen.provisioners.jenkins.JenkinsProvisioner',
         state='OK',
-        location='-',
-        access_id='demo',
-        access_key='Demo123'
+        engine='kqueen.engines.jenkins.JenkinsEngine',
+        parameters={
+            'username': 'demo',
+            'password': 'Demo123'
+        }
     )
-    provisioner.save()
+    provisioner.save(check_status=False)
 except:
     print('Adding aws provisioner failed')
 
@@ -45,13 +46,11 @@
     provisioner = Provisioner(
         id=uuid_provisioner_local,
         name='Manual',
-        engine='local',
         state='OK',
-        location='-',
-        access_id='',
-        access_key=''
+        engine='local',
+        parameters={}
     )
-    provisioner.save()
+    provisioner.save(check_status=False)
 except:
     print('Adding aws provisioner failed')
 

docs/kqueen.engines.rst

@@ -0,0 +1,22 @@
+Engines
+=======
+
+Submodules
+----------
+
+JenkinsEngine
+-------------
+
+.. automodule:: kqueen.engines.jenkins
+    :members:
+    :undoc-members:
+    :show-inheritance:
+    :exclude-members: app
+
+
+Module contents
+---------------
+
+.. automodule:: kqueen.engines.base
+    :members:
+    :undoc-members:

docs/kqueen.rst

@@ -1,7 +1,6 @@
 kqueen package
 ==============
 
-
 Models
 -------
 
@@ -58,7 +57,7 @@ Subpackages
 .. toctree::
 
     kqueen.blueprints
-    kqueen.provisioners
+    kqueen.engines
     kqueen.storages
 
 

gulpfile.js

@@ -190,6 +190,9 @@ gulp.task('watch', function () {
         gulp.start('javascript-all');
         gulp.start('run-server');
     });
+    watch('./kqueen/**/**/*.py', function() {
+        gulp.start('run-server');
+    });
 });
 
 gulp.task('dev', ['run-server', 'watch']);

kqueen/blueprints/ui/forms.py

@@ -3,7 +3,7 @@
 from wtforms import PasswordField, SelectField, StringField
 from wtforms.validators import DataRequired
 
-PROVISIONER_ENGINES = [('kqueen.provisioners.jenkins.JenkinsProvisioner', 'JenkinsProvisioner')]
+PROVISIONER_ENGINES = [('kqueen.engines.JenkinsEngine', 'Jenkins')]
 
 
 class LoginForm(FlaskForm):
@@ -14,8 +14,8 @@ class LoginForm(FlaskForm):
 class ProvisionerCreateForm(FlaskForm):
     name = StringField('Name', validators=[DataRequired()])
     engine = SelectField('Engine', choices=PROVISIONER_ENGINES)
-    access_id = StringField('Access ID', validators=[DataRequired()])
-    access_key = PasswordField('Access key', validators=[DataRequired()])
+    username = StringField('Username', validators=[DataRequired()])
+    password = PasswordField('Password', validators=[DataRequired()])
 
 
 def _get_provisioners():

kqueen/blueprints/ui/tables.py

@@ -39,7 +39,6 @@ class ProvisionerTable(Table):
     # Table fields
     name = Col('Name')
     engine_name = Col('Engine')
-    location = Col('Location')
     state = StatusCol('Status')
     delete = DeleteCol(
         'Delete',

kqueen/blueprints/ui/views.py

@@ -4,7 +4,7 @@
 from .tables import ProvisionerTable
 from flask import abort
 from flask import Blueprint
-from flask import current_app
+from flask import current_app as app
 from flask import flash
 from flask import jsonify
 from flask import redirect
@@ -18,7 +18,6 @@
 from uuid import UUID, uuid4
 
 import logging
-import time
 
 logging.basicConfig(level=logging.DEBUG)
 logger = logging.getLogger(__name__)
@@ -30,13 +29,13 @@
 @ui.route('/')
 @login_required
 def index():
-    username = current_app.config['USERNAME']
+    username = app.config['USERNAME']
     clusters = []
     healthy = 0
     for cluster in list(Cluster.list(return_objects=True).values()):
         data = cluster.get_dict()
         if data and 'state' in data:
-            if 'Error' not in data['state']:
+            if app.config['CLUSTER_ERROR_STATE'] not in data['state']:
                 healthy = healthy + 1
 
             # TODO: teach ORM to get related objects for us
@@ -67,9 +66,9 @@ def index():
 def login():
     error = None
     if request.method == 'POST':
-        if request.form['username'] != current_app.config['USERNAME']:
+        if request.form['username'] != app.config['USERNAME']:
             error = 'Invalid username'
-        elif request.form['password'] != current_app.config['PASSWORD']:
+        elif request.form['password'] != app.config['PASSWORD']:
             error = 'Invalid password'
         else:
             session['logged_in'] = True
@@ -107,18 +106,16 @@ def provisioner_create():
             provisioner = Provisioner(
                 name=form.name.data,
                 engine=form.engine.data,
-                state='Not Available',
-                location='-',
-                access_id=form.access_id.data,
-                access_key=form.access_key
+                state=app.config['PROVISIONER_UNKNOWN_STATE'],
+                parameters={
+                    'username': form.username.data,
+                    'password': form.password.data
+                }
             )
-            # Check if provisioner lives
-            if provisioner.alive():
-                provisioner.state = 'OK'
             provisioner.save()
             flash('Provisioner %s successfully created.' % provisioner.name, 'success')
         except Exception as e:
-            logging.error('Could not create provisioner: %s' % repr(e))
+            logger.error('Could not create provisioner: %s' % repr(e))
             flash('Could not create provisioner.', 'danger')
         return redirect('/')
     return render_template('ui/provisioner_create.html', form=form)
@@ -145,7 +142,7 @@ def provisioner_delete(provisioner_id):
     except NameError:
         abort(404)
     except Exception as e:
-        logging.error(e)
+        logger.error(e)
         abort(500)
 
 
@@ -161,28 +158,28 @@ def cluster_deploy():
             cluster = Cluster(
                 id=cluster_id,
                 name=form.name.data,
-                state='Deploying',
+                state=app.config['CLUSTER_PROVISIONING_STATE'],
                 provisioner=form.provisioner.data,
                 kubeconfig={},
             )
             cluster.save()
         except Exception as e:
             flash('Could not create cluster %s.' % form.name.data, 'danger')
-            logging.error('Creating cluster %s failed with following reason: %s' % (form.name.data, repr(e)))
+            logger.error('Creating cluster %s failed with following reason: %s' % (form.name.data, repr(e)))
             return redirect('/')
 
         # Actually provision cluster
-        res = False
+        result = False
         try:
-            prv = Provisioner.load(form.provisioner.data).engine_cls()
-            res = prv.provision(cluster_id)
+            result, err = cluster.engine.provision()
         except Exception as e:
             flash('Could not create cluster %s.' % form.name.data, 'danger')
-            logging.error('Creating cluster %s failed with following reason: %s' % (form.name.data, repr(e)))
+            logger.error('Creating cluster %s failed with following reason: %s' % (form.name.data, repr(e)))
             return redirect('/')
-        if res:
+        if result:
             flash('Provisioning of cluster %s is in progress.' % form.name.data, 'success')
         else:
+            logger.error('Creating cluster %s failed with following reason: %s' % (form.name.data, str(err)))
             flash('Could not create cluster %s.' % form.name.data, 'danger')
         return redirect('/')
     return render_template('ui/cluster_deploy.html', form=form)
@@ -211,7 +208,7 @@ def cluster_detail(cluster_id):
         flash('Unable to load cluster', 'danger')
 
     status = {}
-    if obj.get_state() == 'OK':
+    if obj.get_state() == app.config['CLUSTER_OK_STATE']:
         try:
             status = obj.status()
         except:
@@ -235,32 +232,22 @@ def cluster_delete(cluster_id):
 @login_required
 def cluster_deployment_status(cluster_id):
     try:
-        object_id = UUID(cluster_id, version=5)
+        object_id = UUID(cluster_id, version=4)
     except ValueError:
-        logging.debug('%s not valid UUID' % cluster_id)
+        logger.debug('%s not valid UUID' % cluster_id)
         abort(404)
 
     # load object
     try:
-        obj = Cluster.load(object_id)
+        cluster = Cluster.load(object_id)
     except NameError:
-        logging.debug('Cluster with UUID %s not found' % cluster_id)
+        logger.debug('Cluster with UUID %s not found' % cluster_id)
         abort(404)
 
-    res = 0
-    progress = 1
-    result = 'UNKNOWN'
     try:
-        prv = obj.get_provisioner()
-        if prv:
-            data = prv.engine_cls().get(cluster_id)
-            result = data['state']
-            if data['state'] == 'Deploying':
-                progress = int((((time.time() * 1000) - data['build_timestamp']) / data['build_estimated_duration']) * 100)
-                if progress > 99:
-                    progress = 99
-            else:
-                progress = 100
-    except:
-        res = 1
-    return jsonify({'response': res, 'progress': progress, 'result': result})
+        status = cluster.engine.get_progress()
+    except Exception as e:
+        logger.error('Error occured while getting provisioning status for cluster %s: %s' % (cluster_id, repr(e)))
+        abort(500)
+
+    return jsonify(status)

kqueen/config_dev.py

@@ -7,9 +7,22 @@
 # App secret
 SECRET_KEY = 'secret'
 
-# Jenkins provisioner settings
+# Cluster statuses
+CLUSTER_ERROR_STATE = 'Error'
+CLUSTER_OK_STATE = 'OK'
+CLUSTER_PROVISIONING_STATE = 'Deploying'
+CLUSTER_DEPROVISIONING_STATE = 'Destroying'
+CLUSTER_UNKNOWN_STATE = 'Unknown'
+
+# Provisioner statuses
+PROVISIONER_ERROR_STATE = 'Error'
+PROVISIONER_OK_STATE = 'OK'
+PROVISIONER_UNKNOWN_STATE = 'Not Reachable'
+
+# Jenkins engine settings
 JENKINS_API_URL = 'https://ci.mcp.mirantis.net'
-JENKINS_JOB_NAME = 'deploy-aws-k8s_ha_calico_sm'
-JENKINS_JOB_CTX = {}
+JENKINS_PROVISION_JOB_NAME = 'deploy-aws-k8s_ha_calico_sm'
+JENKINS_PROVISION_JOB_CTX = {}
+JENKINS_ANCHOR_PARAMETER = 'STACK_NAME'
 JENKINS_USERNAME = None
 JENKINS_PASSWORD = None

kqueen/engines/__init__.py

@@ -0,0 +1,3 @@
+from .jenkins import JenkinsEngine
+
+__all__ = ['JenkinsEngine']