Merge pull request #193 from Mirantis/cluster-timeout

Configurable timeout for Provisioner

kqueen/config/base.py

@@ -39,6 +39,8 @@ class BaseConfig:
 
     PROVISIONER_ENGINE_WHITELIST = None
 
+    # Timeout for cluster operations (in seconds)
+    PROVISIONER_TIMEOUT = 3600
     PROMETHEUS_WHITELIST = '127.0.0.0/8'
 
     @classmethod

kqueen/conftest.py

@@ -57,7 +57,7 @@ def cluster():
         'provisioner': prov,
         'state': config.get('CLUSTER_UNKNOWN_STATE'),
         'kubeconfig': yaml.load(open('kubeconfig_localhost', 'r').read()),
-        'created_at': datetime.datetime(2017, 11, 15, 13, 36, 24),
+        'created_at': datetime.datetime.utcnow().replace(microsecond=0),
         'owner': _user
     }
 

kqueen/models.py

@@ -11,6 +11,7 @@
 from kqueen.storages.etcd import RelationField
 from kqueen.storages.etcd import StringField
 from tempfile import mkstemp
+from datetime import datetime, timedelta
 
 import logging
 import os
@@ -51,6 +52,11 @@ def get_state(self):
             self.state = config.get('CLUSTER_UNKNOWN_STATE')
             self.save()
 
+        # check for stale clusters
+        max_age = timedelta(seconds=config.get('PROVISIONER_TIMEOUT'))
+        if self.state == config.get('CLUSTER_PROVISIONING_STATE') and datetime.utcnow() - self.created_at > max_age:
+            self.state = config.get('CLUSTER_ERROR_STATE')
+
         return self.state
 
     @property

kqueen/tests/test_manual_cluster.py

@@ -1,6 +1,7 @@
 from flask import url_for
 from kqueen.conftest import auth_header
 from kqueen.models import User
+from datetime import datetime
 
 import json
 import pytest
@@ -58,6 +59,7 @@ def create_cluster(self):
             'provisioner': 'Provisioner:{}'.format(self.provisioner_id),
             'kubeconfig': yaml.load(open('kubeconfig_localhost', 'r').read()),
             'owner': 'User:{}'.format(self.user.id),
+            'created_at': str(datetime.utcnow()),
         }
 
         response = self.client.post(

kqueen/tests/test_models.py

@@ -1,5 +1,8 @@
 from datetime import datetime
+from datetime import timedelta
+from kqueen.config import current_config
 from kqueen.engines import __all__ as all_engines
+from kqueen.engines import ManualEngine
 from kqueen.models import Cluster
 from kqueen.models import Provisioner
 from kqueen.storages.etcd import Field
@@ -9,6 +12,8 @@
 import subprocess
 import yaml
 
+config = current_config()
+
 
 class TestModelMethods:
     @pytest.mark.parametrize('model_class,req', [
@@ -200,3 +205,24 @@ def test_load_provisioner(self, user):
         loaded = Provisioner.load(user.namespace, provisioner.id)
 
         assert loaded.get_dict(True) == provisioner.get_dict(True)
+
+
+class TestClusterState:
+    @pytest.fixture(autouse=True)
+    def prepare(self, cluster, monkeypatch):
+        def fake_cluster_get(self):
+            return {'state': config.get('CLUSTER_PROVISIONING_STATE')}
+
+        monkeypatch.setattr(ManualEngine, 'cluster_get', fake_cluster_get)
+
+        stale_date = datetime.utcnow() - timedelta(days=365)
+        cluster.created_at = stale_date
+        cluster.save()
+
+        self.cluster = cluster
+
+    def test_stale_cluster(self):
+        cluster_state = self.cluster.get_state()
+        print(self.cluster.get_state())
+
+        assert cluster_state == config.get('CLUSTER_ERROR_STATE')