save cluster state after timeout

Mirantis · Jan 15, 2018 · a0ea987 · a0ea987
1 parent e8eff0a
commit a0ea987
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 7 deletions.
diff --git a/kqueen/engines/gce.py b/kqueen/engines/gce.py
@@ -202,7 +202,12 @@ def cluster_get(self):
         First we try to get cluster by external_id, because its much more efficient in this
         implementation. If its not possible yet, we return from the slower method
         """
-        request = self.client.projects().zones().clusters().get(projectId=self.project, zone=self.zone, clusterId=self.cluster_id)
+        request = self.client.projects().zones().clusters().get(
+            projectId=self.project,
+            zone=self.zone,
+            clusterId=self.cluster_id
+        )
+
         try:
             response = request.execute()
         except Exception as e:

diff --git a/kqueen/models.py b/kqueen/models.py
@@ -1,3 +1,5 @@
+from datetime import datetime
+from datetime import timedelta
 from importlib import import_module
 from kqueen.config import current_config
 from kqueen.kubeapi import KubernetesAPI
@@ -11,7 +13,6 @@
 from kqueen.storages.etcd import RelationField
 from kqueen.storages.etcd import StringField
 from tempfile import mkstemp
-from datetime import datetime, timedelta
 
 import logging
 import os
@@ -38,15 +39,16 @@ class Cluster(Model, metaclass=ModelMeta):
 
     def get_state(self):
         try:
-            cluster = self.engine.cluster_get()
+            remote_cluster = self.engine.cluster_get()
         except Exception as e:
             logger.error('Unable to get data from backend for cluster {}'.format(self.name))
-            cluster = {}
+            remote_cluster = {}
 
-        if 'state' in cluster:
-            if cluster['state'] == self.state:
+        if 'state' in remote_cluster:
+            if remote_cluster['state'] == self.state:
                 return self.state
-            self.state = cluster['state']
+
+            self.state = remote_cluster['state']
             self.save()
         else:
             self.state = config.get('CLUSTER_UNKNOWN_STATE')
@@ -56,6 +58,7 @@ def get_state(self):
         max_age = timedelta(seconds=config.get('PROVISIONER_TIMEOUT'))
         if self.state == config.get('CLUSTER_PROVISIONING_STATE') and datetime.utcnow() - self.created_at > max_age:
             self.state = config.get('CLUSTER_ERROR_STATE')
+            self.save()
 
         return self.state