diff --git a/README.rst b/README.rst index 65240f80..1f1dfb64 100644 --- a/README.rst +++ b/README.rst @@ -11,6 +11,9 @@ KQueen - Kubernetes cluster manager .. image:: https://coveralls.io/repos/github/Mirantis/kqueen/badge.svg?branch=master :target: https://coveralls.io/github/Mirantis/kqueen?branch=master +.. image:: https://readthedocs.org/projects/kqueen/badge/?version=master + :target: http://kqueen.readthedocs.io/en/master/?badge=master + Overview -------- diff --git a/kqueen/auth.py b/kqueen/auth.py index 1eebb879..922f4fd4 100644 --- a/kqueen/auth.py +++ b/kqueen/auth.py @@ -16,7 +16,7 @@ def authenticate(username, password): Args: username (str): Username to login - password (str): Passwore + password (str): Password Returns: user: authenticated user @@ -25,10 +25,11 @@ def authenticate(username, password): users = list(User.list(None, return_objects=True).values()) username_table = {u.username: u for u in users} user = username_table.get(username) - user_password = user.password.encode('utf-8') - given_password = password.encode('utf-8') - if user and user.active and bcrypt.checkpw(given_password, user_password): - return user + if user: + user_password = user.password.encode('utf-8') + given_password = password.encode('utf-8') + if user.active and bcrypt.checkpw(given_password, user_password): + return user def identity(payload): @@ -90,19 +91,25 @@ def is_authorized(_user, policy_value, resource=None): ORGANIZATION = user['organization'].id # noqa: F841 ROLE = user['role'] if resource: - if not resource.validate(): + validation, _ = resource.validate() + if not validation: invalid = True + # test if we are validating on create view and if so, patch missing object id if not resource.id: resource.id = uuid4() - if resource.validate(): + validation, _ = resource.validate() + if validation: invalid = False resource.id = None + # if invalid resource is passed, let's just continue dispatch_request # so it can properly fail with 500 response code if invalid: logger.error('Cannot evaluate policy for invalid object: {}'.format(str(resource.get_dict()))) return True + + # TODO: check owner has id and, organization ... if hasattr(resource, 'owner'): OWNER = resource.owner.id # noqa: F841 OWNER_ORGANIZATION = resource.owner.organization.id # noqa: F841 diff --git a/kqueen/blueprints/api/generic_views.py b/kqueen/blueprints/api/generic_views.py index 7679a180..85207c96 100644 --- a/kqueen/blueprints/api/generic_views.py +++ b/kqueen/blueprints/api/generic_views.py @@ -68,6 +68,7 @@ def check_authorization(self): def set_object(self, *args, **kwargs): self.obj = get_object(self.get_class(), kwargs['pk'], current_identity) + # check authorization for given object self.check_authorization() @@ -143,6 +144,7 @@ def set_object(self, *args, **kwargs): namespace = current_identity.namespace except AttributeError: namespace = None + self.obj = list(self.get_class().list(namespace, return_objects=True).values()) self.check_authorization() @@ -175,6 +177,7 @@ def get_content(self, *args, **kwargs): def dispatch_request(self, *args, **kwargs): self.check_authentication() + if not request.json: abort(400, description='JSON data expected') else: diff --git a/kqueen/conftest.py b/kqueen/conftest.py index 8d7a670f..da708f10 100644 --- a/kqueen/conftest.py +++ b/kqueen/conftest.py @@ -130,7 +130,7 @@ def auth_header(client): token=token, ), 'X-Test-Namespace': _user.namespace, - 'X-User': str(_user), + 'X-User': str(_user.id), } diff --git a/kqueen/engines/test_manual.py b/kqueen/engines/test_manual.py index 7521b9bd..04f60a2e 100644 --- a/kqueen/engines/test_manual.py +++ b/kqueen/engines/test_manual.py @@ -107,7 +107,8 @@ def test_create_over_api(self): # load cluster_id = response.json['id'] obj = Cluster.load(self.namespace, cluster_id) - assert obj.validate() + validation, _ = obj.validate() + assert validation # check parameters assert obj.kubeconfig == data['kubeconfig'] diff --git a/kqueen/middleware.py b/kqueen/middleware.py index 1895ba01..f1985472 100644 --- a/kqueen/middleware.py +++ b/kqueen/middleware.py @@ -31,12 +31,19 @@ def record_request_data(response): return response -def setup_metrics(app): - # TODO: detect multiprocess mode and raise only when needed +def check_prometheus(): + is_gunicorn = "gunicorn" in os.environ.get("SERVER_SOFTWARE", "") + if 'prometheus_multiproc_dir' in os.environ: os.makedirs(os.environ['prometheus_multiproc_dir'], exist_ok=True) - else: - raise Exception('Please set prometheus_multiproc_dir variable') + + elif is_gunicorn: + raise Exception('Please set prometheus_multiproc_dir variable using `export prometheus_multiproc_dir=$(mktemp -d)`') + + +def setup_metrics(app): + + check_prometheus() app.before_request(start_timer) app.after_request(record_request_data) diff --git a/kqueen/models.py b/kqueen/models.py index d6f1e2d1..430535d8 100644 --- a/kqueen/models.py +++ b/kqueen/models.py @@ -323,6 +323,7 @@ def get_engine_cls(self): except Exception as e: logger.error(repr(e)) _class = None + return _class def engine_status(self, save=True): diff --git a/kqueen/storages/etcd.py b/kqueen/storages/etcd.py index 69b657eb..e34c83f3 100644 --- a/kqueen/storages/etcd.py +++ b/kqueen/storages/etcd.py @@ -137,7 +137,7 @@ def encrypt(self): if not self.encrypted: return serialized - if self.value is not None: + if serialized is not None: key = self._get_encryption_key() padded = self._pad(str(serialized)) @@ -162,6 +162,7 @@ def decrypt(self, crypted, **kwargs): decrypted_decoded = decrypted.decode('utf-8') serialized = self._unpad(decrypted_decoded) + self.deserialize(serialized, **kwargs) def __str__(self): @@ -264,7 +265,7 @@ def set_value(self, value, **kwargs): self.value = value def serialize(self): - if self.value and isinstance(self.value, dict): + if isinstance(self.value, dict): return json.dumps(self.value) else: return None @@ -563,8 +564,9 @@ def save(self, validate=True, assign_id=True): if assign_id: self.verify_id() - if validate and not self.validate(): - raise ValueError('Validation for model failed') + validation_status, validation_msg = self.validate() + if validate and not validation_status: + raise ValueError('Validation for model failed with: {}'.format(validation_msg)) key = self.get_db_key() logger.debug('Writing {} to {}'.format(self, key)) @@ -600,12 +602,12 @@ def validate(self): # validation # TODO: move to validate method of Field if field_object.required and field_object.value is None: - return False + return False, 'Required field {} is None'.format(field) if field_object.value and not field_object.validate(): - return False + return False, 'Field {} validation failed'.format(field) - return True + return True, None def _expand(self, obj): expanded = obj.get_dict() diff --git a/kqueen/storages/test_model_fields.py b/kqueen/storages/test_model_fields.py index a40a8949..5f1327f6 100644 --- a/kqueen/storages/test_model_fields.py +++ b/kqueen/storages/test_model_fields.py @@ -37,6 +37,12 @@ class TestModel(Model, metaclass=ModelMeta): else: _namespace = namespace + print('Creating model with: required: {}, global_ns: {}, encrypted: {}'.format( + required, + global_ns, + encrypted + )) + return TestModel @@ -79,12 +85,13 @@ def get_model(request): def create_object(required=False, global_ns=False, encrypted=False): - model = create_model() + model = create_model(required, global_ns, encrypted) obj1 = model(namespace, **model_kwargs) obj2 = model(namespace, **model_kwargs) - obj2.save() + # don't validate first object because we don't have relation field + obj2.save(False) obj1.relation = obj2 @@ -119,7 +126,9 @@ def setup(self): self.obj = model(namespace) def test_model_invalid(self): - assert not self.obj.validate() + validation, _ = self.obj.validate() + + assert not validation def test_save_raises(self): with pytest.raises(ValueError, match='Validation for model failed'): @@ -146,7 +155,8 @@ def test_required(self, required): model = create_model(required=required) obj = model(namespace, **model_kwargs) - assert obj.validate() != required + validation, _ = obj.validate() + assert validation != required class TestGetFieldNames: @@ -188,6 +198,9 @@ class TestSerialization: def test_serizalization(self, get_object): serialized = get_object.serialize() + if get_object.__class__._encrypted: + pytest.skip('Unable to check serialization for encrypted class') + assert serialized == model_serialized(related=get_object.relation) def test_deserialization(self, get_object, monkeypatch): @@ -200,7 +213,7 @@ def fake(self, class_name): get_object.save() new_object = object_class.deserialize(get_object.serialize(), namespace=namespace) - assert new_object.get_dict() == get_object.get_dict() + assert new_object.get_dict(True) == get_object.get_dict(True) class TestGetDict: @@ -449,7 +462,7 @@ def test_encrypt_none(self, field_name): assert field.encrypt() is None @pytest.mark.parametrize('field_value', [i * 'a' for i in range(35)]) - def test_string_various_lenght(self, field_value): + def test_string_various_length(self, field_value): field_name = 'string' cls = create_model(False, False, True) obj = cls(namespace, **model_kwargs) @@ -475,3 +488,29 @@ def test_serialized_model_dont_contain_value(self, field_name, field_value): assert '"{}"'.format(field.value) not in serialized assert '"{}"'.format(field.serialize()) not in serialized + + +class TestModelEncryptionWithNone: + def test_serialization(self, monkeypatch, get_object): + def fake(self, class_name): + return get_object.__class__ + + monkeypatch.setattr(RelationField, '_get_related_class', fake) + + # load information about test setup + namespace = get_object.__class__._namespace + required = get_object.__class__._required + + # set None to fields if possible + if not required: + for field_name in get_object.__class__.get_fields().keys(): + field = getattr(get_object, '_{}'.format(field_name)) + field.value = None + + assert field.value is None + + get_object.save() + + loaded = get_object.__class__.load(namespace, get_object.id) + + assert get_object.get_dict(True) == loaded.get_dict(True) diff --git a/kqueen/tests/test_auth.py b/kqueen/tests/test_auth.py new file mode 100644 index 00000000..1bb1f0d8 --- /dev/null +++ b/kqueen/tests/test_auth.py @@ -0,0 +1,15 @@ +from kqueen.auth import authenticate + + +def test_nonexisting_user(): + """ + Try authenticate with non-existing user + It is expected to return None but not fail + """ + + username = "non_existing_user" + password = "password" + + result = authenticate(username, password) + + assert result is None diff --git a/kqueen/tests/test_manual_cluster.py b/kqueen/tests/test_manual_cluster.py new file mode 100644 index 00000000..7b3e9377 --- /dev/null +++ b/kqueen/tests/test_manual_cluster.py @@ -0,0 +1,80 @@ +from flask import url_for +from kqueen.conftest import auth_header +from kqueen.models import User + +import json +import pytest +import yaml + + +@pytest.mark.usefixtures('client_class') +class TestInsertManualCluster: + def setup(self): + self.auth_header = auth_header(self.client) + self.namespace = self.auth_header['X-Test-Namespace'] + self.user = User.load(None, self.auth_header['X-User']) + + self.provisioner_id = None + + def test_run(self): + self.create_provisioner() + self.get_provisioners() + self.create_cluster() + self.get_cluster() + + def create_provisioner(self): + data = { + 'name': 'Manual provisioner', + 'engine': 'kqueen.engines.ManualEngine', + 'owner': 'User:{}'.format(self.user.id), + } + + response = self.client.post( + url_for('api.provisioner_create'), + data=json.dumps(data), + headers=self.auth_header, + content_type='application/json', + ) + + assert response.status_code == 200 + + self.provisioner_id = response.json['id'] + + def get_provisioners(self): + response = self.client.get( + url_for('api.provisioner_list'), + headers=self.auth_header, + content_type='application/json', + ) + + assert response.status_code == 200 + + content = response.data.decode(response.charset) + assert self.provisioner_id in content + + def create_cluster(self): + data = { + 'name': 'Manual cluster', + 'provisioner': 'Provisioner:{}'.format(self.provisioner_id), + 'kubeconfig': yaml.load(open('kubeconfig_localhost', 'r').read()), + 'owner': 'User:{}'.format(self.user.id), + } + + response = self.client.post( + url_for('api.cluster_create'), + data=json.dumps(data), + headers=self.auth_header, + content_type='application/json', + ) + + assert response.status_code == 200 + self.cluster_id = response.json['id'] + + def get_cluster(self): + response = self.client.get( + url_for('api.cluster_get', pk=self.cluster_id), + headers=self.auth_header, + content_type='application/json', + ) + + assert response.status_code == 200 diff --git a/kqueen/tests/test_models.py b/kqueen/tests/test_models.py index 867d44f4..c22ae758 100644 --- a/kqueen/tests/test_models.py +++ b/kqueen/tests/test_models.py @@ -1,3 +1,4 @@ +from datetime import datetime from kqueen.engines import __all__ as all_engines from kqueen.models import Cluster from kqueen.models import Provisioner @@ -22,7 +23,9 @@ def test_get_model_name(self, model_class, req): class TestClusterModel: def test_create(self, cluster): - assert cluster.validate() + validation, _ = cluster.validate() + + assert validation assert cluster.save() def test_load(self, cluster): @@ -178,3 +181,22 @@ def test_list_engines(self): engines = Provisioner.list_engines() assert engines == all_engines + + +class TestProvisionerSerialization: + def test_load_provisioner(self, user): + user.save() + provisioner = Provisioner( + user.namespace, + name='Manual provisioner', + state='OK', + engine='kqueen.engines.ManualEngine', + parameters={}, + created_at=datetime.utcnow().replace(microsecond=0), + owner=user + ) + provisioner.save() + + loaded = Provisioner.load(user.namespace, provisioner.id) + + assert loaded.get_dict(True) == provisioner.get_dict(True)