SNO2-1-upgrade-validator (#2)

setup.py

@@ -18,7 +18,7 @@
     "elasticsearch-dsl==5.4.0",
     "elasticsearch==5.4.0",
     "future==0.18.2",
-    "jsonschema_serialize_fork==2.1.1",
+    "jsonschema[format]==4.4.0",
     "lucenequery==0.1",
     "passlib==1.7.2",
     "psutil==5.6.7",
@@ -33,7 +33,7 @@
     "rdflib-jsonld==0.6.0",
     "rdflib==4.2.2",
     "redis==3.5.3",
-    "requests==2.22.0",
+    "requests==2.27.1",
     "simplejson==3.17.0",
     "snovault-search==1.0.4",
     "transaction==3.0.0",

src/snovault/schema_utils.py

@@ -8,12 +8,10 @@
 import codecs
 import collections
 import copy
-from jsonschema_serialize_fork import (
-    Draft4Validator,
-    FormatChecker,
-    RefResolver,
-)
-from jsonschema_serialize_fork.exceptions import ValidationError
+from snovault.schema_validation import SerializingSchemaValidator
+from jsonschema import FormatChecker
+from jsonschema import RefResolver
+from jsonschema.exceptions import ValidationError
 from uuid import UUID
 from .util import ensurelist
 
@@ -95,7 +93,6 @@ def linkTo(validator, linkTo, instance, schema):
         error = "%r is not of type %s" % (instance, ", ".join(reprs))
         yield ValidationError(error)
         return
-
     linkEnum = schema.get('linkEnum')
     if linkEnum is not None:
         if not validator.is_type(linkEnum, "array"):
@@ -123,10 +120,6 @@ def linkTo(validator, linkTo, instance, schema):
                 yield ValidationError(error)
                 return
 
-    # And normalize the value to a uuid
-    if validator._serialize:
-        validator._validated[-1] = str(item.uuid)
-
 
 def linkFrom(validator, linkFrom, instance, schema):
     # avoid circular import
@@ -154,9 +147,6 @@ def linkFrom(validator, linkFrom, instance, schema):
             yield ValidationError(error)
             return
     else:
-        if validator._serialize:
-            lv = len(validator._validated)
-
         # Look for an existing item;
         # if found use the schema for its type,
         # which may be a subtype of an abstract linkType
@@ -210,16 +200,13 @@ def linkFrom(validator, linkFrom, instance, schema):
         for error in validator.descend(instance, subschema):
             yield error
 
-        if validator._serialize:
-            validated_instance = validator._validated[lv]
-            del validator._validated[lv:]
-            if uuid is not None:
-                validated_instance['uuid'] = uuid
-            elif 'uuid' in validated_instance:  # where does this come from?
-                del validated_instance['uuid']
-            if new_type is not None:
-                validated_instance['@type'] = [new_type]
-            validator._validated[-1] = validated_instance
+        validated_instance = instance
+        if uuid is not None:
+            validated_instance['uuid'] = uuid
+        elif 'uuid' in validated_instance:  # where does this come from?
+            del validated_instance['uuid']
+        if new_type is not None:
+            validated_instance['@type'] = [new_type]
 
 
 class IgnoreUnchanged(ValidationError):
@@ -250,17 +237,6 @@ def permission(validator, permission, instance, schema):
         yield IgnoreUnchanged(error)
 
 
-orig_uniqueItems = Draft4Validator.VALIDATORS['uniqueItems']
-
-
-def uniqueItems(validator, uI, instance, schema):
-    # Use serialized items if available
-    # (this gives the linkTo validator a chance to normalize paths into uuids)
-    if validator._serialize and validator._validated[-1]:
-        instance = validator._validated[-1]
-    yield from orig_uniqueItems(validator, uI, instance, schema)
-
-
 VALIDATOR_REGISTRY = {}
 
 
@@ -281,16 +257,15 @@ def notSubmittable(validator, linkTo, instance, schema):
     yield ValidationError('submission disallowed')
 
 
-class SchemaValidator(Draft4Validator):
-    VALIDATORS = Draft4Validator.VALIDATORS.copy()
+class SchemaValidator(SerializingSchemaValidator):
+    VALIDATORS = SerializingSchemaValidator.VALIDATORS.copy()
     VALIDATORS['notSubmittable'] = notSubmittable
     # for backwards-compatibility
     VALIDATORS['calculatedProperty'] = notSubmittable
     VALIDATORS['linkTo'] = linkTo
     VALIDATORS['linkFrom'] = linkFrom
     VALIDATORS['permission'] = permission
     VALIDATORS['requestMethod'] = requestMethod
-    VALIDATORS['uniqueItems'] = uniqueItems
     VALIDATORS['validators'] = validators
     SERVER_DEFAULTS = SERVER_DEFAULTS
 
@@ -311,13 +286,13 @@ def load_schema(filename):
     schema = mixinProperties(schema, resolver)
 
     # SchemaValidator is not thread safe for now
-    SchemaValidator(schema, resolver=resolver, serialize=True)
+    SchemaValidator(schema, resolver=resolver)
     return schema
 
 
 def validate(schema, data, current=None):
     resolver = NoRemoteResolver.from_schema(schema)
-    sv = SchemaValidator(schema, resolver=resolver, serialize=True, format_checker=format_checker)
+    sv = SchemaValidator(schema, resolver=resolver, format_checker=format_checker)
     validated, errors = sv.serialize(data)
 
     filtered_errors = []
@@ -336,6 +311,18 @@ def validate(schema, data, current=None):
                     validated_value = validated_value[key]
                 if validated_value == current_value:
                     continue
+        # Also ignore requestMethod and permission errors from defaults.
+        if isinstance(error, IgnoreUnchanged):
+            current_value = data
+            try:
+                for key in error.path:
+                    # If it's in original data then either user passed it in
+                    # or it's from PATCH object with unchanged data. If it's
+                    # unchanged then it's already been skipped above.
+                    current_value = current_value[key]
+            except KeyError:
+                # If it's not in original data then it's filled in by defaults.
+                continue
         filtered_errors.append(error)
 
     return validated, filtered_errors

src/snovault/schema_validation.py

@@ -0,0 +1,143 @@
+from copy import deepcopy
+from jsonschema import Draft202012Validator
+from jsonschema import validators
+from jsonschema.exceptions import ValidationError
+from pyramid.threadlocal import get_current_request
+from pyramid.traversal import find_resource
+
+
+NO_DEFAULT = object()
+
+
+def get_resource_base(validator, linkTo):
+    from snovault import COLLECTIONS
+    request = get_current_request()
+    collections = request.registry[COLLECTIONS]
+    if validator.is_type(linkTo, 'string'):
+        resource_base = collections.get(linkTo, request.root)
+    else:
+        resource_base = request.root
+    return resource_base
+
+
+def normalize_links(validator, links, linkTo):
+    resource_base = get_resource_base(validator, linkTo)
+    normalized_links = []
+    errors = []
+    for link in links:
+        try:
+            normalized_links.append(
+                str(find_resource(resource_base, link.replace(':', '%3A')).uuid)
+            )
+        except KeyError:
+            errors.append(
+                ValidationError(f'Unable to resolve link: {link}')
+            )
+            normalized_links.append(
+                link
+            )
+    return normalized_links, errors
+
+
+def should_mutate_properties(validator, instance):
+    if validator.is_type(instance, 'object'):
+        return True
+    return False
+
+
+def get_items_or_empty_object(validator, subschema):
+    items = subschema.get('items', {})
+    if validator.is_type(items, 'object'):
+        return items
+    return {}
+
+
+def maybe_normalize_links_to_uuids(validator, property, subschema, instance):
+    errors = []
+    if 'linkTo' in subschema:
+        link = instance.get(property)
+        if link:
+            normalized_links, errors = normalize_links(
+                validator,
+                [link],
+                subschema.get('linkTo'),
+            )
+            instance[property] = normalized_links[0]
+    if 'linkTo' in get_items_or_empty_object(validator, subschema):
+        links = instance.get(property, [])
+        if links:
+            normalized_links, errors = normalize_links(
+                validator,
+                links,
+                subschema.get('items').get('linkTo'),
+            )
+            instance[property] = normalized_links
+    for error in errors:
+        yield error
+
+
+def set_defaults(validator, property, subschema, instance):
+    if 'default' in subschema:
+        instance.setdefault(
+            property,
+            deepcopy(subschema['default'])
+        )
+    if 'serverDefault' in subschema:
+        server_default = validator.server_default(
+            instance,
+            subschema
+        )
+        if server_default is not NO_DEFAULT:
+            instance.setdefault(
+                property,
+                server_default
+            )
+
+
+def extend_with_default(validator_class):
+    validate_properties = validator_class.VALIDATORS['properties']
+
+    def mutate_properties(validator, properties, instance, schema):
+        for property, subschema in properties.items():
+            if not validator.is_type(subschema, 'object'):
+                continue
+            yield from maybe_normalize_links_to_uuids(validator, property, subschema, instance)
+            set_defaults(validator, property, subschema, instance)
+
+    def before_properties_validation_hook(validator, properties, instance, schema):
+        if should_mutate_properties(validator, instance):
+            yield from mutate_properties(validator, properties, instance, schema)
+        yield from validate_properties(validator, properties, instance, schema)
+
+    return validators.extend(
+        validator_class, {'properties': before_properties_validation_hook},
+    )
+
+
+ExtendedValidator = extend_with_default(Draft202012Validator)
+
+
+class SerializingSchemaValidator(ExtendedValidator):
+
+    SERVER_DEFAULTS = {}
+
+    def add_server_defaults(self, server_defaults):
+        self.SERVER_DEFAULTS.update(server_defaults)
+        return self
+
+    def serialize(self, instance):
+        self._original_instance = instance
+        self._mutated_instance = deepcopy(
+            self._original_instance
+        )
+        errors = list(
+            self.iter_errors(
+                self._mutated_instance
+            )
+        )
+        return self._mutated_instance, errors
+
+    def server_default(self, instance, subschema):
+        factory_name = subschema['serverDefault']
+        factory = self.SERVER_DEFAULTS[factory_name]
+        return factory(instance, subschema)

src/snovault/schema_views.py

@@ -78,8 +78,8 @@ def schemas_map(context, request):
     types = request.registry[TYPES]
     profiles_map = {}
     for type_info in types.by_item_type.values():
-        if 'id' in type_info.schema:
-            profiles_map[type_info.name] = type_info.schema['id']
+        if '$id' in type_info.schema:
+            profiles_map[type_info.name] = type_info.schema['$id']
     profiles_map['@type'] = ['JSONSchemas']
     return profiles_map
 

src/snovault/tests/test_post_put_patch.py

@@ -233,7 +233,10 @@ def test_put_object_child_validation(content_with_child, testapp):
         }]
     }
     res = testapp.put_json(content_with_child['@id'], edit, status=422)
-    assert res.json['errors'][0]['name'] == [u'reverse', 0, u'target']
+    assert ['reverse', 0, 'target'] in [
+        x['name']
+        for x in res.json['errors']
+    ]
 
 
 def test_put_object_validates_child_references(content_with_child, testapp):

src/snovault/tests/test_schema_utils.py

@@ -16,17 +16,21 @@ def content(testapp):
 
 def test_uniqueItems_validates_normalized_links(content, threadlocals):
     schema = {
-        'uniqueItems': True,
-        'items': {
-            'linkTo': 'TestingLinkTarget',
+        'properties': {
+            'some_links': {
+                'uniqueItems': True,
+                'items': {
+                    'linkTo': 'TestingLinkTarget',
+                }
+            }
         }
     }
     uuid = targets[0]['uuid']
     data = [
         uuid,
         '/testing-link-targets/{}'.format(uuid),
     ]
-    validated, errors = validate(schema, data)
+    validated, errors = validate(schema, {'some_links': data})
     assert len(errors) == 1
     assert (
         errors[0].message == "['{}', '{}'] has non-unique elements".format(