Skip to content

Commit

Permalink
move get_flattened_*_schema funcs to util.schema
Browse files Browse the repository at this point in the history
  • Loading branch information
@cognifloyd
cognifloyd committed Jul 23, 2022
1 parent 7e0211c commit 60f9362
Show file tree
Hide file tree
Showing 2 changed files with 125 additions and 119 deletions.
127 changes: 8 additions & 119 deletions st2common/st2common/util/config_loader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@

from __future__ import absolute_import
import copy
import re

import six

Expand All @@ -27,6 +26,10 @@
from st2common.persistence.pack import Config
from st2common.content import utils as content_utils
from st2common.util import jinja as jinja_utils
from st2common.util.schema import (
get_flattened_array_items_schema,
get_flattened_object_properties_schema,
)
from st2common.util.templating import render_template_with_system_and_user_context
from st2common.util.config_parser import ContentPackConfigParser
from st2common.exceptions.db import StackStormDBObjectNotFoundError
Expand Down Expand Up @@ -101,120 +104,6 @@ def _get_values_for_config(self, config_schema_db, config_db):
)
return config

@staticmethod
def _get_flattened_object_properties_schema(object_schema, object_keys=None):
"""
Create a schema for an object property using all of: properties,
patternProperties, and additionalProperties.
This 'flattens' properties, patternProperties, and additionalProperties
so that we can handle patternProperties and additionalProperties
as if they were defined in properties.
So, every key in object_keys will be assigned a schema
from properties, patternProperties, or additionalProperties.
NOTE: order of precedence: properties, patternProperties, additionalProperties
So, the additionalProperties schema is only used for keys that are not in
properties and that do not match any of the patterns in patternProperties.
And, patternProperties schemas only apply to keys missing from properties.
:rtype: ``dict``
"""
# preserve the order in object_keys
flattened_properties_schema = {key: {} for key in object_keys}

# properties takes precedence over patternProperties and additionalProperties
properties_schema = object_schema.get("properties", {})
flattened_properties_schema.update(properties_schema)

# extra_keys has keys that may use patternProperties or additionalProperties
# we remove keys when they have been assigned a schema
extra_keys = set(object_keys) - set(properties_schema.keys())

if not extra_keys:
# nothing to check. Don't look at patternProperties or additionalProperties.
return flattened_properties_schema

# match each key against patternPropetties
pattern_properties = object_schema.get("patternProperties", {})
# patternProperties should be a dict if defined
if pattern_properties and isinstance(pattern_properties, dict):
# we need to match all extra_keys against all patterns
# and then compose the per-property schema from all
# the matched patterns' properties.
pattern_properties = {
re.compile(raw_pattern): pattern_schema
for raw_pattern, pattern_schema in pattern_properties.items()
}
for key in list(extra_keys):
key_schemas = []
for pattern, pattern_schema in pattern_properties.items():
if pattern.search(key):
key_schemas.append(pattern_schema)
if key_schemas:
# This naive schema composition approximates allOf.
# We can improve this later if someone provides examples that need
# a better allOf schema implementation for patternProperties.
composed_schema = {}
for schema in key_schemas:
composed_schema.update(schema)
# update matched key
flattened_properties_schema[key] = composed_schema
# don't overwrite matched key's schema with additionalProperties
extra_keys.remove(key)

if not extra_keys:
# nothing else to check. Don't look at additionalProperties.
return flattened_properties_schema

# fill in any remaining keys with additionalProperties
additional_properties = object_schema.get("additionalProperties", {})
# additionalProperties can be a boolean or a dict
if additional_properties and isinstance(additional_properties, dict):
# ensure that these keys are present in the object
for key in extra_keys:
flattened_properties_schema[key] = additional_properties

return flattened_properties_schema

@staticmethod
def _get_flattened_array_items_schema(array_schema, items_count=0):
"""
Create a schema for array items using both additionalItems and items.
This 'flattens' items and additionalItems so that we can handle additionalItems
as if each additional item was defined in items.
The additionalItems schema will only be used if the items schema is shorter
than items_count. So, when additionalItems is defined, the items schema will be
extended to be at least as long as items_count.
:rtype: ``list``
"""
flattened_items_schema = []
items_schema = array_schema.get("items", [])
if isinstance(items_schema, dict):
# with only one schema for all items, additionalItems will be ignored.
flattened_items_schema.extend([items_schema] * items_count)
else:
# items is a positional array of schemas
flattened_items_schema.extend(items_schema)

flattened_items_schema_count = len(flattened_items_schema)
if flattened_items_schema_count >= items_count:
# no additional items to account for.
return flattened_items_schema

additional_items = array_schema.get("additionalItems", {})
# additionalItems can be a boolean or a dict
if additional_items and isinstance(additional_items, dict):
# ensure that these indexes are present in the array
flattened_items_schema.extend(
[additional_items] * (items_count - flattened_items_schema_count)
)

return flattened_items_schema

def _assign_dynamic_config_values(self, schema, config, parent_keys=None):
"""
Assign dynamic config value for a particular config item if the ite utilizes a Jinja
Expand Down Expand Up @@ -255,7 +144,7 @@ def _assign_dynamic_config_values(self, schema, config, parent_keys=None):

# Inspect nested object properties
if is_dictionary:
properties_schema = self._get_flattened_object_properties_schema(
properties_schema = get_flattened_object_properties_schema(
schema_item,
object_keys=config_item_value.keys(),
)
Expand All @@ -266,7 +155,7 @@ def _assign_dynamic_config_values(self, schema, config, parent_keys=None):
)
# Inspect nested list items
elif is_list:
items_schema = self._get_flattened_array_items_schema(
items_schema = get_flattened_array_items_schema(
schema_item,
items_count=len(config[config_item_key]),
)
Expand Down Expand Up @@ -359,7 +248,7 @@ def _assign_default_values(self, schema, instance, can_replace_instance=False):
if not instance:
instance = {}

properties_schema = self._get_flattened_object_properties_schema(
properties_schema = get_flattened_object_properties_schema(
schema,
object_keys=instance.keys(),
)
Expand All @@ -377,7 +266,7 @@ def _assign_default_values(self, schema, instance, can_replace_instance=False):
if not instance:
instance = []

items_schema = self._get_flattened_array_items_schema(
items_schema = get_flattened_array_items_schema(
schema,
items_count=len(instance),
)
Expand Down
117 changes: 117 additions & 0 deletions st2common/st2common/util/schema/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from __future__ import absolute_import

import os
import re
from typing import Mapping, Sequence

import six
Expand All @@ -42,6 +43,8 @@
"is_attribute_type_array",
"is_attribute_type_object",
"validate",
"get_flattened_array_items_schema",
"get_flattened_object_properties_schema",
]

# https://github.com/json-schema/json-schema/blob/master/draft-04/schema
Expand Down Expand Up @@ -467,3 +470,117 @@ def normalize(x):
schema["additionalProperties"] = allow_additional_properties

return schema


def get_flattened_object_properties_schema(object_schema, object_keys=None):
"""
Create a schema for an object property using all of: properties,
patternProperties, and additionalProperties.
This 'flattens' properties, patternProperties, and additionalProperties
so that we can handle patternProperties and additionalProperties
as if they were defined in properties.
So, every key in object_keys will be assigned a schema
from properties, patternProperties, or additionalProperties.
NOTE: order of precedence: properties, patternProperties, additionalProperties
So, the additionalProperties schema is only used for keys that are not in
properties and that do not match any of the patterns in patternProperties.
And, patternProperties schemas only apply to keys missing from properties.
:rtype: ``dict``
"""
# preserve the order in object_keys
flattened_properties_schema = {key: {} for key in object_keys}

# properties takes precedence over patternProperties and additionalProperties
properties_schema = object_schema.get("properties", {})
flattened_properties_schema.update(properties_schema)

# extra_keys has keys that may use patternProperties or additionalProperties
# we remove keys when they have been assigned a schema
extra_keys = set(object_keys) - set(properties_schema.keys())

if not extra_keys:
# nothing to check. Don't look at patternProperties or additionalProperties.
return flattened_properties_schema

# match each key against patternPropetties
pattern_properties = object_schema.get("patternProperties", {})
# patternProperties should be a dict if defined
if pattern_properties and isinstance(pattern_properties, dict):
# we need to match all extra_keys against all patterns
# and then compose the per-property schema from all
# the matched patterns' properties.
pattern_properties = {
re.compile(raw_pattern): pattern_schema
for raw_pattern, pattern_schema in pattern_properties.items()
}
for key in list(extra_keys):
key_schemas = []
for pattern, pattern_schema in pattern_properties.items():
if pattern.search(key):
key_schemas.append(pattern_schema)
if key_schemas:
# This naive schema composition approximates allOf.
# We can improve this later if someone provides examples that need
# a better allOf schema implementation for patternProperties.
composed_schema = {}
for schema in key_schemas:
composed_schema.update(schema)
# update matched key
flattened_properties_schema[key] = composed_schema
# don't overwrite matched key's schema with additionalProperties
extra_keys.remove(key)

if not extra_keys:
# nothing else to check. Don't look at additionalProperties.
return flattened_properties_schema

# fill in any remaining keys with additionalProperties
additional_properties = object_schema.get("additionalProperties", {})
# additionalProperties can be a boolean or a dict
if additional_properties and isinstance(additional_properties, dict):
# ensure that these keys are present in the object
for key in extra_keys:
flattened_properties_schema[key] = additional_properties

return flattened_properties_schema


def get_flattened_array_items_schema(array_schema, items_count=0):
"""
Create a schema for array items using both additionalItems and items.
This 'flattens' items and additionalItems so that we can handle additionalItems
as if each additional item was defined in items.
The additionalItems schema will only be used if the items schema is shorter
than items_count. So, when additionalItems is defined, the items schema will be
extended to be at least as long as items_count.
:rtype: ``list``
"""
flattened_items_schema = []
items_schema = array_schema.get("items", [])
if isinstance(items_schema, dict):
# with only one schema for all items, additionalItems will be ignored.
flattened_items_schema.extend([items_schema] * items_count)
else:
# items is a positional array of schemas
flattened_items_schema.extend(items_schema)

flattened_items_schema_count = len(flattened_items_schema)
if flattened_items_schema_count >= items_count:
# no additional items to account for.
return flattened_items_schema

additional_items = array_schema.get("additionalItems", {})
# additionalItems can be a boolean or a dict
if additional_items and isinstance(additional_items, dict):
# ensure that these indexes are present in the array
flattened_items_schema.extend(
[additional_items] * (items_count - flattened_items_schema_count)
)

return flattened_items_schema

0 comments on commit 60f9362

Please sign in to comment.