Skip to content

Commit

Permalink
Support default type schemas with no extras
Browse files Browse the repository at this point in the history
We found that when dataset with the 'default' schema type is created
there are no 'extras' returned and so the code to build the 'geometry'
field failed with a key error.

Resolve this by catching key error and allowing the geometry field to
default to all 0's (which appears to already be the default unknown
value behaviour), eg:

<georss:box>0.000000 0.000000 0.000000 0.000000</georss:box>
  • Loading branch information
tomd committed Apr 24, 2024
1 parent a30dad8 commit 0deff7b
Show file tree
Hide file tree
Showing 6 changed files with 237 additions and 3 deletions.
9 changes: 7 additions & 2 deletions ckanext/mapactiongeorss/plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,16 @@ def get_item_additional_fields(self, package):
""" Get the geometry from either extras (for legacy datasets) or dataset
fields """
geometry_fields = ('ymin', 'xmin', 'ymax', 'xmax')
geometry_source = {}

if all(o in package for o in geometry_fields):
geometry_source = package
else:
# look for geometry in extras
geometry_source = {e['key']: e['value'] for e in package['extras']}
try:
# look for geometry in extras
geometry_source = {e['key']: e['value'] for e in package['extras']}
except KeyError:
pass

box = tuple(
float(geometry_source.get(n, '0'))
Expand Down
10 changes: 10 additions & 0 deletions ckanext/mapactiongeorss/tests/controllers/test_feed.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
import ckanext.mapactiongeorss.tests.helpers as helpers
import ckanext.mapactiongeorss.tests.factories as custom_factories

from ckanext.mapactionimporter.plugin import create_product_themes

assert_equals = nose.tools.assert_equals
assert_true = nose.tools.assert_true
assert_regexp_matches = nose.tools.assert_regexp_matches
Expand Down Expand Up @@ -117,6 +119,14 @@ def test_feed_contains_published(self):
expected_published = self.convert_date(dataset['metadata_created'])
assert_equals(published, expected_published)

def test_feed_handles_default_schema_without_extras(self):
create_product_themes()
dataset = custom_factories.DefaultDataset(product_themes=['Agriculture'])

published = self.find_in_rss('xmlns:entry/xmlns:published').text
expected_published = self.convert_date(dataset['metadata_created'])
assert_equals(published, expected_published)

def test_main_feed_title_is_mapaction_georss(self):
title = self.find_in_rss('xmlns:title').text

Expand Down
81 changes: 81 additions & 0 deletions ckanext/mapactiongeorss/tests/default.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
{
"scheming_version": 1,
"dataset_type": "default",
"about": "MapAction Default",
"about_url": "",
"dataset_fields": [
{
"field_name": "title",
"label": "Title",
"preset": "title",
"form_placeholder": "eg. A descriptive title"
},
{
"field_name": "name",
"label": "URL",
"preset": "dataset_slug",
"form_placeholder": "eg. my-dataset"
},
{
"field_name": "notes",
"label": "Description",
"form_snippet": "markdown.html",
"form_placeholder": "eg. Some useful notes about the data"
},
{
"field_name": "license_id",
"label": "License",
"form_snippet": "license.html",
"help_text": "License definitions and additional information can be found at http://opendefinition.org/"
},
{
"field_name": "owner_org",
"label": "Organization",
"preset": "dataset_organization"
},
{
"field_name": "url",
"label": "Source",
"form_placeholder": "http://example.com/dataset.json",
"display_property": "foaf:homepage",
"display_snippet": "link.html"
},
{
"field_name": "version",
"label": "Version",
"validators": "ignore_missing unicode package_version_validator",
"form_placeholder": "1.0"
},
{
"field_name": "product_themes",
"label": "Themes",
"preset": "product_themes",
"form_placeholder": "eg. economy, mental health, government",
"vocabulary": "product_themes",
"choices_helper": "mapactionschemas_vocabulary_choices"
}
],
"resource_fields": [
{
"field_name": "url",
"label": "URL",
"preset": "resource_url_upload"
},
{
"field_name": "name",
"label": "Name",
"form_placeholder": "eg. January 2011 Gold Prices"
},
{
"field_name": "description",
"label": "Description",
"form_snippet": "markdown.html",
"form_placeholder": "Some useful notes about the data"
},
{
"field_name": "format",
"label": "Format",
"preset": "resource_format_autocomplete"
}
]
}
8 changes: 8 additions & 0 deletions ckanext/mapactiongeorss/tests/factories.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
import ckan.tests.factories as factories


class Dataset(factories.Dataset):
type = 'test_schema'

class DefaultDataset(factories.Dataset):
type = 'default'
130 changes: 130 additions & 0 deletions ckanext/mapactiongeorss/tests/test_schema.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
{
"scheming_version": 1,
"dataset_type": "test-schema",
"about_url": "http://github.com/ckan/ckanext-scheming",
"dataset_fields": [
{
"field_name": "title",
"label": "Title",
"preset": "title",
"form_placeholder": "eg. Larry, Peter, Susan"
},
{
"field_name": "name",
"label": "URL",
"preset": "dataset_slug",
"form_placeholder": "eg. camel-no-5"
},
{
"field_name": "humps",
"label": "Humps",
"validators": "ignore_missing int_validator",
"form_placeholder": "eg. 2"
},
{
"field_name": "category",
"label": "Category",
"help_text": "Make and model",
"help_inline": true,
"preset": "select",
"choices": [
{
"value": "bactrian",
"label": "Bactrian Camel"
},
{
"value": "hybrid",
"label": "Hybrid Camel"
},
{
"value": "f2hybrid",
"label": "F2 Hybrid Camel"
},
{
"value": "snowwhite",
"label": "Snow-white Dromedary"
},
{
"value": "black",
"label": "Black Camel"
}
]
},
{
"field_name": "personality",
"label": "Personality",
"preset": "multiple_checkbox",
"choices_helper": "scheming_test_schema_choices"
},
{
"field_name": "a_relevant_date",
"label": "A relevant date",
"preset": "date"
},
{
"field_name": "a_relevant_datetime",
"label": "Date+Time",
"label_time": "Time (combined with above)",
"preset": "datetime"
},
{
"field_name": "a_relevant_datetime_tz",
"label": "Date+Time+Tz",
"label_time": "Time (for above)",
"label_tz": "Timezone (for above)",
"preset": "datetime_tz"
},
{
"field_name": "other",
"label": {"en": "Other information"},
"output_validators": "ignore_missing"
},
{
"label": "Example JSON",
"field_name": "a_json_field",
"preset": "json_object"
}
],
"resource_fields": [
{
"field_name": "url",
"label": "Photo",
"preset": "resource_url_upload",
"form_placeholder": "http://example.com/my-camel-photo.jpg",
"upload_label": "Photo"
},
{
"field_name": "camels_in_photo",
"label": "Camels in Photo",
"validators": "ignore_missing int_validator",
"form_placeholder": "eg. 2"
},
{
"field_name": "others_in_photo",
"label": "Other Thing in Photo",
"output_validators": "ignore_missing"
},
{
"field_name": "date",
"label": "Date",
"preset": "date"
},
{
"field_name": "datetime",
"label": "Date Taken",
"label_time": "Time Taken",
"preset": "datetime"
},
{
"field_name": "datetime_tz",
"label": "Date Taken",
"label_time": "Time Taken",
"preset": "datetime_tz"
},
{
"label": "Example JSON Resource",
"field_name": "a_resource_json_field",
"preset": "json_object"
}
]
}
2 changes: 1 addition & 1 deletion test.ini
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ use = config:../ckan/test-core.ini
# tests here.

ckan.plugins = scheming_datasets mapactionschemas
scheming.dataset_schemas = ckanext.mapactionschemas:dataset.json ckanext.mapactiongeorss.tests:test_schema.json
scheming.dataset_schemas = ckanext.mapactionschemas:dataset.json ckanext.mapactiongeorss.tests:test_schema.json ckanext.mapactiongeorss.tests:default.json
scheming.presets = ckanext.scheming:presets.json ckanext.mapactionschemas:presets.json
scheming.dataset_fallback = false

Expand Down

0 comments on commit 0deff7b

Please sign in to comment.