From 8f7d23d54e43a3ad0840e825e5b24407355ff3e2 Mon Sep 17 00:00:00 2001 From: Karen Turner Date: Thu, 2 Nov 2017 09:35:05 +1300 Subject: [PATCH 1/2] Adding validator and parser for default_groups and tags - Work done by Tobias Schulmann - Adding the validation that is done in ckanext-harvestor to here for consistencey - originally done Wed Dec 14 09:46:10 2016 +1300 --- ckanext/spatial/harvesters/base.py | 33 ++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/ckanext/spatial/harvesters/base.py b/ckanext/spatial/harvesters/base.py index 4bac371e..86a25ecf 100644 --- a/ckanext/spatial/harvesters/base.py +++ b/ckanext/spatial/harvesters/base.py @@ -145,11 +145,20 @@ def validate_config(self, source_config): if 'default_tags' in source_config_obj: if not isinstance(source_config_obj['default_tags'],list): raise ValueError('default_tags must be a list') + elif not isinstance(source_config_obj['default_tags'][0], dict): + # This check is taken from ckanext-harvest CKANHarvester to ensure consistency + # between harversters. + raise ValueError('default_tags must be a list of dicts like {"name": "name-of-tag"}') if 'default_extras' in source_config_obj: if not isinstance(source_config_obj['default_extras'],dict): raise ValueError('default_extras must be a dictionary') + if 'default_groups' in source_config_obj: + # Also taken from CKANHarvester to ensure consistency. + if not isinstance(source_config_obj['default_groups'], list): + raise ValueError('default_groups must be a *list* of group names/ids') + for key in ('override_extras', 'clean_tags'): if key in source_config_obj: if not isinstance(source_config_obj[key],bool): @@ -211,17 +220,29 @@ def get_package_dict(self, context, data_dict): tags_val = [munge_tag(tag) if do_clean else tag[:100] for tag in iso_values['tags']] tags = [{'name': tag} for tag in tags_val] - # Add default_tags from config - default_tags = self.source_config.get('default_tags', []) - if default_tags: - for tag in default_tags: - tags.append({'name': tag}) + + # Add default_tags from config. + # For consistency with CKANHarvester `default_tags` is now a list of + # dicts, which we can add to tags without further parsing. + tags.extend(self.source_config.get('default_tags', [])) + + # Adding default_groups from config. This was previously not supported + # by ckanext-spatial. + context = {'model': model, 'user': p.toolkit.c.user} + groups = [] + for group_name_or_id in self.source_config.get('default_groups', []): + try: + group = p.toolkit.get_action('group_show')(context, {'id': group_name_or_id}) + groups.append({'id': group['id'], 'name': group['name']}) + except p.toolkit.ObjectNotFound, e: + logging.error('Default group %s not found, proceeding without.' % group_name_or_id) package_dict = { 'title': iso_values['title'], 'notes': iso_values['abstract'], - 'tags': tags, + 'tags': dict((tag['name'], tag) for tag in tags).values(), 'resources': [], + 'groups': dict((group['id'], group) for group in groups).values(), } # We need to get the owner organization (if any) from the harvest From 5315a69c00f01481e26817e1e093daf0866a0116 Mon Sep 17 00:00:00 2001 From: ebuckley Date: Wed, 3 Oct 2018 14:51:33 +1300 Subject: [PATCH 2/2] support queries across anti-meridian line - the envelope query would have values above 180, which is not valid CQL --- ckanext/spatial/plugin.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ckanext/spatial/plugin.py b/ckanext/spatial/plugin.py index e2d73df3..78777905 100644 --- a/ckanext/spatial/plugin.py +++ b/ckanext/spatial/plugin.py @@ -317,6 +317,9 @@ def _params_for_solr_spatial_field_search(self, bbox, search_params): +spatial_geom:"Intersects(ENVELOPE({minx}, {miny}, {maxx}, {maxy})) ''' + if bbox['maxx'] > 180: + bbox['maxx'] = -180 + (bbox['maxx'] - 180) + search_params['fq_list'] = search_params.get('fq_list', []) search_params['fq_list'].append('+spatial_geom:"Intersects(ENVELOPE({minx}, {maxx}, {maxy}, {miny}))"' .format(minx=bbox['minx'], miny=bbox['miny'], maxx=bbox['maxx'], maxy=bbox['maxy']))