Merge pull request #529 from GovDataOfficial/fix-search-index-rebuild

Fix a problem with data-dictization of lists to fix search-index rebuild

CHANGELOG.rst

@@ -11,6 +11,8 @@ and this project adheres to `Semantic Versioning <http://semver.org/>`_
 Unreleased_
 ***********
 
+- Fix a problem with data-dictization when using sqlalchemy 1.4+
+
 ***********
 1.4.2_ - 2023-01-12
 ***********

ckanext/harvest/logic/dictization.py

@@ -68,15 +68,16 @@ def harvest_job_dictize(job, context):
             .group_by(HarvestObjectError.message) \
             .order_by(text('error_count desc')) \
             .limit(context.get('error_summmary_limit', 20))
-        out['object_error_summary'] = q.all()
+        out['object_error_summary'] = harvest_error_dictize(q.all(), context)
         q = model.Session.query(
             HarvestGatherError.message,
             func.count(HarvestGatherError.message).label('error_count')) \
             .filter(HarvestGatherError.harvest_job_id == job.id) \
             .group_by(HarvestGatherError.message) \
             .order_by(text('error_count desc')) \
             .limit(context.get('error_summmary_limit', 20))
-        out['gather_error_summary'] = q.all()
+        out['gather_error_summary'] = harvest_error_dictize(q.all(), context)
+
     return out
 
 
@@ -106,6 +107,13 @@ def harvest_log_dictize(obj, context):
     return out
 
 
+def harvest_error_dictize(obj, context):
+    out = []
+    for elem in obj:
+        out.append(elem._asdict())
+    return out
+
+
 def _get_source_status(source, context):
     '''
     TODO: Deprecated, use harvest_source_show_status instead

ckanext/harvest/tests/test_action.py

@@ -755,3 +755,33 @@ def test_harvest_job_create_as_admin(self):
         assert job['status'] == 'Running'
         assert job['gather_started'] is None
         assert 'stats' in job.keys()
+
+    def test_harvest_source_show_status(self):
+
+        source = factories.HarvestSourceObj(**SOURCE_DICT.copy())
+        job = factories.HarvestJobObj(source=source)
+        dataset = ckan_factories.Dataset()
+        obj = factories.HarvestObjectObj(
+            job=job, source=source, package_id=dataset['id'])
+
+        harvest_gather_error = harvest_model.HarvestGatherError(message="Unexpected gather error", job=job)
+        harvest_gather_error.save()
+        harvest_object_error = harvest_model.HarvestObjectError(message="Unexpected object error", object=obj)
+        harvest_object_error.save()
+
+        context = {'model': model}
+        data_dict = {'id': source.id}
+
+        source_status = get_action('harvest_source_show_status')(context, data_dict)
+
+        # verifiy that the response is dictized properly
+        json.dumps(source_status)
+
+        last_job = source_status['last_job']
+        assert last_job['source_id'] == source.id
+        assert last_job['status'] == 'New'
+        assert last_job['stats']['errored'] == 2
+        assert len(last_job['object_error_summary']) == 1
+        assert last_job['object_error_summary'][0]['message'] == harvest_object_error.message
+        assert len(last_job['gather_error_summary']) == 1
+        assert last_job['gather_error_summary'][0]['message'] == harvest_gather_error.message