Merge pull request #7 from qld-gov-au/develop

CHDEXCHAE-34, CHDEXCHAE-40: patches

.github/workflows/test.yml

@@ -12,7 +12,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.6'
+          python-version: '3.x'
       - name: Cache pip
         uses: actions/cache@v2
         with:

README.md

@@ -73,9 +73,13 @@ To install ckanext-validation-schema-generator:
     ckanext.validation_schema_generator.resource_schema_field_name = schema
 
     # Field name for dataset schema field
-    # (optionak, default: schema)
+    # (optional, default: schema)
     ckanext.validation_schema_generator.package_schema_field_name = default_data_schema
 
+    # Allow edit generated schema before apply
+    # (optional, defaukt: False)
+    ckanext.validation_schema_generator.allow_edit_generated_schema = False
+
 
 ## Developer installation
 
@@ -166,15 +170,10 @@ The extension has next endpoints to manipulate the schema generation process.
     **Returns**
     Updated task data, same as `vsg_status`
 
-4. `vsg_apply` - Apply a generated scheme or a new one. The scheme can be applied only if the generation process is successfully completed.
-    **params**:
+4. `vsg_apply` - Apply a generated schema for a resource or dataset. The schema can be applied only if the generation process is successfully completed.
+    **Params**:
     - `id` (required) - ID of the resource
     - `apply_for` _(required)_- apply for entity, must be one of `["dataset", "resource"]`
-    - `schema` _(optional)_ - a table schema. If not provided, the generated one will be used.
-
-5. `vsg_unapply` - Unapply the schema. Automatically clears the dataset/resource schema if it was using the generated schema.
-    **params**:
-    - `id` (required) - ID of the resource
 
 ## License
 

ckanext/validation_schema_generator/constants.py

@@ -14,8 +14,6 @@
 
 CF_JOB_TIMEOUT = u"ckanext.validation_schema_generator.job_timeout"
 CF_JOB_TIMEOUT_DF = 3600
-CF_PASS_AUTH = u"ckanext.validation_schema_generator.pass_api_key"
-CF_PASS_AUTH_DF = True
 CF_API_KEY = u"ckanext.validation_schema_generator.api_key"
 
 APPLY_FOR_OPTIONS = (u"dataset", u"resource")

ckanext/validation_schema_generator/jobs.py

@@ -1,15 +1,13 @@
 import requests
 
 from tableschema import infer as ts_infer, TableSchemaException
-from tabulator.exceptions import FormatError
+from tabulator.exceptions import TabulatorException
 
 import ckan.model as model
 import ckan.plugins.toolkit as tk
 import ckan.lib.uploader as uploader
 
 from ckanext.validation_schema_generator.constants import (
-    CF_PASS_AUTH,
-    CF_PASS_AUTH_DF,
     CF_API_KEY,
     TASK_STATE_FINISHED,
     TASK_STATE_ERROR,
@@ -22,9 +20,6 @@ def generate_schema_from_resource(input):
     resource = tk.get_action(u'resource_show')(context, {
         'id': input[u'resource_id']
     })
-    dataset = tk.get_action('package_show')(context, {
-        'id': resource[u'package_id']
-    })
 
     errors = {}
     options = {}
@@ -37,8 +32,7 @@ def generate_schema_from_resource(input):
         if isinstance(upload, uploader.ResourceUpload):
             source = upload.get_path(resource[u'id'])
         else:
-            if dataset[u'private'] and _pass_auth_header():
-                options[u'http_session'] = _make_session()
+            options[u'http_session'] = _make_session()
 
     if not source:
         source = resource[u'url']
@@ -52,7 +46,7 @@ def generate_schema_from_resource(input):
             schema = ts_infer(source=source, **options)
     except TableSchemaException as e:
         errors[u'schema'] = str(e)
-    except FormatError as e:
+    except TabulatorException as e:
         errors[u'format'] = str(e)
     except Exception as e:
         errors[u'undefined'] = str(e)
@@ -79,10 +73,6 @@ def _get_site_user():
     }, {})
 
 
-def _pass_auth_header():
-    return tk.asbool(tk.config.get(CF_PASS_AUTH, CF_PASS_AUTH_DF))
-
-
 def _make_session():
     s = requests.Session()
     s.headers.update({u'Authorization': _get_api_key()})

ckanext/validation_schema_generator/logic/action.py

@@ -4,8 +4,8 @@
 from ckan.logic import validate
 from ckan.lib.jobs import enqueue as enqueue_job
 
-from ckanext.validation_schema_generator import jobs, utils as vsg_utils, constants as const
 import ckanext.validation_schema_generator.logic.schema as vsg_schema
+from ckanext.validation_schema_generator import jobs, utils as vsg_utils, constants as const
 
 
 def _get_actions():
@@ -34,7 +34,8 @@ def vsg_generate(context, data_dict):
 
     data = {"resource_id": data_dict["id"], "task_id": task["id"]}
 
-    timeout = tk.asint(tk.config.get(const.CF_JOB_TIMEOUT, const.CF_JOB_TIMEOUT_DF))
+    timeout = tk.asint(tk.config.get(
+        const.CF_JOB_TIMEOUT, const.CF_JOB_TIMEOUT_DF))
     job = enqueue_job(jobs.generate_schema_from_resource, [data],
                       rq_kwargs={"timeout": timeout})
 
@@ -83,27 +84,26 @@ def vsg_update(context, data_dict):
     task['state'] = status
     task['last_updated'] = vsg_utils.get_current_time()
     task['error'] = error
-    task['value']['schema'] = json.loads(data_dict.get("schema", const.EMPTY_SCHEMA))
+
+    if data_dict.get("schema"):
+        task['value']['schema'] = json.loads(data_dict["schema"])
 
     return vsg_utils.update_task(context, task)
 
 
 @validate(vsg_schema.vsg_apply_schema)
 def vsg_apply(context, data_dict):
-    """Apply a generated scheme or a new one. The scheme can be applied only
+    """Apply a generated schema for a resource or dataset. The schema can be applied only
     if the generation process is successfully completed
 
     :param id: resource ID
     :type id: string
     :param apply_for: apply schema agaisnt the whole `dataset` or `resource` only.
     :type apply_for: string
-    :param schema: if the schema is provided, it will replace the generated one (optional)
-    :type schema: string
     """
     tk.check_access('vsg_generate', context, data_dict)
 
     apply_for = data_dict.get(const.APPLY_FOR_FIELD)
-    schema = data_dict.get('schema')
 
     task = tk.get_action('vsg_status')(context, data_dict)
 
@@ -114,14 +114,9 @@ def vsg_apply(context, data_dict):
         raise tk.ValidationError(
             tk._(u"Schema generation failed. Check status."))
 
-    current_schema = task['value']['schema']
-
-    if current_schema != schema:
-        task['last_updated'] = vsg_utils.get_current_time()
-
+    task['last_updated'] = vsg_utils.get_current_time()
     task['value'][const.APPLY_FOR_FIELD] = apply_for
-
-    task['value']['schema'] = json.loads(schema or current_schema)
+    schema = vsg_utils.dump_schema(task['value']['schema'])
 
     if apply_for == const.APPLY_FOR_DATASET:
         _apply_pkg_schema(schema, data_dict['id'])
@@ -140,11 +135,9 @@ def _apply_res_schema(schema, resource_id):
 
     tk.get_action(u'resource_update')(context, res)
 
-    _unapply_package_schema(resource_id)
-
 
 def _apply_pkg_schema(schema, resource_id):
-    """Apply the generated schema for package and unapply for resource"""
+    """Apply the generated schema for package and resource"""
     context = {"user": "", "ignore_auth": True}
 
     res = tk.get_action(u'resource_show')(context, {u'id': resource_id})
@@ -155,23 +148,17 @@ def _apply_pkg_schema(schema, resource_id):
     if schema:
         for resource in pkg.get('resources', []):
             if resource['id'] == resource_id:
-                resource[const.RES_SCHEMA_FIELD] = ''
+                resource[const.RES_SCHEMA_FIELD] = schema
 
     tk.get_action(u'package_update')(context, pkg)
 
 
 @validate(vsg_schema.vsg_default_schema)
 def vsg_unapply(context, data_dict):
-    """Unapply the schema. Automatically clears the dataset/resource schema if
-    it was using the generated schema.
-
-    :param id: resource ID
-    :type id: string
+    """Unapply the schema
     """
     tk.check_access('vsg_generate', context, data_dict)
 
-    apply_for = data_dict.get(const.APPLY_FOR_FIELD)
-
     task = tk.get_action('vsg_status')(context, data_dict)
 
     if task['state'] in (const.TASK_STATE_NOT_GENERATED,
@@ -185,21 +172,6 @@ def vsg_unapply(context, data_dict):
     if not applied_for:
         raise tk.ValidationError(u"The schema is not applied yet")
 
-    if apply_for == const.APPLY_FOR_DATASET:
-        _unapply_package_schema(data_dict['id'])
-    else:
-        _unapply_resource_schema(data_dict['id'])
-
     task['value'][const.APPLY_FOR_FIELD] = ''
 
     return vsg_utils.update_task(context, task)
-
-
-def _unapply_resource_schema(resource_id):
-    """Unapply resource schema actually means applying an empty one"""
-    _apply_res_schema(const.EMPTY_SCHEMA, resource_id)
-
-
-def _unapply_package_schema(resource_id):
-    """Unapply package schema actually means applying an empty one"""
-    _apply_pkg_schema(const.EMPTY_SCHEMA, resource_id)

ckanext/validation_schema_generator/logic/schema.py

@@ -21,12 +21,10 @@ def vsg_generate_schema(not_missing, vsg_is_resource_supportable,
 
 
 @validator_args
-def vsg_apply_schema(not_missing, resource_id_exists, one_of, ignore_missing,
-                     vsg_validate_schema, unicode_safe):
+def vsg_apply_schema(not_missing, resource_id_exists, one_of):
     return {
         "id": [not_missing, resource_id_exists],
-        "apply_for": [not_missing, one_of(const.APPLY_FOR_OPTIONS)],
-        "schema": [ignore_missing, unicode_safe, vsg_validate_schema]
+        "apply_for": [not_missing, one_of(const.APPLY_FOR_OPTIONS)]
     }
 
 

ckanext/validation_schema_generator/logic/validators.py

@@ -41,12 +41,15 @@ def vsg_is_resource_supportable(key, data, errors, context):
     res = session.query(model.Resource).get(data[key])
 
     if not res.extras.get('datastore_active'):
-        errors[key].append(tk._('Schema couldn\'t be generated for this resource'))
+        errors[key].append(
+            tk._('Schema couldn\'t be generated for this resource'))
         raise tk.StopOnError()
 
 
 def vsg_validate_schema(value, context):
     """Validate table schema"""
+    if not value:
+        return ""
 
     errors = validate_schema(value)
 
@@ -63,5 +66,6 @@ def vsg_generation_started(key, data, errors, context):
     result = tk.get_action('vsg_status')(context, {"id": data[key]})
 
     if result['state'] == const.TASK_STATE_NOT_GENERATED:
-        errors[key].append(tk._('The schema generation procecss isn\'t started yet.'))
+        errors[key].append(
+            tk._('The schema generation procecss isn\'t started yet.'))
         raise tk.StopOnError()

ckanext/validation_schema_generator/templates/vsg/index.html

@@ -29,7 +29,7 @@
     <form method="post" action="{{ h.url_for('validation_schema_generator.index', dataset_id=pkg.name, resource_id=res.id) }}" class="generate-schema-form">
         <button class="btn btn-primary btn-generate" name="generate" type="submit">
             <i class="fa fa-plus"></i>
-            {{ _('Generate JSON data scheme') }}
+            {{ _('Generate JSON data schema') }}
         </button>
 
         <table class="table table-bordered table-schema">
@@ -57,7 +57,7 @@
                     <th>{{ _('JSON data schema') }}</th>
 
                     <td class="with-textarea">
-                        {{ form.textarea('schema', id='field-schema', value=task.value.schema, error=error) }}
+                        {{ form.textarea('schema', id='field-schema', value=task.value.schema, error=error, attrs={"disabled": true}) }}
                     </td>
                 </tr>
             {% endif %}

ckanext/validation_schema_generator/tests/test_action.py

@@ -112,13 +112,12 @@ def test_apply_for_package(self, table_schema):
                             schema=table_schema)
         helpers.call_action('vsg_apply',
                             id=resource["id"],
-                            apply_for=const.APPLY_FOR_DATASET,
-                            schema=table_schema)
+                            apply_for=const.APPLY_FOR_DATASET)
 
         pkg = helpers.call_action("package_show", id=resource["package_id"])
         assert pkg[const.PKG_SCHEMA_FIELD]
 
-    def test_apply_resource_must_unapply_pkg_schema(self, table_schema):
+    def test_apply_resource_must_not_unapply_pkg_schema(self, table_schema):
         resource = factories.Resource(datastore_active=True)
         helpers.call_action('vsg_generate', id=resource['id'])
         helpers.call_action('vsg_update',
@@ -141,9 +140,12 @@ def test_apply_resource_must_unapply_pkg_schema(self, table_schema):
 
         pkg = helpers.call_action("package_show", id=resource["package_id"])
         assert pkg[const.PKG_SCHEMA_FIELD]
-        assert not pkg['resources'][0][const.RES_SCHEMA_FIELD]
+        assert pkg['resources'][0][const.RES_SCHEMA_FIELD]
 
-    def test_apply_pkg_must_unapply_resource_schema(self, table_schema):
+    def test_apply_pkg_must_apply_this_schema_for_resource(self, table_schema):
+        """Applying generated scheam as a default_schema must apply it to the
+        respective resource too (the one where we've generated this schema)
+        """
         resource = factories.Resource(datastore_active=True)
 
         helpers.call_action('vsg_generate', id=resource['id'])
@@ -168,14 +170,16 @@ def test_apply_pkg_must_unapply_resource_schema(self, table_schema):
         resource = helpers.call_action("resource_show", id=resource["id"])
         pkg = helpers.call_action("package_show", id=resource["package_id"])
 
-        assert not pkg[const.PKG_SCHEMA_FIELD]
-        assert resource[const.RES_SCHEMA_FIELD]
+        assert pkg[const.PKG_SCHEMA_FIELD] == resource[const.RES_SCHEMA_FIELD]
 
 
 @pytest.mark.usefixtures("clean_db", "with_plugins")
 class TestActionUnapply(object):
 
     def test_unapply(self, table_schema):
+        """Unapply doesn't remove the schema from resource or dataset, just
+        changing the task status to indicate, that the generated schema doesn't
+        attached to any entity..."""
         resource = factories.Resource(datastore_active=True)
 
         helpers.call_action('vsg_generate', id=resource['id'])
@@ -194,8 +198,11 @@ def test_unapply(self, table_schema):
 
         helpers.call_action('vsg_unapply', id=resource["id"])
 
+        task_status = helpers.call_action('vsg_status', id=resource["id"])
+        assert not task_status["value"]["apply_for"]
+
         resource = helpers.call_action("resource_show", id=resource["id"])
-        assert not resource[const.RES_SCHEMA_FIELD]
+        assert resource[const.RES_SCHEMA_FIELD]
 
     def test_unapply_not_applied(self, table_schema):
         resource = factories.Resource(datastore_active=True)

ckanext/validation_schema_generator/tests/test_auth.py

@@ -17,14 +17,13 @@ def test_anon_not_allowed(self):
         with pytest.raises(tk.NotAuthorized):
             helpers.call_auth("vsg_generate", context, id=resource["id"])
 
-    # TODO: Problem with auth for regualar user
-    # def test_regular_user_not_allowed(self):
-    #     resource = factories.Resource()
-    #     user = factories.User()
-    #     context = {"user": user["name"], "model": model}
-
-    #     with pytest.raises(tk.NotAuthorized):
-    #         helpers.call_auth("vsg_generate", context, id=resource["id"])
+    def test_regular_user_not_allowed(self):
+        resource = factories.Resource()
+        user = factories.User()
+        context = {"user": user["name"], "model": model}
+
+        with pytest.raises(tk.NotAuthorized):
+            helpers.call_auth("vsg_generate", context, id=resource["id"])
 
     def test_sysadmin(self):
         user = factories.Sysadmin()