Merge branch 'develop' into feature/sem-search-routing

backend/pytest.ini

@@ -1,2 +1,3 @@
 [pytest]
 DJANGO_SETTINGS_MODULE = readit.test_settings
+log_level = INFO

backend/rdf/renderers.py

@@ -31,3 +31,28 @@ class TurtleRenderer(RDFLibRenderer):
     rdflib_args = {
         'format': 'turtle',
     }
+
+
+class RdfXMLRenderer(RDFLibRenderer):
+    media_type = 'application/rdf+xml'
+    format = 'xml'
+    rdflib_args = {
+        'format': 'xml',
+    }
+
+
+class JsonLdRenderer(RDFLibRenderer):
+    media_type = 'application/ld+json'
+    format = 'jsonld'
+    rdflib_args = {
+        'format': 'json-ld',
+    }
+
+
+class NTriplesRenderer(RDFLibRenderer):
+    media_type = 'application/n-triples'
+    format = 'nt'
+    rdflib_args = {
+        'format': 'nt',
+        'encoding': 'ascii'  # N-triples are always ascii encoded
+    }

backend/sparql/conftest.py

@@ -5,7 +5,7 @@
 from rdflib import Literal
 
 from nlp_ontology import namespace as my
-from nlp_ontology.graph import graph
+from sources.graph import graph
 from rdf.ns import RDF, SCHEMA
 from rdf.utils import graph_from_triples
 from sources.constants import SOURCES_NS
@@ -124,7 +124,11 @@ def accept_headers():
         'turtle': 'text/turtle',
         'sparql_json': 'application/sparql-results+json',
         'sparql_xml': 'application/sparql-results+xml',
-        'json': 'application/json'
+        'sparql_csv': 'text/csv',
+        'json': 'application/json',
+        'rdfxml': 'application/rdf+xml',
+        'ntriples': 'application/n-triples',
+        'jsonld': 'application/ld+json'
     }
     return SimpleNamespace(**values)
 

backend/sparql/negotiation.py

@@ -1,24 +1,27 @@
+from rdf.renderers import (JsonLdRenderer, NTriplesRenderer, RdfXMLRenderer,
+                           TurtleRenderer)
 from rest_framework.negotiation import DefaultContentNegotiation
 
-from rdf.renderers import TurtleRenderer
-
-from .renderers import QueryResultsJSONRenderer, QueryResultsTurtleRenderer, QueryResultsXMLRenderer
+from .renderers import (QueryResultsCSVRenderer, QueryResultsJSONRenderer,
+                        QueryResultsXMLRenderer)
 
 
 class SPARQLContentNegotiator(DefaultContentNegotiation):
-    results_renderers = (QueryResultsJSONRenderer,
-                         QueryResultsXMLRenderer, QueryResultsTurtleRenderer)
-    rdf_renderers = (TurtleRenderer,)
+    results_renderers = [QueryResultsJSONRenderer, QueryResultsXMLRenderer,
+                         QueryResultsCSVRenderer]
+    rdf_renderers = [TurtleRenderer, RdfXMLRenderer,
+                     JsonLdRenderer, NTriplesRenderer]
+    querytype_accepts = {
+        'SELECT': results_renderers,
+        'ASK': results_renderers,
+        'CONSTRUCT': rdf_renderers,
+        'EMPTY': rdf_renderers
+    }
 
     def select_renderer(self, request, renderers, format_suffix=None):
         query_type = request.data.get('query_type', None)
-
-        if query_type in ('ASK', 'SELECT'):
-            renderers = [renderer() for renderer in self.results_renderers]
-        elif query_type in ('EMPTY', 'CONSTRUCT'):
-            renderers = [renderer() for renderer in self.rdf_renderers]
-        else:
-            renderers = [renderer() for renderer in set(
-                self.rdf_renderers+self.results_renderers)]
+        renderer_classes = self.querytype_accepts.get(
+            query_type, self.rdf_renderers + self.results_renderers)
+        renderers = [renderer() for renderer in renderer_classes]
 
         return super().select_renderer(request, renderers, format_suffix)

backend/sparql/renderers.py

@@ -1,12 +1,13 @@
 import json
 from io import StringIO
 
+from rdf.ns import HTTP, HTTPSC, RDF
+from rdf.renderers import RDFLibRenderer, TurtleRenderer
+from rdf.utils import graph_from_triples
+from rdflib import BNode, Literal
 from rdflib.plugins.sparql.results.jsonresults import JSONResultSerializer
 from rest_framework.renderers import JSONRenderer
 
-from rdf.renderers import TurtleRenderer, RDFLibRenderer
-from rdf.utils import graph_from_triples
-
 
 class QueryResultsTurtleRenderer(TurtleRenderer):
     ''' Renders turtle from rdflib SPARQL query results'''
@@ -37,3 +38,11 @@ class QueryResultsXMLRenderer(RDFLibRenderer):
 
     def render(self, query_results, media_type=None, renderer_context=None):
         return query_results.serialize(format='xml')
+
+
+class QueryResultsCSVRenderer(RDFLibRenderer):
+    media_type = 'text/csv'
+    format = 'csv'
+    rdflib_args = {
+        'format': 'csv'
+    }

backend/sparql/views.py

@@ -9,7 +9,7 @@
 from rdflib import BNode, Literal
 from rdflib.plugins.sparql.parser import parseUpdate
 from requests.exceptions import HTTPError
-from rest_framework.exceptions import APIException
+from rest_framework.exceptions import APIException, NotAcceptable, ParseError
 from rest_framework.response import Response
 from rest_framework.views import APIView
 
@@ -38,7 +38,7 @@ def check_supported(self, updatestring):
                 raise UnsupportedUpdateError(
                     'Update operation is not supported.'
                 )
-                
+
         # Do a quick check for blank nodes
         if re.search(BLANK_NODE_PATTERN, updatestring):
             parse_request = parseUpdate(updatestring).request
@@ -57,8 +57,8 @@ def execute_update(self, updatestring):
 
         try:
             self.check_supported(updatestring)
-            return graph.update(updatestring)
-        except ParseException as p_e:
+            graph.update(updatestring)
+        except (ParseException, ParseError) as p_e:
             # Raised when SPARQL syntax is not valid, or parsing fails
             graph.rollback()
             raise ParseSPARQLError(p_e)
@@ -80,7 +80,6 @@ def post(self, request, **kwargs):
         if not sparql_string:
             # POST must contain an update
             raise NoParamError()
-
         blank = BNode()
         status = 200
         response = graph_from_triples(
@@ -91,48 +90,28 @@ def post(self, request, **kwargs):
                 (blank, HTTP.sc, HTTPSC.OK),
             )
         )
-
         self.execute_update(sparql_string)
+
         return Response(response)
 
     def graph(self):
         raise NotImplementedError
 
 
 class SPARQLQueryAPIView(APIView):
-    renderer_classes = (TurtleRenderer,)
+    renderer_classes = SPARQLContentNegotiator.rdf_renderers + \
+        SPARQLContentNegotiator.results_renderers
     content_negotiation_class = SPARQLContentNegotiator
 
     def get_exception_handler(self):
+        ''' Errors are returned as turtle, even when the original querytype
+        does not satisfy could not be rendered in this format. A hard
+        overwrite of renderer is necessary. Ideally, this should render errors
+        in a querytype-compatibleway. '''
+        self.request.accepted_renderer = TurtleRenderer()
+        self.request.accepted_media_type = TurtleRenderer.media_type
         return turtle_exception_handler
 
-    def finalize_response(self, request, response, *args, **kwargs):
-        """
-        Adapts APIView method to additionaly perform content negotation
-        when a query type was set
-        """
-        assert isinstance(response, HttpResponseBase), (
-            "Expected a `Response`, `HttpResponse` or `HttpStreamingResponse` "
-            "to be returned from the view, but received a `%s`" % type(
-                response)
-        )
-
-        if isinstance(response, Response):
-            # re-perform content negotiation if a query type was set
-            if not getattr(request, "accepted_renderer", None) or \
-                    self.request.data.get("query_type", None):
-                neg = self.perform_content_negotiation(request, force=True)
-                request.accepted_renderer, request.accepted_media_type = neg
-
-            response.accepted_renderer = request.accepted_renderer
-            response.accepted_media_type = request.accepted_media_type
-            response.renderer_context = self.get_renderer_context()
-
-        for key, value in self.headers.items():
-            response[key] = value
-
-        return response
-
     def execute_query(self, querystring):
         """ Attempt to query a graph with a SPARQL-Query string
             Sets query type on succes
@@ -146,6 +125,11 @@ def execute_query(self, querystring):
                 query_results = graph.query(querystring)
                 query_type = query_results.type
             self.request.data["query_type"] = query_type
+
+            # re-perform content negotiation to determine if
+            # querytype satisfies accept header
+            neg = self.perform_content_negotiation(self.request)
+            self.request.accepted_renderer, self.request.accepted_media_type = neg
             return query_results
 
         except ParseException as p_e:
@@ -158,6 +142,9 @@ def execute_query(self, querystring):
                 raise ParseSPARQLError(h_e)
             else:
                 raise APIException(h_e)
+        except NotAcceptable:
+            graph.rollback()
+            raise
         except Exception as n_e:
             graph.rollback()
             raise APIException(n_e)
@@ -170,6 +157,9 @@ def get(self, request, **kwargs):
             on query type and header 'Accept.
         """
         sparql_string = request.query_params.get("query")
+        if request.data:
+            raise ParseSPARQLError(
+                "GET request should provide query in parameter, not request body.")
         query_results = self.execute_query(sparql_string)
 
         return Response(query_results)

backend/sparql/views_test.py

@@ -6,8 +6,8 @@
 from .exceptions import BlankNodeError
 from .views import SPARQLUpdateAPIView
 
-QUERY_URL = '/sparql/nlp-ontology/query'
-UPDATE_URL = '/sparql/nlp-ontology/update'
+QUERY_URL = '/sparql/source/query'
+UPDATE_URL = '/sparql/source/update'
 
 
 def check_content_type(response, content_type):
@@ -63,29 +63,6 @@ def test_malformed(sparql_client, sparqlstore):
     assert malformed_update.status_code == 400
 
 
-def test_negotiation(client, ontologygraph_db, accept_headers, test_queries):
-    empty_get = client.get(QUERY_URL)
-    assert empty_get.status_code == 200
-    assert check_content_type(empty_get, accept_headers.turtle)
-
-    sparql_json_get = client.get(
-        QUERY_URL, {'query': test_queries.SELECT},
-        HTTP_ACCEPT=accept_headers.sparql_json)
-    assert sparql_json_get.status_code == 200
-    assert check_content_type(sparql_json_get, accept_headers.sparql_json)
-
-    sparql_xml_get = client.get(
-        QUERY_URL, {'query': test_queries.SELECT},
-        HTTP_ACCEPT=accept_headers.sparql_xml)
-    assert sparql_xml_get.status_code == 200
-    assert check_content_type(sparql_xml_get, accept_headers.sparql_xml)
-
-    json_get = client.get(
-        QUERY_URL, {'query': test_queries.SELECT},
-        HTTP_ACCEPT=accept_headers.json)
-    assert json_get.status_code == 406
-
-
 def test_permissions(client, sparql_user, test_queries, sparqlstore):
     res = client.post(UPDATE_URL, {'update': test_queries.INSERT})
     assert res.status_code == 403
@@ -121,30 +98,75 @@ def test_delete(sparql_client, test_queries, ontologygraph_db):
 
 def test_select_from(sparql_client, test_queries, ontologygraph_db, ontologygraph, accept_headers):
     # Should not return results if querying a different endpoint
-    # Note that any triples in SOURCES_NS graph would be returned here
-    res = sparql_client.post('/sparql/source/query',
-                             {'query': test_queries.SELECT_FROM_NLP},
-                             HTTP_ACCEPT=accept_headers.turtle)
+    # Note that any triples in VOCAB_NS graph would be returned here
+    res = sparql_client.post('/sparql/vocab/query',
+                             {'query': test_queries.SELECT_FROM_NLP})
     assert res.status_code == 200
-    assert len(Graph().parse(data=res.content, format='turtle')) == 0
+    assert len(res.data) == 0
 
     # Should ignore FROM clause if endpoint and graph don't match
     res = sparql_client.post(QUERY_URL,
-                             {'query': test_queries.SELECT_FROM_SOURCES},
-                             HTTP_ACCEPT=accept_headers.turtle)
+                             {'query': test_queries.SELECT_FROM_SOURCES})
     assert res.status_code == 200
-    assert len(Graph().parse(data=res.content, format='turtle')) == 3
+    assert len(res.data) == 3
 
     # Should return results if FROM and endpoint match
     res = sparql_client.post(QUERY_URL,
-                             {'query': test_queries.SELECT_FROM_NLP},
-                             HTTP_ACCEPT=accept_headers.turtle)
+                             {'query': test_queries.SELECT_FROM_NLP})
     assert res.status_code == 200
-    assert len(Graph().parse(data=res.content, format='turtle')) == 3
+    assert len(res.data) == 3
 
 
 def test_blanknodes(blanknode_queries):
     view = SPARQLUpdateAPIView()
     for q in blanknode_queries:
         with pytest.raises(BlankNodeError):
             view.check_supported(q)
+
+
+def test_GET_body(sparql_client):
+    res = sparql_client.get(
+        QUERY_URL,
+        data={'query': 'some query'}
+    )
+    assert res.status_code == 400
+
+
+def test_graph_negotiation(sparql_client, accept_headers, test_queries):
+    ''' result headers should succeed for CONSTRUCT/DESRIBE/empty, fail for SELECT/ASK'''
+    graph_headers = [accept_headers.rdfxml, accept_headers.ntriples,
+                     accept_headers.jsonld, accept_headers.turtle]
+
+    for h in graph_headers:
+        accept = sparql_client.get(
+            QUERY_URL, {'query': test_queries.CONSTRUCT}, HTTP_ACCEPT=h)
+        deny = sparql_client.get(
+            QUERY_URL, {'query': test_queries.SELECT}, HTTP_ACCEPT=h)
+        empty = sparql_client.get(QUERY_URL, HTTP_ACCEPT=h)
+        assert accept.status_code == 200
+        assert check_content_type(accept, h)
+        assert deny.status_code == 406
+        assert empty.status_code == 200
+        assert check_content_type(empty, h)
+
+
+def test_result_negotiation(sparql_client, accept_headers, test_queries):
+    ''' result headers should succeed for SELECT/ASK, fail for CONSTRUCT/DESCRIBE/empty'''
+    result_headers = [accept_headers.sparql_json, accept_headers.sparql_xml,
+                      accept_headers.sparql_csv]
+    for h in result_headers:
+        accept = sparql_client.get(
+            QUERY_URL, {'query': test_queries.SELECT}, HTTP_ACCEPT=h)
+        deny = sparql_client.get(
+            QUERY_URL, {'query': test_queries.CONSTRUCT}, HTTP_ACCEPT=h)
+        empty = sparql_client.get(QUERY_URL, HTTP_ACCEPT=h)
+        assert accept.status_code == 200
+        assert check_content_type(accept, h)
+        assert deny.status_code == 406
+        assert empty.status_code == 406
+
+    # application/json should not work
+    regular_json = sparql_client.get(
+        QUERY_URL, {'query': test_queries.SELECT},
+        HTTP_ACCEPT=accept_headers.json)
+    assert regular_json.status_code == 406