Suggestions v2

django_files/atlas/urls.py

@@ -4,6 +4,7 @@
 #from django.views.generic.simple import redirect_to
 from django.conf.urls import patterns, include
 from django.views.generic import TemplateView, RedirectView
+from django.views.generic.simple import direct_to_template
 
 # sitemap
 from django.conf.urls.defaults import *
@@ -125,8 +126,8 @@ def render_to_response(self, context, **kwargs):
 
   (r'^api/near/(?P<country>\w{3})/(?P<year>[0-9\.]+)/(?P<num_prods>\d+)/$', 'observatory.views_exhibit.api_near'),
 
-  (r'^api/search/$', 'observatory.views.api_search'),
-  (r'^search/$', 'observatory.views.search'),
+  (r'^api/search/$', 'observatory.views_search.api_search'),
+  (r'^search/$', direct_to_template, {'template': 'searchresults.html'}),
 
   # Overview (Countries) ######################################################
   (r'^country/(?P<country>\w{2,3})/$', 'observatory.views_overview.country2'),
@@ -156,6 +157,8 @@ def render_to_response(self, context, **kwargs):
   url(r'^favicon\.ico$', RedirectView.as_view(url='/media/img/favicon.ico')),
 
   url(r'^sitemap\.xml$', RedirectView.as_view(url='/media/sitemaps/sitemap_index.xml')),
-  #(r'^sitemap\.xml$', 'django.contrib.sitemaps.views.sitemap', {'sitemaps': sitemaps}),
+  url(r'^opensearch.xml$', direct_to_template, {'template': 'opensearch.xml',
+                                                'mimetype':
+                                                'application/opensearchdescription+xml'}),
 
 )

django_files/observatory/helpers.py

@@ -1,7 +1,5 @@
 from observatory.models import Hs4_cpy, Sitc4_cpy, Country, Hs4, Sitc4
 
-import re
-
 
 # make sure app name is in the list of possible apps
 def get_app_name(app_name):
@@ -220,22 +218,3 @@ def params_to_url(api_name=None, app_name=None, country_codes=None,
         url += "%s/" % years
 
     return url
-
-year_expressions = [
-    re.compile(r'between (\d{4}) and (\d{4})', re.IGNORECASE),
-    re.compile(r'from (\d{4}) to (\d{4})', re.IGNORECASE),
-    re.compile(r'(\d{4}).*(\d{4})'),
-    re.compile(r'(?:in|at|during) (\d{4})', re.IGNORECASE),
-    re.compile(r'(\d{4})')
-]
-
-
-def extract_years(input_str):
-    """Extract things that look like years out of a given plaintext."""
-    results = (exp.search(input_str) for exp in year_expressions)
-    results = [result for result in results if result is not None]
-
-    if len(results) == 0:
-        return None, None
-    else:
-        return results[0].span(), results[0].groups()

django_files/observatory/management/commands/index_questions.py

@@ -15,10 +15,17 @@ def handle(self, *args, **options):
 
         # TODO: "the" in country names
         trade_flows = ["import", "export"]
-        countries_flat = list(Country.objects.get_valid().only('name_en',
-                                                               'name_3char'))
+        countries_flat = list(Country.objects.get_valid()
+                              .only('name_en',
+                                    'name_3char',
+                                    'region__name',
+                                    ))
         countries = [[c] for c in countries_flat]
-        products = list(Hs4.objects.get_low_level().only('name_en', 'code'))
+        products = list(Hs4.objects.get_low_level()
+                        .only('name_en',
+                              'code',
+                              'community__name',
+                              ))
 
         # Which products are feasible for Latvia?
         casy_questions = self.generate_index_entries(['casy'], ['pie_scatter'],
@@ -135,16 +142,22 @@ def generate_index_entry(args):
             )
             index["url"] = url
 
+            regions = None
+            if args[2] is not None:
+                regions = [c.region.name for c in args[2]]
+
             # Add in params into elasticsearch in case we need them later
             kwargs = dict(
                 api_name=args[0],
                 app_name=args[1],
                 country_names=country_names,
                 country_codes=country_codes,
+                regions=regions,
                 trade_flow=args[3],
                 years=args[4],
                 product_name=args[5].name_en if args[5] is not None else None,
-                product_code=args[5].code if args[5] is not None else None
+                product_code=args[5].code if args[5] is not None else None,
+                product_community=args[5].community.name if args[5] is not None else None,
             )
             kwargs = {k: v for k, v in kwargs.iteritems() if v is not None}
             index.update(kwargs)

django_files/observatory/tests.py

@@ -1,16 +1,74 @@
-"""
-This file demonstrates writing tests using the unittest module. These will pass
-when you run "manage.py test".
+from django.utils.unittest import TestCase
 
-Replace this with more appropriate tests for your application.
-"""
+from observatory.views_search import make_extractor, remove_spans
 
-from django.test import TestCase
 
+class TestRemoveSpans(TestCase):
 
-class SimpleTest(TestCase):
-    def test_basic_addition(self):
-        """
-        Tests that 1 + 1 always equals 2.
-        """
-        self.assertEqual(1 + 1, 2)
+    def test_one_span(self):
+        s = "I am a test string! Hahaaa! Narwhals!"
+        self.assertEquals("I am a string! Hahaaa! Narwhals!",
+                          remove_spans(s, [(6, 11)]))
+
+    def test_two_spans(self):
+        s = "I am a test string! Hahaaa! Narwhals!"
+        self.assertEquals("I am a string! Narwhals!",
+                          remove_spans(s, [(6, 11), (19, 27)]))
+
+    def test_empty_span(self):
+        s = "I am a test string! Hahaaa! Narwhals!"
+        self.assertEquals(s, remove_spans(s, []))
+
+    def test_consecutive_span(self):
+        s = "I am a test string! Hahaaa! Narwhals!"
+        self.assertEquals("I am a test string!",
+                          remove_spans(s, [(19, 27), (27, 37)]))
+
+    def test_one_gap_span(self):
+        s = "I am a test string! Hahaaa! Narwhals!"
+        self.assertEquals("I am a test string!!",
+                          remove_spans(s, [(19, 26), (27, 37)]))
+
+    def test_beginning_span(self):
+        s = "I am a test string! Hahaaa! Narwhals!"
+        self.assertEquals("a test string! Hahaaa! Narwhals!",
+                          remove_spans(s, [(0, 5)]))
+
+    def test_end_span(self):
+        s = "I am a test string! Hahaaa! Narwhals!"
+        self.assertEquals("I am a test string! Hahaaa!",
+                          remove_spans(s, [(27, 37)]))
+
+
+class SearchExtractorTest(TestCase):
+
+    def test_one_match_one_group(self):
+        string = "I have a cat."
+        extractor = make_extractor(r"(cat)")
+        result, processed_string = extractor(string)
+        self.assertEqual(processed_string, "I have a .")
+        self.assertEqual(result, [(("cat",), ((9, 12),))])
+
+    def test_two_match_one_group(self):
+        string = "I have a cat and another cat."
+
+        extractor = make_extractor(r"(cat|dog)")
+        result, processed_string = extractor(string)
+        self.assertEqual(processed_string, "I have a  and another .")
+        self.assertEqual(result, [(("cat", ), ((9, 12),)),
+                                  (("cat", ), ((25, 28),))])
+
+    def test_one_match_two_group(self):
+        string = "I have a cat and another dog."
+
+        extractor = make_extractor(r"(cat).*?(dog)")
+        result, processed_string = extractor(string)
+        self.assertEqual(processed_string, "I have a .")
+        self.assertEqual(result, [(("cat", "dog"), ((9, 28),))])
+
+        extractor2 = make_extractor(r"(cat).*?(dog)", remove_only_matches=True)
+        result, processed_string = extractor2(string)
+        self.assertEqual(processed_string, "I have a  and another .")
+        self.assertEqual(result, [(("cat", "dog"), ((9, 12), (25, 28)))])
+
+        #TODO: maybe match output and spans together?

django_files/observatory/views.py

@@ -18,7 +18,6 @@
 import json
 from django.core import serializers
 from django.core.urlresolvers import reverse
-from elasticsearch import Elasticsearch
 # Project specific
 from django.utils.translation import gettext as _
 # App specific
@@ -2546,63 +2545,3 @@ def get_country_lookup():
     lookup[c.id] = [c.name_en, c.name_3char]
   return lookup
 
-def api_search(request):
-
-    query = request.GET.get("term", None)
-    if query == None:
-        return HttpResponse("[]")
-
-    span, years = helpers.extract_years(query)
-    if span is not None:
-        # Strip out year expression from query since elasticsearch doesn't
-        # contain year data
-        query = query[:span[0]] + query[span[1]:]
-
-    if years is None:
-        year_string = ""
-        year_url_param = ""
-    elif len(years) == 1:
-        year_string = " (%s)" % years[0]
-        year_url_param = "%s/" % years[0]
-    else:
-        year_string = " (%s to %s)" % (years[0], years[1])
-        year_url_param = "%s.%s/" % (years[0], years[1])
-
-    es = Elasticsearch()
-    result = es.search(
-        index="questions",
-        body={
-            "query": {
-                "filtered": {
-                    "query": {
-                        "fuzzy_like_this": {
-                            "like_text": query,
-                            "fields": ["title"],
-                            "fuzziness": 3,
-                            "max_query_terms": 15,
-                            "prefix_length": 4
-                        }
-                    }
-                }
-            },
-            # "highlight": {
-            #     "pre_tags": ["<div class=highlighted>"],
-            #     "fields": {"title": {}},
-            #     "post_tags": ["</div>"]
-            # },
-            "size": 8
-        })
-    result_list = []
-    for x in result['hits']['hits']:
-        label = x['_source']['title'] + year_string
-        url = x['_source']['url'] + year_url_param
-        # TODO: This is a hack, the correct way is to generate the url here
-        # instead of pregenerating it. See issue # 134
-        if years and len(years) > 1:
-            url = url.replace("tree_map", "stacked")
-        result_list.append(dict(label=label, value=url))
-    return HttpResponse(json.dumps(result_list))
-
-
-def search(request):
-    return render_to_response("test_search.html")