diff --git a/django_files/atlas/urls.py b/django_files/atlas/urls.py index 26dcbb24..a06da876 100644 --- a/django_files/atlas/urls.py +++ b/django_files/atlas/urls.py @@ -125,6 +125,9 @@ def render_to_response(self, context, **kwargs): (r'^api/near/(?P\w{3})/(?P[0-9\.]+)/(?P\d+)/$', 'observatory.views_exhibit.api_near'), + (r'^api/search/$', 'observatory.views.api_search'), + (r'^search/$', 'observatory.views.search'), + # Overview (Countries) ###################################################### (r'^country/(?P\w{2,3})/$', 'observatory.views_overview.country2'), (r'^hs4/(?P\d{4})/$', 'observatory.views_overview.product'), diff --git a/django_files/observatory/helpers.py b/django_files/observatory/helpers.py index a7301fc1..157d4e62 100644 --- a/django_files/observatory/helpers.py +++ b/django_files/observatory/helpers.py @@ -1,155 +1,241 @@ -from observatory.models import * -import random +from observatory.models import Hs4_cpy, Sitc4_cpy, Country, Hs4, Sitc4 + +import re + # make sure app name is in the list of possible apps def get_app_name(app_name): - possible_apps = ["tree_map", "stacked", "product_space", "map"] + possible_apps = ["tree_map", "stacked", "product_space", "map"] - # if the app_name requested is not in the list of possibilities - if app_name not in possible_apps: - app_name = None + # if the app_name requested is not in the list of possibilities + if app_name not in possible_apps: + app_name = None - return app_name + return app_name # make sure this is accepted trade_flow def get_trade_flow(trade_flow): - possible_yoga_flows = ["export", "import", "net_export", "net_import"] + possible_yoga_flows = ["export", "import", "net_export", "net_import"] - if trade_flow not in possible_yoga_flows: - trade_flow = None + if trade_flow not in possible_yoga_flows: + trade_flow = None - return trade_flow + return trade_flow def get_years(classification): - # get distince years from db, different for diff product classifications + # get distince years from db, different for diff product classifications - if classification == "sitc4": - years_available = list(Sitc4_cpy.objects.values_list("year", flat=True).distinct()) - elif classification == "hs4": - years_available = list(Hs4_cpy.objects.values_list("year", flat=True).distinct()) + if classification == "sitc4": + years_available = list( + Sitc4_cpy.objects.values_list( + "year", + flat=True).distinct()) + elif classification == "hs4": + years_available = list( + Hs4_cpy.objects.values_list( + "year", + flat=True).distinct()) - return years_available + return years_available # Returns app type in CCPY format def get_app_type(country1, country2, product, year): - # country / all / show / year - if country2 == "all" and product == "show": - return "casy" + # country / all / show / year + if country2 == "all" and product == "show": + return "casy" - # country / show / all / year - elif country2 == "show" and product == "all": - return "csay" + # country / show / all / year + elif country2 == "show" and product == "all": + return "csay" - # show / all / product / year - elif country1 == "show" and country2 == "all": - return "sapy" + # show / all / product / year + elif country1 == "show" and country2 == "all": + return "sapy" - # country / country / show / year - elif product == "show": - return "ccsy" + # country / country / show / year + elif product == "show": + return "ccsy" - # country / show / product / year - else: - return "cspy" + # country / show / product / year + else: + return "cspy" # Returns the Country object or None def get_country(country): - # first try looking up based on 3 character code - try: - c = Country.objects.get(name_3char=country) - except Country.DoesNotExist: - # next try 2 character code + # first try looking up based on 3 character code try: - c = Country.objects.get(name_2char=country) + c = Country.objects.get(name_3char=country) except Country.DoesNotExist: - c = None - return c + # next try 2 character code + try: + c = Country.objects.get(name_2char=country) + except Country.DoesNotExist: + c = None + return c # Returns the Product object or None def get_product(product, classification): - # first try looking up based on 3 character code - if classification == "hs4": - try: - p = Hs4.objects.get(code=product) - except Hs4.DoesNotExist: - # next try SITC4 - try: - conv_code = Sitc4.objects.get(code=product).conversion_code - p = Hs4.objects.get(code=conv_code) - except Sitc4.DoesNotExist: - p = None - else: - try: - p = Sitc4.objects.get(code=product) - except Sitc4.DoesNotExist: - # next try SITC4 - try: - conv_code = Hs4.objects.get(code=product).conversion_code - p = Sitc4.objects.get(code=conv_code) - except Hs4.DoesNotExist: - p = None - return p - - -# Returns the question to display on the page describing the URL requested -def get_question(app_type, **kwargs): - - trade_flow = kwargs["trade_flow"] - - if app_type == "casy": - origin = kwargs["origin"] - - if trade_flow in ['net_export','net_import']: - title = "What does %s %s in net terms?" % (origin.name, trade_flow.replace("_", " ").split()[1]) + # first try looking up based on 3 character code + if classification == "hs4": + try: + p = Hs4.objects.get(code=product) + except Hs4.DoesNotExist: + # next try SITC4 + try: + conv_code = Sitc4.objects.get(code=product).conversion_code + p = Hs4.objects.get(code=conv_code) + except Sitc4.DoesNotExist: + p = None else: - title = "What does %s %s?" % (origin.name, trade_flow.replace("_", " ")) - - # Country but showing other country trade partners - elif app_type == "csay": - origin = kwargs["origin"] - - article = "to" if trade_flow in ["export","net_export"] else "from" - if trade_flow in ['net_export','net_import']: - title = "Where does %s %s %s in net terms? " % (origin.name, trade_flow.replace("_", " ").split()[1], article) + try: + p = Sitc4.objects.get(code=product) + except Sitc4.DoesNotExist: + # next try SITC4 + try: + conv_code = Hs4.objects.get(code=product).conversion_code + p = Sitc4.objects.get(code=conv_code) + except Hs4.DoesNotExist: + p = None + return p + + +def get_time_clause(years): + """Generate a string like 'between 2005 and 2007' or 'in 2011' from a list + of years. Beginning space is included to allow for empty time clause.""" + if years is None or len(years) == 0: + return "" + elif len(years) == 1: + return " in %d" % years[0] else: - title = "Where does %s %s %s?" % (origin.name, trade_flow.replace("_", " "), article) + return " between %d and %d" % (years[0], years[1]) + + +def get_title(api_name, app_name, country_names=None, trade_flow=None, + years=None, product_name=None): + """Fetch the natural-languageized title of a page based on the data being + displayed. + + :param api_name: One of: casy, cspy, csay, ccsy, sapy + :param app_name: E.g. pie_scatter, stacked, product_space, rings ... + :param list country_names: List of country name strings. If multiple, first + is "from" country and second is the "to" country. + :param str trade_flow: import, export, net_import, net_export + :param list years: List of years. If multiple, first is the start year and + second is the end year. + :param str product_name: Localized name of product + """ + + # e.g. What did Burundi export in 2013? Which products are feasible for + # Latvia? + if api_name == "casy": + if app_name == "pie_scatter": + return "Which products are feasible for %s?" % country_names[0] + else: + return "What did %s %s%s?" % (country_names[0], + trade_flow, + get_time_clause(years)) + + # e.g. Where did Albania export to in 2009? + elif api_name == "csay": + article = "to" if trade_flow == "export" else "from" + return "Where did %s %s %s%s?" % (country_names[0], + trade_flow, + article, + get_time_clause(years)) + + # e.g. Who exported Petroleum in 1990? + elif api_name == "sapy": + return "Who %sed %s%s?" % (trade_flow, product_name, + get_time_clause(years)) + + # e.g. What did Germany import from Turkey in 2011? + elif api_name == "ccsy": + article = "to" if trade_flow == "export" else "from" + return "What did %s %s %s %s%s?" % (country_names[0], trade_flow, + article, country_names[1], + get_time_clause(years)) + + # e.g. Where did France export wine to in 2012? + elif api_name == "cspy": + article = "to" if trade_flow == "export" else "from" + return "Where did %s %s %s %s%s?" % (country_names[0], trade_flow, + product_name, article, + get_time_clause(years)) - # Product - elif app_type == "sapy": - product = kwargs["product"] + else: + raise ValueError("Unknown API name when trying to generate title: %s" % + api_name) + + +def params_to_url(api_name=None, app_name=None, country_codes=None, + trade_flow=None, years=None, product_code=None): + """Generate explore/ urls from specific parameters. Same parameter syntax + as get_title, but product code instead of product name and 3 letter country + codees instead of country names.""" + + if app_name is None: + # Treemap is a safe default that works with almost all of our data + app_name = 'tree_map' + + if api_name == 'casy': + # What did Germany import in 2012? + # Which products are feasible for Latvia? + # Looks like explore/tree_map/import/deu/all/show/2012/ + country_codes.append('all') + product_code = "show" + + elif api_name == 'cspy': + # Where did Germany import Swine from in 2012? + # Looks like explore/tree_map/import/deu/show/0103/2012/ + country_codes.append('show') + + elif api_name == 'csay': + # Where does germany import from? + # Looks like explore/tree_map/import/deu/show/all/2012/ + country_codes.append('show') + product_code = 'all' + + elif api_name == 'ccsy': + # What did Germany import from Congo in 2012? + # Looks like explore/tree_map/import/deu/cog/show/2012/ + product_code = 'show' + + elif api_name == 'sapy': + # Who exports potatoes? + # Looks like explore/tree_map/export/show/all/0101/2012/ + country_codes = ("show", "all") - if trade_flow in ['net_export','net_import']: - title = "Who %ss %s in net terms?" % (trade_flow.replace("_", " ").split()[1], product.name_en) else: - title = "Who %ss %s?" % (trade_flow.replace("_", " "), product.name_en) + raise ValueError("Unknown API name : %s" % api_name) - # Bilateral Country x Country - elif app_type == "ccsy": - origin = kwargs["origin"] - destination = kwargs["destination"] + url = "explore/%s/%s/%s/%s/%s/" % (app_name, trade_flow, country_codes[0], + country_codes[1], product_code) + if years is not None: + url += "%s/" % years - article = "to" if trade_flow in ["export","net_export"] else "from" - if trade_flow in ['net_export','net_import']: - title = "What does %s %s %s %s in net terms?" % (origin.name, trade_flow.replace("_", " ").split()[1], article, destination.name) - else: - title = "What does %s %s %s %s?" % (origin.name, trade_flow, article, destination.name) + return url - # Bilateral Country / Show / Product / Year - elif app_type == "cspy": - origin = kwargs["origin"] - product = kwargs["product"] +year_expressions = [ + re.compile(r'between (\d{4}) and (\d{4})', re.IGNORECASE), + re.compile(r'from (\d{4}) to (\d{4})', re.IGNORECASE), + re.compile(r'(\d{4}).*(\d{4})'), + re.compile(r'(?:in|at|during) (\d{4})', re.IGNORECASE), + re.compile(r'(\d{4})') +] - article = "to" if trade_flow in ["export","net_export"] else "from" - if trade_flow in ['net_export','net_import']: - title = "Where does %s %s %s %s in net terms?" % (origin.name, trade_flow.replace("_", " ").split()[1], product.name_en, article) - else: - title = "Where does %s %s %s %s?" % (origin.name, trade_flow, product.name_en, article) - return title +def extract_years(input_str): + """Extract things that look like years out of a given plaintext.""" + results = (exp.search(input_str) for exp in year_expressions) + results = [result for result in results if result is not None] + if len(results) == 0: + return None, None + else: + return results[0].span(), results[0].groups() diff --git a/django_files/observatory/management/commands/index_questions.py b/django_files/observatory/management/commands/index_questions.py new file mode 100644 index 00000000..aad12e87 --- /dev/null +++ b/django_files/observatory/management/commands/index_questions.py @@ -0,0 +1,159 @@ +from django.core.management.base import BaseCommand +from elasticsearch import Elasticsearch +from elasticsearch.helpers import streaming_bulk + +from observatory.models import Country, Hs4 +from observatory.helpers import get_title, params_to_url + +import itertools + + +class Command(BaseCommand): + help = 'Generate elasticsearch indices for questions in homepage' + + def handle(self, *args, **options): + + # TODO: "the" in country names + trade_flows = ["import", "export"] + countries_flat = list(Country.objects.get_valid().only('name_en', + 'name_3char')) + countries = [[c] for c in countries_flat] + products = list(Hs4.objects.get_low_level().only('name_en', 'code')) + + # Which products are feasible for Latvia? + casy_questions = self.generate_index_entries(['casy'], ['pie_scatter'], + countries, ['export']) + + # What did Burundi export in 2013? + casy_questions2 = self.generate_index_entries(['casy'], [None], + countries, trade_flows, + [None]) + + # Where did Albania export to in 2009? + csay_questions = self.generate_index_entries(['csay'], [None], + countries, trade_flows, + [None]) + + # Who exported Petroleum in 1990? + sapy_questions = self.generate_index_entries(['sapy'], [None], [None], + trade_flows, [None], + products) + + # What did Germany import from Turkey in 2011? + country_pairs = itertools.ifilter( + # Germany importing from Germany etc makes no sense + lambda country_pair: country_pair[0].id != country_pair[1].id, + itertools.product( + countries_flat, + countries_flat), + ) + ccsy_questions = self.generate_index_entries(['ccsy'], [None], + country_pairs, + trade_flows, [None]) + + # Where did France export wine to in 2012? + cspy_questions = self.generate_index_entries(['cspy'], [None], + countries, trade_flows, + [None], products) + + all_questions = itertools.chain(casy_questions, casy_questions2, + csay_questions, sapy_questions, + ccsy_questions, cspy_questions) + + all_questions = enumerate(all_questions) + + es = Elasticsearch() + for ok, result in streaming_bulk( + es, + all_questions, + index='questions', + expand_action_callback=self.convert_to_elasticsearch_command + ): + if not ok: + print 'Failed: %s' % result + + @staticmethod + def convert_to_elasticsearch_command(data): + """Convert the question list into an elasticsearch `index` command + consumable by the elasticsearch bulk api.""" + doc_id = data[0] + doc_body = data[1] + + action = {'index': + {'_id': doc_id, + '_index': 'questions', + '_type': 'question', + }} + return action, doc_body + + @staticmethod + def generate_index_entries(api_name=[None], app_name=[None], + countries=[None], trade_flow=[None], + years=[None], product=[None]): + """Given a list of possible parameters for the get_title() and + params_to_url() functions, generate index entries for all permutations + of titles. Parameters must be in order for get_title(). Each parameter + should contain a list of possibilities. For example: api_name=["casy", + "ccsy"] or countries = [["USA", "DEU"], ["USA", "TUR"]]. + + :return: An iterator yielding titles and arguments e.g: + [{'title': 'foo', 'url':'qux', 'country_name': 'Germany'}, + {'title': 'bar', 'url':'quux', 'country_name': 'Sweden'}] + """ + + def generate_index_entry(args): + + index = {} + + # Generate title + country_names = None + if args[2] is not None: + country_names = [c.name_en for c in args[2]] + + title = get_title( + api_name=args[0], + app_name=args[1], + country_names=country_names, + trade_flow=args[3], + years=args[4], + product_name=args[5].name_en if args[5] is not None else None + ) + index["title"] = title + + # Generate url + country_codes = None + if args[2] is not None: + country_codes = [c.name_3char.lower() for c in args[2]] + + url = params_to_url( + api_name=args[0], + app_name=args[1], + country_codes=country_codes, + trade_flow=args[3], + years=args[4], + product_code=args[5].code if args[5] is not None else None + ) + index["url"] = url + + # Add in params into elasticsearch in case we need them later + kwargs = dict( + api_name=args[0], + app_name=args[1], + country_names=country_names, + country_codes=country_codes, + trade_flow=args[3], + years=args[4], + product_name=args[5].name_en if args[5] is not None else None, + product_code=args[5].code if args[5] is not None else None + ) + kwargs = {k: v for k, v in kwargs.iteritems() if v is not None} + index.update(kwargs) + + return index + + parameter_possibilities = [api_name, app_name, countries, trade_flow, + years, product] + return itertools.imap( + generate_index_entry, + itertools.product(*parameter_possibilities) + ) diff --git a/django_files/observatory/models.py b/django_files/observatory/models.py index ace05494..f784a42f 100644 --- a/django_files/observatory/models.py +++ b/django_files/observatory/models.py @@ -132,15 +132,18 @@ def get_all(self, lang): "region__text_color", )) + def get_valid(self): + return self.filter( + name_3char__isnull=False, + name_2char__isnull=False, + region__isnull=False + ) + def get_random(self): - """Grab a random country that has a 3char name. This uses the 'ORDER BY - RAND()' method which is fine for this purpose but slow in mysql for - larger tables so beware.""" - return Country.objects.filter( - name_3char__isnull=False, - name_2char__isnull=False, - region__isnull=False - ).order_by('?')[1] + """Grab a random country. This uses the 'ORDER BY RAND()' method which + is fine for this purpose but slow in mysql for larger tables so + beware.""" + return self.get_valid().order_by('?')[1] class Country(models.Model): @@ -371,6 +374,13 @@ def get_all(self, lang): "ps_y", "ps_size" )) + + def get_low_level(self): + """Only get low level, detailed products, and don't get the high level + aggregate categories like: Foodstuffs or Beverages, Spirits & + Vinegar.""" + return self.filter(id__lte=1241) + # HS4 Products class Hs4(models.Model): diff --git a/django_files/observatory/views.py b/django_files/observatory/views.py index 4553a4f1..9da91d6a 100644 --- a/django_files/observatory/views.py +++ b/django_files/observatory/views.py @@ -18,11 +18,13 @@ import json from django.core import serializers from django.core.urlresolvers import reverse +from elasticsearch import Elasticsearch # Project specific from django.utils.translation import gettext as _ # App specific from observatory.models import * from observatory.models import storychapter +from observatory import helpers from django.db.models import Max from django.forms import ModelForm import msgpack @@ -1217,18 +1219,17 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 if app_name == "stacked" and year == "2009": year = "1969.2011.10" + if "." in year: y = [int(x) for x in year.split(".")] - # year = range(y[0], y[1]+1, y[2]) year_start = y[0] year_end = y[1] - year_interval = y[2] year2_list = year1_list year_interval_list = range(1, 11) - # year_interval = year[1] - year[0] else: - year_start, year_end, year_interval = None, None, None + year_start, year_end = None, None year = int(year) + # Check that year is within bounds if year > years_available[len(years_available)-1]: year = years_available[len(years_available)-1] elif year < years_available[0]: @@ -1251,86 +1252,53 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 data_as_text["columns"] = view_response[2] - app_type = get_app_type(country1, country2, product, year) - - if product != "show" and product != "all": - p_code = product - product = clean_product(p_code, prod_class) - # TODO: check if product exists - #if product: - # if product.__class__ == Sitc4: - # product_list = Sitc4.objects.get_all(lang) - # request.session['product_classification'] = "sitc4" - # else: - # product_list = Hs4.objects.get_all(lang) - # request.session['product_classification'] = "hs4" - #else: - # alert = {"title": "Product could not be found", "text": "There was no product with the 4 digit code %s. Please double check the list of HS4 products."%(p_code)} - - - list_countries_the = ["Cayman Islands", "Central African Republic", "Channel Islands", "Congo, Dem. Rep.", "Czech Republic", "Dominican Republic", "Faeroe Islands", "Falkland Islands", "Fm Yemen Dm", "Lao PDR", "Marshall Islands", "Philippines", "Seychelles", "Slovak Republic", "Syrian Arab Republic", "Turks and Caicos Islands", "United Arab Emirates", "United Kingdom", "Virgin Islands, U.S.", "United States"] + app_type = helpers.get_app_type(country1, country2, product, year) + # Some countries need "the" before their names + list_countries_the = set(("Cayman Islands", "Central African Republic", + "Channel Islands", "Congo, Dem. Rep.", + "Czech Republic", "Dominican Republic", + "Faeroe Islands", "Falkland Islands", "Fm Yemen Dm", + "Lao PDR", "Marshall Islands", "Philippines", + "Seychelles", "Slovak Republic", + "Syrian Arab Republic", "Turks and Caicos Islands", + "United Arab Emirates", "United Kingdom", + "Virgin Islands, U.S.", "United States")) if countries[0] and countries[0].name in list_countries_the: countries[0].name = "the "+countries[0].name - prod_or_partner = "partner" # quick fix should be merged with item_type + #p_code, product = None, None + if product not in ("show", "all"): + p_code = product + product = clean_product(p_code, prod_class) if not alert: - if app_type == "casy": - # raise Exception(app_name) - if app_name == "pie_scatter": - title = "Which products are feasible for %s?" % countries[0].name - elif (app_name == "product_space" or app_name == "rings"): - title = "What did %s export in %s?" % (countries[0].name, year) # INSERTED NEW TITLE HERE - elif app_name == "stacked": - title = "What did %s %s between %s and %s?" % (countries[0].name, trade_flow.replace("_", " "), year_start, year_end) # NEW TITLE HERE - prod_or_partner = "product" - else: - title = "What did %s %s in %s?" % (countries[0].name, trade_flow.replace("_", " "), year) # NEW TITLE HERE - prod_or_partner = "product" - # Country but showing other country trade partners - elif app_type == "csay": - item_type = "countries" - article = "to" if trade_flow == "export" else "from" - if app_name == "stacked": - title = "Where did %s %s %s between %s and %s?" % (countries[0].name, trade_flow.replace("_", " "), article, year_start, year_end) - else: - title = "Where did %s %s %s in %s?" % (countries[0].name, trade_flow.replace("_", " "), article, year) - - # Product - elif app_type == "sapy": - item_type = "countries" - if app_name == "stacked": - title = "Who %sed %s between %s and %s?" % (trade_flow.replace("_", " "), product.name_en, year_start, year_end) - else: - title = "Who %sed %s in %s?" % (trade_flow.replace("_", " "), product.name_en, year) - - prod_or_partner = "product" - - # Bilateral Country x Country - elif app_type == "ccsy": - # trade_flow_list = ["export", "import"] + # Generate page title depending on visualization being used + trade_flow = trade_flow.replace('_', ' ') + years = [year_start, year_end] if year_start is not None else [year] + product_name = product.name_en if not isinstance(product, basestring) else product + country_names = [getattr(x, "name", None) for x in countries] + title = helpers.get_title(app_type, app_name, + country_names=country_names, + trade_flow=trade_flow, + years=years, + product_name=product_name + ) + + if app_type in ("ccsy", "cspy"): if _("net_export") in trade_flow_list: del trade_flow_list[trade_flow_list.index(_("net_export"))] if _("net_import") in trade_flow_list: del trade_flow_list[trade_flow_list.index(_("net_import"))] - article = "to" if trade_flow == "export" else "from" - if app_name == "stacked": - title = "What did %s %s %s %s between %s and %s?" % (countries[0].name, trade_flow.replace("_", " "), article, countries[1].name, year_start, year_end) - else: - title = "What did %s %s %s %s in %s?" % (countries[0].name, trade_flow.replace("_", " "), article, countries[1].name, year) - - # Bilateral Country / Show / Product / Year - elif app_type == "cspy": - if "net_export" in trade_flow_list: del trade_flow_list[trade_flow_list.index("net_export")] - if "net_import" in trade_flow_list: del trade_flow_list[trade_flow_list.index("net_import")] - item_type = "countries" - article = "to" if trade_flow == "export" else "from" - if app_name == "stacked": - title = "Where did %s %s %s %s between %s and %s?" % (countries[0].name, trade_flow.replace("_", " "), product.name_en, article, year_start, year_end) - else: - title = "Where did %s %s %s %s in %s?" % (countries[0].name, trade_flow.replace("_", " "), product.name_en, article, year) + #trade_flow_list.pop(_("net_export"), None) + # Should we show the product or partner tab pane? + prod_or_partner = "partner" # quick fix should be merged with item_type + if app_type in ["cspy", "sapy"]: prod_or_partner = "product" + elif app_type == "casy": + if app_name in ("stacked", "map", "tree_map"): + prod_or_partner = "product" + # Return page without visualization data @@ -1372,25 +1340,24 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 "warning": warning, "alert": alert, "prod_class": prod_class, - "years_available": years_available, "data_as_text": data_as_text, "app_name": app_name, - "title": title,#get_question(app_type, trade_flow=trade_flow,origin=countries[0],destination=countries[1],product=product), + "title": title, "trade_flow": trade_flow, "country1": countries[0] or country1, "country2": countries[1] or country2, "product": product, + "years_available": years_available, "year": year, "year_start": year_start, "year_end": year_end, - "year_interval": year_interval, + "year1_list": year1_list, + "year2_list": year2_list, + "year_interval_list": year_interval_list, "trade_flow_list": trade_flow_list, "country1_list": country_lists[0], "country2_list": country_lists[1], "product_list": product_list, - "year1_list": year1_list, - "year2_list": year2_list, - "year_interval_list": year_interval_list, "api_uri": api_uri, "app_type": app_type, "redesign_api_uri": redesign_api_uri, @@ -2579,23 +2546,63 @@ def get_country_lookup(): lookup[c.id] = [c.name_en, c.name_3char] return lookup -def get_app_type(country1, country2, product, year): - # country / all / show / year - if country2 == "all" and product == "show": - return "casy" - - # country / show / all / year - elif country2 == "show" and product == "all": - return "csay" +def api_search(request): - # show / all / product / year - elif country1 == "show" and country2 == "all": - return "sapy" + query = request.GET.get("term", None) + if query == None: + return HttpResponse("[]") - # country / country / show / year - elif product == "show": - return "ccsy" + span, years = helpers.extract_years(query) + if span is not None: + # Strip out year expression from query since elasticsearch doesn't + # contain year data + query = query[:span[0]] + query[span[1]:] - # country / show / product / year - else: - return "cspy" + if years is None: + year_string = "" + year_url_param = "" + elif len(years) == 1: + year_string = " (%s)" % years[0] + year_url_param = "%s/" % years[0] + else: + year_string = " (%s to %s)" % (years[0], years[1]) + year_url_param = "%s.%s/" % (years[0], years[1]) + + es = Elasticsearch() + result = es.search( + index="questions", + body={ + "query": { + "filtered": { + "query": { + "fuzzy_like_this": { + "like_text": query, + "fields": ["title"], + "fuzziness": 3, + "max_query_terms": 15, + "prefix_length": 4 + } + } + } + }, + # "highlight": { + # "pre_tags": ["
"], + # "fields": {"title": {}}, + # "post_tags": ["
"] + # }, + "size": 8 + }) + result_list = [] + for x in result['hits']['hits']: + label = x['_source']['title'] + year_string + url = x['_source']['url'] + year_url_param + # TODO: This is a hack, the correct way is to generate the url here + # instead of pregenerating it. See issue # 134 + if years and len(years) > 1: + url = url.replace("tree_map", "stacked") + result_list.append(dict(label=label, value=url)) + return HttpResponse(json.dumps(result_list)) + + +def search(request): + return render_to_response("test_search.html") diff --git a/html/test_search.html b/html/test_search.html new file mode 100644 index 00000000..08b61455 --- /dev/null +++ b/html/test_search.html @@ -0,0 +1,62 @@ + + + + +jQuery UI Autocomplete - Remote with caching + + + + + + + + +
+ + +
+ + diff --git a/requirements.txt b/requirements.txt index 7fc2e1ba..ec516170 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,5 @@ Django==1.4.5 MySQL-python numpy msgpack-python -pycrypto==2.6 \ No newline at end of file +pycrypto==2.6 +elasticsearch>=1.0.0,<2.0.0