From 7969ff7eda462d91fcc4d7d77c42c79b01964379 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Fri, 25 Apr 2014 11:44:22 -0400 Subject: [PATCH 01/52] Remove junk --- django_files/observatory/views.py | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/django_files/observatory/views.py b/django_files/observatory/views.py index affec991..5c3f3c1e 100644 --- a/django_files/observatory/views.py +++ b/django_files/observatory/views.py @@ -1198,21 +1198,6 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 app_type = get_app_type(country1, country2, product, year) - if product != "show" and product != "all": - p_code = product - product = clean_product(p_code, prod_class) - # TODO: check if product exists - #if product: - # if product.__class__ == Sitc4: - # product_list = Sitc4.objects.get_all(lang) - # request.session['product_classification'] = "sitc4" - # else: - # product_list = Hs4.objects.get_all(lang) - # request.session['product_classification'] = "hs4" - #else: - # alert = {"title": "Product could not be found", "text": "There was no product with the 4 digit code %s. Please double check the list of HS4 products."%(p_code)} - - list_countries_the = ["Cayman Islands", "Central African Republic", "Channel Islands", "Congo, Dem. Rep.", "Czech Republic", "Dominican Republic", "Faeroe Islands", "Falkland Islands", "Fm Yemen Dm", "Lao PDR", "Marshall Islands", "Philippines", "Seychelles", "Slovak Republic", "Syrian Arab Republic", "Turks and Caicos Islands", "United Arab Emirates", "United Kingdom", "Virgin Islands, U.S.", "United States"] if countries[0] and countries[0].name in list_countries_the: @@ -1220,6 +1205,10 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 prod_or_partner = "partner" # quick fix should be merged with item_type + if product not in ("show", "all"): + p_code = product + product = clean_product(p_code, prod_class) + if not alert: if app_type == "casy": # raise Exception(app_name) From 73ed4e0d570963d13a0328f77394404913c82046 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Fri, 25 Apr 2014 11:44:50 -0400 Subject: [PATCH 02/52] New title generation function. --- django_files/observatory/helpers.py | 74 +++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/django_files/observatory/helpers.py b/django_files/observatory/helpers.py index a7301fc1..03d51f29 100644 --- a/django_files/observatory/helpers.py +++ b/django_files/observatory/helpers.py @@ -153,3 +153,77 @@ def get_question(app_type, **kwargs): return title + +def get_title(app_name, api_name, country_names=None, trade_flow=None, years=None, product_name=None): + """ + Fetch the natural-languageized title of a page based on the data being + displayed. + + :param app_name: E.g. pie_scatter, stacked, product_space, rings ... + :param api_name: One of: casy, cspy, csay, ccsy, sapy + :param list country_names: List of country name strings. If multiple, first + is "from" country and second is the "to" country. + :param str trade_flow: import, export, net_import, net_export + :param list years: List of years. If multiple, first is the start year and + second is the end year. + :param str product_name: Localized name of product + """ + + # e.g. What did Burundi export in 2013? Which products are feasible for + # Latvia? + if api_name == "casy": + if app_name == "pie_scatter": + return "Which products are feasible for %s?" % country_names[0] + elif (app_name == "product_space" or app_name == "rings"): + return "What did %s export in %s?" % (country_names[0], years[0]) + elif app_name == "stacked": + return "What did %s %s between %s and %s?" % (country_names[0], + trade_flow, years[0], + years[1]) + else: + return "What did %s %s in %s?" % (country_names[0], trade_flow, year) + + # e.g. Where did Albania export to in 2009? + elif app_type == "csay": + article = "to" if trade_flow == "export" else "from" + if app_name == "stacked": + return "Where did %s %s %s between %s and %s?" % (country_names[0], + trade_flow, + article, + years[0], + years[1]) + else: + return "Where did %s %s %s in %s?" % (country_names[0], trade_flow, + article, years[0]) + + # e.g. Who exported Petroleum in 1990? + elif app_type == "sapy": + if app_name == "stacked": + return "Who %sed %s between %s and %s?" % (trade_flow, product_name, + years[0], years[1]) + else: + return "Who %sed %s in %s?" % (trade_flow, product_name, years[0]) + + # e.g. What did Germany import from Turkey in 2011? + elif app_type == "ccsy": + article = "to" if trade_flow == "export" else "from" + if app_name == "stacked": + return "What did %s %s %s %s between %s and %s?" % \ + (country_names[0], trade_flow, article, country_names[1], + years[0], years[1]) + else: + return "What did %s %s %s %s in %s?" % (country_names[0], + trade_flow, article, + country_names[1], year) + + # e.g. Where did France export wine to in 2012? + elif app_type == "cspy": + article = "to" if trade_flow == "export" else "from" + if app_name == "stacked": + return "Where did %s %s %s %s between %s and %s?" % \ + (country_names[0], trade_flow, product_name, article, years[0], + years[1]) + else: + return "Where did %s %s %s %s in %s?" % (country_names[0], + trade_flow, product_name, + article, year) From d3ee517e01dd16a1f9693d73627a77ba98bd6920 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Fri, 25 Apr 2014 11:53:02 -0400 Subject: [PATCH 03/52] Fix line length --- django_files/observatory/helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django_files/observatory/helpers.py b/django_files/observatory/helpers.py index 03d51f29..57a4efe9 100644 --- a/django_files/observatory/helpers.py +++ b/django_files/observatory/helpers.py @@ -153,8 +153,8 @@ def get_question(app_type, **kwargs): return title - -def get_title(app_name, api_name, country_names=None, trade_flow=None, years=None, product_name=None): +def get_title(app_name, api_name, country_names=None, trade_flow=None, + years=None, product_name=None): """ Fetch the natural-languageized title of a page based on the data being displayed. From 299d3d26a853f1a063ee681c70b1bff4fa38fc57 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Fri, 25 Apr 2014 15:52:38 -0400 Subject: [PATCH 04/52] Refactor time clause generation out of title code. --- django_files/observatory/helpers.py | 65 ++++++++++++----------------- 1 file changed, 26 insertions(+), 39 deletions(-) diff --git a/django_files/observatory/helpers.py b/django_files/observatory/helpers.py index 57a4efe9..c0297d09 100644 --- a/django_files/observatory/helpers.py +++ b/django_files/observatory/helpers.py @@ -153,10 +153,19 @@ def get_question(app_type, **kwargs): return title +def get_time_clause(years): + """Generate a string like 'between 2005 and 2007' or 'in 2011' from a list + of years.""" + if len(years) == 0: + return "" + elif len(years) == 1: + return "in %d" % years[0] + else: + return "between %d and %d" + def get_title(app_name, api_name, country_names=None, trade_flow=None, years=None, product_name=None): - """ - Fetch the natural-languageized title of a page based on the data being + """Fetch the natural-languageized title of a page based on the data being displayed. :param app_name: E.g. pie_scatter, stacked, product_space, rings ... @@ -174,56 +183,34 @@ def get_title(app_name, api_name, country_names=None, trade_flow=None, if api_name == "casy": if app_name == "pie_scatter": return "Which products are feasible for %s?" % country_names[0] - elif (app_name == "product_space" or app_name == "rings"): - return "What did %s export in %s?" % (country_names[0], years[0]) - elif app_name == "stacked": - return "What did %s %s between %s and %s?" % (country_names[0], - trade_flow, years[0], - years[1]) else: - return "What did %s %s in %s?" % (country_names[0], trade_flow, year) + return "What did %s %s %s?" % (country_names[0], + trade_flow, + get_time_clause(years)) # e.g. Where did Albania export to in 2009? elif app_type == "csay": article = "to" if trade_flow == "export" else "from" - if app_name == "stacked": - return "Where did %s %s %s between %s and %s?" % (country_names[0], - trade_flow, - article, - years[0], - years[1]) - else: - return "Where did %s %s %s in %s?" % (country_names[0], trade_flow, - article, years[0]) + return "Where did %s %s %s %s?" % (country_names[0], + trade_flow, + article, + get_time_clause(years)) # e.g. Who exported Petroleum in 1990? elif app_type == "sapy": - if app_name == "stacked": - return "Who %sed %s between %s and %s?" % (trade_flow, product_name, - years[0], years[1]) - else: - return "Who %sed %s in %s?" % (trade_flow, product_name, years[0]) + return "Who %sed %s %s?" % (trade_flow, product_name, + get_time_clause(years)) # e.g. What did Germany import from Turkey in 2011? elif app_type == "ccsy": article = "to" if trade_flow == "export" else "from" - if app_name == "stacked": - return "What did %s %s %s %s between %s and %s?" % \ - (country_names[0], trade_flow, article, country_names[1], - years[0], years[1]) - else: - return "What did %s %s %s %s in %s?" % (country_names[0], - trade_flow, article, - country_names[1], year) + return "What did %s %s %s %s %s?" % (country_names[0], trade_flow, + article, country_names[1], + get_time_clause(years)) # e.g. Where did France export wine to in 2012? elif app_type == "cspy": article = "to" if trade_flow == "export" else "from" - if app_name == "stacked": - return "Where did %s %s %s %s between %s and %s?" % \ - (country_names[0], trade_flow, product_name, article, years[0], - years[1]) - else: - return "Where did %s %s %s %s in %s?" % (country_names[0], - trade_flow, product_name, - article, year) + return "Where did %s %s %s %s %s?" % (country_names[0], trade_flow, + product_name, article, + get_time_clause(years)) From d2b9b3b75e26494d8f8b7b3866c079fdb8ac2db7 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Fri, 25 Apr 2014 17:42:26 -0400 Subject: [PATCH 05/52] Fix some formatting --- django_files/observatory/views.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/django_files/observatory/views.py b/django_files/observatory/views.py index 5c3f3c1e..179fc861 100644 --- a/django_files/observatory/views.py +++ b/django_files/observatory/views.py @@ -1198,8 +1198,16 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 app_type = get_app_type(country1, country2, product, year) - list_countries_the = ["Cayman Islands", "Central African Republic", "Channel Islands", "Congo, Dem. Rep.", "Czech Republic", "Dominican Republic", "Faeroe Islands", "Falkland Islands", "Fm Yemen Dm", "Lao PDR", "Marshall Islands", "Philippines", "Seychelles", "Slovak Republic", "Syrian Arab Republic", "Turks and Caicos Islands", "United Arab Emirates", "United Kingdom", "Virgin Islands, U.S.", "United States"] - + # Some countries need "the" before their names + list_countries_the = ["Cayman Islands", "Central African Republic", + "Channel Islands", "Congo, Dem. Rep.", + "Czech Republic", "Dominican Republic", + "Faeroe Islands", "Falkland Islands", "Fm Yemen Dm", + "Lao PDR", "Marshall Islands", "Philippines", + "Seychelles", "Slovak Republic", + "Syrian Arab Republic", "Turks and Caicos Islands", + "United Arab Emirates", "United Kingdom", + "Virgin Islands, U.S.", "United States"] if countries[0] and countries[0].name in list_countries_the: countries[0].name = "the "+countries[0].name From 0899c2908347150242cfbcbfc21be475450bc4a8 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Fri, 25 Apr 2014 17:54:34 -0400 Subject: [PATCH 06/52] Typo app_type -> api_name --- django_files/observatory/helpers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/django_files/observatory/helpers.py b/django_files/observatory/helpers.py index c0297d09..da271c7f 100644 --- a/django_files/observatory/helpers.py +++ b/django_files/observatory/helpers.py @@ -189,7 +189,7 @@ def get_title(app_name, api_name, country_names=None, trade_flow=None, get_time_clause(years)) # e.g. Where did Albania export to in 2009? - elif app_type == "csay": + elif api_name == "csay": article = "to" if trade_flow == "export" else "from" return "Where did %s %s %s %s?" % (country_names[0], trade_flow, @@ -197,19 +197,19 @@ def get_title(app_name, api_name, country_names=None, trade_flow=None, get_time_clause(years)) # e.g. Who exported Petroleum in 1990? - elif app_type == "sapy": + elif api_name == "sapy": return "Who %sed %s %s?" % (trade_flow, product_name, get_time_clause(years)) # e.g. What did Germany import from Turkey in 2011? - elif app_type == "ccsy": + elif api_name == "ccsy": article = "to" if trade_flow == "export" else "from" return "What did %s %s %s %s %s?" % (country_names[0], trade_flow, article, country_names[1], get_time_clause(years)) # e.g. Where did France export wine to in 2012? - elif app_type == "cspy": + elif api_name == "cspy": article = "to" if trade_flow == "export" else "from" return "Where did %s %s %s %s %s?" % (country_names[0], trade_flow, product_name, article, From bf616ab370cb8a9aca55b4f487cae02a1295ba3a Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Fri, 25 Apr 2014 17:56:59 -0400 Subject: [PATCH 07/52] Make list of "the" countries a set --- django_files/observatory/views.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/django_files/observatory/views.py b/django_files/observatory/views.py index 179fc861..427acf83 100644 --- a/django_files/observatory/views.py +++ b/django_files/observatory/views.py @@ -1199,15 +1199,15 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 app_type = get_app_type(country1, country2, product, year) # Some countries need "the" before their names - list_countries_the = ["Cayman Islands", "Central African Republic", - "Channel Islands", "Congo, Dem. Rep.", - "Czech Republic", "Dominican Republic", - "Faeroe Islands", "Falkland Islands", "Fm Yemen Dm", - "Lao PDR", "Marshall Islands", "Philippines", - "Seychelles", "Slovak Republic", - "Syrian Arab Republic", "Turks and Caicos Islands", - "United Arab Emirates", "United Kingdom", - "Virgin Islands, U.S.", "United States"] + list_countries_the = set(("Cayman Islands", "Central African Republic", + "Channel Islands", "Congo, Dem. Rep.", + "Czech Republic", "Dominican Republic", + "Faeroe Islands", "Falkland Islands", "Fm Yemen Dm", + "Lao PDR", "Marshall Islands", "Philippines", + "Seychelles", "Slovak Republic", + "Syrian Arab Republic", "Turks and Caicos Islands", + "United Arab Emirates", "United Kingdom", + "Virgin Islands, U.S.", "United States")) if countries[0] and countries[0].name in list_countries_the: countries[0].name = "the "+countries[0].name From c6ae836dfb985578ee0c3e963d0e1a7b3d2decee Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Fri, 25 Apr 2014 17:57:56 -0400 Subject: [PATCH 08/52] Initial attempt at untangling title generation code. --- django_files/observatory/views.py | 75 ++++++++++--------------------- 1 file changed, 23 insertions(+), 52 deletions(-) diff --git a/django_files/observatory/views.py b/django_files/observatory/views.py index 427acf83..331b26be 100644 --- a/django_files/observatory/views.py +++ b/django_files/observatory/views.py @@ -23,6 +23,7 @@ # App specific from observatory.models import * from observatory.models import storychapter +from observatory import helpers from django.db.models import Max from django.forms import ModelForm import msgpack @@ -1211,68 +1212,38 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 if countries[0] and countries[0].name in list_countries_the: countries[0].name = "the "+countries[0].name - prod_or_partner = "partner" # quick fix should be merged with item_type - + #p_code, product = None, None if product not in ("show", "all"): p_code = product product = clean_product(p_code, prod_class) if not alert: - if app_type == "casy": - # raise Exception(app_name) - if app_name == "pie_scatter": - title = "Which products are feasible for %s?" % countries[0].name - elif (app_name == "product_space" or app_name == "rings"): - title = "What did %s export in %s?" % (countries[0].name, year) # INSERTED NEW TITLE HERE - elif app_name == "stacked": - title = "What did %s %s between %s and %s?" % (countries[0].name, trade_flow.replace("_", " "), year_start, year_end) # NEW TITLE HERE - prod_or_partner = "product" - else: - title = "What did %s %s in %s?" % (countries[0].name, trade_flow.replace("_", " "), year) # NEW TITLE HERE - prod_or_partner = "product" - - # Country but showing other country trade partners - elif app_type == "csay": - item_type = "countries" - article = "to" if trade_flow == "export" else "from" - if app_name == "stacked": - title = "Where did %s %s %s between %s and %s?" % (countries[0].name, trade_flow.replace("_", " "), article, year_start, year_end) - else: - title = "Where did %s %s %s in %s?" % (countries[0].name, trade_flow.replace("_", " "), article, year) - - # Product - elif app_type == "sapy": - item_type = "countries" - if app_name == "stacked": - title = "Who %sed %s between %s and %s?" % (trade_flow.replace("_", " "), product.name_en, year_start, year_end) - else: - title = "Who %sed %s in %s?" % (trade_flow.replace("_", " "), product.name_en, year) - prod_or_partner = "product" - - # Bilateral Country x Country - elif app_type == "ccsy": - # trade_flow_list = ["export", "import"] + # Generate page title depending on visualization being used + trade_flow = trade_flow.replace('_', ' ') + years = [] #TODO: implement + product_name = product.name_en if not isinstance(product, basestring) else product + country_names = [x.name for x in countries if x is not None], #TODO: test + title = helpers.get_title(app_name, app_type, + country_names=country_names, + trade_flow=trade_flow, + years=years, + product_name=product_name + ) + + if app_type in ("ccsy", "cspy"): if _("net_export") in trade_flow_list: del trade_flow_list[trade_flow_list.index(_("net_export"))] if _("net_import") in trade_flow_list: del trade_flow_list[trade_flow_list.index(_("net_import"))] - article = "to" if trade_flow == "export" else "from" - if app_name == "stacked": - title = "What did %s %s %s %s between %s and %s?" % (countries[0].name, trade_flow.replace("_", " "), article, countries[1].name, year_start, year_end) - else: - title = "What did %s %s %s %s in %s?" % (countries[0].name, trade_flow.replace("_", " "), article, countries[1].name, year) - - # Bilateral Country / Show / Product / Year - elif app_type == "cspy": - if "net_export" in trade_flow_list: del trade_flow_list[trade_flow_list.index("net_export")] - if "net_import" in trade_flow_list: del trade_flow_list[trade_flow_list.index("net_import")] - item_type = "countries" - article = "to" if trade_flow == "export" else "from" - if app_name == "stacked": - title = "Where did %s %s %s %s between %s and %s?" % (countries[0].name, trade_flow.replace("_", " "), product.name_en, article, year_start, year_end) - else: - title = "Where did %s %s %s %s in %s?" % (countries[0].name, trade_flow.replace("_", " "), product.name_en, article, year) + #trade_flow_list.pop(_("net_export"), None) + # Should we show the product or partner tab pane? + prod_or_partner = "partner" # quick fix should be merged with item_type + if app_type in ["cspy", "sapy"]: prod_or_partner = "product" + elif app_type == "casy": + if app_name in ("stacked", "map", "tree_map"): + prod_or_partner = "product" + # Return page without visualization data From 86d194f61a69547d5ac7f3099621e5b049080258 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Mon, 28 Apr 2014 09:42:43 -0400 Subject: [PATCH 09/52] Bugfix in generating list of country names --- django_files/observatory/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_files/observatory/views.py b/django_files/observatory/views.py index 331b26be..8eee77be 100644 --- a/django_files/observatory/views.py +++ b/django_files/observatory/views.py @@ -1223,7 +1223,7 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 trade_flow = trade_flow.replace('_', ' ') years = [] #TODO: implement product_name = product.name_en if not isinstance(product, basestring) else product - country_names = [x.name for x in countries if x is not None], #TODO: test + country_names = [getattr(x, "name", None), for x in countries] title = helpers.get_title(app_name, app_type, country_names=country_names, trade_flow=trade_flow, From 468f3bbfc8b8df4d6ad03416025e2c02a000d038 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Mon, 28 Apr 2014 10:13:36 -0400 Subject: [PATCH 10/52] Remove comment --- django_files/observatory/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_files/observatory/views.py b/django_files/observatory/views.py index 8eee77be..9525b798 100644 --- a/django_files/observatory/views.py +++ b/django_files/observatory/views.py @@ -1275,7 +1275,7 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 "years_available": years_available, "data_as_text": data_as_text, "app_name": app_name, - "title": title,#get_question(app_type, trade_flow=trade_flow,origin=countries[0],destination=countries[1],product=product), + "title": title, "trade_flow": trade_flow, "country1": countries[0] or country1, "country2": countries[1] or country2, From ebf6aa75471b435d06b116cc473f7b65253576af Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Mon, 28 Apr 2014 10:18:45 -0400 Subject: [PATCH 11/52] Move around year-related return values together --- django_files/observatory/views.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/django_files/observatory/views.py b/django_files/observatory/views.py index 9525b798..9fdd8361 100644 --- a/django_files/observatory/views.py +++ b/django_files/observatory/views.py @@ -1272,7 +1272,6 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 "warning": warning, "alert": alert, "prod_class": prod_class, - "years_available": years_available, "data_as_text": data_as_text, "app_name": app_name, "title": title, @@ -1280,17 +1279,18 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 "country1": countries[0] or country1, "country2": countries[1] or country2, "product": product, + "years_available": years_available, "year": year, "year_start": year_start, "year_end": year_end, "year_interval": year_interval, + "year1_list": year1_list, + "year2_list": year2_list, + "year_interval_list": year_interval_list, "trade_flow_list": trade_flow_list, "country1_list": country_lists[0], "country2_list": country_lists[1], "product_list": product_list, - "year1_list": year1_list, - "year2_list": year2_list, - "year_interval_list": year_interval_list, "api_uri": api_uri, "app_type": app_type, "redesign_api_uri": redesign_api_uri, From c089577a5b9689731ebb8842eee0377a9ecc683e Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Mon, 28 Apr 2014 10:22:38 -0400 Subject: [PATCH 12/52] Remove year_interval which is not being used anymore. --- django_files/observatory/views.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/django_files/observatory/views.py b/django_files/observatory/views.py index 9fdd8361..3ac0354a 100644 --- a/django_files/observatory/views.py +++ b/django_files/observatory/views.py @@ -1168,12 +1168,10 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 # year = range(y[0], y[1]+1, y[2]) year_start = y[0] year_end = y[1] - year_interval = y[2] year2_list = year1_list year_interval_list = range(1, 11) - # year_interval = year[1] - year[0] else: - year_start, year_end, year_interval = None, None, None + year_start, year_end = None, None year = int(year) if year > years_available[len(years_available)-1]: year = years_available[len(years_available)-1] @@ -1283,7 +1281,6 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 "year": year, "year_start": year_start, "year_end": year_end, - "year_interval": year_interval, "year1_list": year1_list, "year2_list": year2_list, "year_interval_list": year_interval_list, From 2a444b4c572041321e2ac208b21bfbaac88455de Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Mon, 28 Apr 2014 11:05:11 -0400 Subject: [PATCH 13/52] Add missing parameterized arguments in get_time_clause --- django_files/observatory/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_files/observatory/helpers.py b/django_files/observatory/helpers.py index da271c7f..2cbb96d4 100644 --- a/django_files/observatory/helpers.py +++ b/django_files/observatory/helpers.py @@ -161,7 +161,7 @@ def get_time_clause(years): elif len(years) == 1: return "in %d" % years[0] else: - return "between %d and %d" + return "between %d and %d" % (years[0], years[1]) def get_title(app_name, api_name, country_names=None, trade_flow=None, years=None, product_name=None): From 34f9783cca8dd62e5733d73af13ef22061e8898c Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Mon, 28 Apr 2014 11:06:09 -0400 Subject: [PATCH 14/52] Cleanup --- django_files/observatory/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django_files/observatory/views.py b/django_files/observatory/views.py index 3ac0354a..fbba5ab3 100644 --- a/django_files/observatory/views.py +++ b/django_files/observatory/views.py @@ -1163,9 +1163,9 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 if app_name == "stacked" and year == "2009": year = "1969.2011.10" + if "." in year: y = [int(x) for x in year.split(".")] - # year = range(y[0], y[1]+1, y[2]) year_start = y[0] year_end = y[1] year2_list = year1_list @@ -1173,6 +1173,7 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 else: year_start, year_end = None, None year = int(year) + # Check that year is within bounds if year > years_available[len(years_available)-1]: year = years_available[len(years_available)-1] elif year < years_available[0]: From 02967dcdd342a5943005fe0a300cab905e6b703f Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Mon, 28 Apr 2014 11:06:44 -0400 Subject: [PATCH 15/52] Fix year parameters --- django_files/observatory/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_files/observatory/views.py b/django_files/observatory/views.py index fbba5ab3..39937640 100644 --- a/django_files/observatory/views.py +++ b/django_files/observatory/views.py @@ -1220,7 +1220,7 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 # Generate page title depending on visualization being used trade_flow = trade_flow.replace('_', ' ') - years = [] #TODO: implement + years = [year_start, year_end] if year_start is not None else [year] product_name = product.name_en if not isinstance(product, basestring) else product country_names = [getattr(x, "name", None), for x in countries] title = helpers.get_title(app_name, app_type, From dbe9a854657306a794bd16f2ed85de4189f205c6 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Mon, 28 Apr 2014 11:06:58 -0400 Subject: [PATCH 16/52] Fix syntax error --- django_files/observatory/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_files/observatory/views.py b/django_files/observatory/views.py index 39937640..5230b8ed 100644 --- a/django_files/observatory/views.py +++ b/django_files/observatory/views.py @@ -1222,7 +1222,7 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 trade_flow = trade_flow.replace('_', ' ') years = [year_start, year_end] if year_start is not None else [year] product_name = product.name_en if not isinstance(product, basestring) else product - country_names = [getattr(x, "name", None), for x in countries] + country_names = [getattr(x, "name", None) for x in countries] title = helpers.get_title(app_name, app_type, country_names=country_names, trade_flow=trade_flow, From 5cf3c20e2318f61da60b8b892cd3cb231caa39c3 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Tue, 29 Apr 2014 10:06:56 -0400 Subject: [PATCH 17/52] Add error case to get_title(). Fixes #79. --- django_files/observatory/helpers.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/django_files/observatory/helpers.py b/django_files/observatory/helpers.py index 2cbb96d4..549f00f7 100644 --- a/django_files/observatory/helpers.py +++ b/django_files/observatory/helpers.py @@ -214,3 +214,8 @@ def get_title(app_name, api_name, country_names=None, trade_flow=None, return "Where did %s %s %s %s %s?" % (country_names[0], trade_flow, product_name, article, get_time_clause(years)) + + else: + raise ValueError("Unknown API name when trying to generate title: %s", + api_name) + From c6399bb3038ed17d882e2396ea88da249755e72f Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Wed, 30 Apr 2014 12:16:55 -0400 Subject: [PATCH 18/52] Change get_title param order. --- django_files/observatory/helpers.py | 4 ++-- django_files/observatory/views.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/django_files/observatory/helpers.py b/django_files/observatory/helpers.py index 549f00f7..fb3a214e 100644 --- a/django_files/observatory/helpers.py +++ b/django_files/observatory/helpers.py @@ -163,13 +163,13 @@ def get_time_clause(years): else: return "between %d and %d" % (years[0], years[1]) -def get_title(app_name, api_name, country_names=None, trade_flow=None, +def get_title(api_name, app_name, country_names=None, trade_flow=None, years=None, product_name=None): """Fetch the natural-languageized title of a page based on the data being displayed. - :param app_name: E.g. pie_scatter, stacked, product_space, rings ... :param api_name: One of: casy, cspy, csay, ccsy, sapy + :param app_name: E.g. pie_scatter, stacked, product_space, rings ... :param list country_names: List of country name strings. If multiple, first is "from" country and second is the "to" country. :param str trade_flow: import, export, net_import, net_export diff --git a/django_files/observatory/views.py b/django_files/observatory/views.py index 6436f992..28de4f5d 100644 --- a/django_files/observatory/views.py +++ b/django_files/observatory/views.py @@ -1223,7 +1223,7 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 years = [year_start, year_end] if year_start is not None else [year] product_name = product.name_en if not isinstance(product, basestring) else product country_names = [getattr(x, "name", None) for x in countries] - title = helpers.get_title(app_name, app_type, + title = helpers.get_title(app_type, app_name, country_names=country_names, trade_flow=trade_flow, years=years, From 4cfe5a3f60543c0a670dc8c4f252de67d976165d Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Wed, 30 Apr 2014 12:17:44 -0400 Subject: [PATCH 19/52] Factor out valid country filters to get_valid() --- django_files/observatory/models.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/django_files/observatory/models.py b/django_files/observatory/models.py index ace05494..557923a4 100644 --- a/django_files/observatory/models.py +++ b/django_files/observatory/models.py @@ -132,15 +132,18 @@ def get_all(self, lang): "region__text_color", )) + def get_valid(self): + return self.filter( + name_3char__isnull=False, + name_2char__isnull=False, + region__isnull=False + ) + def get_random(self): - """Grab a random country that has a 3char name. This uses the 'ORDER BY - RAND()' method which is fine for this purpose but slow in mysql for - larger tables so beware.""" - return Country.objects.filter( - name_3char__isnull=False, - name_2char__isnull=False, - region__isnull=False - ).order_by('?')[1] + """Grab a random country. This uses the 'ORDER BY RAND()' method which + is fine for this purpose but slow in mysql for larger tables so + beware.""" + return self.get_valid().order_by('?')[1] class Country(models.Model): From 5736966adbaaadb3c7d0ea70ac4a2eb3c7c353a6 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Wed, 30 Apr 2014 17:36:14 -0400 Subject: [PATCH 20/52] Management command to generate question types --- .../management/commands/index_questions.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 django_files/observatory/management/commands/index_questions.py diff --git a/django_files/observatory/management/commands/index_questions.py b/django_files/observatory/management/commands/index_questions.py new file mode 100644 index 00000000..ffe498d0 --- /dev/null +++ b/django_files/observatory/management/commands/index_questions.py @@ -0,0 +1,62 @@ +from django.core.management.base import BaseCommand + +from observatory.models import Country, Hs4 +from observatory.helpers import get_title + +import itertools + + +class Command(BaseCommand): + help = 'Generate elasticsearch indices for questions in homepage' + + def handle(self, *args, **options): + + # TODO: "the" in country names + country_names = Country.objects.get_valid()\ + .values_list('name_en') + trade_flows = ["import", "export"] + # TODO: should this be name_en? + product_names = Hs4.objects.values_list('name', flat=True) + + # e.g. What did Burundi export in 2013? Which products are feasible for + # Latvia? + self.generate_titles(['casy'], ['pie_scatter'], country_names) + self.generate_titles(['casy'], + ['stacked', 'treemap', + 'product_space', 'rings'], + country_names, trade_flows, [[]]) + + # e.g. Where did Albania export to in 2009? + self.generate_titles(['csay'], [None], country_names, trade_flows, + [[]]) + + # e.g. Who exported Petroleum in 1990? + self.generate_titles(['sapy'], [None], [None], trade_flows, [[]], + product_names) + + # e.g. What did Germany import from Turkey in 2011? + country_names_flat = Country.objects.get_valid()\ + .values_list('name_en', flat=True) + self.generate_titles(['ccsy'], [None], + itertools.product(country_names_flat, + country_names_flat), + trade_flows, [[]]) + # TODO: remove dupes where antarctica is exporting to antarctica + + # e.g. Where did France export wine to in 2012? + self.generate_titles(['cspy'], [None], country_names, trade_flows, + [[]], product_names) + + @staticmethod + def generate_titles(*possible_parameters): + """Given a list of possible parameters for the get_title() function, + generate all permutations of outputs. Parameters must be in order for + get_title(). + + :param possible_parameters: List of lists of possible parameters. + Should look like: [[param1_possibility1, param1_possibility2], + [param2_possibility1, param2_possibility2]] etc. + """ + return itertools.imap( + lambda args: get_title(*args), + itertools.product(*possible_parameters)) From 5234dd6a489e1a0498f17a428de90765d57e9021 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Wed, 30 Apr 2014 18:11:41 -0400 Subject: [PATCH 21/52] Chain all questions together --- .../management/commands/index_questions.py | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/django_files/observatory/management/commands/index_questions.py b/django_files/observatory/management/commands/index_questions.py index ffe498d0..444e9e65 100644 --- a/django_files/observatory/management/commands/index_questions.py +++ b/django_files/observatory/management/commands/index_questions.py @@ -20,32 +20,40 @@ def handle(self, *args, **options): # e.g. What did Burundi export in 2013? Which products are feasible for # Latvia? - self.generate_titles(['casy'], ['pie_scatter'], country_names) - self.generate_titles(['casy'], - ['stacked', 'treemap', - 'product_space', 'rings'], - country_names, trade_flows, [[]]) + casy_questions = self.generate_titles(['casy'], ['pie_scatter'], + country_names) + casy_questions2 = self.generate_titles(['casy'], + ['stacked', 'treemap', + 'product_space', 'rings'], + country_names, trade_flows, + [[]]) # e.g. Where did Albania export to in 2009? - self.generate_titles(['csay'], [None], country_names, trade_flows, - [[]]) + csay_questions = self.generate_titles(['csay'], [None], country_names, + trade_flows, [[]]) # e.g. Who exported Petroleum in 1990? - self.generate_titles(['sapy'], [None], [None], trade_flows, [[]], - product_names) + sapy_questions = self.generate_titles(['sapy'], [None], [None], + trade_flows, [[]], product_names) # e.g. What did Germany import from Turkey in 2011? country_names_flat = Country.objects.get_valid()\ .values_list('name_en', flat=True) - self.generate_titles(['ccsy'], [None], - itertools.product(country_names_flat, - country_names_flat), - trade_flows, [[]]) + ccsy_questions = self.generate_titles(['ccsy'], [None], + itertools.product( + country_names_flat, + country_names_flat), + trade_flows, [[]]) # TODO: remove dupes where antarctica is exporting to antarctica # e.g. Where did France export wine to in 2012? - self.generate_titles(['cspy'], [None], country_names, trade_flows, - [[]], product_names) + cspy_questions = self.generate_titles(['cspy'], [None], country_names, + trade_flows, [[]], product_names) + + all_questions = itertools.chain(casy_questions, casy_questions2, + csay_questions, sapy_questions, + ccsy_questions, cspy_questions) + @staticmethod def generate_titles(*possible_parameters): From 7528557dd442bec2cbd3be1dbacf9b35a9e67639 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Thu, 1 May 2014 14:14:09 -0400 Subject: [PATCH 22/52] Add in elasticsearch --- .../management/commands/index_questions.py | 26 +++++++++++++++++++ requirements.txt | 3 ++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/django_files/observatory/management/commands/index_questions.py b/django_files/observatory/management/commands/index_questions.py index 444e9e65..7b8f983c 100644 --- a/django_files/observatory/management/commands/index_questions.py +++ b/django_files/observatory/management/commands/index_questions.py @@ -1,4 +1,6 @@ from django.core.management.base import BaseCommand +from elasticsearch import Elasticsearch +from elasticsearch.helpers import streaming_bulk from observatory.models import Country, Hs4 from observatory.helpers import get_title @@ -54,6 +56,30 @@ def handle(self, *args, **options): csay_questions, sapy_questions, ccsy_questions, cspy_questions) + all_questions = enumerate(all_questions) + + es = Elasticsearch() + for ok, result in streaming_bulk( + es, + all_questions, + index='questions', + expand_action_callback=self.convert_to_elasticsearch_command + ): + print ok, result + + @staticmethod + def convert_to_elasticsearch_command(data): + """Convert the question list into an elasticsearch `index` command + consumable by the elasticsearch bulk api.""" + doc_id = data[0] + doc_body = data[1] + + action = {'index': + {'_id': doc_id, + '_index': 'questions', + '_type': 'question', + }} + return action, {'body': doc_body} @staticmethod def generate_titles(*possible_parameters): diff --git a/requirements.txt b/requirements.txt index 7fc2e1ba..ec516170 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,5 @@ Django==1.4.5 MySQL-python numpy msgpack-python -pycrypto==2.6 \ No newline at end of file +pycrypto==2.6 +elasticsearch>=1.0.0,<2.0.0 From d2f6a6fd56003ec903b500bdf4514ea1ea442140 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Thu, 1 May 2014 14:32:54 -0400 Subject: [PATCH 23/52] Remove now-redundant get_question --- django_files/observatory/helpers.py | 58 +---------------------------- 1 file changed, 1 insertion(+), 57 deletions(-) diff --git a/django_files/observatory/helpers.py b/django_files/observatory/helpers.py index fb3a214e..a1eeb875 100644 --- a/django_files/observatory/helpers.py +++ b/django_files/observatory/helpers.py @@ -97,62 +97,6 @@ def get_product(product, classification): return p -# Returns the question to display on the page describing the URL requested -def get_question(app_type, **kwargs): - - trade_flow = kwargs["trade_flow"] - - if app_type == "casy": - origin = kwargs["origin"] - - if trade_flow in ['net_export','net_import']: - title = "What does %s %s in net terms?" % (origin.name, trade_flow.replace("_", " ").split()[1]) - else: - title = "What does %s %s?" % (origin.name, trade_flow.replace("_", " ")) - - # Country but showing other country trade partners - elif app_type == "csay": - origin = kwargs["origin"] - - article = "to" if trade_flow in ["export","net_export"] else "from" - if trade_flow in ['net_export','net_import']: - title = "Where does %s %s %s in net terms? " % (origin.name, trade_flow.replace("_", " ").split()[1], article) - else: - title = "Where does %s %s %s?" % (origin.name, trade_flow.replace("_", " "), article) - - # Product - elif app_type == "sapy": - product = kwargs["product"] - - if trade_flow in ['net_export','net_import']: - title = "Who %ss %s in net terms?" % (trade_flow.replace("_", " ").split()[1], product.name_en) - else: - title = "Who %ss %s?" % (trade_flow.replace("_", " "), product.name_en) - - # Bilateral Country x Country - elif app_type == "ccsy": - origin = kwargs["origin"] - destination = kwargs["destination"] - - article = "to" if trade_flow in ["export","net_export"] else "from" - if trade_flow in ['net_export','net_import']: - title = "What does %s %s %s %s in net terms?" % (origin.name, trade_flow.replace("_", " ").split()[1], article, destination.name) - else: - title = "What does %s %s %s %s?" % (origin.name, trade_flow, article, destination.name) - - # Bilateral Country / Show / Product / Year - elif app_type == "cspy": - origin = kwargs["origin"] - product = kwargs["product"] - - article = "to" if trade_flow in ["export","net_export"] else "from" - if trade_flow in ['net_export','net_import']: - title = "Where does %s %s %s %s in net terms?" % (origin.name, trade_flow.replace("_", " ").split()[1], product.name_en, article) - else: - title = "Where does %s %s %s %s?" % (origin.name, trade_flow, product.name_en, article) - - return title - def get_time_clause(years): """Generate a string like 'between 2005 and 2007' or 'in 2011' from a list of years.""" @@ -163,6 +107,7 @@ def get_time_clause(years): else: return "between %d and %d" % (years[0], years[1]) + def get_title(api_name, app_name, country_names=None, trade_flow=None, years=None, product_name=None): """Fetch the natural-languageized title of a page based on the data being @@ -218,4 +163,3 @@ def get_title(api_name, app_name, country_names=None, trade_flow=None, else: raise ValueError("Unknown API name when trying to generate title: %s", api_name) - From 110796520167de0385c883ecd5e271f3630def8b Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Thu, 1 May 2014 14:35:34 -0400 Subject: [PATCH 24/52] Run autopep8 on helpers.py, indentation now fixed --- django_files/observatory/helpers.py | 128 +++++++++++++++------------- 1 file changed, 68 insertions(+), 60 deletions(-) diff --git a/django_files/observatory/helpers.py b/django_files/observatory/helpers.py index a1eeb875..1b379dff 100644 --- a/django_files/observatory/helpers.py +++ b/django_files/observatory/helpers.py @@ -2,99 +2,107 @@ import random # make sure app name is in the list of possible apps + + def get_app_name(app_name): - possible_apps = ["tree_map", "stacked", "product_space", "map"] + possible_apps = ["tree_map", "stacked", "product_space", "map"] - # if the app_name requested is not in the list of possibilities - if app_name not in possible_apps: - app_name = None + # if the app_name requested is not in the list of possibilities + if app_name not in possible_apps: + app_name = None - return app_name + return app_name # make sure this is accepted trade_flow def get_trade_flow(trade_flow): - possible_yoga_flows = ["export", "import", "net_export", "net_import"] + possible_yoga_flows = ["export", "import", "net_export", "net_import"] - if trade_flow not in possible_yoga_flows: - trade_flow = None + if trade_flow not in possible_yoga_flows: + trade_flow = None - return trade_flow + return trade_flow def get_years(classification): - # get distince years from db, different for diff product classifications + # get distince years from db, different for diff product classifications - if classification == "sitc4": - years_available = list(Sitc4_cpy.objects.values_list("year", flat=True).distinct()) - elif classification == "hs4": - years_available = list(Hs4_cpy.objects.values_list("year", flat=True).distinct()) + if classification == "sitc4": + years_available = list( + Sitc4_cpy.objects.values_list( + "year", + flat=True).distinct()) + elif classification == "hs4": + years_available = list( + Hs4_cpy.objects.values_list( + "year", + flat=True).distinct()) - return years_available + return years_available # Returns app type in CCPY format def get_app_type(country1, country2, product, year): - # country / all / show / year - if country2 == "all" and product == "show": - return "casy" + # country / all / show / year + if country2 == "all" and product == "show": + return "casy" - # country / show / all / year - elif country2 == "show" and product == "all": - return "csay" + # country / show / all / year + elif country2 == "show" and product == "all": + return "csay" - # show / all / product / year - elif country1 == "show" and country2 == "all": - return "sapy" + # show / all / product / year + elif country1 == "show" and country2 == "all": + return "sapy" - # country / country / show / year - elif product == "show": - return "ccsy" + # country / country / show / year + elif product == "show": + return "ccsy" - # country / show / product / year - else: - return "cspy" + # country / show / product / year + else: + return "cspy" # Returns the Country object or None def get_country(country): - # first try looking up based on 3 character code - try: - c = Country.objects.get(name_3char=country) - except Country.DoesNotExist: - # next try 2 character code + # first try looking up based on 3 character code try: - c = Country.objects.get(name_2char=country) + c = Country.objects.get(name_3char=country) except Country.DoesNotExist: - c = None - return c + # next try 2 character code + try: + c = Country.objects.get(name_2char=country) + except Country.DoesNotExist: + c = None + return c # Returns the Product object or None def get_product(product, classification): - # first try looking up based on 3 character code - if classification == "hs4": - try: - p = Hs4.objects.get(code=product) - except Hs4.DoesNotExist: - # next try SITC4 - try: - conv_code = Sitc4.objects.get(code=product).conversion_code - p = Hs4.objects.get(code=conv_code) - except Sitc4.DoesNotExist: - p = None - else: - try: - p = Sitc4.objects.get(code=product) - except Sitc4.DoesNotExist: - # next try SITC4 - try: - conv_code = Hs4.objects.get(code=product).conversion_code - p = Sitc4.objects.get(code=conv_code) - except Hs4.DoesNotExist: - p = None - return p + # first try looking up based on 3 character code + if classification == "hs4": + try: + p = Hs4.objects.get(code=product) + except Hs4.DoesNotExist: + # next try SITC4 + try: + conv_code = Sitc4.objects.get(code=product).conversion_code + p = Hs4.objects.get(code=conv_code) + except Sitc4.DoesNotExist: + p = None + else: + try: + p = Sitc4.objects.get(code=product) + except Sitc4.DoesNotExist: + # next try SITC4 + try: + conv_code = Hs4.objects.get(code=product).conversion_code + p = Sitc4.objects.get(code=conv_code) + except Hs4.DoesNotExist: + p = None + return p def get_time_clause(years): From a4cd2f28e7c832907ce8f032cb428d58884a442d Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Thu, 1 May 2014 14:38:04 -0400 Subject: [PATCH 25/52] Clean up imports --- django_files/observatory/helpers.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/django_files/observatory/helpers.py b/django_files/observatory/helpers.py index 1b379dff..f584608e 100644 --- a/django_files/observatory/helpers.py +++ b/django_files/observatory/helpers.py @@ -1,9 +1,7 @@ -from observatory.models import * -import random - -# make sure app name is in the list of possible apps +from observatory.models import Hs4_cpy, Sitc4_cpy, Country, Hs4, Sitc4 +# make sure app name is in the list of possible apps def get_app_name(app_name): possible_apps = ["tree_map", "stacked", "product_space", "map"] From 7bb2f63f70aac1afa1cc329361a1b7ab47b792dd Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Thu, 1 May 2014 15:46:17 -0400 Subject: [PATCH 26/52] Report index entries that errored. --- .../observatory/management/commands/index_questions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django_files/observatory/management/commands/index_questions.py b/django_files/observatory/management/commands/index_questions.py index 7b8f983c..788adc48 100644 --- a/django_files/observatory/management/commands/index_questions.py +++ b/django_files/observatory/management/commands/index_questions.py @@ -65,7 +65,8 @@ def handle(self, *args, **options): index='questions', expand_action_callback=self.convert_to_elasticsearch_command ): - print ok, result + if not ok: + print 'Failed: %s' % result @staticmethod def convert_to_elasticsearch_command(data): From 4846b45ebd10ed14b1c13bd83405bc6d252fe280 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Thu, 1 May 2014 15:51:18 -0400 Subject: [PATCH 27/52] Better support empty time clauses --- django_files/observatory/helpers.py | 40 ++++++++++++++--------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/django_files/observatory/helpers.py b/django_files/observatory/helpers.py index f584608e..3daf3a89 100644 --- a/django_files/observatory/helpers.py +++ b/django_files/observatory/helpers.py @@ -105,13 +105,13 @@ def get_product(product, classification): def get_time_clause(years): """Generate a string like 'between 2005 and 2007' or 'in 2011' from a list - of years.""" - if len(years) == 0: + of years. Beginning space is included to allow for empty time clause.""" + if years is None or len(years) == 0: return "" elif len(years) == 1: - return "in %d" % years[0] + return " in %d" % years[0] else: - return "between %d and %d" % (years[0], years[1]) + return " between %d and %d" % (years[0], years[1]) def get_title(api_name, app_name, country_names=None, trade_flow=None, @@ -135,37 +135,37 @@ def get_title(api_name, app_name, country_names=None, trade_flow=None, if app_name == "pie_scatter": return "Which products are feasible for %s?" % country_names[0] else: - return "What did %s %s %s?" % (country_names[0], - trade_flow, - get_time_clause(years)) + return "What did %s %s%s?" % (country_names[0], + trade_flow, + get_time_clause(years)) # e.g. Where did Albania export to in 2009? elif api_name == "csay": article = "to" if trade_flow == "export" else "from" - return "Where did %s %s %s %s?" % (country_names[0], - trade_flow, - article, - get_time_clause(years)) + return "Where did %s %s %s%s?" % (country_names[0], + trade_flow, + article, + get_time_clause(years)) # e.g. Who exported Petroleum in 1990? elif api_name == "sapy": - return "Who %sed %s %s?" % (trade_flow, product_name, - get_time_clause(years)) + return "Who %sed %s%s?" % (trade_flow, product_name, + get_time_clause(years)) # e.g. What did Germany import from Turkey in 2011? elif api_name == "ccsy": article = "to" if trade_flow == "export" else "from" - return "What did %s %s %s %s %s?" % (country_names[0], trade_flow, - article, country_names[1], - get_time_clause(years)) + return "What did %s %s %s %s%s?" % (country_names[0], trade_flow, + article, country_names[1], + get_time_clause(years)) # e.g. Where did France export wine to in 2012? elif api_name == "cspy": article = "to" if trade_flow == "export" else "from" - return "Where did %s %s %s %s %s?" % (country_names[0], trade_flow, - product_name, article, - get_time_clause(years)) + return "Where did %s %s %s %s%s?" % (country_names[0], trade_flow, + product_name, article, + get_time_clause(years)) else: - raise ValueError("Unknown API name when trying to generate title: %s", + raise ValueError("Unknown API name when trying to generate title: %s" % api_name) From 1ff4a1f3a0c93fd9b1b2a56422501dfee29e40d7 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Thu, 1 May 2014 16:45:48 -0400 Subject: [PATCH 28/52] Make use of None in get_time_clause --- .../management/commands/index_questions.py | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/django_files/observatory/management/commands/index_questions.py b/django_files/observatory/management/commands/index_questions.py index 788adc48..17fa4cf3 100644 --- a/django_files/observatory/management/commands/index_questions.py +++ b/django_files/observatory/management/commands/index_questions.py @@ -20,37 +20,40 @@ def handle(self, *args, **options): # TODO: should this be name_en? product_names = Hs4.objects.values_list('name', flat=True) - # e.g. What did Burundi export in 2013? Which products are feasible for - # Latvia? + # Which products are feasible for Latvia? casy_questions = self.generate_titles(['casy'], ['pie_scatter'], country_names) + + # What did Burundi export in 2013? casy_questions2 = self.generate_titles(['casy'], ['stacked', 'treemap', 'product_space', 'rings'], country_names, trade_flows, - [[]]) + [None]) - # e.g. Where did Albania export to in 2009? + # Where did Albania export to in 2009? csay_questions = self.generate_titles(['csay'], [None], country_names, - trade_flows, [[]]) + trade_flows, [None]) - # e.g. Who exported Petroleum in 1990? + # Who exported Petroleum in 1990? sapy_questions = self.generate_titles(['sapy'], [None], [None], - trade_flows, [[]], product_names) + trade_flows, [None], + product_names) - # e.g. What did Germany import from Turkey in 2011? + # What did Germany import from Turkey in 2011? country_names_flat = Country.objects.get_valid()\ .values_list('name_en', flat=True) ccsy_questions = self.generate_titles(['ccsy'], [None], itertools.product( country_names_flat, country_names_flat), - trade_flows, [[]]) + trade_flows, [None]) # TODO: remove dupes where antarctica is exporting to antarctica - # e.g. Where did France export wine to in 2012? + # Where did France export wine to in 2012? cspy_questions = self.generate_titles(['cspy'], [None], country_names, - trade_flows, [[]], product_names) + trade_flows, [None], + product_names) all_questions = itertools.chain(casy_questions, casy_questions2, csay_questions, sapy_questions, From a0945899d36fed8c0dd658053f07d30bd83e4539 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Thu, 1 May 2014 16:51:45 -0400 Subject: [PATCH 29/52] Fix one source of dupe questions "What did country_name import?" --- .../observatory/management/commands/index_questions.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/django_files/observatory/management/commands/index_questions.py b/django_files/observatory/management/commands/index_questions.py index 17fa4cf3..021b9008 100644 --- a/django_files/observatory/management/commands/index_questions.py +++ b/django_files/observatory/management/commands/index_questions.py @@ -26,10 +26,8 @@ def handle(self, *args, **options): # What did Burundi export in 2013? casy_questions2 = self.generate_titles(['casy'], - ['stacked', 'treemap', - 'product_space', 'rings'], - country_names, trade_flows, - [None]) + [None], country_names, + trade_flows, [None]) # Where did Albania export to in 2009? csay_questions = self.generate_titles(['csay'], [None], country_names, From 91f4e1b519afbb6588b6d422986f7c6b91066bba Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Thu, 1 May 2014 17:14:35 -0400 Subject: [PATCH 30/52] Remove more dupes in indexer --- .../management/commands/index_questions.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/django_files/observatory/management/commands/index_questions.py b/django_files/observatory/management/commands/index_questions.py index 021b9008..94b3c84b 100644 --- a/django_files/observatory/management/commands/index_questions.py +++ b/django_files/observatory/management/commands/index_questions.py @@ -41,12 +41,15 @@ def handle(self, *args, **options): # What did Germany import from Turkey in 2011? country_names_flat = Country.objects.get_valid()\ .values_list('name_en', flat=True) - ccsy_questions = self.generate_titles(['ccsy'], [None], - itertools.product( - country_names_flat, - country_names_flat), + country_pairs = itertools.ifilter( + # Germany importing from Germany etc makes no sense + lambda country1, country2: country1 != country2, + itertools.product( + country_names_flat, + country_names_flat), + ) + ccsy_questions = self.generate_titles(['ccsy'], [None], country_pairs, trade_flows, [None]) - # TODO: remove dupes where antarctica is exporting to antarctica # Where did France export wine to in 2012? cspy_questions = self.generate_titles(['cspy'], [None], country_names, From f5f00717357256b17e15f9fc4b7d5bd0990b6b20 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Thu, 1 May 2014 17:17:24 -0400 Subject: [PATCH 31/52] Fix bug with filter / lambda number of params --- django_files/observatory/management/commands/index_questions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_files/observatory/management/commands/index_questions.py b/django_files/observatory/management/commands/index_questions.py index 94b3c84b..eefbf6dd 100644 --- a/django_files/observatory/management/commands/index_questions.py +++ b/django_files/observatory/management/commands/index_questions.py @@ -43,7 +43,7 @@ def handle(self, *args, **options): .values_list('name_en', flat=True) country_pairs = itertools.ifilter( # Germany importing from Germany etc makes no sense - lambda country1, country2: country1 != country2, + lambda country_pair: country_pair[0] != country_pair[1], itertools.product( country_names_flat, country_names_flat), From a39ff907987c3bc76768646e0beceddc5232866f Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Fri, 2 May 2014 13:45:08 -0400 Subject: [PATCH 32/52] Also index arguments that were used to generate title. Fixes #113 --- .../management/commands/index_questions.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/django_files/observatory/management/commands/index_questions.py b/django_files/observatory/management/commands/index_questions.py index eefbf6dd..751fc590 100644 --- a/django_files/observatory/management/commands/index_questions.py +++ b/django_files/observatory/management/commands/index_questions.py @@ -84,18 +84,30 @@ def convert_to_elasticsearch_command(data): '_index': 'questions', '_type': 'question', }} - return action, {'body': doc_body} + return action, doc_body @staticmethod def generate_titles(*possible_parameters): """Given a list of possible parameters for the get_title() function, - generate all permutations of outputs. Parameters must be in order for + generate all permutations of titles. Parameters must be in order for get_title(). :param possible_parameters: List of lists of possible parameters. Should look like: [[param1_possibility1, param1_possibility2], [param2_possibility1, param2_possibility2]] etc. + :return: An iterator yielding titles and arguments e.g: + [{'title': 'foo', 'country_name': 'Germany'}, {'title': 'bar', + 'country_name': 'Sweden'}] """ + + arg_names = ['api_name', 'app_name', 'country_names', 'trade_flow', + 'years', 'product_name'] + + def generate_title(args): + kwargs = dict(zip(arg_names, args)) + kwargs["title"] = get_title(**kwargs) + return kwargs + return itertools.imap( - lambda args: get_title(*args), + generate_title, itertools.product(*possible_parameters)) From 47d59d70f7e813105a639a2b52c64edf509a6507 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Fri, 2 May 2014 14:53:15 -0400 Subject: [PATCH 33/52] Initial version of search api #112 --- django_files/atlas/urls.py | 2 ++ django_files/observatory/views.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/django_files/atlas/urls.py b/django_files/atlas/urls.py index 4cc91eb3..c5054f52 100644 --- a/django_files/atlas/urls.py +++ b/django_files/atlas/urls.py @@ -124,6 +124,8 @@ def render_to_response(self, context, **kwargs): (r'^api/near/(?P\w{3})/(?P[0-9\.]+)/(?P\d+)/$', 'observatory.views_exhibit.api_near'), + (r'^api/search/$', 'observatory.views.api_search'), + # Overview (Countries) ###################################################### (r'^country/(?P\w{2,3})/$', 'observatory.views_overview.country2'), (r'^hs4/(?P\d{4})/$', 'observatory.views_overview.product'), diff --git a/django_files/observatory/views.py b/django_files/observatory/views.py index 28de4f5d..d937317c 100644 --- a/django_files/observatory/views.py +++ b/django_files/observatory/views.py @@ -18,6 +18,7 @@ import json from django.core import serializers from django.core.urlresolvers import reverse +from elasticsearch import Elasticsearch # Project specific from django.utils.translation import gettext as _ # App specific @@ -2495,3 +2496,32 @@ def get_app_type(country1, country2, product, year): # country / show / product / year else: return "cspy" + +def api_search(request): + + query = request.GET["term"] + if query == None: + return HttpResponse(json.dumps("{}")) + + es = Elasticsearch() + result = es.search( + index="questions", + body={"query": { + "filtered": { + "query": { + "fuzzy_like_this": { + "like_text": query, + "fields": [ + "title" + ], + "max_query_terms": 15, + "prefix_length": 1 + } + } + } + }, + "size": 10 + }) + result_list = [x['_source']['title'] for x in result['hits']['hits']] + return HttpResponse(json.dumps(result_list)) + From 81101f85ce8c3b10309a4c2e9322ab500bbfc733 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Fri, 2 May 2014 16:04:59 -0400 Subject: [PATCH 34/52] Twiddle with parameters --- django_files/observatory/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django_files/observatory/views.py b/django_files/observatory/views.py index d937317c..cbdb5ed1 100644 --- a/django_files/observatory/views.py +++ b/django_files/observatory/views.py @@ -2514,8 +2514,9 @@ def api_search(request): "fields": [ "title" ], + "fuzziness": 0.5, "max_query_terms": 15, - "prefix_length": 1 + "prefix_length": 4 } } } From 9d153b09426a96c37ac48882cfa279a4283d1077 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Fri, 2 May 2014 17:12:21 -0400 Subject: [PATCH 35/52] Remove exact dupe of get_app_type #100 --- django_files/observatory/views.py | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/django_files/observatory/views.py b/django_files/observatory/views.py index cbdb5ed1..eb8edfa0 100644 --- a/django_files/observatory/views.py +++ b/django_files/observatory/views.py @@ -1197,7 +1197,7 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 data_as_text["columns"] = view_response[2] - app_type = get_app_type(country1, country2, product, year) + app_type = helpers.get_app_type(country1, country2, product, year) # Some countries need "the" before their names list_countries_the = set(("Cayman Islands", "Central African Republic", @@ -2476,27 +2476,6 @@ def get_country_lookup(): lookup[c.id] = [c.name_en, c.name_3char] return lookup -def get_app_type(country1, country2, product, year): - # country / all / show / year - if country2 == "all" and product == "show": - return "casy" - - # country / show / all / year - elif country2 == "show" and product == "all": - return "csay" - - # show / all / product / year - elif country1 == "show" and country2 == "all": - return "sapy" - - # country / country / show / year - elif product == "show": - return "ccsy" - - # country / show / product / year - else: - return "cspy" - def api_search(request): query = request.GET["term"] From 6538b213892182aed0cf118a3839361860bb10d8 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Tue, 6 May 2014 09:57:23 -0400 Subject: [PATCH 36/52] Checkpoint --- django_files/observatory/helpers.py | 57 +++++++++++++++++++ .../management/commands/index_questions.py | 7 ++- 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/django_files/observatory/helpers.py b/django_files/observatory/helpers.py index 3daf3a89..faaa719a 100644 --- a/django_files/observatory/helpers.py +++ b/django_files/observatory/helpers.py @@ -169,3 +169,60 @@ def get_title(api_name, app_name, country_names=None, trade_flow=None, else: raise ValueError("Unknown API name when trying to generate title: %s" % api_name) + +def params_to_url(api_name=None, app_name=None, country_names=None, + trade_flow=None, years=None, product_name=None): + """Generate explore/ urls from specific parameters. Same parameter syntax + as get_title.""" + + if app_name is None: + # Treemap is a safe default that works with almost all of our data + app_name = 'tree_map' + + country_codes = [] + if country_names is not None: + for name in country_names: + country_codes.append( + Country.objects\ + .filter(name=name)\ + .values_list('name_3char', flat=True)[0].lower()) + + if api_name == 'casy': + # What did Germany import in 2012? + # Which products are feasible for Latvia? + # Looks like explore/tree_map/import/deu/all/show/2012/ + country_codes.append('all') + product_code = "show" + + elif api_name == 'cspy': + # Where did Germany import Swine from in 2012? + # Looks like explore/tree_map/import/deu/show/0103/2012/ + country_codes.append('show') + product_code = '0409' # TODO + + elif api_name == 'csay': + # Where does germany import from? + # Looks like explore/tree_map/import/deu/show/all/2012/ + country_codes.append('show') + product_code = 'all' + + elif api_name == 'ccsy': + # What did Germany import from Congo in 2012? + # Looks like explore/tree_map/import/deu/cog/show/2012/ + product_code = 'show' + + elif api_name == 'sapy': + # Who exports potatoes? + # Looks like explore/tree_map/export/show/all/0101/2012/ + country_codes = ("show", "all") + product_code = 'all' + + else: + raise ValueError("Unknown API name : %s" % api_name) + + url = "explore/%s/%s/%s/%s/%s/" % (app_name, trade_flow, country_codes[0], + country_codes[1], product_code) + if years is not None: + url += "%s/" % years + + return url diff --git a/django_files/observatory/management/commands/index_questions.py b/django_files/observatory/management/commands/index_questions.py index 751fc590..2a419ac4 100644 --- a/django_files/observatory/management/commands/index_questions.py +++ b/django_files/observatory/management/commands/index_questions.py @@ -3,7 +3,7 @@ from elasticsearch.helpers import streaming_bulk from observatory.models import Country, Hs4 -from observatory.helpers import get_title +from observatory.helpers import get_title, params_to_url import itertools @@ -105,7 +105,10 @@ def generate_titles(*possible_parameters): def generate_title(args): kwargs = dict(zip(arg_names, args)) - kwargs["title"] = get_title(**kwargs) + title = get_title(**kwargs) + url = params_to_url(**kwargs) + kwargs["title"] = title + kwargs["url"] = url return kwargs return itertools.imap( From e83d80cbdb1af6409d303abbc210f78762877a8b Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Tue, 6 May 2014 16:08:09 -0400 Subject: [PATCH 37/52] Pep8 fixes --- django_files/observatory/helpers.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/django_files/observatory/helpers.py b/django_files/observatory/helpers.py index faaa719a..62dabab5 100644 --- a/django_files/observatory/helpers.py +++ b/django_files/observatory/helpers.py @@ -170,6 +170,7 @@ def get_title(api_name, app_name, country_names=None, trade_flow=None, raise ValueError("Unknown API name when trying to generate title: %s" % api_name) + def params_to_url(api_name=None, app_name=None, country_names=None, trade_flow=None, years=None, product_name=None): """Generate explore/ urls from specific parameters. Same parameter syntax @@ -183,8 +184,8 @@ def params_to_url(api_name=None, app_name=None, country_names=None, if country_names is not None: for name in country_names: country_codes.append( - Country.objects\ - .filter(name=name)\ + Country.objects + .filter(name=name) .values_list('name_3char', flat=True)[0].lower()) if api_name == 'casy': @@ -198,7 +199,7 @@ def params_to_url(api_name=None, app_name=None, country_names=None, # Where did Germany import Swine from in 2012? # Looks like explore/tree_map/import/deu/show/0103/2012/ country_codes.append('show') - product_code = '0409' # TODO + product_code = '0409' # TODO elif api_name == 'csay': # Where does germany import from? From 88d5554253898d7a8b638e7ef553445ba0662f2d Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Tue, 6 May 2014 16:08:32 -0400 Subject: [PATCH 38/52] Bugfix: use hs4 data name_en instead of name to make things more human-readable. Fixes #117. --- .../observatory/management/commands/index_questions.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/django_files/observatory/management/commands/index_questions.py b/django_files/observatory/management/commands/index_questions.py index 2a419ac4..6a3951dc 100644 --- a/django_files/observatory/management/commands/index_questions.py +++ b/django_files/observatory/management/commands/index_questions.py @@ -17,8 +17,7 @@ def handle(self, *args, **options): country_names = Country.objects.get_valid()\ .values_list('name_en') trade_flows = ["import", "export"] - # TODO: should this be name_en? - product_names = Hs4.objects.values_list('name', flat=True) + product_names = Hs4.objects.values_list('name_en', flat=True) # Which products are feasible for Latvia? casy_questions = self.generate_titles(['casy'], ['pie_scatter'], From 5d638dfb7fff768f0287a74d26312c2d83466277 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Wed, 7 May 2014 16:30:39 -0400 Subject: [PATCH 39/52] Temporarily add search test UI. --- django_files/atlas/urls.py | 1 + django_files/observatory/views.py | 2 ++ html/test_search.html | 53 +++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 html/test_search.html diff --git a/django_files/atlas/urls.py b/django_files/atlas/urls.py index c5054f52..626b20aa 100644 --- a/django_files/atlas/urls.py +++ b/django_files/atlas/urls.py @@ -125,6 +125,7 @@ def render_to_response(self, context, **kwargs): (r'^api/near/(?P\w{3})/(?P[0-9\.]+)/(?P\d+)/$', 'observatory.views_exhibit.api_near'), (r'^api/search/$', 'observatory.views.api_search'), + (r'^search/$', 'observatory.views.search'), # Overview (Countries) ###################################################### (r'^country/(?P\w{2,3})/$', 'observatory.views_overview.country2'), diff --git a/django_files/observatory/views.py b/django_files/observatory/views.py index eb8edfa0..c57d2874 100644 --- a/django_files/observatory/views.py +++ b/django_files/observatory/views.py @@ -2505,3 +2505,5 @@ def api_search(request): result_list = [x['_source']['title'] for x in result['hits']['hits']] return HttpResponse(json.dumps(result_list)) +def search(request): + return render_to_response("test_search.html") diff --git a/html/test_search.html b/html/test_search.html new file mode 100644 index 00000000..87860d21 --- /dev/null +++ b/html/test_search.html @@ -0,0 +1,53 @@ + + + + +jQuery UI Autocomplete - Remote with caching + + + + + + + + +
+ + +
+ + From c263247bdd21b7bf0354be3162ec13ecde8082dc Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Wed, 7 May 2014 16:32:25 -0400 Subject: [PATCH 40/52] Add URL values to search api result --- django_files/observatory/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/django_files/observatory/views.py b/django_files/observatory/views.py index c57d2874..a3b7523b 100644 --- a/django_files/observatory/views.py +++ b/django_files/observatory/views.py @@ -2502,7 +2502,9 @@ def api_search(request): }, "size": 10 }) - result_list = [x['_source']['title'] for x in result['hits']['hits']] + result_list = [{'label': x['_source']['title'], + 'value': x['_source']['url']} + for x in result['hits']['hits']] return HttpResponse(json.dumps(result_list)) def search(request): From 1a38a64954d3d2fb39adb92489e6810349cca7e8 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Wed, 7 May 2014 18:18:55 -0400 Subject: [PATCH 41/52] Major refactor to properly generate titles without repeating tons of db queries. --- django_files/observatory/helpers.py | 17 +--- .../management/commands/index_questions.py | 95 +++++++++++-------- 2 files changed, 62 insertions(+), 50 deletions(-) diff --git a/django_files/observatory/helpers.py b/django_files/observatory/helpers.py index 62dabab5..311fa2b9 100644 --- a/django_files/observatory/helpers.py +++ b/django_files/observatory/helpers.py @@ -171,23 +171,16 @@ def get_title(api_name, app_name, country_names=None, trade_flow=None, api_name) -def params_to_url(api_name=None, app_name=None, country_names=None, - trade_flow=None, years=None, product_name=None): +def params_to_url(api_name=None, app_name=None, country_codes=None, + trade_flow=None, years=None, product_code=None): """Generate explore/ urls from specific parameters. Same parameter syntax - as get_title.""" + as get_title, but product code instead of product name and 3 letter country + codees instead of country names.""" if app_name is None: # Treemap is a safe default that works with almost all of our data app_name = 'tree_map' - country_codes = [] - if country_names is not None: - for name in country_names: - country_codes.append( - Country.objects - .filter(name=name) - .values_list('name_3char', flat=True)[0].lower()) - if api_name == 'casy': # What did Germany import in 2012? # Which products are feasible for Latvia? @@ -199,7 +192,6 @@ def params_to_url(api_name=None, app_name=None, country_names=None, # Where did Germany import Swine from in 2012? # Looks like explore/tree_map/import/deu/show/0103/2012/ country_codes.append('show') - product_code = '0409' # TODO elif api_name == 'csay': # Where does germany import from? @@ -216,7 +208,6 @@ def params_to_url(api_name=None, app_name=None, country_names=None, # Who exports potatoes? # Looks like explore/tree_map/export/show/all/0101/2012/ country_codes = ("show", "all") - product_code = 'all' else: raise ValueError("Unknown API name : %s" % api_name) diff --git a/django_files/observatory/management/commands/index_questions.py b/django_files/observatory/management/commands/index_questions.py index 6a3951dc..fd406b23 100644 --- a/django_files/observatory/management/commands/index_questions.py +++ b/django_files/observatory/management/commands/index_questions.py @@ -14,46 +14,45 @@ class Command(BaseCommand): def handle(self, *args, **options): # TODO: "the" in country names - country_names = Country.objects.get_valid()\ - .values_list('name_en') trade_flows = ["import", "export"] - product_names = Hs4.objects.values_list('name_en', flat=True) + countries_flat = list(Country.objects.get_valid().only('name_en', + 'name_3char')) + countries = [[c] for c in countries_flat] + products = list(Hs4.objects.only('name_en', 'code')) # Which products are feasible for Latvia? casy_questions = self.generate_titles(['casy'], ['pie_scatter'], - country_names) + countries) # What did Burundi export in 2013? casy_questions2 = self.generate_titles(['casy'], - [None], country_names, + [None], countries, trade_flows, [None]) # Where did Albania export to in 2009? - csay_questions = self.generate_titles(['csay'], [None], country_names, + csay_questions = self.generate_titles(['csay'], [None], countries, trade_flows, [None]) # Who exported Petroleum in 1990? sapy_questions = self.generate_titles(['sapy'], [None], [None], trade_flows, [None], - product_names) + products) # What did Germany import from Turkey in 2011? - country_names_flat = Country.objects.get_valid()\ - .values_list('name_en', flat=True) country_pairs = itertools.ifilter( # Germany importing from Germany etc makes no sense - lambda country_pair: country_pair[0] != country_pair[1], + lambda country_pair: country_pair[0].id != country_pair[1].id, itertools.product( - country_names_flat, - country_names_flat), + countries_flat, + countries_flat), ) ccsy_questions = self.generate_titles(['ccsy'], [None], country_pairs, trade_flows, [None]) # Where did France export wine to in 2012? - cspy_questions = self.generate_titles(['cspy'], [None], country_names, + cspy_questions = self.generate_titles(['cspy'], [None], countries, trade_flows, [None], - product_names) + products) all_questions = itertools.chain(casy_questions, casy_questions2, csay_questions, sapy_questions, @@ -86,30 +85,52 @@ def convert_to_elasticsearch_command(data): return action, doc_body @staticmethod - def generate_titles(*possible_parameters): - """Given a list of possible parameters for the get_title() function, - generate all permutations of titles. Parameters must be in order for - get_title(). - - :param possible_parameters: List of lists of possible parameters. - Should look like: [[param1_possibility1, param1_possibility2], - [param2_possibility1, param2_possibility2]] etc. + def generate_index_entries(api_name=None, app_name=None, countries=None, + trade_flow=None, years=None, product=None): + """Given a list of possible parameters for the get_title() and + params_to_url() functions, generate index entries for all permutations + of titles. Parameters must be in order for get_title(). Each parameter + should contain a list of possibilities. For example: api_name=["casy", + "ccsy"] or countries = [["USA", "DEU"], ["USA", "TUR"]]. + :return: An iterator yielding titles and arguments e.g: - [{'title': 'foo', 'country_name': 'Germany'}, {'title': 'bar', - 'country_name': 'Sweden'}] + [{'title': 'foo', 'url':'qux', 'country_name': 'Germany'}, + {'title': 'bar', 'url':'quux', 'country_name': 'Sweden'}] """ - arg_names = ['api_name', 'app_name', 'country_names', 'trade_flow', - 'years', 'product_name'] - - def generate_title(args): - kwargs = dict(zip(arg_names, args)) - title = get_title(**kwargs) - url = params_to_url(**kwargs) - kwargs["title"] = title - kwargs["url"] = url - return kwargs - + def generate_index_entry(args): + + index = {} + + # Generate title + title = get_title( + api_name=args[0], + app_name=args[1], + country_names=[c.name_en for c in args[2]], + trade_flow=args[3], + years=args[4], + product_names=[p.name_en for p in args[5]], + ) + index["title"] = title + + # Generate url + url = params_to_url( + api_name=args[0], + app_name=args[1], + country_codes=[c.name_3char for c in args[2]], + trade_flow=args[3], + years=args[4], + product_codes=[p.code for p in args[5]], + ) + index["url"] = url + + # TODO: Add in params into elasticsearch in case we need them later + + return index + + parameter_possibilities = [api_name, app_name, countries, trade_flow, + years, product] return itertools.imap( - generate_title, - itertools.product(*possible_parameters)) + generate_index_entry, + itertools.product(*parameter_possibilities) + ) From 6e8092feae8354dacd1f5c80cab664aaf2fdce02 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Wed, 7 May 2014 18:44:57 -0400 Subject: [PATCH 42/52] Minor bugfixes wrangling parameters. Fixes #112. --- .../management/commands/index_questions.py | 53 +++++++++++-------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/django_files/observatory/management/commands/index_questions.py b/django_files/observatory/management/commands/index_questions.py index fd406b23..1a9e6eb2 100644 --- a/django_files/observatory/management/commands/index_questions.py +++ b/django_files/observatory/management/commands/index_questions.py @@ -21,22 +21,23 @@ def handle(self, *args, **options): products = list(Hs4.objects.only('name_en', 'code')) # Which products are feasible for Latvia? - casy_questions = self.generate_titles(['casy'], ['pie_scatter'], - countries) + casy_questions = self.generate_index_entries(['casy'], ['pie_scatter'], + countries) # What did Burundi export in 2013? - casy_questions2 = self.generate_titles(['casy'], - [None], countries, - trade_flows, [None]) + casy_questions2 = self.generate_index_entries(['casy'], [None], + countries, trade_flows, + [None]) # Where did Albania export to in 2009? - csay_questions = self.generate_titles(['csay'], [None], countries, - trade_flows, [None]) + csay_questions = self.generate_index_entries(['csay'], [None], + countries, trade_flows, + [None]) # Who exported Petroleum in 1990? - sapy_questions = self.generate_titles(['sapy'], [None], [None], - trade_flows, [None], - products) + sapy_questions = self.generate_index_entries(['sapy'], [None], [None], + trade_flows, [None], + products) # What did Germany import from Turkey in 2011? country_pairs = itertools.ifilter( @@ -46,13 +47,14 @@ def handle(self, *args, **options): countries_flat, countries_flat), ) - ccsy_questions = self.generate_titles(['ccsy'], [None], country_pairs, - trade_flows, [None]) + ccsy_questions = self.generate_index_entries(['ccsy'], [None], + country_pairs, + trade_flows, [None]) # Where did France export wine to in 2012? - cspy_questions = self.generate_titles(['cspy'], [None], countries, - trade_flows, [None], - products) + cspy_questions = self.generate_index_entries(['cspy'], [None], + countries, trade_flows, + [None], products) all_questions = itertools.chain(casy_questions, casy_questions2, csay_questions, sapy_questions, @@ -85,8 +87,9 @@ def convert_to_elasticsearch_command(data): return action, doc_body @staticmethod - def generate_index_entries(api_name=None, app_name=None, countries=None, - trade_flow=None, years=None, product=None): + def generate_index_entries(api_name=[None], app_name=[None], + countries=[None], trade_flow=[None], + years=[None], product=[None]): """Given a list of possible parameters for the get_title() and params_to_url() functions, generate index entries for all permutations of titles. Parameters must be in order for get_title(). Each parameter @@ -103,24 +106,32 @@ def generate_index_entry(args): index = {} # Generate title + country_names = None + if args[2] is not None: + country_names = [c.name_en for c in args[2]] + title = get_title( api_name=args[0], app_name=args[1], - country_names=[c.name_en for c in args[2]], + country_names=country_names, trade_flow=args[3], years=args[4], - product_names=[p.name_en for p in args[5]], + product_name=args[5].name_en if args[5] is not None else None ) index["title"] = title # Generate url + country_codes = None + if args[2] is not None: + country_codes = [c.name_3char for c in args[2]] + url = params_to_url( api_name=args[0], app_name=args[1], - country_codes=[c.name_3char for c in args[2]], + country_codes=country_codes, trade_flow=args[3], years=args[4], - product_codes=[p.code for p in args[5]], + product_code=args[5].code if args[5] is not None else None ) index["url"] = url From 3331f92db6236d90b4946ae95689a2584650dd65 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Thu, 8 May 2014 11:51:53 -0400 Subject: [PATCH 43/52] Add in question arguments to index entries --- .../management/commands/index_questions.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/django_files/observatory/management/commands/index_questions.py b/django_files/observatory/management/commands/index_questions.py index 1a9e6eb2..5499b896 100644 --- a/django_files/observatory/management/commands/index_questions.py +++ b/django_files/observatory/management/commands/index_questions.py @@ -135,7 +135,19 @@ def generate_index_entry(args): ) index["url"] = url - # TODO: Add in params into elasticsearch in case we need them later + # Add in params into elasticsearch in case we need them later + kwargs = dict( + api_name=args[0], + app_name=args[1], + country_names=country_names, + country_codes=country_codes, + trade_flow=args[3], + years=args[4], + product_name=args[5].name_en if args[5] is not None else None, + product_code=args[5].code if args[5] is not None else None + ) + kwargs = {k: v for k, v in kwargs.iteritems() if v is not None} + index.update(kwargs) return index From b62d1983870839068b60a987ba2fa6be243e4483 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Thu, 8 May 2014 11:56:35 -0400 Subject: [PATCH 44/52] Fiddle with delay --- html/test_search.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/html/test_search.html b/html/test_search.html index 87860d21..a5a88a29 100644 --- a/html/test_search.html +++ b/html/test_search.html @@ -17,7 +17,7 @@ var cache = {}; $("#searchbar").autocomplete({ minLength: 3, -delay: 240, +delay: 260, source: function(request, response) { var term = request.term; if (term in cache) { From e7dfbdc8f149b9e665c37506178e301b8a6e2a9a Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Thu, 8 May 2014 11:56:58 -0400 Subject: [PATCH 45/52] Adjust fuzziness and number of search results --- django_files/observatory/views.py | 34 ++++++++++++++++++------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/django_files/observatory/views.py b/django_files/observatory/views.py index a3b7523b..edbda433 100644 --- a/django_files/observatory/views.py +++ b/django_files/observatory/views.py @@ -2485,27 +2485,33 @@ def api_search(request): es = Elasticsearch() result = es.search( index="questions", - body={"query": { - "filtered": { - "query": { - "fuzzy_like_this": { - "like_text": query, - "fields": [ - "title" - ], - "fuzziness": 0.5, - "max_query_terms": 15, - "prefix_length": 4 + explain=True, + body={ + "query": { + "filtered": { + "query": { + "fuzzy_like_this": { + "like_text": query, + "fields": ["title"], + "fuzziness": 3, + "max_query_terms": 15, + "prefix_length": 4 + } } } - } - }, - "size": 10 + }, + # "highlight": { + # "pre_tags": ["
"], + # "fields": {"title": {}}, + # "post_tags": ["
"] + # }, + "size": 8 }) result_list = [{'label': x['_source']['title'], 'value': x['_source']['url']} for x in result['hits']['hits']] return HttpResponse(json.dumps(result_list)) + def search(request): return render_to_response("test_search.html") From fc390265420a75a7b3310ed8031c38cc9d76c89f Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Thu, 8 May 2014 11:59:06 -0400 Subject: [PATCH 46/52] Bugfix when term is not specified. --- django_files/observatory/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django_files/observatory/views.py b/django_files/observatory/views.py index edbda433..370b575c 100644 --- a/django_files/observatory/views.py +++ b/django_files/observatory/views.py @@ -2478,9 +2478,9 @@ def get_country_lookup(): def api_search(request): - query = request.GET["term"] + query = request.GET.get("term", None) if query == None: - return HttpResponse(json.dumps("{}")) + return HttpResponse("[]") es = Elasticsearch() result = es.search( From d43c4cfd96fe2fd72a11c778c7328db0b443a4a6 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Thu, 8 May 2014 14:58:56 -0400 Subject: [PATCH 47/52] Fix wrong url redirection bug and add jquery highlight search results. --- html/test_search.html | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/html/test_search.html b/html/test_search.html index a5a88a29..08b61455 100644 --- a/html/test_search.html +++ b/html/test_search.html @@ -14,6 +14,15 @@