diff --git a/FAQ.md b/FAQ.md new file mode 100644 index 00000000..8d275d1f --- /dev/null +++ b/FAQ.md @@ -0,0 +1,11 @@ + +Install +=============== + +**Q: How to install the cairo module?** + +- Get [XQuartz](https://xquartz.macosforge.org/landing/) +- Get [homebrew](http://brew.sh/) if you don't have it already +- Run `brew install cairo` +- Run `brew install py2cairo` + diff --git a/django_files/atlas/settings.py b/django_files/atlas/settings.py index 1ad18df6..5bb55e0b 100644 --- a/django_files/atlas/settings.py +++ b/django_files/atlas/settings.py @@ -114,6 +114,11 @@ VERSION = '1.0.4' +CACHE_VERY_SHORT = 60*10 # 10 minutes +CACHE_SHORT = 60*60 # 1 hour +CACHE_LONG = 60*60*24 # 1 day +CACHE_VERY_LONG = 60*60*24*7 # 1 week + try: from settings_local import * except ImportError: diff --git a/django_files/atlas/urls.py b/django_files/atlas/urls.py index e4be2221..75de38da 100644 --- a/django_files/atlas/urls.py +++ b/django_files/atlas/urls.py @@ -1,13 +1,11 @@ #from django.conf.urls import patterns, include, url #from django.conf.urls import patterns, url -from django.conf.urls.defaults import * #from django.views.generic.simple import redirect_to from django.conf.urls import patterns, include from django.views.generic import TemplateView, RedirectView -from django.views.generic.simple import direct_to_template # sitemap -from django.conf.urls.defaults import * +from django.conf.urls import * from django.contrib.sitemaps import FlatPageSitemap, GenericSitemap from django.conf import settings @@ -70,7 +68,7 @@ def render_to_response(self, context, **kwargs): (r'^api/apps/$', 'observatory.views_infopages.api_apps'), (r'^api/data/$', 'observatory.views_infopages.api_data'), (r'^api/views/$', 'observatory.views.api_views'), - (r'^api/dropdowns/products/$', 'observatory.views_dropdown.api_dropdown_products'), + (r'^api/dropdowns/products/(?P[a-z0-9A-Z=_]+)/$', 'observatory.views_dropdown.api_dropdown_products'), (r'^api/dropdowns/countries/$', 'observatory.views_dropdown.api_dropdown_countries'), # Story ##### @@ -97,29 +95,22 @@ def render_to_response(self, context, **kwargs): (r'^browseStoryPrev/$', 'observatory.views_stories.browseStoryPrev'), (r'^createStory/$', 'observatory.views_stories.createStory'), - # Explore (App) ############################################################# - # Legacy app redirect - (r'^app/(?P[a-z0-9_]+)/(?P\w{6,10})/(?P[a-z0-9\.]+)/(?P[0-9\.]+)/$', 'observatory.views.app_redirect'), - # New app URL structure (r'^explore/$', 'observatory.views.explore_random'), (r'^explore/(?P[a-z_]+)/(?P\w{6,10})/(?P\w{3,4})/(?P\w{3,4})/(?P\w{3,4})/(?P[0-9\.]+)/$', 'observatory.views.explore'), (r'^explore/(?P[a-z_]+)/(?P\w{6,10})/(?P\w{3,4})/(?P\w{3,4})/(?P\w{3,4})/$', 'observatory.views.explore'), - # Embed URL - (r'^embed/(?P[a-z_]+)/(?P\w{6,10})/(?P\w{3,4})/(?P\w{3,4})/(?P\w{3,4})/(?P[0-9\.]+)/$', 'observatory.views.embed'), - # API ####################################################################### - (r'^api/(?P[a-z_]{6,10})/(?P\w{3})/all/show/(?P[0-9\.]+)/$', 'observatory.views.api_casy'), - (r'^api/(?P[a-z_]{6,10})/(?P\w{3})/show/all/(?P[0-9\.]+)/$', 'observatory.views.api_csay'), - (r'^api/(?P[a-z_]{6,10})/(?P\w{3})/(?P\w{3})/show/(?P[0-9\.]+)/$', 'observatory.views.api_ccsy'), - (r'^api/(?P[a-z_]{6,10})/(?P\w{3})/show/(?P\w{4})/(?P[0-9\.]+)/$', 'observatory.views.api_cspy'), - (r'^api/(?P[a-z_]{6,10})/show/all/(?P\w{4})/(?P[0-9\.]+)/$', 'observatory.views.api_sapy'), + (r'^api/(?P[a-z_]{6,10})/(?P\w{3})/all/show/(?P[0-9\.]+)/$', 'observatory.views_api.api_casy'), + (r'^api/(?P[a-z_]{6,10})/(?P\w{3})/show/all/(?P[0-9\.]+)/$', 'observatory.views_api.api_csay'), + (r'^api/(?P[a-z_]{6,10})/(?P\w{3})/(?P\w{3})/show/(?P[0-9\.]+)/$', 'observatory.views_api.api_ccsy'), + (r'^api/(?P[a-z_]{6,10})/(?P\w{3})/show/(?P\w{4})/(?P[0-9\.]+)/$', 'observatory.views_api.api_cspy'), + (r'^api/(?P[a-z_]{6,10})/show/all/(?P\w{4})/(?P[0-9\.]+)/$', 'observatory.views_api.api_sapy'), (r'^api/near/(?P\w{3})/(?P[0-9\.]+)/(?P\d+)/$', 'observatory.views_exhibit.api_near'), (r'^api/search/$', 'observatory.views_search.api_search'), - (r'^search/$', direct_to_template, {'template': 'searchresults.html'}), + (r'^search/$', TemplateView.as_view(template_name='searchresults.html')), (r'^api/views/$', 'observatory.views.api_views'), @@ -145,8 +136,7 @@ def render_to_response(self, context, **kwargs): url(r'^favicon\.ico$', RedirectView.as_view(url='/media/img/favicon.ico')), url(r'^sitemap\.xml$', RedirectView.as_view(url='/media/sitemaps/sitemap_index.xml')), - url(r'^opensearch.xml$', direct_to_template, {'template': 'opensearch.xml', - 'mimetype': - 'application/opensearchdescription+xml'}), + url(r'^opensearch.xml$', TemplateView.as_view(template_name='opensearch.xml', + content_type='application/opensearchdescription+xml')) ) diff --git a/django_files/observatory/helpers.py b/django_files/observatory/helpers.py index 311fa2b9..11c1331c 100644 --- a/django_files/observatory/helpers.py +++ b/django_files/observatory/helpers.py @@ -1,4 +1,10 @@ -from observatory.models import Hs4_cpy, Sitc4_cpy, Country, Hs4, Sitc4 +from django.conf import settings +from django.utils.translation import get_language_info + +from cache_utils.decorators import cached + +from observatory.models import (Hs4_cpy, Sitc4_cpy, Country, Hs4, Sitc4, + Sitc4_py, Hs4_py, Cy, Country_region) # make sure app name is in the list of possible apps @@ -77,28 +83,28 @@ def get_country(country): return c -# Returns the Product object or None -def get_product(product, classification): - # first try looking up based on 3 character code +def get_product_by_code(product_code, classification="hs4"): + """Look up a product code in a given product code with fallback to + another.""" if classification == "hs4": try: - p = Hs4.objects.get(code=product) + p = Hs4.objects.get(code=product_code) except Hs4.DoesNotExist: - # next try SITC4 try: - conv_code = Sitc4.objects.get(code=product).conversion_code + conv_code = Sitc4.objects\ + .get(code=product_code).conversion_code p = Hs4.objects.get(code=conv_code) - except Sitc4.DoesNotExist: + except (Hs4.DoesNotExist, Sitc4.DoesNotExist): p = None else: try: - p = Sitc4.objects.get(code=product) + p = Sitc4.objects.get(code=product_code) except Sitc4.DoesNotExist: - # next try SITC4 try: - conv_code = Hs4.objects.get(code=product).conversion_code + conv_code = Hs4.objects\ + .get(code=product_code).conversion_code p = Sitc4.objects.get(code=conv_code) - except Hs4.DoesNotExist: + except (Hs4.DoesNotExist, Sitc4.DoesNotExist): p = None return p @@ -218,3 +224,116 @@ def params_to_url(api_name=None, app_name=None, country_codes=None, url += "%s/" % years return url + + +@cached(settings.CACHE_VERY_LONG) +def get_world_trade(prod_class="hs4"): + """Get world trade volume for every product in a classification.""" + if prod_class == "sitc4": + return list( + Sitc4_py.objects.all().values( + 'year', + 'product_id', + 'world_trade')) + elif prod_class == "hs4": + return list( + Hs4_py.objects.all().values( + 'year', + 'product_id', + 'world_trade')) + + +@cached(settings.CACHE_VERY_LONG) +def get_attrs(prod_class="hs4", name="name_en"): + """Get extraneous attributes (like color and code) for each product in a + classification.""" + if prod_class == "sitc4": + attr_list = list( + Sitc4.objects.all().values( + 'code', + name, + 'id', + 'color')) + attr = {} + for i in attr_list: + attr[i['code']] = { + 'code': i['code'], + 'name': i[name], + 'color': i['color']} + elif prod_class == "hs4": + attr_list = list( + Hs4.objects.all().values( + 'code', + name, + 'id', + 'community_id__color')) + attr = {} + for i in attr_list: + attr[ + i['code']] = { + 'code': i['code'], + 'name': i[name], + 'item_id': i['id'], + 'color': i['community_id__color']} + return attr + + +@cached(settings.CACHE_VERY_LONG) +def get_years_available(prod_class="hs4"): + """Get years available for a given classification.""" + if prod_class == "sitc4": + years_available = Sitc4_cpy.objects\ + .values_list("year", flat=True).distinct() + else: + years_available = Hs4_cpy.objects\ + .values_list("year", flat=True).distinct() + return sorted(list(years_available)) + + +@cached(settings.CACHE_VERY_LONG) +def get_inflation_adjustment(country, first_year, last_year): + """For a given country and year range, get inflation adjustment + constants.""" + inflation_constants = Cy.objects\ + .filter(country=country.id, + year__range=(first_year, + last_year))\ + .values('year', + 'pc_constant', + 'pc_current', + 'notpc_constant') + magic_numbers = {} + for year in inflation_constants: + magic_numbers[year['year']] = { + "pc_constant": year['pc_constant'], + "pc_current": year['pc_current'], + "notpc_constant": year["notpc_constant"]} + return magic_numbers + + +@cached(settings.CACHE_VERY_LONG) +def get_region_list(): + region_list = list(Country_region.objects.all().values()) + region = {} + for i in region_list: + region[i['id']] = i + return region + + +@cached(settings.CACHE_VERY_LONG) +def get_continent_list(): + continent_list = list(Country.objects.all().distinct().values('continent')) + continents = {} + for i, k in enumerate(continent_list): + continents[k['continent']] = i*1000 + return continents + + +def get_language(request): + """Given a request, check the GET params and then the session to find + language info specified in 2 char ISO code form, and then make sure it's + valid, and return a tuple in the format of django's get_language_info. + Nonexistent languages raise KeyError.""" + lang = request.GET.get("lang", + request.session.get('django_language', 'en')) + return get_language_info(lang) diff --git a/django_files/observatory/models.py b/django_files/observatory/models.py index d09bddbd..cc2ac690 100644 --- a/django_files/observatory/models.py +++ b/django_files/observatory/models.py @@ -3,6 +3,7 @@ from django.forms import ModelForm from django.conf import settings +from cache_utils.decorators import cached if not settings.DB_PREFIX: DB_PREFIX = '' @@ -242,6 +243,7 @@ def filter_lang(self, lang): lang = lang.replace("-", "_") return self.extra(select={"name": "name_"+lang}) + @cached(settings.CACHE_VERY_LONG) def get_all(self, lang): products = self.filter_lang(lang) @@ -309,7 +311,9 @@ class Meta: db_table = DB_PREFIX+"observatory_sitc4_py" product = models.ForeignKey(Sitc4) - year = models.PositiveSmallIntegerField(max_length=4) + # The unique=True here is a workaround that somehow makes the ForeignObject + # on cpy work. + year = models.PositiveSmallIntegerField(max_length=4, unique=True) pci = models.FloatField(null=True) pci_rank = models.PositiveSmallIntegerField(max_length=4) world_trade = models.FloatField(null=True) @@ -323,7 +327,9 @@ class Meta: db_table = DB_PREFIX+"observatory_hs4_py" product = models.ForeignKey('Hs4') - year = models.PositiveSmallIntegerField(max_length=4) + # The unique=True here is a workaround that somehow makes the ForeignObject + # on cpy work. + year = models.PositiveSmallIntegerField(max_length=4, unique=True) pci = models.FloatField(null=True) pci_rank = models.PositiveSmallIntegerField(max_length=4) world_trade = models.FloatField(null=True) @@ -358,6 +364,7 @@ def filter_lang(self, lang): lang = lang.replace("-", "_") return self.extra(select={"name": "name_"+lang}) + @cached(settings.CACHE_VERY_LONG) def get_all(self, lang): products = self.filter_lang(lang) products = products.filter(community__isnull=False)#, ps_size__isnull=False) @@ -438,6 +445,8 @@ class Meta: distance = models.FloatField(null=True) opp_gain = models.FloatField(null=True) + product_year = models.ForeignObject(Sitc4_py, ('product', 'year'), ('product', 'year')) + def __unicode__(self): return "CPY: %s.%s.%d" % (self.country.name, self.product.code, self.year) @@ -459,6 +468,8 @@ class Meta: distance = models.FloatField(null=True) opp_gain = models.FloatField(null=True) + product_year = models.ForeignObject(Hs4_py, ('product', 'year'), ('product', 'year')) + def __unicode__(self): return "CPY: %s.%s.%d" % (self.country.name, self.product.code, self.year) diff --git a/django_files/observatory/views.py b/django_files/observatory/views.py index e3ed7473..f7101019 100644 --- a/django_files/observatory/views.py +++ b/django_files/observatory/views.py @@ -5,511 +5,407 @@ import collections import json import time -from urlparse import urlparse - # Django from django.shortcuts import render_to_response, redirect -from django.http import (HttpResponse, Http404, HttpResponsePermanentRedirect, - HttpResponseRedirect) +from django.http import (HttpResponse, HttpResponseRedirect) from django.template import RequestContext from django.conf import settings +from django.core.cache import cache from django.core.urlresolvers import reverse, resolve +from django.utils import translation from django.utils.translation import gettext as _ import cairosvg +from redis_cache.cache import RedisCache # Local -from observatory.models import (Country, Country_region, Cy, Hs4, Hs4_py, - Hs4_cpy, Sitc4, Sitc4_py, Sitc4_cpy) -from observatory.models import raw_q +from observatory.models import (Country, Hs4_cpy, Sitc4_cpy) from observatory import helpers -# Conditional things -if settings.REDIS: - import redis - import msgpack - if not settings.DB_PREFIX: - DB_PREFIX = '' + DB_PREFIX = '' else: - DB_PREFIX = settings.DB_PREFIX + DB_PREFIX = settings.DB_PREFIX if not settings.VERSION: - VERSION = '1.0.0' + VERSION = '1.0.0' else: - VERSION = settings.VERSION + VERSION = settings.VERSION if not settings.HTTP_HOST: - HTTP_HOST = '/' + HTTP_HOST = '/' else: - HTTP_HOST = settings.HTTP_HOST - - -#Static Images - -def url_to_filename(request,Url): - # set language (if session data available use that as default) - lang = request.session['django_language'] if 'django_language' in request.session else "en" - lang = request.GET.get("lang", lang) - # set product classification (if session data available use that as default) - prod_class = request.session['product_classification'] if 'product_classification' in request.session else "hs4" - prod_class = request.GET.get("product_classification", prod_class) - #Get session parameters - language=lang - try: - view, args, kwargs = resolve(urlparse(Url)[2]) - app_name = kwargs["app_name"] - trade_flow = kwargs["trade_flow"] - country1 = kwargs["country1"] - product = kwargs["product"] - country2 = kwargs["country2"] - year = kwargs["year"] - url = reverse(view, kwargs=kwargs) - product_file_bit = country2 + "_" + product - file_name = app_name + "_" + language + "_" + prod_class + "_" + trade_flow + "_" + country1 + "_" + product_file_bit + "_" + str(year) - response = file_name - return response - except Http404: - print "Invalid Url" - -def filename_to_url(request, fileName,): - # set language (if session data available use that as default) - lang = request.session['django_language'] if 'django_language' in request.session else "en" - lang = request.GET.get("lang", lang) - # set product classification (if session data available use that as default) - prod_class = request.session['product_classification'] if 'product_classification' in request.session else "hs4" - prod_class = request.GET.get("product_classification", prod_class) - #Get session parameters - language=lang - # Split the array to get the parts we want - data=fileName.split('_') - app_name1=data[0] - app_name2=data[1] - if app_name1 == "stacked": - app_name= data[0] - lanuage = data[1] - prod_class = data[2] - trade_flow = data[3] - country1 = data[4] - country2 = data[5] - product = data[6] - year= data[7] - # Build the url - #url = app_name + "/" + trade_flow + "/" + country1 + "/" + country2 + "/" + product + "/" + year - url = reverse('observatory.views.explore', kwargs={'app_name': app_name,'trade_flow':trade_flow,'country1':country1,'country2':country2,'product':product,'year':year}) - full_url = request.META['HTTP_HOST'] + '/explore/' + url - response = full_url - return response - if app_name1 == "map": - app_name= data[0] - lanuage = data[1] - prod_class = data[2] - trade_flow = data[3] - country1 = data[4] - country2 = data[5] - product = data[6] - year= data[7] - # Build the url - #url = app_name + "/" + trade_flow + "/" + country1 + "/" + country2 + "/" + product + "/" + year - url = reverse('observatory.views.explore', kwargs={'app_name': app_name,'trade_flow':trade_flow,'country1':country1,'country2':country2,'product':product,'year':year}) - full_url = request.META['HTTP_HOST'] + '/explore/' + url - response = full_url - return response - if app_name1 == "tree" or "pie" or "product": - app_name = data[0] + "_" + data[1] - language = data[2] - prod_class = data[3] - trade_flow = data[4] - country1 = data[5] - country2 = data[6] - product = data[7] - year= data[8] - # Build the url - #url = app_name + "/" + trade_flow + "/" + country1 + "/" + country2 + "/" + product + "/" + year - url = reverse('observatory.views.explore', kwargs={'app_name': app_name,'trade_flow':trade_flow,'country1':country1,'country2':country2,'product':product,'year':year}) - print url - full_url = request.META['HTTP_HOST'] + '/explore/' + url - response = full_url - return response + HTTP_HOST = settings.HTTP_HOST def home(request): - iscreatemode=False - request.session['create']=iscreatemode - isbrowsemode=False - request.session['retrieve']=isbrowsemode - try: - ip = request.META["HTTP_X_FORWARDED_FOR"] - except KeyError: - ip = request.META["REMOTE_ADDR"] c = Country.objects.get(name_2char="us") - return render_to_response("home.html", - {"default_country": c}, - context_instance=RequestContext(request)) + return render_to_response("home.html", + {"default_country": c}, + context_instance=RequestContext(request)) def set_language(request, lang): - next = request.REQUEST.get('next', None) - if not next: - next = request.META.get('HTTP_REFERER', None) - if not next: - next = '/' - response = HttpResponseRedirect(next) - # if request.method == 'GET': - # lang_code = request.GET.get('language', None) - lang_code = lang - if lang_code: - if hasattr(request, 'session'): - request.session['django_language'] = lang_code - else: - response.set_cookie(settings.LANGUAGE_COOKIE_NAME, lang_code) - translation.activate(lang_code) - return response - -def set_product_classification(request, prod_class): - next = request.REQUEST.get('next', None) - if not next: - next = request.META.get('HTTP_REFERER', None) - if not next: - next = '/' - response = HttpResponseRedirect(next) - if prod_class: - if hasattr(request, 'session'): - request.session['product_classification'] = prod_class - request.session['classification'] = prod_class - return response - -def download(request): - - content = request.POST.get("file_content") - title = request.POST.get("file_name").replace(" ", "_") - file_format = request.POST.get("file_format").lower() - - if len(content) == 0 or "" not in content: - return HttpResponse(status=500, - content="Invalid svg image.") - - if file_format == "svg": - response = HttpResponse(content.encode("utf-8"), - mimetype="application/octet-stream") + next = request.REQUEST.get('next', None) + if not next: + next = request.META.get('HTTP_REFERER', None) + if not next: + next = '/' + response = HttpResponseRedirect(next) + # if request.method == 'GET': + # lang_code = request.GET.get('language', None) + lang_code = lang + if lang_code: + if hasattr(request, 'session'): + request.session['django_language'] = lang_code + else: + response.set_cookie(settings.LANGUAGE_COOKIE_NAME, lang_code) + translation.activate(lang_code) + return response - elif file_format == "pdf": - response = HttpResponse(cairosvg.svg2pdf(bytestring=content), - mimetype="application/pdf") - elif file_format == "png": - response = HttpResponse(cairosvg.svg2png(bytestring=content), - mimetype="image/png") +def set_product_classification(request, prod_class): + next = request.REQUEST.get('next', None) + if not next: + next = request.META.get('HTTP_REFERER', None) + if not next: + next = '/' + response = HttpResponseRedirect(next) + if prod_class: + if hasattr(request, 'session'): + request.session['product_classification'] = prod_class + request.session['classification'] = prod_class + return response - else: - return HttpResponse(status=500, content="Wrong image format.") - response["Content-Disposition"]= "attachment; filename=%s.%s" % (title, file_format) +def download(request): - return response + content = request.POST.get("file_content") + title = request.POST.get("file_name").replace(" ", "_") + file_format = request.POST.get("file_format").lower() + if len(content) == 0 or "" not in content: + return HttpResponse(status=500, + content="Invalid svg image.") -def app_redirect(request, app_name, trade_flow, filter, year): - # Corrent for old spelling of tree map as one word - if app_name == "treemap": - app_name = "tree_map" + if file_format == "svg": + response = HttpResponse(content.encode("utf-8"), + mimetype="application/octet-stream") - # Bilateral - if "." in filter: - bilateral_filters = filter.split(".") + elif file_format == "pdf": + response = HttpResponse(cairosvg.svg2pdf(bytestring=content), + mimetype="application/pdf") - # Country x Product - if len(bilateral_filters[1]) > 3: - country1, country2, product = bilateral_filters[0], "show", bilateral_filters[1] + elif file_format == "png": + response = HttpResponse(cairosvg.svg2png(bytestring=content), + mimetype="image/png") - # Country x Country else: - country1, country2, product = bilateral_filters[0], bilateral_filters[1], "show" - - # Product - elif len(filter) > 3: - country1, country2, product = "show", "all", filter - - # Country - else: - country1, country2, product = filter, "all", "show" - # raise Exception("/explore/%s/%s/%s/%s/%s/%s/" % (app_name, trade_flow, country1, country2, product, year)) - return HttpResponsePermanentRedirect(HTTP_HOST+"explore/%s/%s/%s/%s/%s/%s/" % (app_name, trade_flow, country1, country2, product, year)) - - -def explore(request, app_name, trade_flow, country1, country2, product, year="2011"): - iscreatemode=False - iscreatemode = request.session['create'] if 'create' in request.session else False - isbrowsemode=False - isbrowsemode = request.session['retrieve'] if 'retrieve' in request.session else False - NoOfChapter=request.session['BrowseStoryChapNos'] if 'BrowseStoryChapNos' in request.session else "" - browseStoryName=request.session['browseStoryName'] if 'browseStoryName' in request.session else "" - browseStoryDesc=request.session['browseStoryDesc'] if 'browseStoryDesc' in request.session else "" - browseChapterName=request.session['browseStoryChapName'] if 'browseStoryChapName' in request.session else "" - browseChapterDesc=request.session['browseStoryChapterDesc'] if 'browseStoryChapterDesc'in request.session else "" - browseModeJScript=request.session['browseStoryJScript'] if 'browseStoryJScript' in request.session else "" - NoOfChpt=request.session['NoOfchap'] if 'NoOfchap' in request.session else "" - userName=request.session['username'] if 'username' in request.session else "" - userId=request.session['userid'] if 'userid' in request.session else 0 - likeBtnEnable=request.session['likeBtnEnable'] if 'likeBtnEnable' in request.session else False - likeCount=request.session['likeCount'] if 'likeCount' in request.session else "" - #Set app_name to session - request.session['app_name']=app_name - # raise Exception(country1, country2, product, year) - # Get URL query parameters - was_redirected = request.GET.get("redirect", False) - crawler = request.GET.get("_escaped_fragment_", False) - options = request.GET.copy() - # set language (if session data available use that as default) - lang = request.session['django_language'] if 'django_language' in request.session else "en" - lang = request.GET.get("lang", lang) - options["lang"] = lang - # set product classification (if session data available use that as default) - prod_class = request.session['product_classification'] if 'product_classification' in request.session else "hs4" - prod_class = request.GET.get("product_classification", prod_class) - options["product_classification"] = prod_class - options = options.urlencode() - #Get session parameters - language=lang - # Setup the hash dictionary - request_hash_dictionary = collections.OrderedDict() - - # Add prod class to request hash dictionary - request_hash_dictionary['app_name'] = app_name - request_hash_dictionary['lang'] = lang - request_hash_dictionary['prod_class'] = prod_class - - # Add the arguments to the request hash dictionary - request_hash_dictionary['trade_flow'] = trade_flow - request_hash_dictionary['country1'] = country1 - request_hash_dictionary['country2'] = country2 - request_hash_dictionary['product_type'] = product - request_hash_dictionary['year'] = year - - # We are here, so let us store this data somewhere - request_hash_string = "_".join( request_hash_dictionary.values() ) - - # Check staic image mode - if( settings.STATIC_IMAGE_MODE == "SVG" ): - # Check if we have a valid PNG image to display for this - if os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".png"): - # display the static images - displayviz = True - displayImage = settings.STATIC_URL + "data/" + request_hash_string + ".png" + return HttpResponse(status=500, content="Wrong image format.") + + response[ + "Content-Disposition"] = "attachment; filename=%s.%s" % (title, file_format) + + return response + + +def explore( + request, + app_name, + trade_flow, + country1, + country2, + product, + year="2011"): + + request.session['app_name'] = app_name + + was_redirected = request.GET.get("redirect", False) + lang = helpers.get_language(request)['code'] + crawler = request.GET.get("_escaped_fragment_", False) + + prod_class = request.session['product_classification'] if\ + 'product_classification' in request.session else "hs4" + prod_class = request.GET.get("product_classification", prod_class) + + options = request.GET.copy() + options["lang"] = lang + options["product_classification"] = prod_class + options = options.urlencode() + + # Setup the hash dictionary + request_hash_dictionary = collections.OrderedDict() + + # Add prod class to request hash dictionary + request_hash_dictionary['app_name'] = app_name + request_hash_dictionary['lang'] = lang + request_hash_dictionary['prod_class'] = prod_class + + # Add the arguments to the request hash dictionary + request_hash_dictionary['trade_flow'] = trade_flow + request_hash_dictionary['country1'] = country1 + request_hash_dictionary['country2'] = country2 + request_hash_dictionary['product_type'] = product + request_hash_dictionary['year'] = year + + # We are here, so let us store this data somewhere + request_hash_string = "_".join(request_hash_dictionary.values()) + + # Check staic image mode + if(settings.STATIC_IMAGE_MODE == "SVG"): + # Check if we have a valid PNG image to display for this + if os.path.exists(settings.DATA_FILES_PATH + "/" + request_hash_string + ".png"): + # display the static images + displayviz = True + displayImage = settings.STATIC_URL + \ + "data/" + request_hash_string + ".png" + else: + displayviz = False + displayImage = settings.STATIC_URL + "img/all/loader.gif" else: - displayviz=False + displayviz = False displayImage = settings.STATIC_URL + "img/all/loader.gif" - else: - displayviz=False - displayImage = settings.STATIC_URL + "img/all/loader.gif" - - # Verify countries from DB - countries = [None, None] - alert = None - for i, country in enumerate([country1, country2]): - if country != "show" and country != "all": - try: - countries[i] = Country.objects.get(name_3char=country) - except Country.DoesNotExist: - alert = {"title": "Country could not be found", - "text": "There was no country with the 3 letter abbreviation %s. Please double check the list of countries."%(country)} - # The years of data available tends to vary based on the dataset used (Hs4 - # vs Sitc4) and the specific country. - years_available_model = Sitc4_cpy if prod_class == "sitc4" else Hs4_cpy - years_available = years_available_model.objects\ - .values_list("year", flat=True)\ - .order_by("year")\ - .distinct() - # Sometimes the query is not about a specific country (e.g. "all countries" - # queries) in which case filtering by country is not necessary - if countries[0]: - years_available = years_available.filter(country=countries[0].id) - # Force lazy queryset to hit the DB to reduce number of DB queries later - years_available = list(years_available) - - if len(years_available) == 0: - alert = {"title": """The product classification you're using (%s) does + # Verify countries from DB + countries = [None, None] + alert = None + for i, country in enumerate([country1, country2]): + if country != "show" and country != "all": + try: + countries[i] = Country.objects.get(name_3char=country) + except Country.DoesNotExist: + alert = { + "title": "Country could not be found", + "text": """There was no country with the 3 letter + abbreviation %s. Please double check the + list of countries.""" % + (country)} + + # The years of data available tends to vary based on the dataset used (Hs4 + # vs Sitc4) and the specific country. + years_available_model = Sitc4_cpy if prod_class == "sitc4" else Hs4_cpy + years_available = years_available_model.objects\ + .values_list("year", flat=True)\ + .order_by("year")\ + .distinct() + # Sometimes the query is not about a specific country (e.g. "all countries" + # queries) in which case filtering by country is not necessary + if countries[0]: + years_available = years_available.filter(country=countries[0].id) + # Force lazy queryset to hit the DB to reduce number of DB queries later + years_available = list(years_available) + + if len(years_available) == 0: + alert = {"title": """The product classification you're using (%s) does not seem to include data for the country code (%s) you selected. Please try a different product classification or country.""" % - (prod_class, countries[0].name)} - years_available = range(1995, 2012) #Dummy - - country1_list, product_list, year1_list, year2_list, year_interval = None, None, None, None, None - warning, title = None, None - data_as_text = {} - # What is actually being shown on the page - item_type = "product" - prod_or_partner = "partner" - - # To make sure it cannot be another product class - if prod_class != "hs4" and prod_class != "sitc4": - prod_class = "sitc4" - - # Test for country exceptions - if prod_class == "hs4": - # redirect if and exception country - if country1 == "bel" or country1 == "lux": - return redirect(HTTP_HOST+'explore/%s/%s/blx/%s/%s/%s/?redirect=true' % (app_name, trade_flow, country2, product, year)) - if country1 == "bwa" or country1 == "lso" or country1 == "nam" or country1 == "swz": - return redirect(HTTP_HOST+'explore/%s/%s/zaf/%s/%s/%s/?redirect=true' % (app_name, trade_flow, country2, product, year)) - if was_redirected: - # display warning is redirected from exception - if country1 == "blx": - warning = {"title": "Country Substitution", - "text": "In the Harmonized System (HS) classification, trade for Belgium and Luxembourg is reported as 'Belgium-Luxembourg'."} - if country1 == "zaf": - warning = {"title": "Country Substitution", - "text": "In the Harmonized System (HS) classification, trade for Namibia, Republic of South Africa, Botswana, Lesotho and Swaziland is reported under 'South African Customs Union'."} - - trade_flow_list = [("export", _("Export")), ("import", _("Import")), ("net_export", _("Net Export")), ("net_import", _("Net Import"))] - if (app_name == "product_space" or app_name == "rings"): - trade_flow_list = [trade_flow_list[0]] - - year1_list = range(years_available[0], years_available[len(years_available)-1]+1, 1) - - if app_name == "stacked" and year == "2009": - year = "1969.2011.10" - - if "." in year: - y = [int(x) for x in year.split(".")] - year_start = y[0] - year_end = y[1] - year2_list = year1_list - else: - year_start, year_end = None, None - year = int(year) - # Check that year is within bounds - if year > years_available[len(years_available)-1]: - year = years_available[len(years_available)-1] - elif year < years_available[0]: - year = years_available[0] - - api_uri = "api/%s/%s/%s/%s/%s/?%s" % (trade_flow, country1, country2, product, year, options) - - redesign_api_uri = "redesign/api/%s/%s/%s/%s/%s/%s" % (prod_class, trade_flow, country1, country2, product, year) - - country_code = None - if country1 != "show" and country1 != "all": country_code = country1 - - if crawler == "": - view, args, kwargs = resolve("api/%s/%s/%s/%s/%s/" % (trade_flow, country1, country2, product, year)) - kwargs['request'] = request - view_response = view(*args, **kwargs) - raise Exception(view_response) - data_as_text["data"] = view_response[0] - data_as_text["total_value"] = view_response[1] - data_as_text["columns"] = view_response[2] - - - app_type = helpers.get_app_type(country1, country2, product, year) - - # Some countries need "the" before their names - list_countries_the = set(("Cayman Islands", "Central African Republic", - "Channel Islands", "Congo, Dem. Rep.", - "Czech Republic", "Dominican Republic", - "Faeroe Islands", "Falkland Islands", "Fm Yemen Dm", - "Lao PDR", "Marshall Islands", "Philippines", - "Seychelles", "Slovak Republic", - "Syrian Arab Republic", "Turks and Caicos Islands", - "United Arab Emirates", "United Kingdom", - "Virgin Islands, U.S.", "United States")) - if countries[0] and countries[0].name in list_countries_the: - countries[0].name = "the "+countries[0].name - - #p_code, product = None, None - if product not in ("show", "all"): - p_code = product - product = clean_product(p_code, prod_class) - - if not alert: - - # Generate page title depending on visualization being used - trade_flow = trade_flow.replace('_', ' ') - years = [year_start, year_end] if year_start is not None else [year] - product_name = product.name_en if not isinstance(product, basestring) else product - country_names = [getattr(x, "name", None) for x in countries] - title = helpers.get_title(app_type, app_name, - country_names=country_names, - trade_flow=trade_flow, - years=years, - product_name=product_name - ) - - if app_type in ("ccsy", "cspy"): - if _("net_export") in trade_flow_list: del trade_flow_list[trade_flow_list.index(_("net_export"))] - if _("net_import") in trade_flow_list: del trade_flow_list[trade_flow_list.index(_("net_import"))] - #trade_flow_list.pop(_("net_export"), None) - - # Should we show the product or partner tab pane? - # quick fix should be merged with item_type - if app_type in ["cspy", "sapy"]: - prod_or_partner = "product" - elif app_type == "casy": - if app_name in ("stacked", "map", "tree_map","pie_scatter", "product_space"): - prod_or_partner = "product" - - - # Return page without visualization data - - # Record views in redis for "newest viewed pages" visualization - if settings.REDIS: - views_image_path = settings.STATIC_URL + "data/" + request_hash_string + ".png" - view_data = { - "timestamp": time.time(), - "image": views_image_path, - "title": title, - "url": request.build_absolute_uri() - } - raw = redis.Redis("localhost", db=1) - raw.rpush("views", json.dumps(view_data)) + (prod_class, countries[0].name)} + years_available = range(1995, 2012) # Dummy + + year1_list, year2_list = None, None + warning, title = None, None + data_as_text = {} + # What is actually being shown on the page + item_type = "product" + prod_or_partner = "partner" + + # To make sure it cannot be another product class + if prod_class != "hs4" and prod_class != "sitc4": + prod_class = "sitc4" + + # Test for country exceptions + if prod_class == "hs4": + # redirect if and exception country + if country1 == "bel" or country1 == "lux": + return redirect( + HTTP_HOST+'explore/%s/%s/blx/%s/%s/%s/?redirect=true' % + (app_name, trade_flow, country2, product, year)) + if country1 == "bwa" or country1 == "lso" or country1 == "nam" or country1 == "swz": + return redirect( + HTTP_HOST+'explore/%s/%s/zaf/%s/%s/%s/?redirect=true' % + (app_name, trade_flow, country2, product, year)) + if was_redirected: + # display warning is redirected from exception + if country1 == "blx": + warning = { + "title": "Country Substitution", "text": + "In the Harmonized System (HS) classification, trade for Belgium and Luxembourg is reported as 'Belgium-Luxembourg'."} + if country1 == "zaf": + warning = { + "title": "Country Substitution", "text": + "In the Harmonized System (HS) classification, trade for Namibia, Republic of South Africa, Botswana, Lesotho and Swaziland is reported under 'South African Customs Union'."} + + trade_flow_list = [ + ("export", _("Export")), ("import", _("Import")), + ("net_export", _("Net Export")), ("net_import", _("Net Import"))] + if (app_name == "product_space" or app_name == "rings"): + trade_flow_list = [trade_flow_list[0]] + + year1_list = range( + years_available[0], + years_available[ + len(years_available) - + 1] + + 1, + 1) + + if app_name == "stacked" and year == "2009": + year = "1969.2011.10" + + if "." in year: + y = [int(x) for x in year.split(".")] + year_start = y[0] + year_end = y[1] + year2_list = year1_list + else: + year_start, year_end = None, None + year = int(year) + # Check that year is within bounds + if year > years_available[len(years_available)-1]: + year = years_available[len(years_available)-1] + elif year < years_available[0]: + year = years_available[0] + + api_uri = "api/%s/%s/%s/%s/%s/?%s" % (trade_flow, + country1, + country2, + product, + year, + options) + + redesign_api_uri = "redesign/api/%s/%s/%s/%s/%s/%s" % (prod_class, + trade_flow, + country1, + country2, + product, + year) + + country_code = None + if country1 != "show" and country1 != "all": + country_code = country1 + if crawler == "": + view, args, kwargs = resolve( + "api/%s/%s/%s/%s/%s/" % + (trade_flow, country1, country2, product, year)) + kwargs['request'] = request + view_response = view(*args, **kwargs) + raise Exception(view_response) + data_as_text["data"] = view_response[0] + data_as_text["total_value"] = view_response[1] + data_as_text["columns"] = view_response[2] + + app_type = helpers.get_app_type(country1, country2, product, year) + + # Some countries need "the" before their names + list_countries_the = set( + ("Cayman Islands", + "Central African Republic", + "Channel Islands", + "Congo, Dem. Rep.", + "Czech Republic", + "Dominican Republic", + "Faeroe Islands", + "Falkland Islands", + "Fm Yemen Dm", + "Lao PDR", + "Marshall Islands", + "Philippines", + "Seychelles", + "Slovak Republic", + "Syrian Arab Republic", + "Turks and Caicos Islands", + "United Arab Emirates", + "United Kingdom", + "Virgin Islands, U.S.", + "United States")) + if countries[0] and countries[0].name in list_countries_the: + countries[0].name = "the "+countries[0].name + + if product not in ("show", "all"): + p_code = product + product = helpers.get_product_by_code(p_code, prod_class) + + if not alert: + + # Generate page title depending on visualization being used + years = [year_start, year_end] if year_start is not None else [year] + product_name = product.name_en if not isinstance( + product, + basestring) else product + country_names = [getattr(x, "name", None) for x in countries] + title = helpers.get_title(app_type, app_name, + country_names=country_names, + trade_flow=trade_flow.replace('_', ' '), + years=years, + product_name=product_name + ) + + if app_type in ("ccsy", "cspy"): + if _("net_export") in trade_flow_list: + del trade_flow_list[trade_flow_list.index(_("net_export"))] + if _("net_import") in trade_flow_list: + del trade_flow_list[trade_flow_list.index(_("net_import"))] + + # Should we show the product or partner tab pane? + # quick fix should be merged with item_type + if app_type in ["cspy", "sapy"]: + prod_or_partner = "product" + elif app_type == "casy": + if app_name in ("stacked", "map", "tree_map", "pie_scatter", "product_space"): + prod_or_partner = "product" + + # Record views in redis for "newest viewed pages" visualization + if isinstance(cache, RedisCache): + views_image_path = settings.STATIC_URL + \ + "data/" + request_hash_string + ".png" + view_data = { + "timestamp": time.time(), + "image": views_image_path, + "title": title, + "url": request.build_absolute_uri() + } + r = cache.raw_client + r.rpush("views", json.dumps(view_data)) + + return render_to_response( + "explore/index.html", + {"lang": lang, + "warning": warning, + "alert": alert, + "prod_class": prod_class, + "data_as_text": data_as_text, + "app_name": app_name, + "title": title, + "trade_flow": trade_flow, + "country1": countries[0] or country1, + "country2": countries[1] or country2, + "country1_3char": countries[0].name_3char if countries[0] else "", + "country2_3char": countries[1].name_3char if countries[1] else "", + "product": product, + "product_code": product.code if not isinstance(product, basestring) else product, + "years_available": years_available, + "year": year, + "year_start": year_start, + "year_end": year_end, + "year1_list": year1_list, + "year2_list": year2_list, + "trade_flow_list": trade_flow_list, + "api_uri": api_uri, + "app_type": app_type, + "redesign_api_uri": redesign_api_uri, + "country_code": country_code, + "prod_or_partner": prod_or_partner, + "version": VERSION, + "previous_page": request.META.get('HTTP_REFERER', + None), + "item_type": item_type, + "displayviz": displayviz, + "displayImage": displayImage + }, + context_instance=RequestContext(request)) - return render_to_response("explore/index.html", { - "displayviz":displayviz, - "displayImage":displayImage, - "likeBtnEnable":likeBtnEnable, - "browseModeJScript": browseModeJScript, - "browseChapterDesc" : browseChapterDesc, - "likeCount":likeCount, - "browseChapterName": browseChapterName, - "NoOfChapter" : NoOfChapter, - "browseStoryName": browseStoryName, - "browseStoryDesc" : browseStoryDesc, - "isbrowsemode": isbrowsemode, - "iscreatemode": iscreatemode, - "userName":userName, - "userId":userId, - "language":language, - "warning": warning, - "alert": alert, - "prod_class": prod_class, - "data_as_text": data_as_text, - "app_name": app_name, - "title": title, - "trade_flow": trade_flow, - "country1": countries[0] or country1, - "country2": countries[1] or country2, - "country1_3char": countries[0].name_3char if countries[0] else "", - "country2_3char": countries[1].name_3char if countries[1] else "", - "product": product, - "product_code": product.code if not isinstance(product, basestring) else product, - "years_available": years_available, - "year": year, - "year_start": year_start, - "year_end": year_end, - "year1_list": year1_list, - "year2_list": year2_list, - "trade_flow_list": trade_flow_list, - "api_uri": api_uri, - "app_type": app_type, - "redesign_api_uri": redesign_api_uri, - "country_code": country_code, - "prod_or_partner": prod_or_partner, - "version": VERSION, - "previous_page": request.META.get('HTTP_REFERER', None), - "item_type": item_type}, context_instance=RequestContext(request)) def explore_random(request): """Pick a random country and explore that, for the /explore link on the top @@ -525,1086 +421,12 @@ def explore_random(request): ))) -''' / all / show / ''' -def api_casy(request, trade_flow, country1, year): - # Setup the hash dictionary - request_hash_dictionary = collections.OrderedDict() - - # Store the country code - country_code = country1 - '''Init variables''' - prod_class = request.session['product_classification'] if 'product_classification' in request.session else "hs4" - prod_class = request.GET.get("prod_class", prod_class) - lang = request.session['django_language'] if 'django_language' in request.session else "en" - lang = request.GET.get("lang", lang) - crawler = request.GET.get("_escaped_fragment_", False) - country1 = Country.objects.get(name_3char=country1) - single_year = 'single_year' in request.GET - - '''Set query params with our changes''' - query_params = request.GET.copy() - query_params["lang"] = lang - query_params["product_classification"] = prod_class - #Get app_name from session - app_name = request.session.get( "app_name", "tree_map" ) #request.session['app_name'] if 'app_name' in request.session else "tree_map" - # See if we have an app name passed as part of the request URL - forced_app_name = request.GET.get( "use_app_name", None ) - # If we have an app name passed, override and use that - if ( forced_app_name is not None ): - # override the app_name in this case, since generate_svg will pass app names specifically - app_name = forced_app_name - '''Grab extraneous details''' - ## Clasification & Django Data Call - name = "name_%s" % lang - # Add prod class to request hash dictionary - request_hash_dictionary['app_name'] = app_name - request_hash_dictionary['lang'] = lang - request_hash_dictionary['prod_class'] = prod_class - - # Add the arguments to the request hash dictionary - request_hash_dictionary['trade_flow'] = trade_flow - request_hash_dictionary['country'] = country_code - - #refererUrl = request.META['HTTP_REFERER'] - #view, args, kwargs = resolve(urlparse(refererUrl)[2]) - #request_hash_string = kwargs['app_name'] + '_' + lang + '_' + prod_class + '_' + kwargs['trade_flow'] + '_' + kwargs['country1'] + '_' + kwargs['country2'] + '_' + kwargs['product'] + '_' + year - - # Set the product stuff based on the app - if ( app_name in [ "product_space", "pie_scatter" ] ): - request_hash_dictionary['product_type'] = 'all' - request_hash_dictionary['product_display'] = 'show' - else: - request_hash_dictionary['product_type'] = 'all' - request_hash_dictionary['product_display'] = 'show' - request_hash_dictionary['year'] = year - - # We are here, so let us store this data somewhere - request_hash_string = "_".join( request_hash_dictionary.values() ) #base64.b64encode( request_unique_hash ) - # Setup the store data - store_data = request.build_absolute_uri().replace( "product_classification", "prod_class" ) + "||" - store_page_url = request.build_absolute_uri().replace( "/api/", "/explore/" + app_name + "/" ) - store_page_url = store_page_url.replace( "data_type=json", "headless=true" ) - store_page_url = store_page_url.replace( "product_classification", "prod_class" ) - store_data = store_data + store_page_url + "||" - store_data = store_data + request_hash_string - - # Write the store data to file - store_file = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".store", "w+" ) - store_file.write( store_data ) - store_file.close() - - # Set proper permissions since we want the cron to remove the file as well - os.chmod( settings.DATA_FILES_PATH + "/" + request_hash_string + ".store", 0777 ) - - if ( os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".svg" ) is True ): - # Check the request data type - if ( request.GET.get( 'data_type', None ) is None ): - # Let us get the data from the file - response_json_data = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".svg", "r" ) - - # Set the return data - returnData = response_json_data.read() - - #"""Return to browser as JSON for AJAX request""" - return HttpResponse( returnData ) - elif ( request.GET.get( 'data_type', '' ) == 'json' ): - # Check if we have the valid json file - if ( os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json" ) is True ): - # Let us get the data from the file - response_json_data = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json", "r" ) - - # Set the return data - returnData = response_json_data.read() - - #"""Return to browser as JSON for AJAX request""" - return HttpResponse( returnData ) - - # Get attribute information - if prod_class == "sitc4": - world_trade = list(Sitc4_py.objects.all().values('year','product_id','world_trade')) - attr_list = list(Sitc4.objects.all().values('code',name,'id','color')) - attr = {} - for i in attr_list: - attr[i['code']] = {'code':i['code'],'name':i[name],'color':i['color']} - #.extra(where=['CHAR_LENGTH(code) = 2']) - - elif prod_class == "hs4": - world_trade = list(Hs4_py.objects.all().values('year','product_id','world_trade')) - attr_list = list(Hs4.objects.all().values('code',name,'id','community_id__color')) - attr = {} - for i in attr_list: - attr[i['code']] = {'code':i['code'],'name':i[name],'item_id':i['id'],'color':i['community_id__color']} - #.extra(where=['CHAR_LENGTH(code) = 2']) - - # get distince years from db, different for diff product classifications - years_available = list(Sitc4_cpy.objects.values_list("year", flat=True).distinct()) if prod_class == "sitc4" else list(Hs4_cpy.objects.values_list("year", flat=True).distinct()) - years_available.sort() - - # Inflation adjustment - magic = Cy.objects.filter(country=country1.id, - year__range=(years_available[0], - years_available[-1])).values('year', - 'pc_constant', - 'pc_current', - 'notpc_constant') - magic_numbers = {} - for i in magic: - magic_numbers[i['year']] = {"pc_constant":i['pc_constant'], - "pc_current":i['pc_current'], - "notpc_constant":i["notpc_constant"]} - - - '''Define parameters for query''' - if crawler == True or single_year == True: - year_where = "AND cpy.year = %s" % (year,) - else: - year_where = " " - rca_col = "null" - if trade_flow == "net_export": - val_col = "export_value - import_value as val" - rca_col = "export_rca" - elif trade_flow == "net_import": - val_col = "import_value - export_value as val" - elif trade_flow == "export": - val_col = "export_value as val" - rca_col = "export_rca" - else: - val_col = "import_value as val" - - """Create query [year, id, abbrv, name_lang, val, export_rca]""" - q = """ - SELECT cpy.year, p.id, p.code, p.name_%s, p.community_id, c.color,c.name, %s, %s, distance, opp_gain, py.pci - FROM %sobservatory_%s_cpy as cpy, %sobservatory_%s as p, %sobservatory_%s_community as c, %sobservatory_%s_py as py - WHERE country_id=%s and cpy.product_id = p.id %s and p.community_id = c.id and py.product_id=p.id and cpy.year=py.year - HAVING val > 0 - ORDER BY val DESC - """ % (lang, val_col, rca_col, DB_PREFIX, prod_class, DB_PREFIX, prod_class, DB_PREFIX, prod_class, DB_PREFIX, prod_class, country1.id, year_where) - - """Prepare JSON response""" - json_response = {} - - """Check cache""" - if settings.REDIS: - raw = redis.Redis("localhost") - key = "%s:%s:%s:%s:%s" % (country1.name_3char, "all", "show", prod_class, trade_flow) - if single_year: - key += ":%d" % int(year) - # See if this key is already in the cache - cache_query = raw.get(key) - if (cache_query == None): - - rows = raw_q(query=q, params=None) - total_val = sum([r[4] for r in rows]) - """Add percentage value to return vals""" - rows = [{"year":r[0], "item_id":r[1], "abbrv":r[2], "name":r[3], "value":r[7], "rca":r[8], - "distance":r[9],"opp_gain":r[10], "pci": r[11], "share": (r[7] / total_val)*100, - "community_id": r[4], "color": r[5], "community_name":r[6], "code":r[2], "id": r[2]} for r in rows] - - if crawler == "": - return [rows, total_val, ["#", "Year", "Abbrv", "Name", "Value", "RCA", "%"]] - - # SAVE key in cache. - raw.set(key, msgpack.dumps(rows))#, 'data', json.dumps(rows)) - json_response["data"] = rows - - else: - # If already cached, now simply retrieve - encoded = cache_query - decoded = msgpack.loads(encoded) - json_response["data"] = decoded - - else: - rows = raw_q(query=q, params=None) - total_val = sum([r[7] for r in rows]) - """Add percentage value to return vals""" - rows = [{"year":r[0], "item_id":r[1], "abbrv":r[2], "name":r[3], "value":r[7], "rca":r[8], - "distance":r[9],"opp_gain":r[10], "pci": r[11], "share": (r[7] / total_val)*100, - "community_id": r[4], "color": r[5], "community_name":r[6], "code":r[2], "id": r[2]} for r in rows] - - if crawler == "": - return [rows, total_val, ["#", "Year", "Abbrv", "Name", "Value", "RCA", "%"]] - - json_response["data"] = rows - - json_response["attr"] = attr - json_response["attr_data"] = Sitc4.objects.get_all(lang) if prod_class == "sitc4" else Hs4.objects.get_all(lang) - json_response["country1"] = country1.to_json() - json_response["title"] = "What does %s %s?" % (country1.name, trade_flow.replace("_", " ")) - json_response["year"] = year - json_response["item_type"] = "product" - json_response["app_type"] = "casy" - json_response["magic_numbers"] = magic_numbers - json_response["world_trade"] = world_trade - json_response["prod_class"] = prod_class - json_response["other"] = query_params - - if not os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json" ): - response_json_file = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json", "w+" ) - response_json_file.write( json.dumps( json_response ) ) - response_json_file.close() - - # Check the request data type - if ( request.GET.get( 'data_type', None ) is None ): - return HttpResponse( "" ) - elif ( request.GET.get( 'data_type', '' ) == 'json' ): - return HttpResponse(json.dumps(json_response)) - -def api_sapy(request, trade_flow, product, year): - # Setup the hash dictionary - #request_hash_dictionary = { 'trade_flow': trade_flow, 'product': product, 'year': year } - """Init variables""" - prod_class = request.session['product_classification'] if 'product_classification' in request.session else "hs4" - prod_class = request.GET.get("prod_class", prod_class) - lang = request.session['django_language'] if 'django_language' in request.session else "en" - lang = request.GET.get("lang", lang) - crawler = request.GET.get("_escaped_fragment_", False) - product = clean_product(product, prod_class) - #Set product code to product - product_code = product.code - """Set query params with our changes""" - query_params = request.GET.copy() - query_params["lang"] = lang - query_params["product_classification"] = prod_class - #Get app_name from session - app_name = request.session['app_name'] if 'app_name' in request.session else "" - # Setup the hash dictionary - request_hash_dictionary = collections.OrderedDict() - # Setup the hash dictionary - request_hash_dictionary = collections.OrderedDict() - # Add prod class to request hash dictionary - request_hash_dictionary['app_name'] = app_name - request_hash_dictionary['lang'] = lang - request_hash_dictionary['prod_class'] = prod_class - # Add the arguments to the request hash dictionary - request_hash_dictionary['trade_flow'] = trade_flow - request_hash_dictionary['country1'] = 'show' - request_hash_dictionary['country2'] = 'all' - request_hash_dictionary['product_dispaly'] = product_code - request_hash_dictionary['year'] = year - # We are here, so let us store this data somewhere - request_hash_string = "_".join(request_hash_dictionary.values()) - - # Setup the store data - store_data = request.build_absolute_uri().replace( "product_classification", "prod_class" ) + "||" - store_page_url = request.build_absolute_uri().replace( "/api/", "/explore/" + app_name + "/" ) - store_page_url = store_page_url.replace( "data_type=json", "headless=true" ) - store_page_url = store_page_url.replace( "product_classification", "prod_class" ) - store_data = store_data + store_page_url + "||" - store_data = store_data + request_hash_string - - # Write the store data to file - store_file = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".store", "w+" ) - store_file.write( store_data ) - store_file.close() - - - if ( os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".svg" ) is True ): - # Check the request data type - if ( request.GET.get( 'data_type', None ) is None ): - # Let us get the data from the file - response_json_data = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".svg", "r" ) - - # Set the return data - returnData = response_json_data.read() - - #"""Return to browser as JSON for AJAX request""" - return HttpResponse( returnData ) - elif ( request.GET.get( 'data_type', '' ) == 'json' ): - # Check if we have the valid json file - if ( os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json" ) is True ): - # Let us get the data from the file - response_json_data = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json", "r" ) - - # Set the return data - returnData = response_json_data.read() - - #"""Return to browser as JSON for AJAX request""" - return HttpResponse( returnData ) - - '''Grab extraneous details''' - ## Clasification & Django Data Call - name = "name_%s" % lang - - '''Grab extraneous details''' - if prod_class == "sitc4": - # attr_list = list(Sitc4.objects.all().values('code','name','color')) - attr_list = list(Sitc4.objects.all().values('code',name,'id','color')) - attr = {} - for i in attr_list: - attr[i['code']] = {'code':i['code'],'name':i[name],'color':i['color']} - #.extra(where=['CHAR_LENGTH(code) = 2']) - elif prod_class == "hs4": - # attr_list = list(Hs4.objects.all().values('code','name')) #.extra(where=['CHAR_LENGTH(code) = 2']) - attr_list = list(Hs4.objects.all().values('code',name,'id','community_id__color')) - attr = {} - for i in attr_list: - attr[i['code']] = {'code':i['code'],'name':i[name],'item_id':i['id'],'color':i['community_id__color']} - - - # Create dictionary of region codes - region_list = list(Country_region.objects.all().values()) #.extra(where=['CHAR_LENGTH(code) = 2']) - region = {} - for i in region_list: - region[i['id']] = i - - # Create dictinoary for continent groupings - continent_list = list(Country.objects.all().distinct().values('continent')) - continents = {} - for i,k in enumerate(continent_list): - continents[k['continent']] = i*1000 - - """Define parameters for query""" - year_where = "AND year = %s" % (year,) if crawler == "" else " " - rca_col = "null" - if trade_flow == "net_export": - val_col = "export_value - import_value as val" - rca_col = "export_rca" - elif trade_flow == "net_import": - val_col = "import_value - export_value as val" - elif trade_flow == "export": - val_col = "export_value as val" - rca_col = "export_rca" - else: - val_col = "import_value as val" - - """Create query [year, id, abbrv, name_lang, val, export_rca]""" - q = """ - SELECT year, c.id, c.name_3char, c.name_%s, c.region_id, c.continent, %s, %s - FROM %sobservatory_%s_cpy as cpy, %sobservatory_country as c - WHERE product_id=%s and cpy.country_id = c.id %s - HAVING val > 0 - ORDER BY val DESC - """ % (lang, val_col, rca_col, DB_PREFIX, prod_class, DB_PREFIX, product.id, year_where) - - """Prepare JSON response""" - json_response = {} - - """Check cache""" - if settings.REDIS: - # raw = get_redis_connection('default') - raw = redis.Redis("localhost") - key = "%s:%s:%s:%s:%s" % ("show", "all", product.id, prod_class, trade_flow) - # See if this key is already in the cache - cache_query = raw.get(key) - if (cache_query == None): - rows = raw_q(query=q, params=None) - total_val = sum([r[6] for r in rows]) - # raise Exception(total_val) - """Add percentage value to return vals""" - # rows = [list(r) + [(r[4] / total_val)*100] for r in rows] - rows = [{"year":r[0], "item_id":r[1], "abbrv":r[2], "name":r[3], "value":r[6], "rca":r[7], "share": (r[6] / total_val)*100, - "id": r[1], "region_id":r[4],"continent":r[5]} for r in rows] - - if crawler == "": - return [rows, total_val, ["#", "Year", "Abbrv", "Name", "Value", "RCA", "%"]] - - json_response["data"] = rows - - # SAVE key in cache. - raw.set(key, msgpack.dumps(rows)) - - else: - # If already cached, now simply retrieve - encoded = cache_query - decoded = msgpack.loads(encoded) - json_response["data"] = decoded - - else: - rows = raw_q(query=q, params=None) - total_val = sum([r[6] for r in rows]) - # raise Exception(total_val) - """Add percentage value to return vals""" - # rows = [list(r) + [(r[4] / total_val)*100] for r in rows] - rows = [{"year":r[0], "item_id":r[1], "abbrv":r[2], "name":r[3], "value":r[6], "rca":r[7], "share": (r[6] / total_val)*100, - "id": r[1], "region_id":r[4],"continent":r[5]} for r in rows] - - if crawler == "": - return [rows, total_val, ["#", "Year", "Abbrv", "Name", "Value", "RCA", "%"]] - - json_response["data"] = rows - - json_response["attr_data"] = Country.objects.get_all(lang) - json_response["product"] = product.to_json() - json_response["title"] = "Who %sed %s?" % (trade_flow.replace("_", " "), product.name_en) - json_response["year"] = year - json_response["item_type"] = "country" - json_response["app_type"] = "sapy" - json_response["attr"] = attr - json_response["region"]= region - json_response["continents"] = continents - json_response["other"] = query_params - - if not os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json" ): - response_json_file = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json", "w+" ) - response_json_file.write( json.dumps( json_response ) ) - response_json_file.close() - - # raise Exception(time.time() - start) - # Check the request data type - if ( request.GET.get( 'data_type', None ) is None ): - #"""Return to browser as JSON for AJAX request""" - return HttpResponse( "" ) - elif ( request.GET.get( 'data_type', '' ) == 'json' ): - """Return to browser as JSON for AJAX request""" - return HttpResponse(json.dumps(json_response)) - - -''' / show / product / ''' -def api_csay(request, trade_flow, country1, year): - """Init variables""" - prod_class = request.session['product_classification'] if 'product_classification' in request.session else "hs4" - prod_class = request.GET.get("prod_class", prod_class) - lang = request.session['django_language'] if 'django_language' in request.session else "en" - lang = request.GET.get("lang", lang) - crawler = request.GET.get("_escaped_fragment_", False) - country1 = Country.objects.get(name_3char=country1) - #Get app_name from session - app_name = request.session['app_name'] if 'app_name' in request.session else "" - """Set query params with our changes""" - query_params = request.GET.copy() - query_params["lang"] = lang - query_params["product_classification"] = prod_class - # Setup the hash dictionary - request_hash_dictionary = collections.OrderedDict() - # Setup the hash dictionary - request_hash_dictionary = collections.OrderedDict() - # Add prod class to request hash dictionary - request_hash_dictionary['app_name'] = app_name - request_hash_dictionary['lang'] = lang - request_hash_dictionary['prod_class'] = prod_class - # Add the arguments to the request hash dictionary - request_hash_dictionary['trade_flow'] = trade_flow - request_hash_dictionary['country1'] = country1.name_3char.lower() - request_hash_dictionary['product_dispaly'] = 'show' - request_hash_dictionary['country2'] = 'all' - request_hash_dictionary['year'] = year - - # We are here, so let us store this data somewhere - request_hash_string = "_".join(request_hash_dictionary.values()) - - # Setup the store data - store_data = request.build_absolute_uri().replace( "product_classification", "prod_class" ) + "||" - store_page_url = request.build_absolute_uri().replace( "/api/", "/explore/" + app_name + "/" ) - store_page_url = store_page_url.replace( "data_type=json", "headless=true" ) - store_page_url = store_page_url.replace( "product_classification", "prod_class" ) - store_data = store_data + store_page_url + "||" - store_data = store_data + request_hash_string - - # Write the store data to file - store_file = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".store", "w+" ) - store_file.write( store_data ) - store_file.close() - - if ( os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".svg" ) is True ): - # Check the request data type - if ( request.GET.get( 'data_type', None ) is None ): - # Let us get the data from the file - response_json_data = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".svg", "r" ) - - # Set the return data - returnData = response_json_data.read() - - #"""Return to browser as JSON for AJAX request""" - return HttpResponse( returnData ) - elif ( request.GET.get( 'data_type', '' ) == 'json' ): - # Check if we have the valid json file - if ( os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json" ) is True ): - # Let us get the data from the file - response_json_data = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json", "r" ) - - # Set the return data - returnData = response_json_data.read() - - #"""Return to browser as JSON for AJAX request""" - return HttpResponse( returnData ) - '''Grab extraneous details''' - region_list = list(Country_region.objects.all().values()) #.extra(where=['CHAR_LENGTH(code) = 2']) - region = {} - for i in region_list: - region[i['id']] = i - - continent_list = list(Country.objects.all().distinct().values('continent')) - continents = {} - for i,k in enumerate(continent_list): - continents[k['continent']] = i*1000 - - """Define parameters for query""" - year_where = "AND year = %s" % (year,) if crawler == "" else " " - rca_col = "null" - if trade_flow == "net_export": - val_col = "SUM(export_value - import_value) as val" - elif trade_flow == "net_import": - val_col = "SUM(import_value - export_value) as val" - elif trade_flow == "export": - val_col = "SUM(export_value) as val" - else: - val_col = "SUM(import_value) as val" - - '''Create query [year, id, abbrv, name_lang, val, rca]''' - q = """ - SELECT year, c.id, c.name_3char, c.name_%s, c.region_id, c.continent, %s, %s - FROM %sobservatory_%s_ccpy as ccpy, %sobservatory_country as c - WHERE origin_id=%s and ccpy.destination_id = c.id %s - GROUP BY year, destination_id - HAVING val > 0 - ORDER BY val DESC - """ % (lang, val_col, rca_col, DB_PREFIX, prod_class, DB_PREFIX, country1.id, year_where) - - """Prepare JSON response""" - json_response = {} - - """Check cache""" - if settings.REDIS: - raw = redis.Redis("localhost") - key = "%s:%s:%s:%s:%s" % (country1.name_3char, "show", "all", prod_class, trade_flow) - # See if this key is already in the cache - cache_query = raw.get(key) - if (cache_query == None): - rows = raw_q(query=q, params=None) - total_val = sum([r[6] for r in rows]) - - """Add percentage value to return vals""" - # rows = [list(r) + [(r[4] / total_val)*100] for r in rows] - rows = [{"year":r[0], "item_id":r[1], "abbrv":r[2], "name":r[3], "value":r[6], "rca":r[7], "share": (r[6] / total_val)*100, - "id":r[1], "region_id":r[4], "continent":r[5]} for r in rows] - - if crawler == "": - return [rows, total_val, ["#", "Year", "Abbrv", "Name", "Value", "RCA", "%"]] - - json_response["data"] = rows - - # SAVE key in cache. - raw.set(key, msgpack.dumps(rows)) - - else: - # If already cached, now simply retrieve - encoded = cache_query - decoded = msgpack.loads(encoded) - json_response["data"] = decoded - - else: - rows = raw_q(query=q, params=None) - total_val = sum([r[6] for r in rows]) - - """Add percentage value to return vals""" - # rows = [list(r) + [(r[4] / total_val)*100] for r in rows] - rows = [{"year":r[0], "item_id":r[1], "abbrv":r[2], "name":r[3], "value":r[6], "rca":r[7], "share": (r[6] / total_val)*100, - "id":r[1], "region_id":r[4], "continent":r[5]} for r in rows] - - if crawler == "": - return [rows, total_val, ["#", "Year", "Abbrv", "Name", "Value", "RCA", "%"]] - - json_response["data"] = rows - - # get distince years from db, different for diff product classifications - years_available = list(Sitc4_cpy.objects.values_list("year", flat=True).distinct()) if prod_class == "sitc4" else list(Hs4_cpy.objects.values_list("year", flat=True).distinct()) - years_available.sort() - - magic = Cy.objects.filter(country=country1.id, - year__range=(years_available[0], - years_available[-1])).values('year', - 'pc_constant', - 'pc_current', - 'notpc_constant') - magic_numbers = {} - for i in magic: - magic_numbers[i['year']] = {"pc_constant":i['pc_constant'], - "pc_current":i['pc_current'], - "notpc_constant":i["notpc_constant"]} - - """Set article variable for question """ - article = "to" if trade_flow == "export" else "from" - - json_response["attr_data"] = Country.objects.get_all(lang) - json_response["country1"] = country1.to_json() - json_response["title"] = "Where does %s %s %s?" % (country1.name, trade_flow, article) - json_response["year"] = year - json_response["item_type"] = "country" - json_response["app_type"] = "csay" - json_response["region"]= region - json_response["continents"]= continents - json_response["prod_class"] = prod_class - json_response["magic_numbers"] = magic_numbers - json_response["other"] = query_params - - # raise Exception(time.time() - start) - # Check the request data type - if ( request.GET.get( 'data_type', None ) is None ): - #"""Return to browser as JSON for AJAX request""" - return HttpResponse( "" ) - elif ( request.GET.get( 'data_type', '' ) == 'json' ): - """Return to browser as JSON for AJAX request""" - return HttpResponse(json.dumps(json_response)) - - -def api_ccsy(request, trade_flow, country1, country2, year): - # import time - # start = time.time() - '''Init variables''' - prod_class = request.session['product_classification'] if 'product_classification' in request.session else "hs4" - prod_class = request.GET.get("prod_class", prod_class) - lang = request.session['django_language'] if 'django_language' in request.session else "en" - lang = request.GET.get("lang", lang) - crawler = request.GET.get("_escaped_fragment_", False) - country1 = Country.objects.get(name_3char=country1) - country2 = Country.objects.get(name_3char=country2) - article = "to" if trade_flow == "export" else "from" - # Setup the hash dictionary - request_hash_dictionary = collections.OrderedDict() - country_code1=Country.objects.filter(name_3char=country1) - #Get app_name from session - app_name = request.session['app_name'] if 'app_name' in request.session else "" - '''Set query params with our changes''' - query_params = request.GET.copy() - query_params["lang"] = lang - query_params["product_classification"] = prod_class - # Get Name in proper lang - name = "name_%s" % lang - # Setup the hash dictionary - request_hash_dictionary1 = collections.OrderedDict() - # Add prod class to request hash dictionary - request_hash_dictionary['app_name'] = app_name - request_hash_dictionary['lang'] = lang - request_hash_dictionary['prod_class'] = prod_class - - #Set country_code to Country - country_code1=Country.objects.get(name=country1) - country_code2=Country.objects.get(name=country2) - country_code_one=country_code1.name_3char.lower() - country_code_two=country_code2.name_3char.lower() - # Add the arguments to the request hash dictionary - request_hash_dictionary['trade_flow'] = trade_flow - request_hash_dictionary['country1'] = country_code_one - request_hash_dictionary['country2'] = country_code_two - request_hash_dictionary['product_dispaly'] = 'show' - request_hash_dictionary['year'] = year - - # We are here, so let us store this data somewhere - request_hash_string = "_".join(request_hash_dictionary.values()) - - # Setup the store data - store_data = request.build_absolute_uri().replace( "product_classification", "prod_class" ) + "||" - store_page_url = request.build_absolute_uri().replace( "/api/", "/explore/" + app_name + "/" ) - store_page_url = store_page_url.replace( "data_type=json", "headless=true" ) - store_page_url = store_page_url.replace( "product_classification", "prod_class" ) - store_data = store_data + store_page_url + "||" - store_data = store_data + request_hash_string - - # Write the store data to file - store_file = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".store", "w+" ) - store_file.write( store_data ) - store_file.close() - - if ( os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".svg" ) is True ): - # Check the request data type - if ( request.GET.get( 'data_type', None ) is None ): - # Let us get the data from the file - response_json_data = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".svg", "r" ) - - # Set the return data - returnData = response_json_data.read() - - #"""Return to browser as JSON for AJAX request""" - return HttpResponse( returnData ) - elif ( request.GET.get( 'data_type', '' ) == 'json' ): - # Check if we have the valid json file - if ( os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json" ) is True ): - # Let us get the data from the file - response_json_data = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json", "r" ) - - # Set the return data - returnData = response_json_data.read() - - #"""Return to browser as JSON for AJAX request""" - return HttpResponse( returnData ) - '''Grab extraneous details''' - if prod_class == "sitc4": - # attr_list = list(Sitc4.objects.all().values('code','name','color')) - attr_list = list(Sitc4.objects.all().values('code',name,'id','color')) - attr = {} - for i in attr_list: - attr[i['code']] = {'code':i['code'],'name':i[name],'color':i['color']} - #.extra(where=['CHAR_LENGTH(code) = 2']) - elif prod_class == "hs4": - # attr_list = list(Hs4.objects.all().values('code','name')) #.extra(where=['CHAR_LENGTH(code) = 2']) - attr_list = list(Hs4.objects.all().values('code',name,'id','community_id__color')) - attr = {} - for i in attr_list: - attr[i['code']] = {'code':i['code'],'name':i[name],'item_id':i['id'],'color':i['community_id__color']} - - - - '''Define parameters for query''' - year_where = "AND year = %s" % (year,) if crawler == "" else " " - rca_col = "null" - if trade_flow == "net_export": - val_col = "(export_value - import_value) as val" - elif trade_flow == "net_import": - val_col = "(import_value - export_value) as val" - elif trade_flow == "export": - val_col = "export_value as val" - else: - val_col = "import_value as val" - - '''Create query''' - q = """ - SELECT year, p.id, p.code, p.name_%s, p.community_id, c.name, c.color, %s, %s - FROM %sobservatory_%s_ccpy as ccpy, %sobservatory_%s as p, %sobservatory_%s_community as c - WHERE origin_id=%s and destination_id=%s and ccpy.product_id = p.id and p.community_id = c.id %s - HAVING val > 0 - ORDER BY val DESC - """ % (lang, val_col, rca_col, DB_PREFIX, prod_class, DB_PREFIX, prod_class, DB_PREFIX, prod_class, country1.id, country2.id, year_where) - - """Prepare JSON response""" - json_response = {} - - """Check cache""" - if settings.REDIS: - #raw = get_redis_connection('default') - raw = redis.Redis("localhost") - key = "%s:%s:%s:%s:%s" % (country1.name_3char, country2.name_3char, "show", prod_class, trade_flow) - # See if this key is already in the cache - cache_query = raw.get(key) - if(cache_query == None): - rows = raw_q(query=q, params=None) - total_val = sum([r[7] for r in rows]) - - """Add percentage value to return vals""" - # rows = [list(r) + [(r[4] / total_val)*100] for r in rows] - rows = [{"year":r[0], "item_id":r[1], "abbrv":r[2], "name":r[3], "value":r[7], "rca":r[5], - "share": (r[7] / total_val)*100, - "community_id":r[4],"community_name":r[5],"color":r[6], "code":r[2], "id": r[2]} for r in rows] - - if crawler == "": - return [rows, total_val, ["#", "Year", "Abbrv", "Name", "Value", "RCA", "%"]] - - json_response["data"] = rows - - # SAVE key in cache. - raw.set(key, msgpack.dumps(rows)) - - else: - # If already cached, now simply retrieve - encoded = cache_query - decoded = msgpack.loads(encoded) - json_response["data"] = decoded - - else: - rows = raw_q(query=q, params=None) - total_val = sum([r[7] for r in rows]) - - """Add percentage value to return vals""" - # rows = [list(r) + [(r[4] / total_val)*100] for r in rows] - rows = [{"year":r[0], "item_id":r[1], "abbrv":r[2], "name":r[3], "value":r[7], "rca":r[5], - "share": (r[7] / total_val)*100, - "community_id":r[4],"community_name":r[5],"color":r[6], "code":r[2], "id": r[2]} for r in rows] - - if crawler == "": - return [rows, total_val, ["#", "Year", "Abbrv", "Name", "Value", "RCA", "%"]] - - json_response["data"] = rows - - # get distince years from db, different for diff product classifications - years_available = list(Sitc4_cpy.objects.values_list("year", flat=True).distinct()) if prod_class == "sitc4" else list(Hs4_cpy.objects.values_list("year", flat=True).distinct()) - years_available.sort() - - magic = Cy.objects.filter(country=country1.id, - year__range=(years_available[0], - years_available[-1])).values('year', - 'pc_constant', - 'pc_current', - 'notpc_constant') - magic_numbers = {} - for i in magic: - magic_numbers[i['year']] = {"pc_constant":i['pc_constant'], - "pc_current":i['pc_current'], - "notpc_constant":i["notpc_constant"]} - - json_response["magic_numbers"] = magic_numbers - json_response["attr_data"] = Sitc4.objects.get_all(lang) if prod_class == "sitc4" else Hs4.objects.get_all(lang) - json_response["country1"] = country1.to_json() - json_response["country2"] = country2.to_json() - json_response["title"] = "What does %s %s %s %s?" % (country1.name, trade_flow, article, country2.name) - json_response["year"] = year - json_response["item_type"] = "product" - json_response["app_type"] = "ccsy" - json_response["prod_class"] = prod_class - json_response["attr"] = attr - json_response["class"] = prod_class - json_response["other"] = query_params - - if not os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json" ): - response_json_file = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json", "w+" ) - response_json_file.write( json.dumps( json_response ) ) - response_json_file.close() - - # raise Exception(time.time() - start) - # Check the request data type - if ( request.GET.get( 'data_type', None ) is None ): - #"""Return to browser as JSON for AJAX request""" - return HttpResponse( "" ) - elif ( request.GET.get( 'data_type', '' ) == 'json' ): - """Return to browser as JSON for AJAX request""" - return HttpResponse(json.dumps(json_response)) - -def api_cspy(request, trade_flow, country1, product, year): - '''Init variables''' - prod_class = request.session['product_classification'] if 'product_classification' in request.session else "hs4" - prod_class = request.GET.get("prod_class", prod_class) - lang = request.session['django_language'] if 'django_language' in request.session else "en" - lang = request.GET.get("lang", lang) - crawler = request.GET.get("_escaped_fragment_", False) - country1 = Country.objects.get(name_3char=country1) - #Get app_name from session - app_name = request.session['app_name'] if 'app_name' in request.session else "" - product = clean_product(product, prod_class) - article = "to" if trade_flow == "export" else "from" - - '''Set query params with our changes''' - query_params = request.GET.copy() - query_params["lang"] = lang - query_params["product_classification"] = prod_class - # Setup the hash dictionary - request_hash_dictionary = collections.OrderedDict() - # Add prod class to request hash dictionary - request_hash_dictionary['app_name'] = app_name - request_hash_dictionary['lang'] = lang - request_hash_dictionary['prod_class'] = prod_class - #Set product code to particular product - product_display = product.code - # Add the arguments to the request hash dictionary - request_hash_dictionary['trade_flow'] = trade_flow - request_hash_dictionary['country1'] = country1 - request_hash_dictionary['country1'] = 'show' - request_hash_dictionary['product_display'] = product_display - request_hash_dictionary['year'] = year - # We are here, so let us store this data somewhere - request_hash_string = "_".join(request_hash_dictionary.values()) #base64.b64encode( request_unique_hash ) - - # Setup the store data - store_data = request.build_absolute_uri().replace( "product_classification", "prod_class" ) + "||" - store_page_url = request.build_absolute_uri().replace( "/api/", "/explore/" + app_name + "/" ) - store_page_url = store_page_url.replace( "data_type=json", "headless=true" ) - store_page_url = store_page_url.replace( "product_classification", "prod_class" ) - store_data = store_data + store_page_url + "||" - store_data = store_data + request_hash_string - - # Write the store data to file - store_file = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".store", "w+" ) - store_file.write( store_data ) - store_file.close() - - if ( os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".svg" ) is True ): - # Check the request data type - if ( request.GET.get( 'data_type', None ) is None ): - # Let us get the data from the file - response_json_data = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".svg", "r" ) - - # Set the return data - returnData = response_json_data.read() - - #"""Return to browser as JSON for AJAX request""" - return HttpResponse( returnData ) - elif ( request.GET.get( 'data_type', '' ) == 'json' ): - # Check if we have the valid json file - if ( os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json" ) is True ): - # Let us get the data from the file - response_json_data = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json", "r" ) - - # Set the return data - returnData = response_json_data.read() - - #"""Return to browser as JSON for AJAX request""" - return HttpResponse( returnData ) - - '''Grab extraneous details''' - region_list = list(Country_region.objects.all().values()) #.extra(where=['CHAR_LENGTH(code) = 2']) - region = {} - for i in region_list: - region[i['id']] = i - - continent_list = list(Country.objects.all().distinct().values('continent')) - continents = {} - for i,k in enumerate(continent_list): - continents[k['continent']] = i*1000 - - '''Define parameters for query''' - year_where = "AND year = %s" % (year,) if crawler == "" else " " - rca_col = "null" - if trade_flow == "net_export": - val_col = "(export_value - import_value) as val" - elif trade_flow == "net_import": - val_col = "(import_value - export_value) as val" - elif trade_flow == "export": - val_col = "export_value as val" - else: - val_col = "import_value as val" - - '''Create query''' - q = """ - SELECT year, c.id, c.name_3char, c.name_%s, c.region_id, c.continent, %s, %s - FROM %sobservatory_%s_ccpy as ccpy, %sobservatory_country as c - WHERE origin_id=%s and ccpy.product_id=%s and ccpy.destination_id = c.id %s - GROUP BY year, destination_id - HAVING val > 0 - ORDER BY val DESC - """ % (lang, val_col, rca_col, DB_PREFIX, prod_class, DB_PREFIX, country1.id, product.id, year_where) - - """Prepare JSON response""" - json_response = {} - - """Check cache""" - if settings.REDIS: - raw = redis.Redis("localhost") - key = "%s:%s:%s:%s:%s" % (country1.name_3char, "show", product.id, prod_class, trade_flow) - # See if this key is already in the cache - cache_query = raw.get(key) - - if (cache_query == None): - rows = raw_q(query=q, params=None) - total_val = sum([r[6] for r in rows]) - - """Add percentage value to return vals""" - # rows = [list(r) + [(r[4] / total_val)*100] for r in rows] - rows = [{"year":r[0], "item_id":r[1], "abbrv":r[2], "name":r[3], "value":r[6], "rca":r[7], "share": (r[6] / total_val)*100, - "region_id": r[4], "continent": r[5], "id":r[1]} for r in rows] - - if crawler == "": - return [rows, total_val, ["#", "Year", "Abbrv", "Name", "Value", "RCA", "%"]] - - json_response["data"] = rows - - # SAVE key in cache. - raw.set(key, msgpack.dumps(rows)) - - else: - # If already cached, now simply retrieve - encoded = cache_query - decoded = msgpack.loads(encoded) - json_response["data"] = decoded - - else: - rows = raw_q(query=q, params=None) - total_val = sum([r[6] for r in rows]) - - """Add percentage value to return vals""" - # rows = [list(r) + [(r[4] / total_val)*100] for r in rows] - rows = [{"year":r[0], "item_id":r[1], "abbrv":r[2], "name":r[3], "value":r[6], "rca":r[7], "share": (r[6] / total_val)*100, - "region_id": r[4], "continent": r[5], "id":r[1]} for r in rows] - - if crawler == "": - return [rows, total_val, ["#", "Year", "Abbrv", "Name", "Value", "RCA", "%"]] - - json_response["data"] = rows - - - article = "to" if trade_flow == "export" else "from" - - # get distince years from db, different for diff product classifications - years_available = list(Sitc4_cpy.objects.values_list("year", flat=True).distinct()) if prod_class == "sitc4" else list(Hs4_cpy.objects.values_list("year", flat=True).distinct()) - years_available.sort() - - magic = Cy.objects.filter(country=country1.id, - year__range=(years_available[0], - years_available[-1])).values('year', - 'pc_constant', - 'pc_current', - 'notpc_constant') - magic_numbers = {} - for i in magic: - magic_numbers[i['year']] = {"pc_constant":i['pc_constant'], - "pc_current":i['pc_current'], - "notpc_constant":i["notpc_constant"]} - - json_response["magic_numbers"] = magic_numbers - json_response["attr_data"] = Country.objects.get_all(lang) - json_response["title"] = "Where does %s %s %s %s?" % (country1.name, trade_flow, product.name_en, article) - json_response["country1"] = country1.to_json() - json_response["product"] = product.to_json() - json_response["year"] = year - json_response["item_type"] = "country" - json_response["continents"]= continents - json_response["region"]= region - json_response["app_type"] = "cspy" - json_response["class"] = prod_class - json_response["other"] = query_params - - if not os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json" ): - response_json_file = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json", "w+" ) - response_json_file.write( json.dumps( json_response ) ) - response_json_file.close() - - # raise Exception(time.time() - start) - # Check the request data type - if ( request.GET.get( 'data_type', None ) is None ): - #"""Return to browser as JSON for AJAX request""" - return HttpResponse( "" ) - elif ( request.GET.get( 'data_type', '' ) == 'json' ): - """Return to browser as JSON for AJAX request""" - return HttpResponse(json.dumps(json_response)) - -# Embed for iframe -def embed(request, app_name, trade_flow, country1, country2, product, year): - lang = request.GET.get("lang", "en") - prod_class = request.session['product_classification'] if 'product_classification' in request.session else "hs4" - prod_class = request.GET.get("product_classification", prod_class) - query_string = request.GET.copy() - query_string["product_classification"] = prod_class - # get distince years from db, different for diff product classifications - years_available = list(Sitc4_cpy.objects.values_list("year", flat=True).distinct()) if prod_class == "sitc4" else list(Hs4_cpy.objects.values_list("year", flat=True).distinct()) - years_available.sort() - - return render_to_response("explore/embed.html", {"app":app_name, "trade_flow": trade_flow, "country1":country1, "country2":country2, "product":product, "year":year, "other":json.dumps(query_string),"years_available":json.dumps(years_available), "lang":lang}) - - def api_views(request): - r = redis.Redis(db=1) - recent_views = r.lrange("views", -15, -1) - return HttpResponse("[%s]" % ",".join(recent_views)) - - -############################################################################### -## Helpers -############################################################################### -def clean_country(country): - # first try looking up based on 3 character code - try: - c = Country.objects.get(name_3char=country) - except Country.DoesNotExist: - # next try 2 character code - try: - c = Country.objects.get(name_2char=country) - except Country.DoesNotExist: - c = None - return c - -def clean_product(product, prod_class): - # first try looking up based on 3 character code - if prod_class == "hs4": - try: - p = Hs4.objects.get(code=product) - except Hs4.DoesNotExist: - # next try SITC4 - try: - conv_code = Sitc4.objects.get(code=product).conversion_code - p = Hs4.objects.get(code=conv_code) - except Hs4.DoesNotExist: - p = None - else: - try: - p = Sitc4.objects.get(code=product) - except Sitc4.DoesNotExist: - # next try SITC4 - try: - conv_code = Hs4.objects.get(code=product).conversion_code - p = Sitc4.objects.get(code=conv_code) - except Hs4.DoesNotExist: - p = None - return p + """Return metadata of the last 15 visits to the site, in JSON.""" -def get_country_lookup(): - lookup = {} - for c in Country.objects.all(): - lookup[c.id] = [c.name_en, c.name_3char] - return lookup + if not isinstance(cache, RedisCache): + return HttpResponse("") + r = cache.raw_client + recent_views = r.lrange("views", -15, -1) + return HttpResponse("[%s]" % ",".join(recent_views)) diff --git a/django_files/observatory/views_api.py b/django_files/observatory/views_api.py new file mode 100644 index 00000000..618ddeb7 --- /dev/null +++ b/django_files/observatory/views_api.py @@ -0,0 +1,513 @@ +import json + +from django.conf import settings +from django.core.cache import cache +from django.db import connection +from django.http import HttpResponse +from django.views.decorators.cache import cache_control + +from observatory.models import (Country, Hs4, Hs4_cpy, Sitc4, Sitc4_cpy, + Hs4_ccpy, Sitc4_ccpy) +from observatory import helpers + +import msgpack + + +def calculate_volume(items, trade_flow="export", sum_val=False): + """Given a queryset and trade flow value, append the proper value field + expression to select. + + :param sum_val: If true, aggregate sum the trade flow value + """ + + if trade_flow == "net_export": + val = '`export_value` - `import_value`' + elif trade_flow == "net_import": + val = '`import_value` - `export_value`' + elif trade_flow == "export": + val = '`export_value`' + else: + val = '`import_value`' + + # Don't include negative or 0 trade volumes. Especially useful with net + # import / export since negative values don't mean anything + filter_expr = ["%s > 0" % val] + + if sum_val: + val = "sum(%s)" % val + filter_expr = None + + return items.extra(select={'val': val}, + where=filter_expr) + + +def calculate_rca(items, trade_flow="export"): + """Given a queryset and trade flow value, append the proper rca field to + select.""" + + if trade_flow in ["net_export", "export"]: + select_dict = {'rca': 'export_rca'} + else: + select_dict = {'rca': 'null'} + + return items.extra(select=select_dict) + + +@cache_control(max_age=settings.CACHE_VERY_SHORT) +def api_casy(request, trade_flow, country1, year): + """ / all / show / """ + + # Get session / request vars + prod_class = request.GET.get("prod_class", + request.session.get('product_classification', + 'hs4')) + lang = helpers.get_language(request)['code'] + name = "name_%s" % lang + single_year = 'single_year' in request.GET + country1 = Country.objects.get(name_3char=country1) + + query_params = request.GET.copy() + query_params["lang"] = lang + query_params["product_classification"] = prod_class + + world_trade = helpers.get_world_trade(prod_class=prod_class) + attr = helpers.get_attrs(prod_class=prod_class, name=name) + years_available = helpers.get_years_available(prod_class=prod_class) + magic_numbers = helpers.get_inflation_adjustment(country1, + years_available[0], + years_available[-1]) + + if prod_class == "sitc4": + items = Sitc4_cpy.objects + else: + items = Hs4_cpy.objects + + items = calculate_volume(items, trade_flow=trade_flow) + items = calculate_rca(items, trade_flow=trade_flow) + items = items.extra(select={'name': name}) + items = items.values_list('year', 'product__id', 'product__code', + 'product__name', 'product__community_id', + 'product__community__color', + 'product__community__name', 'val', 'rca', + 'distance', 'opp_gain', 'product_year__pci',) + + if single_year: + items = items.filter(year=year) + + items = items.filter(country_id=country1.id) + items = items.extra(where=["export_value > 0"]) + + + json_response = {} + + # Generate cache key + key = "%s:%s:%s:%s:%s" % (country1.name_3char, "all", "show", + prod_class, trade_flow) + if single_year: + key += ":%d" % int(year) + + # Check cache + cached_data = cache.get(key) + if cached_data is not None: + json_response["data"] = msgpack.loads(cached_data) + else: + rows = list(items) + total_val = sum([r[7] for r in rows]) + rows = [{"year": r[0], "item_id": r[1], "abbrv": r[2], "name": r[3], + "value": r[7], "rca": r[8], "distance": r[9], "opp_gain": + r[10], "pci": r[11], "share": (r[7] / total_val) * 100, + "community_id": r[4], "color": r[5], "community_name": r[6], + "code": r[2], "id": r[2]} for r in rows] + + # Save in cache + cache.set(key, msgpack.dumps(rows), settings.CACHE_LONG) + json_response["data"] = rows + + # Add in remaining metadata + json_response["attr"] = attr + json_response["attr_data"] = Sitc4.objects.get_all( + lang) if prod_class == "sitc4" else Hs4.objects.get_all(lang) + json_response["country1"] = country1.to_json() + json_response["title"] = "What does %s %s?" % ( + country1.name, trade_flow.replace("_", " ")) + json_response["year"] = year + json_response["item_type"] = "product" + json_response["app_type"] = "casy" + json_response["magic_numbers"] = magic_numbers + json_response["world_trade"] = world_trade + json_response["prod_class"] = prod_class + json_response["other"] = query_params + + # Check the request data type + if (request.GET.get('data_type', None) is None): + return HttpResponse("") + elif (request.GET.get('data_type', '') == 'json'): + return HttpResponse(json.dumps(json_response)) + + +@cache_control(max_age=settings.CACHE_VERY_SHORT) +def api_sapy(request, trade_flow, product, year): + """show / all / / """ + + # Get session / request vars + prod_class = request.GET.get("prod_class", + request.session.get('product_classification', + 'hs4')) + lang = helpers.get_language(request)['code'] + name = "name_%s" % lang + single_year = 'single_year' in request.GET + product = helpers.get_product_by_code(product, prod_class) + + query_params = request.GET.copy() + query_params["lang"] = lang + query_params["product_classification"] = prod_class + + region = helpers.get_region_list() + continents = helpers.get_continent_list() + attr = helpers.get_attrs(prod_class=prod_class, name=name) + + if prod_class == "sitc4": + items = Sitc4_cpy.objects + else: + items = Hs4_cpy.objects + + items = calculate_volume(items, trade_flow=trade_flow) + items = calculate_rca(items, trade_flow=trade_flow) + items = items.extra(select={'name': name}) + items = items.values_list('year', 'country__id', 'country__name_3char', + 'name', 'country__region_id', + 'country__continent', 'val', 'rca') + + if single_year: + items = items.filter(year=year) + + items = items.filter(product_id=product.id) + + json_response = {} + + # Generate cache key + key = "%s:%s:%s:%s:%s" % ("show", "all", product.id, prod_class, + trade_flow) + if single_year: + key += ":%d" % int(year) + + # Check cache + cached_data = cache.get(key) + if cached_data is not None: + json_response["data"] = msgpack.loads(cached_data) + else: + rows = list(items) + total_val = sum([r[6] for r in rows]) + rows = [{"year": r[0], "item_id": r[1], "abbrv": r[2], "name": r[3], + "value": r[6], "rca": r[7], "share": (r[6] / total_val) * 100, + "id": r[1], "region_id": r[4], "continent": r[5]} + for r in rows] + + # Save in cache + cache.set(key, msgpack.dumps(rows), settings.CACHE_LONG) + json_response["data"] = rows + + json_response["attr_data"] = Country.objects.get_all(lang) + json_response["product"] = product.to_json() + json_response["title"] = "Who %sed %s?" % ( + trade_flow.replace("_", " "), product.name_en) + json_response["year"] = year + json_response["item_type"] = "country" + json_response["app_type"] = "sapy" + json_response["attr"] = attr + json_response["region"] = region + json_response["continents"] = continents + json_response["other"] = query_params + + if (request.GET.get('data_type', None) is None): + return HttpResponse("") + elif (request.GET.get('data_type', '') == 'json'): + """Return to browser as JSON for AJAX request""" + return HttpResponse(json.dumps(json_response)) + + +@cache_control(max_age=settings.CACHE_VERY_SHORT) +def api_csay(request, trade_flow, country1, year): + """ / show / all / """ + + prod_class = request.GET.get("prod_class", + request.session.get('product_classification', + 'hs4')) + lang = helpers.get_language(request)['code'] + name = "name_%s" % lang + country1 = Country.objects.get(name_3char=country1) + + region = helpers.get_region_list() + continents = helpers.get_continent_list() + + if prod_class == "sitc4": + items = Sitc4_ccpy.objects + else: + items = Hs4_ccpy.objects + + items = calculate_volume(items, trade_flow=trade_flow, + sum_val=True) + items = items.extra(select={'name': name}) + items = items.values_list('year', 'destination__id', + 'destination__name_3char', 'name', + 'destination__region_id', + 'destination__continent', 'val') + + items = items.filter(origin_id=country1.id) + + json_response = {} + + key = "%s:%s:%s:%s:%s" % (country1.name_3char, "show", "all", prod_class, + trade_flow) + + cached_data = cache.get(key) + if cached_data is not None: + json_response["data"] = msgpack.loads(cached_data) + else: + # This might possibly be the most disgusting hack ever made, simply + # because when doing an aggregate (like SUM()) in extra, django does + # not add that stuff correctly into the group by. It's also not + # possible to use annotate() here because it's a complex aggregate that + # uses addition / subtraction. C'est la vie :( + cursor = connection.cursor() + + if trade_flow == "net_export": + val = '`export_value` - `import_value`' + elif trade_flow == "net_import": + val = '`import_value` - `export_value`' + elif trade_flow == "export": + val = '`export_value`' + else: + val = '`import_value`' + + complete_query = str(items.query) + \ + " group by `year`, `destination_id`" + \ + " HAVING sum(%s) > 0" % val + + cursor.execute(complete_query) + rows = cursor.fetchall() + total_val = sum([r[0] for r in rows]) + + rows = [ + {"year": r[2], + "item_id": r[3], + "abbrv": r[4], + "name": r[1], + "value": r[0], + "share": (r[0] / total_val) * 100, + "id": r[3], + "region_id": r[5], + "continent": r[6]} + for r in rows] + + cache.set(key, msgpack.dumps(rows), settings.CACHE_LONG) + json_response["data"] = rows + + years_available = helpers.get_years_available() + magic_numbers = helpers.get_inflation_adjustment(country1, + years_available[0], + years_available[-1]) + + query_params = request.GET.copy() + query_params["lang"] = lang + query_params["product_classification"] = prod_class + + json_response["attr_data"] = Country.objects.get_all(lang) + json_response["country1"] = country1.to_json() + article = "to" if trade_flow == "export" else "from" + json_response["title"] = "Where does %s %s %s?" % ( + country1.name, trade_flow, article) + json_response["year"] = year + json_response["item_type"] = "country" + json_response["app_type"] = "csay" + json_response["region"] = region + json_response["continents"] = continents + json_response["prod_class"] = prod_class + json_response["magic_numbers"] = magic_numbers + json_response["other"] = query_params + + # raise Exception(time.time() - start) + # Check the request data type + if (request.GET.get('data_type', None) is None): + #"""Return to browser as JSON for AJAX request""" + return HttpResponse("") + elif (request.GET.get('data_type', '') == 'json'): + """Return to browser as JSON for AJAX request""" + return HttpResponse(json.dumps(json_response)) + + +@cache_control(max_age=settings.CACHE_VERY_SHORT) +def api_ccsy(request, trade_flow, country1, country2, year): + + # Get session / request vars + prod_class = request.GET.get("prod_class", + request.session.get('product_classification', + 'hs4')) + lang = helpers.get_language(request)['code'] + name = "name_%s" % lang + single_year = 'single_year' in request.GET + country1 = Country.objects.get(name_3char=country1) + country2 = Country.objects.get(name_3char=country2) + + article = "to" if trade_flow == "export" else "from" + + query_params = request.GET.copy() + query_params["lang"] = lang + query_params["product_classification"] = prod_class + + attr = helpers.get_attrs(prod_class=prod_class, name=name) + + if prod_class == "sitc4": + items = Sitc4_ccpy.objects + else: + items = Hs4_ccpy.objects + + items = calculate_volume(items, trade_flow=trade_flow) + items = items.extra(select={'name': name}) + items = items.values_list('year', 'product__id', 'product__code', + 'name', 'product__community_id', + 'product__community__name', + 'product__community__color', 'val') + + if single_year: + items = items.filter(year=year) + + items = items.filter(origin_id=country1.id, destination_id=country2.id) + + json_response = {} + + key = "%s:%s:%s:%s:%s" % (country1.name_3char, country2.name_3char, "show", + prod_class, trade_flow) + + cached_data = cache.get(key) + if cached_data is not None: + json_response["data"] = msgpack.loads(cached_data) + else: + rows = list(items) + total_val = sum([r[7] for r in rows]) + + """Add percentage value to return vals""" + rows = [{"year": r[0], + "item_id":r[1], + "abbrv":r[2], + "name":r[3], + "value":r[7], + "share": (r[7] / total_val)*100, + "community_id":r[4], + "community_name":r[5], + "color":r[6], + "code":r[2], + "id": r[2]} for r in rows] + + json_response["data"] = rows + cache.set(key, msgpack.dumps(rows), settings.CACHE_LONG) + + years_available = helpers.get_years_available(prod_class=prod_class) + magic_numbers = helpers.get_inflation_adjustment(country1, + years_available[0], + years_available[-1]) + + json_response["magic_numbers"] = magic_numbers + json_response["attr_data"] = Sitc4.objects.get_all( + lang) if prod_class == "sitc4" else Hs4.objects.get_all(lang) + json_response["country1"] = country1.to_json() + json_response["country2"] = country2.to_json() + json_response["title"] = "What does %s %s %s %s?" % ( + country1.name, trade_flow, article, country2.name) + json_response["year"] = year + json_response["item_type"] = "product" + json_response["app_type"] = "ccsy" + json_response["prod_class"] = prod_class + json_response["attr"] = attr + json_response["class"] = prod_class + json_response["other"] = query_params + + # Check the request data type + if (request.GET.get('data_type', None) is None): + return HttpResponse("") + elif (request.GET.get('data_type', '') == 'json'): + """Return to browser as JSON for AJAX request""" + return HttpResponse(json.dumps(json_response)) + + +@cache_control(max_age=settings.CACHE_VERY_SHORT) +def api_cspy(request, trade_flow, country1, product, year): + """ country / show / product / year """ + + prod_class = request.GET.get("prod_class", + request.session.get('product_classification', + 'hs4')) + lang = helpers.get_language(request)['code'] + name = "name_%s" % lang + + product = helpers.get_product_by_code(product, prod_class) + country1 = Country.objects.get(name_3char=country1) + single_year = "single_year" in request.GET + + query_params = request.GET.copy() + query_params["lang"] = lang + query_params["product_classification"] = prod_class + + region = helpers.get_region_list() + continents = helpers.get_continent_list() + + if prod_class == "sitc4": + items = Sitc4_ccpy.objects + else: + items = Hs4_ccpy.objects + + items = calculate_volume(items, trade_flow) + items = items.extra(select={'name': name}) + items = items.values_list('year', 'destination__id', + 'destination__name_3char', 'name', + 'destination__region_id', + 'destination__continent', 'val') + items = items.filter(origin_id=country1.id, product_id=product.id) + + if single_year: + items = items.filter(year=year) + + json_response = {} + + key = "%s:%s:%s:%s:%s" % (country1.name_3char, "show", product.id, + prod_class, trade_flow) + + cached_data = cache.get(key) + if cached_data is not None: + json_response['data'] = msgpack.loads(cached_data) + else: + rows = list(items) + total_val = sum([r[6] for r in rows]) + + rows = [ + {"year": r[0], "item_id": r[1], "abbrv": r[2], "name": r[3], + "value": r[6], "share": (r[6] / total_val) * 100, + "region_id": r[4], "continent": r[5], "id": r[1]} + for r in rows] + + json_response["data"] = rows + cache.set(key, msgpack.dumps(rows), settings.CACHE_LONG) + + years_available = helpers.get_years_available(prod_class=prod_class) + inflation_adjustment = helpers.get_inflation_adjustment( + country1, years_available[0], years_available[-1]) + + json_response["magic_numbers"] = inflation_adjustment + json_response["attr_data"] = Country.objects.get_all(lang) + article = "to" if trade_flow == "export" else "from" + json_response["title"] = "Where does %s %s %s %s?" % ( + country1.name, trade_flow, product.name_en, article) + json_response["country1"] = country1.to_json() + json_response["product"] = product.to_json() + json_response["year"] = year + json_response["item_type"] = "country" + json_response["continents"] = continents + json_response["region"] = region + json_response["app_type"] = "cspy" + json_response["class"] = prod_class + json_response["other"] = query_params + + if (request.GET.get('data_type', None) is None): + return HttpResponse("") + elif (request.GET.get('data_type', '') == 'json'): + return HttpResponse(json.dumps(json_response)) diff --git a/django_files/observatory/views_dropdown.py b/django_files/observatory/views_dropdown.py index ffa171ae..e67e0fd2 100644 --- a/django_files/observatory/views_dropdown.py +++ b/django_files/observatory/views_dropdown.py @@ -1,19 +1,20 @@ +from django.conf import settings from django.http import HttpResponse from django.views.decorators.cache import cache_control import json +from observatory import helpers from observatory.models import Hs4, Sitc4, Country -@cache_control(max_age=900) +@cache_control(max_age=settings.CACHE_VERY_SHORT) def api_dropdown_products(request, product_class="hs4"): """API to dynamically fill in a product dropdown, product name to code. Can also set lang=foo to get a specific language, but it'll default to the django user locale. """ - lang = request.session.get('django_language', "en") - lang = request.GET.get("lang", lang) + lang = helpers.get_language(request)['code'] if product_class == "sitc4": products = Sitc4.objects.get_all(lang) @@ -23,15 +24,16 @@ def api_dropdown_products(request, product_class="hs4"): return HttpResponse(json.dumps([(p["name"], p["code"]) for p in products]), content_type="application/json") -@cache_control(max_age=900) + +@cache_control(max_age=settings.CACHE_VERY_SHORT) def api_dropdown_countries(request): """API to dynamically fill in a country dropdown, product name to code. Can also set lang=foo to get a specific language, but it'll default to the django user locale. """ - lang = request.session.get('django_language', "en") - lang = request.GET.get("lang", lang) + lang = helpers.get_language(request)['code'] countries = Country.objects.get_all(lang) - return HttpResponse(json.dumps([(c["name"], c["name_3char"].lower()) for c in countries]), + country_info = [(c["name"], c["name_3char"].lower()) for c in countries] + return HttpResponse(json.dumps(country_info), content_type="application/json") diff --git a/html/about/index.html b/html/about/index.html index 44686e45..b8370d70 100644 --- a/html/about/index.html +++ b/html/about/index.html @@ -205,6 +205,24 @@

Romain Vuillemot

Michele Coscia

Michele currently works as a Growth Lab Fellow at CID. He is trained in data mining and his research is focused primarily on Complex Network analysis, particularly on multidimensional networks, i.e. networks expressing multiple different relations at the same time. Michele has created an algorithm that can search and mine where an employee is positioned on a global skill network, and a data-mining machine that can predict which shop a customer will visit to buy a given product. Michele also designed and developed The Aid Explorer, an online tool to help facilitate better aid coordination.

+ + +
  • + + thumbnail +

    Mali Akmanalp

    +     +

    Mali is a software engineer, and leads backend development of the + Atlas. He believes code is as much craft as engineering, and that + like any craft, it can be honed with practice. He's a big believer in + the open source philosophy. Before working at CID, he's been at a + variety of organizations from Kayak.com to Turkish National Research + Institute of Electronics and Cryptology. When he's not deep in code or lurking on + Hacker News, he enjoys reading, writing, noodling on the + guitar, running and hiking.

    +
  • @@ -337,4 +355,4 @@

    Press

    -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/html/explore/index.html b/html/explore/index.html index 09cab096..fa56f5ac 100644 --- a/html/explore/index.html +++ b/html/explore/index.html @@ -134,28 +134,23 @@

    {% if title %}{{titl
    - Select a Country: -
    + Select a Country +
    - - -
    -     +
    -
    -
    +
    - -
    +
    - - Complexity @@ -553,7 +548,7 @@

    Click Here To Refresh The Visualization

    - Spotlight: + Hide exportations (Spotlight)