Merge pull request #214 from cid-harvard/beta

Django 1.6, View cleanup, caching overhaul, frontend bugfixes

FAQ.md

@@ -0,0 +1,11 @@
+
+Install
+===============
+
+**Q: How to install the cairo module?**
+
+- Get [XQuartz](https://xquartz.macosforge.org/landing/)
+- Get [homebrew](http://brew.sh/) if you don't have it already
+- Run `brew install cairo`
+- Run `brew install py2cairo`
+

django_files/atlas/settings.py

@@ -114,6 +114,11 @@
 
 VERSION = '1.0.4'
 
+CACHE_VERY_SHORT = 60*10  # 10 minutes
+CACHE_SHORT = 60*60  # 1 hour
+CACHE_LONG = 60*60*24  # 1 day
+CACHE_VERY_LONG = 60*60*24*7  # 1 week
+
 try:
   from settings_local import *
 except ImportError:

django_files/atlas/urls.py

@@ -1,13 +1,11 @@
 #from django.conf.urls import patterns, include, url
 #from django.conf.urls import patterns, url
-from django.conf.urls.defaults import *
 #from django.views.generic.simple import redirect_to
 from django.conf.urls import patterns, include
 from django.views.generic import TemplateView, RedirectView
-from django.views.generic.simple import direct_to_template
 
 # sitemap
-from django.conf.urls.defaults import *
+from django.conf.urls import *
 from django.contrib.sitemaps import FlatPageSitemap, GenericSitemap
 from django.conf import settings
 
@@ -70,7 +68,7 @@ def render_to_response(self, context, **kwargs):
   (r'^api/apps/$', 'observatory.views_infopages.api_apps'),
   (r'^api/data/$', 'observatory.views_infopages.api_data'),
   (r'^api/views/$', 'observatory.views.api_views'),
-  (r'^api/dropdowns/products/$', 'observatory.views_dropdown.api_dropdown_products'),
+  (r'^api/dropdowns/products/(?P<product_class>[a-z0-9A-Z=_]+)/$', 'observatory.views_dropdown.api_dropdown_products'),
   (r'^api/dropdowns/countries/$', 'observatory.views_dropdown.api_dropdown_countries'),
 
   # Story #####
@@ -97,29 +95,22 @@ def render_to_response(self, context, **kwargs):
   (r'^browseStoryPrev/$', 'observatory.views_stories.browseStoryPrev'),
   (r'^createStory/$', 'observatory.views_stories.createStory'),
 
-  # Explore (App) #############################################################
-  # Legacy app redirect
-  (r'^app/(?P<app_name>[a-z0-9_]+)/(?P<trade_flow>\w{6,10})/(?P<filter>[a-z0-9\.]+)/(?P<year>[0-9\.]+)/$', 'observatory.views.app_redirect'),
-
   # New app URL structure
   (r'^explore/$', 'observatory.views.explore_random'),
   (r'^explore/(?P<app_name>[a-z_]+)/(?P<trade_flow>\w{6,10})/(?P<country1>\w{3,4})/(?P<country2>\w{3,4})/(?P<product>\w{3,4})/(?P<year>[0-9\.]+)/$', 'observatory.views.explore'),
   (r'^explore/(?P<app_name>[a-z_]+)/(?P<trade_flow>\w{6,10})/(?P<country1>\w{3,4})/(?P<country2>\w{3,4})/(?P<product>\w{3,4})/$', 'observatory.views.explore'),
 
-  # Embed URL
-  (r'^embed/(?P<app_name>[a-z_]+)/(?P<trade_flow>\w{6,10})/(?P<country1>\w{3,4})/(?P<country2>\w{3,4})/(?P<product>\w{3,4})/(?P<year>[0-9\.]+)/$', 'observatory.views.embed'),
-
   # API #######################################################################
-  (r'^api/(?P<trade_flow>[a-z_]{6,10})/(?P<country1>\w{3})/all/show/(?P<year>[0-9\.]+)/$', 'observatory.views.api_casy'),
-  (r'^api/(?P<trade_flow>[a-z_]{6,10})/(?P<country1>\w{3})/show/all/(?P<year>[0-9\.]+)/$', 'observatory.views.api_csay'),
-  (r'^api/(?P<trade_flow>[a-z_]{6,10})/(?P<country1>\w{3})/(?P<country2>\w{3})/show/(?P<year>[0-9\.]+)/$', 'observatory.views.api_ccsy'),
-  (r'^api/(?P<trade_flow>[a-z_]{6,10})/(?P<country1>\w{3})/show/(?P<product>\w{4})/(?P<year>[0-9\.]+)/$', 'observatory.views.api_cspy'),
-  (r'^api/(?P<trade_flow>[a-z_]{6,10})/show/all/(?P<product>\w{4})/(?P<year>[0-9\.]+)/$', 'observatory.views.api_sapy'),
+  (r'^api/(?P<trade_flow>[a-z_]{6,10})/(?P<country1>\w{3})/all/show/(?P<year>[0-9\.]+)/$', 'observatory.views_api.api_casy'),
+  (r'^api/(?P<trade_flow>[a-z_]{6,10})/(?P<country1>\w{3})/show/all/(?P<year>[0-9\.]+)/$', 'observatory.views_api.api_csay'),
+  (r'^api/(?P<trade_flow>[a-z_]{6,10})/(?P<country1>\w{3})/(?P<country2>\w{3})/show/(?P<year>[0-9\.]+)/$', 'observatory.views_api.api_ccsy'),
+  (r'^api/(?P<trade_flow>[a-z_]{6,10})/(?P<country1>\w{3})/show/(?P<product>\w{4})/(?P<year>[0-9\.]+)/$', 'observatory.views_api.api_cspy'),
+  (r'^api/(?P<trade_flow>[a-z_]{6,10})/show/all/(?P<product>\w{4})/(?P<year>[0-9\.]+)/$', 'observatory.views_api.api_sapy'),
 
   (r'^api/near/(?P<country>\w{3})/(?P<year>[0-9\.]+)/(?P<num_prods>\d+)/$', 'observatory.views_exhibit.api_near'),
 
   (r'^api/search/$', 'observatory.views_search.api_search'),
-  (r'^search/$', direct_to_template, {'template': 'searchresults.html'}),
+  (r'^search/$', TemplateView.as_view(template_name='searchresults.html')),
 
   (r'^api/views/$', 'observatory.views.api_views'),
 
@@ -145,8 +136,7 @@ def render_to_response(self, context, **kwargs):
   url(r'^favicon\.ico$', RedirectView.as_view(url='/media/img/favicon.ico')),
 
   url(r'^sitemap\.xml$', RedirectView.as_view(url='/media/sitemaps/sitemap_index.xml')),
-  url(r'^opensearch.xml$', direct_to_template, {'template': 'opensearch.xml',
-                                                'mimetype':
-                                                'application/opensearchdescription+xml'}),
+  url(r'^opensearch.xml$', TemplateView.as_view(template_name='opensearch.xml',
+                                                content_type='application/opensearchdescription+xml'))
 
 )

django_files/observatory/helpers.py

@@ -1,4 +1,10 @@
-from observatory.models import Hs4_cpy, Sitc4_cpy, Country, Hs4, Sitc4
+from django.conf import settings
+from django.utils.translation import get_language_info
+
+from cache_utils.decorators import cached
+
+from observatory.models import (Hs4_cpy, Sitc4_cpy, Country, Hs4, Sitc4,
+                                Sitc4_py, Hs4_py, Cy, Country_region)
 
 
 # make sure app name is in the list of possible apps
@@ -77,28 +83,28 @@ def get_country(country):
     return c
 
 
-# Returns the Product object or None
-def get_product(product, classification):
-    # first try looking up based on 3 character code
+def get_product_by_code(product_code, classification="hs4"):
+    """Look up a product code in a given product code with fallback to
+    another."""
     if classification == "hs4":
         try:
-            p = Hs4.objects.get(code=product)
+            p = Hs4.objects.get(code=product_code)
         except Hs4.DoesNotExist:
-            # next try SITC4
             try:
-                conv_code = Sitc4.objects.get(code=product).conversion_code
+                conv_code = Sitc4.objects\
+                    .get(code=product_code).conversion_code
                 p = Hs4.objects.get(code=conv_code)
-            except Sitc4.DoesNotExist:
+            except (Hs4.DoesNotExist, Sitc4.DoesNotExist):
                 p = None
     else:
         try:
-            p = Sitc4.objects.get(code=product)
+            p = Sitc4.objects.get(code=product_code)
         except Sitc4.DoesNotExist:
-            # next try SITC4
             try:
-                conv_code = Hs4.objects.get(code=product).conversion_code
+                conv_code = Hs4.objects\
+                    .get(code=product_code).conversion_code
                 p = Sitc4.objects.get(code=conv_code)
-            except Hs4.DoesNotExist:
+            except (Hs4.DoesNotExist, Sitc4.DoesNotExist):
                 p = None
     return p
 
@@ -218,3 +224,116 @@ def params_to_url(api_name=None, app_name=None, country_codes=None,
         url += "%s/" % years
 
     return url
+
+
+@cached(settings.CACHE_VERY_LONG)
+def get_world_trade(prod_class="hs4"):
+    """Get world trade volume for every product in a classification."""
+    if prod_class == "sitc4":
+        return list(
+            Sitc4_py.objects.all().values(
+                'year',
+                'product_id',
+                'world_trade'))
+    elif prod_class == "hs4":
+        return list(
+            Hs4_py.objects.all().values(
+                'year',
+                'product_id',
+                'world_trade'))
+
+
+@cached(settings.CACHE_VERY_LONG)
+def get_attrs(prod_class="hs4", name="name_en"):
+    """Get extraneous attributes (like color and code) for each product in a
+    classification."""
+    if prod_class == "sitc4":
+        attr_list = list(
+            Sitc4.objects.all().values(
+                'code',
+                name,
+                'id',
+                'color'))
+        attr = {}
+        for i in attr_list:
+            attr[i['code']] = {
+                'code': i['code'],
+                'name': i[name],
+                'color': i['color']}
+    elif prod_class == "hs4":
+        attr_list = list(
+            Hs4.objects.all().values(
+                'code',
+                name,
+                'id',
+                'community_id__color'))
+        attr = {}
+        for i in attr_list:
+            attr[
+                i['code']] = {
+                'code': i['code'],
+                'name': i[name],
+                'item_id': i['id'],
+                'color': i['community_id__color']}
+    return attr
+
+
+@cached(settings.CACHE_VERY_LONG)
+def get_years_available(prod_class="hs4"):
+    """Get years available for a given classification."""
+    if prod_class == "sitc4":
+        years_available = Sitc4_cpy.objects\
+            .values_list("year", flat=True).distinct()
+    else:
+        years_available = Hs4_cpy.objects\
+            .values_list("year", flat=True).distinct()
+    return sorted(list(years_available))
+
+
+@cached(settings.CACHE_VERY_LONG)
+def get_inflation_adjustment(country, first_year, last_year):
+    """For a given country and year range, get inflation adjustment
+    constants."""
+    inflation_constants = Cy.objects\
+        .filter(country=country.id,
+                year__range=(first_year,
+                             last_year))\
+        .values('year',
+                'pc_constant',
+                'pc_current',
+                'notpc_constant')
+    magic_numbers = {}
+    for year in inflation_constants:
+        magic_numbers[year['year']] = {
+            "pc_constant": year['pc_constant'],
+            "pc_current": year['pc_current'],
+            "notpc_constant": year["notpc_constant"]}
+    return magic_numbers
+
+
+@cached(settings.CACHE_VERY_LONG)
+def get_region_list():
+    region_list = list(Country_region.objects.all().values())
+    region = {}
+    for i in region_list:
+        region[i['id']] = i
+    return region
+
+
+@cached(settings.CACHE_VERY_LONG)
+def get_continent_list():
+    continent_list = list(Country.objects.all().distinct().values('continent'))
+    continents = {}
+    for i, k in enumerate(continent_list):
+        continents[k['continent']] = i*1000
+    return continents
+
+
+def get_language(request):
+    """Given a request, check the GET params and then the session to find
+    language info specified in 2 char ISO code form, and then make sure it's
+    valid, and return a tuple in the format of django's get_language_info.
+    Nonexistent languages raise KeyError."""
+    lang = request.GET.get("lang",
+                           request.session.get('django_language', 'en'))
+    return get_language_info(lang)

django_files/observatory/models.py

@@ -3,6 +3,7 @@
 from django.forms import ModelForm
 from django.conf import settings
 
+from cache_utils.decorators import cached
 
 if not settings.DB_PREFIX:
   DB_PREFIX = ''
@@ -242,6 +243,7 @@ def filter_lang(self, lang):
 			lang = lang.replace("-", "_")
 		return self.extra(select={"name": "name_"+lang})
 
+	@cached(settings.CACHE_VERY_LONG)
 	def get_all(self, lang):
 
 		products = self.filter_lang(lang)
@@ -309,7 +311,9 @@ class Meta:
 		db_table = DB_PREFIX+"observatory_sitc4_py"
 
 	product = models.ForeignKey(Sitc4)
-	year = models.PositiveSmallIntegerField(max_length=4)
+	# The unique=True here is a workaround that somehow makes the ForeignObject
+	# on cpy work.
+	year = models.PositiveSmallIntegerField(max_length=4, unique=True)
 	pci = models.FloatField(null=True)
 	pci_rank = models.PositiveSmallIntegerField(max_length=4)
 	world_trade = models.FloatField(null=True)
@@ -323,7 +327,9 @@ class Meta:
 		db_table = DB_PREFIX+"observatory_hs4_py"
 
 	product = models.ForeignKey('Hs4')
-	year = models.PositiveSmallIntegerField(max_length=4)
+	# The unique=True here is a workaround that somehow makes the ForeignObject
+	# on cpy work.
+	year = models.PositiveSmallIntegerField(max_length=4, unique=True)
 	pci = models.FloatField(null=True)
 	pci_rank = models.PositiveSmallIntegerField(max_length=4)
 	world_trade = models.FloatField(null=True)
@@ -358,6 +364,7 @@ def filter_lang(self, lang):
 			lang = lang.replace("-", "_")
 		return self.extra(select={"name": "name_"+lang})
 
+	@cached(settings.CACHE_VERY_LONG)
 	def get_all(self, lang):
 		products = self.filter_lang(lang)
 		products = products.filter(community__isnull=False)#, ps_size__isnull=False)
@@ -438,6 +445,8 @@ class Meta:
 	distance = models.FloatField(null=True)
 	opp_gain = models.FloatField(null=True)
 
+	product_year = models.ForeignObject(Sitc4_py, ('product', 'year'), ('product', 'year'))
+
 	def __unicode__(self):
 		return "CPY: %s.%s.%d" % (self.country.name, self.product.code, self.year)
 
@@ -459,6 +468,8 @@ class Meta:
 	distance = models.FloatField(null=True)
 	opp_gain = models.FloatField(null=True)
 
+	product_year = models.ForeignObject(Hs4_py, ('product', 'year'), ('product', 'year'))
+
 	def __unicode__(self):
 		return "CPY: %s.%s.%d" % (self.country.name, self.product.code, self.year)