diff --git a/.gitignore b/.gitignore index 0c596319..ecf01ae0 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,8 @@ django_files/atlas/settings_local.py media/js/explore/viz_general.js media/js/explore/viz_general.js + +#virtualenv +env/ +#editor config data for rope +.ropeproject/ diff --git a/django_files/atlas/urls.py b/django_files/atlas/urls.py index 11f0b361..4cc91eb3 100644 --- a/django_files/atlas/urls.py +++ b/django_files/atlas/urls.py @@ -28,46 +28,46 @@ def render_to_response(self, context, **kwargs): urlpatterns = patterns('', - + ## Exploring new patterns #(r'^redesign/', include('redesign.urls')), # (r'^usa/', include('usa.urls')), -# (r'^redesign/', include('redesign.urls')), - #### - ## Revisiting old patterns - #### - +# (r'^redesign/', include('redesign.urls')), + #### + ## Revisiting old patterns + #### + # (r'^new_ps/', 'observatory.views.new_ps'), # internationalization ###################################################### (r'^i18n/', include('django.conf.urls.i18n')), (r'^set_language/(?P[a-z-]{2,5})/$', 'observatory.views.set_language'), - + # product classification #################################################### (r'^set_product_classification/(?P[a-z0-9]{3,5})/$', 'observatory.views.set_product_classification'), - + # general site ############################################################ (r'^$', 'observatory.views.home'), (r'^download/$', 'observatory.views.download'), - + # about section ########################################################### (r'^about/$', 'observatory.views.about'), - (r'^about/data/$', RedirectView.as_view(url='/about/data/sitc4/')), + (r'^about/data/$', RedirectView.as_view(url='/about/data/sitc4/')), (r'^about/data/(?P\w+)/$', "observatory.views.about_data"), (r'^about/permissions/$', "observatory.views.permissions"), - (r'^about/support/$', "observatory.views.support"), - (r'^about/research/$', "observatory.views.research"), - (r'^about/glossary/$', "observatory.views.glossary"), - (r'^about/team/$', "observatory.views.team"), - (r'^about/data/$', "observatory.views.data"), - (r'^about/permissions/$', "observatory.views.permissions"), - (r'^about/privacy/$', "observatory.views.privacy"), + (r'^about/support/$', "observatory.views.support"), + (r'^about/research/$', "observatory.views.research"), + (r'^about/glossary/$', "observatory.views.glossary"), + (r'^about/team/$', "observatory.views.team"), + (r'^about/data/$', "observatory.views.data"), + (r'^about/permissions/$', "observatory.views.permissions"), + (r'^about/privacy/$', "observatory.views.privacy"), # blog (r'^about/blog/$', "blog.views.blog_index"), url(r'^about/blog/(?P\d{4})/(?P\d{2})/(?P\d{2})/(?P[-\w]+)/$', "blog.views.blog_post_detail", name="blog_post"), - + # book ################################################################### (r'^book/$', 'observatory.views.book'), - + # API ####################################################################### (r'^api/$', 'observatory.views.api'), (r'^api/apps/$', 'observatory.views.api_apps'), @@ -90,7 +90,7 @@ def render_to_response(self, context, **kwargs): (r'^publishstory/$','observatory.views.published'), (r'^likeCount/$','observatory.views.likeCount'), (r'^featurestory/$','observatory.views.featured'), - (r'^endbrowsestory/$','observatory.views.endbrowsestory'), + (r'^endbrowsestory/$','observatory.views.endbrowsestory'), (r'^browsestories/(?P\d+)/$','observatory.views.browsestories'), (r'^stories/$','observatory.views.browseStoryForm'), (r'^endSaveStory/$','observatory.views.endSaveStory'), @@ -101,16 +101,16 @@ def render_to_response(self, context, **kwargs): # Explore (App) ############################################################# # Legacy app redirect (r'^app/(?P[a-z0-9_]+)/(?P\w{6,10})/(?P[a-z0-9\.]+)/(?P[0-9\.]+)/$', 'observatory.views.app_redirect'), - + # New app URL structure - (r'^explore/$', RedirectView.as_view(url=HTTP_HOST+'explore/tree_map/export/usa/all/show/2012/')), + (r'^explore/$', 'observatory.views.explore_random'), (r'^explore/(?P[a-z_]+)/(?P\w{6,10})/(?P\w{3,4})/(?P\w{3,4})/(?P\w{3,4})/(?P[0-9\.]+)/$', 'observatory.views.explore'), (r'^explore/(?P[a-z_]+)/(?P\w{6,10})/(?P\w{3,4})/(?P\w{3,4})/(?P\w{3,4})/$', 'observatory.views.explore'), - + # Find similar countries # (r'^similar/(?P\w{2,3})/(?P[0-9\.]+)/$', 'observatory.views.similar'), (r'^similar_wdi/(?P\w{2,3})/(?P\d+)/(?P[0-9\.]+)/$', 'observatory.views.similar_wdi'), - + # Embed URL (r'^embed/(?P[a-z_]+)/(?P\w{6,10})/(?P\w{3,4})/(?P\w{3,4})/(?P\w{3,4})/(?P[0-9\.]+)/$', 'observatory.views.embed'), @@ -121,15 +121,15 @@ def render_to_response(self, context, **kwargs): (r'^api/(?P[a-z_]{6,10})/(?P\w{3})/(?P\w{3})/show/(?P[0-9\.]+)/$', 'observatory.views.api_ccsy'), (r'^api/(?P[a-z_]{6,10})/(?P\w{3})/show/(?P\w{4})/(?P[0-9\.]+)/$', 'observatory.views.api_cspy'), (r'^api/(?P[a-z_]{6,10})/show/all/(?P\w{4})/(?P[0-9\.]+)/$', 'observatory.views.api_sapy'), - + (r'^api/near/(?P\w{3})/(?P[0-9\.]+)/(?P\d+)/$', 'observatory.views_exhibit.api_near'), - + # Overview (Countries) ###################################################### (r'^country/(?P\w{2,3})/$', 'observatory.views_overview.country2'), (r'^hs4/(?P\d{4})/$', 'observatory.views_overview.product'), (r'^sitc4/(?P\d{4})/$', 'observatory.views_overview.product'), # (r'^profile/(?P\w{2,3})/(?P[a-z_]{6})/$', 'observatory.views_overview.country'), - + # Overview (Products) ###################################################### (r'^overview/(?P\d{4})/$', 'observatory.views_overview.product'), (r'^overview/(?P\d{4})/(?P[a-z_]{6})/$', 'observatory.views_overview.product'), @@ -140,7 +140,7 @@ def render_to_response(self, context, **kwargs): (r'^rankings/(?P\w{7})/(?P[0-9\.]+)/$', 'observatory.views_rankings.index'), (r'^rankings/(?P\w{7})/download/$', 'observatory.views_rankings.download'), (r'^rankings/(?P\w{7})/(?P[0-9\.]+)/download/$', 'observatory.views_rankings.download'), - + # Exhibit ################################################################### (r'^exhibit/$', 'observatory.views_exhibit.index'), (r'^exhibit/country_selection/$', 'observatory.views_exhibit.country_selection'), diff --git a/django_files/observatory/helpers.py b/django_files/observatory/helpers.py index 37485d6c..a7301fc1 100644 --- a/django_files/observatory/helpers.py +++ b/django_files/observatory/helpers.py @@ -1,56 +1,57 @@ from observatory.models import * +import random # make sure app name is in the list of possible apps def get_app_name(app_name): possible_apps = ["tree_map", "stacked", "product_space", "map"] - + # if the app_name requested is not in the list of possibilities if app_name not in possible_apps: app_name = None - + return app_name # make sure this is accepted trade_flow def get_trade_flow(trade_flow): possible_yoga_flows = ["export", "import", "net_export", "net_import"] - + if trade_flow not in possible_yoga_flows: trade_flow = None - - return trade_flow + + return trade_flow def get_years(classification): # get distince years from db, different for diff product classifications - + if classification == "sitc4": years_available = list(Sitc4_cpy.objects.values_list("year", flat=True).distinct()) elif classification == "hs4": years_available = list(Hs4_cpy.objects.values_list("year", flat=True).distinct()) - + return years_available -# Returns app type in CCPY format +# Returns app type in CCPY format def get_app_type(country1, country2, product, year): - + # country / all / show / year if country2 == "all" and product == "show": return "casy" - + # country / show / all / year elif country2 == "show" and product == "all": return "csay" - + # show / all / product / year elif country1 == "show" and country2 == "all": return "sapy" - + # country / country / show / year elif product == "show": return "ccsy" - + # country / show / product / year else: return "cspy" @@ -98,56 +99,57 @@ def get_product(product, classification): # Returns the question to display on the page describing the URL requested def get_question(app_type, **kwargs): - + trade_flow = kwargs["trade_flow"] - + if app_type == "casy": - origin = kwargs["origin"] - + origin = kwargs["origin"] + if trade_flow in ['net_export','net_import']: title = "What does %s %s in net terms?" % (origin.name, trade_flow.replace("_", " ").split()[1]) else: title = "What does %s %s?" % (origin.name, trade_flow.replace("_", " ")) - + # Country but showing other country trade partners elif app_type == "csay": - origin = kwargs["origin"] - + origin = kwargs["origin"] + article = "to" if trade_flow in ["export","net_export"] else "from" if trade_flow in ['net_export','net_import']: - title = "Where does %s %s %s in net terms? " % (origin.name, trade_flow.replace("_", " ").split()[1], article) + title = "Where does %s %s %s in net terms? " % (origin.name, trade_flow.replace("_", " ").split()[1], article) else: - title = "Where does %s %s %s?" % (origin.name, trade_flow.replace("_", " "), article) - + title = "Where does %s %s %s?" % (origin.name, trade_flow.replace("_", " "), article) + # Product elif app_type == "sapy": product = kwargs["product"] - + if trade_flow in ['net_export','net_import']: - title = "Who %ss %s in net terms?" % (trade_flow.replace("_", " ").split()[1], product.name_en) - else: + title = "Who %ss %s in net terms?" % (trade_flow.replace("_", " ").split()[1], product.name_en) + else: title = "Who %ss %s?" % (trade_flow.replace("_", " "), product.name_en) - + # Bilateral Country x Country elif app_type == "ccsy": origin = kwargs["origin"] destination = kwargs["destination"] - + article = "to" if trade_flow in ["export","net_export"] else "from" if trade_flow in ['net_export','net_import']: title = "What does %s %s %s %s in net terms?" % (origin.name, trade_flow.replace("_", " ").split()[1], article, destination.name) else: title = "What does %s %s %s %s?" % (origin.name, trade_flow, article, destination.name) - + # Bilateral Country / Show / Product / Year elif app_type == "cspy": - origin = kwargs["origin"] + origin = kwargs["origin"] product = kwargs["product"] - + article = "to" if trade_flow in ["export","net_export"] else "from" if trade_flow in ['net_export','net_import']: title = "Where does %s %s %s %s in net terms?" % (origin.name, trade_flow.replace("_", " ").split()[1], product.name_en, article) else: title = "Where does %s %s %s %s?" % (origin.name, trade_flow, product.name_en, article) - + return title + diff --git a/django_files/observatory/models.py b/django_files/observatory/models.py index 95043399..3828d251 100644 --- a/django_files/observatory/models.py +++ b/django_files/observatory/models.py @@ -25,13 +25,13 @@ class observastory(models.Model): number_of_chapters=models.IntegerField(max_length=11,default=0) class Meta: db_table = "observatory_story" - + def __unicode__(self): return u"%s" % self.story_id class storyform(ModelForm): - + class Meta: model = observastory fields = ('story_name','story_desc') @@ -108,16 +108,16 @@ def __unicode__(self): return self.name class Country_manager(models.Manager): - + def filter_lang(self, lang): return self.extra(select={"name": "name_"+lang}) - + def get_all(self, lang): - + if type(lang) == bool: lang = "en" lang = lang.replace("-", "_") - + countries = self.filter_lang(lang) countries = countries.filter(region__isnull=False, name_3char__isnull=False, name_2char__isnull=False).order_by("name_"+lang) return list(countries.values( @@ -132,6 +132,12 @@ def get_all(self, lang): "region__text_color", )) + def get_random(self): + """Grab a random country that has a 3char name. This uses the 'ORDER BY + RAND()' method which is fine for this purpose but slow in mysql for + larger tables so beware.""" + return Country.objects.filter(name_3char__isnull=False).order_by('?')[1] + class Country(models.Model): class Meta: @@ -163,15 +169,15 @@ class Meta: name_pt = models.TextField(null=True) name_tr = models.TextField(null=True) name_zh_cn = models.TextField(null=True) - + def __unicode__(self): return self.name - + def to_json(self): return { "name": self.name_en, "name_3char": self.name_3char} - + objects = Country_manager() class Cy(models.Model): @@ -222,16 +228,16 @@ def __unicode__(self): # SITC4 products class Sitc4_manager(models.Manager): - + def filter_lang(self, lang): if type(lang) is bool: lang = "en" else: lang = lang.replace("-", "_") return self.extra(select={"name": "name_"+lang}) - + def get_all(self, lang): - + products = self.filter_lang(lang) products = products.filter(community__isnull=False)#, ps_size__isnull=False) return list(products.values( @@ -246,7 +252,7 @@ def get_all(self, lang): "ps_y", "ps_size" )) - + class Sitc4(models.Model): class Meta: @@ -283,12 +289,12 @@ class Meta: def __unicode__(self): return self.code + self.name_en - + def to_json(self): return { "name": self.name_en, "community_id": self.community.id} - + objects = Sitc4_manager() class Sitc4_py(models.Model): @@ -321,7 +327,7 @@ def __unicode__(self): return "%s rank: %d" % (self.product.name, self.pci_rank) -# Colors for HS4 clusters +# Colors for HS4 clusters # http://www.foreign-trade.com/reference/hscode.htm class Hs4_community(models.Model): @@ -393,19 +399,19 @@ class Meta: def __unicode__(self): return self.code + self.name - + def to_json(self): return { "name": self.name_en, "community_id": self.community_id} - + objects = Hs4_manager() ############################################################################### # country - product - year tables ############################################################################### class Sitc4_cpy_manager(models.Manager): - + def set_value(self, trade_flow): if trade_flow == "net_export": return self.extra({'value': 'export_value-import_value'}, where=['export_value-import_value > 0']) @@ -414,14 +420,14 @@ def set_value(self, trade_flow): return self.extra(select={"value": "%s_value" % (trade_flow,)}) def casy(self, country, trade_flow, year=None, lang="en"): - + q_set = self.set_value(trade_flow) - + data = q_set.filter( country = country, product__community__isnull = False, product__ps_x__isnull = False) - + if year: if "." in year: years = [int(x) for x in year.split(".")] @@ -436,23 +442,23 @@ def casy(self, country, trade_flow, year=None, lang="en"): the_sum[y] = data.extra(select={"sum": "sum(import_value-export_value)"}, where=['year=%s'%(y,)]).values()[0]["sum"] else: the_sum[y] = data.filter(year=y).aggregate(Sum("%s_value" % (trade_flow))).values()[0] - + the_data = data.filter(year__in=years).order_by("year", "-value").values_list("year", "product__code", "product__name_%s"%(lang,), "value", "export_rca") columns = ["#", "Year", "SITC4", "Product Name", "Value (USD)", "RCA", "%"] return {"sum":the_sum, "data":the_data, "columns":columns} else: return list(data.extra(select={'item_id': "product_id"}).values("item_id", "year", "value", "export_rca")) - + def sapy(self, product, trade_flow, year=None, lang="en"): - + q_set = self.set_value(trade_flow) - + data = q_set.filter( product = product, country__region__isnull=False, country__name_3char__isnull=False, country__name_2char__isnull=False) - + if year: # put years into array (in case user requests stacked with multiple years of data) if "." in year: @@ -482,19 +488,19 @@ class Meta: export_value = models.FloatField(null=True) import_value = models.FloatField(null=True) export_rca = models.FloatField(null=True) - distance = models.FloatField(null=True) + distance = models.FloatField(null=True) opp_gain = models.FloatField(null=True) - + def __unicode__(self): return "CPY: %s.%s.%d" % (self.country.name, self.product.code, self.year) # def prod(self, lang): # return "%s" % (getattr(self.product, "name_%s" % (lang,))) - + objects = Sitc4_cpy_manager() class Hs4_cpy_manager(models.Manager): - + def set_value(self, trade_flow): if trade_flow == "net_export": return self.extra({'value': 'export_value-import_value'}, where=['export_value-import_value > 0']) @@ -503,14 +509,14 @@ def set_value(self, trade_flow): return self.extra(select={"value": "%s_value" % (trade_flow,)}) def casy(self, country, trade_flow, year=None, lang="en"): - + q_set = self.set_value(trade_flow) - + data = q_set.filter( country = country, product__community__isnull = False, product__ps_x__isnull = False) - + if year: if "." in year: years = [int(x) for x in year.split(".")] @@ -525,17 +531,17 @@ def casy(self, country, trade_flow, year=None, lang="en"): return {"sum":the_sum, "data":the_data, "columns":columns} else: return list(data.extra(select={'item_id': "product_id"}).values("item_id", "year", "value", "export_rca")) - + def sapy(self, product, trade_flow, year=None, lang="en"): - + q_set = self.set_value(trade_flow) - + data = q_set.filter( product = product, country__region__isnull=False, country__name_3char__isnull=False, country__name_2char__isnull=False) - + if year: # put years into array (in case user requests stacked with multiple years of data) if "." in year: @@ -565,12 +571,12 @@ class Meta: export_value = models.FloatField(null=True) import_value = models.FloatField(null=True) export_rca = models.FloatField(null=True) - distance = models.FloatField(null=True) + distance = models.FloatField(null=True) opp_gain = models.FloatField(null=True) - + def __unicode__(self): return "CPY: %s.%s.%d" % (self.country.name, self.product.code, self.year) - + objects = Hs4_cpy_manager() ############################################################################### @@ -586,27 +592,27 @@ def set_value(self, trade_flow): # return self.extra(select={"value": "sum(%s_value)" % (trade_flow,)}) return self.extra(select={"value": "%s_value" % (trade_flow,)}) # qs.extra(select = {'total_amount': 'SUM(one_column + another_column)'}, ) - + def csay(self, country1, trade_flow, year=None, lang="en"): # raw = self.raw("select * from observatory_sitc4_ccpy where year = 2009") # r = [x for x in raw] # raise Exception() # raise Exception(r) # q_set = self.set_value(trade_flow) - + # raise Exception(self.extra(select={"value": "sum(%s_value)" % (trade_flow,)}, where=['year=2009']).values()) - + # raise Exception(q_set.filter(year=2009).aggregate(Sum("value")).values()[0]) # raise Exception(country1) - + data = self.filter( origin = country1, destination__region__isnull=False, destination__name_3char__isnull=False, destination__name_2char__isnull=False) - + # data = self.set_value(trade_flow) - + if year: if "." in year: years = [int(x) for x in year.split(".")] @@ -627,20 +633,20 @@ def csay(self, country1, trade_flow, year=None, lang="en"): else: # if trade_flow == "import": return list(data.extra(select={'item_id': "origin_id"}).values("item_id", "year").annotate(value=Sum('value'))) return list(data.extra(select={'item_id': "destination_id"}).values("item_id", "year").annotate(value=Sum('%s_value'%(trade_flow,)))) - + def ccsy(self, country1, country2, trade_flow, year=None, lang="en"): - + # if trade_flow == "import": # country1, country2 = country2, country1 - + # q_set = self.set_value(trade_flow) - + data = self.filter( destination = country2, origin = country1, product__community__isnull = False, product__ps_x__isnull = False) - + if year: if "." in year: years = [int(x) for x in year.split(".")] @@ -652,11 +658,11 @@ def ccsy(self, country1, country2, trade_flow, year=None, lang="en"): for y in years: # the_sum[y] = data.filter(year=y).aggregate(Sum("value")).values()[0] the_sum[y] = data.filter(year=y).aggregate(Sum("%s_value" % (trade_flow))).values()[0] - + the_data = data.filter(year__in=years).order_by("year", "-value").values_list("year", "product__code", "product__name_%s"%(lang,), "%s_value"%(trade_flow,)) - - - + + + # get the data sorted by year and value # the_data = data.filter(year__in=years).order_by("year", "-value").values_list("year", "product__code", "product__name_%s"%(lang,), "value") columns = ["#", "Year", "SITC4", "Product Name", "Value (USD)", "RCA", "%"] @@ -665,16 +671,16 @@ def ccsy(self, country1, country2, trade_flow, year=None, lang="en"): return list(data.extra(select={'item_id': "product_id"}).values("item_id", "year", "value")) def cspy(self, country1, product, trade_flow, year=None, lang="en"): - + # q_set = self.set_value(trade_flow) - + data = self.filter( product = product, origin = country1, destination__region__isnull=False, destination__name_3char__isnull=False, destination__name_2char__isnull=False).exclude(destination=231) - + if year: if "." in year: years = [int(x) for x in year.split(".")] @@ -695,14 +701,14 @@ def cspy(self, country1, product, trade_flow, year=None, lang="en"): else: # if trade_flow == "import": return list(data.extra(select={'item_id': "origin_id"}).values("item_id", "year", "value")) return list(data.extra(select={'item_id': "destination_id"}).values("item_id", "year", "value")) - - + + # if trade_flow == "export": # data = Sitc4_ccpy.objects.filter(product=p, origin=c, destination__region__isnull=False, destination__name_3char__isnull=False, destination__name_2char__isnull=False).exclude(destination=231).values("destination_id", "year", "value"); # elif trade_flow == "import": # data = Sitc4_ccpy.objects.filter(product=p, destination=c, origin__region__isnull=False, origin__name_3char__isnull=False, origin__name_2char__isnull=False).exclude(origin=231).values("origin_id", "year", "value"); # meta["filter_id"] = "origin_id" - + class Sitc4_ccpy(models.Model): class Meta: @@ -717,18 +723,18 @@ class Meta: def __unicode__(self): return "%s -> %s" % (self.origin.name, self.destination.name) - + objects = Sitc4_ccpy_manager() class Hs4_ccpy_manager(models.Manager): - + def set_value(self, trade_flow): if trade_flow == "net_export": return self.extra({'value': 'export_value-import_value'}, where=['export_value-import_value > 0']) elif trade_flow == "net_import": return self.extra({'value': 'import_value-export_value'}, where=['import_value-export_value > 0']) return self.extra(select={"value": "%s_value" % (trade_flow,)}) - + def csay(self, country1, trade_flow, year=None, lang="en"): data = self.filter( @@ -736,7 +742,7 @@ def csay(self, country1, trade_flow, year=None, lang="en"): destination__region__isnull=False, destination__name_3char__isnull=False, destination__name_2char__isnull=False) - + if year: if "." in year: years = [int(x) for x in year.split(".")] @@ -752,23 +758,23 @@ def csay(self, country1, trade_flow, year=None, lang="en"): the_data = data.filter(year__in=years).order_by("year", "-value").values_list("year", "%s__name_3char"%(to_show),"%s__name_%s"%(to_show,lang)).annotate(value=Sum('value')) columns = ["#", "Year", "Alpha-3", "Country", "Value (USD)", "%"] return {"sum":the_sum, "data":the_data, "columns":columns} - + else: # if trade_flow == "import": return list(data.extra(select={'item_id': "origin_id"}).values("item_id", "year").annotate(value=Sum('%s_value'%(trade_flow)))) return list(data.extra(select={'item_id': "destination_id"}).values("item_id", "year").annotate(value=Sum('%s_value'%(trade_flow)))) - + def ccsy(self, country1, country2, trade_flow, year=None, lang="en"): - + # q_set = self.set_value(trade_flow) - + # if trade_flow == "import": # country1, country2 = country2, country1 - + data = self.filter( destination = country2, origin = country1, product__community__isnull = False) - + if year: if "." in year: years = [int(x) for x in year.split(".")] @@ -789,7 +795,7 @@ def ccsy(self, country1, country2, trade_flow, year=None, lang="en"): def cspy(self, country1, product, trade_flow, year=None, lang="en"): q_set = self.set_value(trade_flow) - + # if trade_flow == "import": # data = q_set.filter( # product = product, @@ -840,7 +846,7 @@ class Meta: def __unicode__(self): return "%s -> %s" % (self.origin.name, self.destination.name) - + objects = Hs4_ccpy_manager() class Wdi(models.Model): @@ -860,7 +866,7 @@ class Wdi_cwy(models.Model): wdi = models.ForeignKey(Wdi) year = models.PositiveSmallIntegerField(max_length=4) value = models.FloatField(null=True) - + def __unicode__(self): return "[%s] %s: %s" % (self.year, self.country.name_3char, self.wdi.name) diff --git a/django_files/observatory/views.py b/django_files/observatory/views.py index 9d7b43da..59be3232 100644 --- a/django_files/observatory/views.py +++ b/django_files/observatory/views.py @@ -52,7 +52,7 @@ #object used to Encrypt/Decrypt ##################################### fpe_obj = fpe.FPEInteger(key=b'mypetsnameeloise', radix=10, width=10) - + ##################################### #create story ##################################### @@ -61,7 +61,7 @@ def createStory(request): request.session['create']=iscreatemode #Get Current URL url = request.META['HTTP_REFERER'] - return redirect(url) + return redirect(url) ###################################### #save story @@ -102,7 +102,7 @@ def endSaveStory(request): sessionParameters={} sessionParameters['product_classification']=prod_class sessionParameters['django_language']=language - sessionParameters = json.dumps(sessionParameters)[1:-1] + sessionParameters = json.dumps(sessionParameters)[1:-1] chapterSerialNo=counter+1 #save session parameter and chapter details to storychapter table saveToChapterTable=storychapter(story_id=storyId,chapter_url=chapterUrl,chapter_title=chapterName,chapter_desc=chapterDescription,chapter_js_details=chapterJs,chapter_details=sessionParameters,serial_number=chapterSerialNo) @@ -140,7 +140,7 @@ def endSaveStory(request): getUserId=observatoryuser.objects.get(user_email=email) request.session['userid']=getUserId.user_id storyId=request.session['storyId'] - updateUserId=observastory.objects.filter(story_id=storyId).update(user_id=getUserId.user_id) + updateUserId=observastory.objects.filter(story_id=storyId).update(user_id=getUserId.user_id) #check user auth_type if source == 'facebook': #check useremail exits @@ -157,9 +157,9 @@ def endSaveStory(request): getUserId=observatoryuser.objects.get(user_email=email) request.session['userid']=getUserId.user_id storyId=request.session['storyId'] - updateUserId=observastory.objects.filter(story_id=storyId).update(user_id=getUserId.user_id) + updateUserId=observastory.objects.filter(story_id=storyId).update(user_id=getUserId.user_id) else: - storyId=request.session['storyId'] + storyId=request.session['storyId'] #update publish value updatePublish=observastory.objects.filter(story_id=storyId).update(published=1) iscreatemode=False @@ -176,7 +176,7 @@ def browseStoryForm(request): isbrowsemode=False request.session['retrieve']=isbrowsemode if 'userid' in request.session: - userUniqueId=request.session['userid'] + userUniqueId=request.session['userid'] elif "socialMediaIntegrationData" in request.POST: socialSitesDetails=request.POST["socialMediaIntegrationData"] if socialSitesDetails: @@ -208,14 +208,14 @@ def browseStoryForm(request): if observatoryuser.objects.filter(user_email=email).exists() == True: #Get userid getUserId=observatoryuser.objects.get(user_email=email) - request.session['userid']=getUserId.user_id + request.session['userid']=getUserId.user_id else: #Create userid and Get userid observaUser=observatoryuser(user_name=name,user_email=email,user_auth_type=source) observaUser.save() getUserId=observatoryuser.objects.get(user_email=email) request.session['userid']=getUserId.user_id - userUniqueId=request.session['userid'] + userUniqueId=request.session['userid'] else: userUniqueId=1 else: @@ -396,7 +396,7 @@ def updateEditForm(request): chapterName=data['chapterTitle'] chapterDesc= data['chapterDesc'] chapterRemove=data['isRemoved'] - chapterID=data['chapterId'] + chapterID=data['chapterId'] if chapterRemove == 'Y': #delete chapter deletechapter=storychapter.objects.filter(chapter_id=chapterID).delete() @@ -426,7 +426,7 @@ def published(request): isPublished=False else: #Update publish value - updatePublishValue=observastory.objects.filter(story_id=browseStoryId).update(published=1) + updatePublishValue=observastory.objects.filter(story_id=browseStoryId).update(published=1) isPublished=True #Load in to json response json_response = {} @@ -436,7 +436,7 @@ def published(request): ##################################################### # delete story ##################################################### -def deleteStory(request): +def deleteStory(request): #Check user id in session if 'userid' in request.session: #Get Story id @@ -485,11 +485,11 @@ def likeCount(request): #Get browse story id browseStoryId=request.session['browseStoryId'] if 'browseStoryIds' in request.session else 0 #Increament like count - updateLikeCount=observastory.objects.filter(story_id=browseStoryId).update(likecount=F('likecount')+1) + updateLikeCount=observastory.objects.filter(story_id=browseStoryId).update(likecount=F('likecount')+1) likeBtnEnable=request.session['likeBtnEnable'] if 'likeBtnEnable' in request.session else False userId=request.session['userid'] if 'userid' in request.session else 0 if userId > 0: - #Save userid and story id to like table + #Save userid and story id to like table saveToLikeTable=observatory_like(user_id=userId,story_id=browseStoryId) saveToLikeTable.save() #Get like count @@ -511,9 +511,9 @@ def logout(request): request.session['retrieve']=isbrowsemode #delete userid and username in session if 'userid' in request.session: - del request.session['userid'] - if 'username' in request.session: - del request.session['username'] + del request.session['userid'] + if 'username' in request.session: + del request.session['username'] return redirect(HTTP_HOST+'explore/') ##################################################### @@ -546,7 +546,7 @@ def browsestories(request,browseStoryId): browseArrayStoryId = browseStoryId request.session['browseStoryIds']=browseStoryId userId=request.session['userid'] if 'userid' in request.session else 0 - #Get like count + #Get like count story=observastory.objects.get(story_id=browseStoryId) request.session['likeCount']=story.likecount #Like Btn @@ -572,7 +572,7 @@ def browsestories(request,browseStoryId): if browseModeIndex < len(browseStoryChapterIds): if browseModeIndex == len(browseStoryChapterIds): browseModeIndex=0 - #Get sessions varibales + #Get sessions varibales chapterDetails=storychapter.objects.get(serial_number=browseStoryChapterIds[browseModeIndex],story_id=browseStoryId) browseStorySesVar=chapterDetails.chapter_details browseStorySesDataToArray=browseStorySesVar.split(',') @@ -613,10 +613,10 @@ def browseStoryNext(request): if index < len(browseStoryChapIds): index=index+1 if index == len(browseStoryChapIds): - index=0 - #Get sessions varibales + index=0 + #Get sessions varibales chapterDetails=storychapter.objects.get(serial_number=browseStoryChapIds[index],story_id=browseStoryId) - request.session['index']=index + request.session['index']=index browseStorySesVar=chapterDetails.chapter_details browseStorySesDataToArray=browseStorySesVar.split(',') for indexvar in range(0, len(browseStorySesDataToArray)): @@ -654,10 +654,10 @@ def browseStoryPrev(request): index=len(browseStoryChapterIds) index=index-1 request.session['index']=index - #Get sessions varibales + #Get sessions varibales chapterDetails=storychapter.objects.get(serial_number=browseStoryChapterIds[index],story_id=browseStoryId) browseStorySesVar=chapterDetails.chapter_details - request.session['index']=index + request.session['index']=index browseStorySesDataToArray=browseStorySesVar.split(',') for indexvar in range(0, len(browseStorySesDataToArray)): browseStorySessionData=browseStorySesDataToArray[indexvar].split(':') @@ -678,14 +678,14 @@ def endbrowse(request): isbrowsemode=False request.session['retrieve']=isbrowsemode return redirect(HTTP_HOST+'explore/') - + def fluid(request): return render_to_response("fluid.html", context_instance=RequestContext(request)) - + ################### ## Abandoned function? Does nothing. ################### -def new_ps(request): +def new_ps(request): ps_nodes = Sitc4.objects.get_all("en") return render_to_response("new_ps.html", {"ps_nodes":json.dumps(ps_nodes, indent=2)},context_instance=RequestContext(request)) @@ -708,7 +708,7 @@ def home(request): # except Country.DoesNotExist: c = Country.objects.get(name_2char="us") # c = Country.objects.order_by('?')[0] # not all countries from this list can be used.. - return render_to_response("home.html", + return render_to_response("home.html", {"default_country": c}, context_instance=RequestContext(request)) @@ -721,11 +721,11 @@ def support(request): def research(request): return render_to_response("about/research.html", context_instance=RequestContext(request)) def glossary(request): - return render_to_response("about/glossary.html", context_instance=RequestContext(request)) + return render_to_response("about/glossary.html", context_instance=RequestContext(request)) def team(request): return render_to_response("about/team.html", context_instance=RequestContext(request)) def data(request): - return render_to_response("about/data.html", context_instance=RequestContext(request)) + return render_to_response("about/data.html", context_instance=RequestContext(request)) def permissions(request): return render_to_response("about/permissions.html", context_instance=RequestContext(request)) def privacy(request): @@ -806,33 +806,33 @@ def download(request): import csv #raise Exception(request.POST) content = request.POST.get("content") - + title = request.POST.get("title") - + format = request.POST.get("format") - + if format == "svg" or format == "pdf" or format == "png": svg = rsvg.Handle(data=content.encode("utf-8")) x = width = svg.props.width y = height = svg.props.height - + if format == "svg": response = HttpResponse(content.encode("utf-8"), mimetype="application/octet-stream") - - elif format == "pdf": + + elif format == "pdf": response = HttpResponse(mimetype='application/pdf') surf = cairo.PDFSurface(response, x, y) cr = cairo.Context(surf) svg.render_cairo(cr) surf.finish() - - elif format == "png": + + elif format == "png": response = HttpResponse(mimetype='image/png') surf = cairo.ImageSurface(cairo.FORMAT_ARGB32, x, y) cr = cairo.Context(surf) svg.render_cairo(cr) surf.write_to_png(response) - + else: response = HttpResponse(mimetype="text/csv;charset=UTF-8") csv_writer = csv.writer(response, delimiter=',', quotechar='"')#, quoting=csv.QUOTE_MINIMAL) @@ -840,10 +840,10 @@ def download(request): # raise Exception(content) for item in item_list: csv_writer.writerow([i.encode("utf-8") for i in item]) - + # Need to change with actual title response["Content-Disposition"]= "attachment; filename=%s.%s" % (title, format) - + return response def app(request, app_name, trade_flow, filter, year): @@ -851,12 +851,12 @@ def app(request, app_name, trade_flow, filter, year): format = request.GET.get("format", False) lang = request.GET.get("lang", False) crawler = request.GET.get("_escaped_fragment_", False) - + country1, country2, product = None, None, None country1_list, country2_list, product_list, year1_list, year2_list, year_interval_list, year_interval = None, None, None, None, None, None, None - + trade_flow_list = ["export", "import", "net_export", "net_import"] - + year1_list = range(1962, 2011, 1) if "." in year: y = [int(x) for x in year.split(".")] @@ -866,7 +866,7 @@ def app(request, app_name, trade_flow, filter, year): year_interval = year[1] - year[0] else: year = int(year) - + json_response = { "year": year, "app": app_name @@ -875,26 +875,26 @@ def app(request, app_name, trade_flow, filter, year): # Bilateral if "." in filter: bilateral_filters = filter.split(".") - + # Country x Product if len(bilateral_filters[1]) > 3: country1 = Country.objects.get(name_3char=bilateral_filters[0]) product = Sitc4.objects.get(code=bilateral_filters[1]) - + # Lists used for control pane country1_list = Country.objects.get_all(lang) product_list = Sitc4.objects.get_all(lang) trade_flow_list = ["export", "import"] - + article = "to" if trade_flow == "export" else "from" title = "Where did %s %s %s %s?" % (country1.name, trade_flow, product.name_en, article) - + # cspy means country1 / countr2 / show / year if crawler == "" or format == "json": json_response["data"] = Sitc4_ccpy.objects.cspy(country1, product, trade_flow) json_response["attr_data"] = Country.objects.get_all(lang) json_response["title"] = title - + # Country x Country else: country1 = Country.objects.get(name_3char=bilateral_filters[0]) @@ -904,46 +904,46 @@ def app(request, app_name, trade_flow, filter, year): country1_list = Country.objects.get_all(lang) country2_list = country1_list trade_flow_list = ["export", "import"] - + article = "to" if trade_flow == "export" else "from" title = "What did %s %s %s %s?" % (country1.name, trade_flow, article, country2.name) - + # ccsy means country1 / countr2 / show / year if crawler == "" or format == "json": json_response["data"] = Sitc4_ccpy.objects.ccsy(country1, country2, trade_flow) json_response["attr_data"] = Sitc4.objects.get_all(lang) json_response["title"] = title - + # Product elif len(filter) > 3: product = Sitc4.objects.get(code=filter) product_list = Sitc4.objects.get_all(lang) - + title = "Who %ss %s?" % (trade_flow.replace("_", " "), product.name_en) - + # sapy means show / all / product / year if crawler == "" or format == "json": json_response["data"] = Sitc4_cpy.objects.sapy(product, trade_flow) json_response["attr_data"] = Country.objects.get_all(lang) json_response["title"] = title - + # Country else: country1 = Country.objects.get(name_3char=filter) country1_list = Country.objects.get_all(lang) - + title = "What did %s %s?" % (country1.name, trade_flow.replace("_", " ")) - + # casy means country1 / all / show / year if crawler == "" or format == "json": json_response["data"] = Sitc4_cpy.objects.casy(country1, trade_flow) json_response["attr_data"] = Sitc4.objects.get_all(lang) json_response["title"] = title - + # Send data as JSON to browser via AJAX if format == "json": return HttpResponse(json.dumps(json_response)) - + # Return page without visualization data return render_to_response("app/index.html", { "title": title, @@ -965,23 +965,23 @@ def app_redirect(request, app_name, trade_flow, filter, year): # Corrent for old spelling of tree map as one word if app_name == "treemap": app_name = "tree_map" - + # Bilateral if "." in filter: bilateral_filters = filter.split(".") - + # Country x Product if len(bilateral_filters[1]) > 3: country1, country2, product = bilateral_filters[0], "show", bilateral_filters[1] - + # Country x Country else: country1, country2, product = bilateral_filters[0], bilateral_filters[1], "show" - + # Product elif len(filter) > 3: country1, country2, product = "show", "all", filter - + # Country else: country1, country2, product = filter, "all", "show" @@ -992,15 +992,15 @@ def generate_png( request ): import rsvg import cairo import cairosvg - + content = request.POST.get('svg_data') - + # Check if we have proper content first if ( content is not None ): # Fix the spaces etc and remove unwanted stuff from the content content = content.strip() content = content + "\n" - + #file_name="test" file_name="tree_map" @@ -1008,20 +1008,20 @@ def generate_png( request ): svg_file = open( settings.DATA_FILES_PATH + "/" + file_name + ".svg", "w+" ) svg_file.write( content ) svg_file.close() - + # Read in the content svg_file = open( settings.DATA_FILES_PATH + "/" + file_name + ".svg", "r" ) - + # Open up the file to be written to png_file = open( settings.DATA_FILES_PATH + "/" + file_name + ".png", "w+" ) - + # Do the export cairosvg.svg2png( file_obj = svg_file, write_to = png_file ) - + # Close the svg & png file svg_file.close() png_file.close() - + # Create the blank image surface #blank_surface = cairo.ImageSurface( cairo.FORMAT_ARGB32, 750, 480 ) @@ -1034,9 +1034,9 @@ def generate_png( request ): # Create the final png image #final_png = blank_surface.write_to_png( settings.DATA_FILES_PATH + "/" + file_name + ".png" ) - - return HttpResponse( "Success" ) - + + return HttpResponse( "Success" ) + def explore(request, app_name, trade_flow, country1, country2, product, year="2011"): iscreatemode=False iscreatemode = request.session['create'] if 'create' in request.session else False @@ -1051,7 +1051,7 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 NoOfChpt=request.session['NoOfchap'] if 'NoOfchap' in request.session else "" userName=request.session['username'] if 'username' in request.session else "" userId=request.session['userid'] if 'userid' in request.session else 0 - likeBtnEnable=request.session['likeBtnEnable'] if 'likeBtnEnable' in request.session else False + likeBtnEnable=request.session['likeBtnEnable'] if 'likeBtnEnable' in request.session else False likeCount=request.session['likeCount'] if 'likeCount' in request.session else "" #Set app_name to session request.session['app_name']=app_name @@ -1073,7 +1073,7 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 language=lang # Setup the hash dictionary request_hash_dictionary = collections.OrderedDict() - + # Add prod class to request hash dictionary request_hash_dictionary['app_name'] = app_name request_hash_dictionary['lang'] = lang @@ -1088,7 +1088,7 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 # We are here, so let us store this data somewhere request_hash_string = "_".join( request_hash_dictionary.values() ) - # Check staic image mode + # Check staic image mode if( settings.STATIC_IMAGE_MODE == "PNG" ): # Check if we have a valid PNG image to display for this if os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".png"): @@ -1101,7 +1101,7 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 else: displayviz=False displayImage = settings.STATIC_URL + "img/all/loader.gif" - + # get distince years from db, different for diff product classifications years_available = list(Sitc4_cpy.objects.values_list("year", flat=True).distinct()) if prod_class == "sitc4" else list(Hs4_cpy.objects.values_list("year", flat=True).distinct()) years_available.sort() @@ -1110,7 +1110,7 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 data_as_text = {} # What is actually being shown on the page item_type = "product" - + # To make sure it cannot be another product class if prod_class != "hs4" and prod_class != "sitc4": prod_class = "sitc4" @@ -1130,11 +1130,11 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 if country1 == "zaf": warning = {"title": "Country Substitution", "text": "In the Harmonized System (HS) classification, trade for Namibia, Republic of South Africa, Botswana, Lesotho and Swaziland is reported under 'South African Customs Union'."} - + trade_flow_list = [("export", _("Export")), ("import", _("Import")), ("net_export", _("Net Export")), ("net_import", _("Net Import"))] if (app_name == "product_space" or app_name == "rings"): trade_flow_list = [trade_flow_list[0]] - + year1_list = range(years_available[0], years_available[len(years_available)-1]+1, 1) if app_name == "stacked" and year == "2009": @@ -1155,14 +1155,14 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 year = years_available[len(years_available)-1] elif year < years_available[0]: year = years_available[0] - + api_uri = "api/%s/%s/%s/%s/%s/?%s" % (trade_flow, country1, country2, product, year, options) - + redesign_api_uri = "redesign/api/%s/%s/%s/%s/%s/%s" % (prod_class, trade_flow, country1, country2, product, year) - + country_code = None if country1 != "show" and country1 != "all": country_code = country1 - + if crawler == "": view, args, kwargs = resolve("api/%s/%s/%s/%s/%s/" % (trade_flow, country1, country2, product, year)) kwargs['request'] = request @@ -1200,23 +1200,23 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 # request.session['product_classification'] = "hs4" #else: # alert = {"title": "Product could not be found", "text": "There was no product with the 4 digit code %s. Please double check the list of HS4 products."%(p_code)} - + if countries[0]: # get distinct years from db, different for diff product classifications # also we need to filter by country1, as not all SITC4 data goes back to '62 years_available = list(Sitc4_cpy.objects.filter(country=countries[0].id).values_list("year", flat=True).distinct()) if prod_class == "sitc4" else list(Hs4_cpy.objects.filter(country=countries[0].id).values_list("year", flat=True).distinct()) else: years_available = list(Sitc4_cpy.objects.values_list("year", flat=True).distinct()) if prod_class == "sitc4" else list(Hs4_cpy.objects.values_list("year", flat=True).distinct()) - + years_available.sort() list_countries_the = ["Cayman Islands", "Central African Republic", "Channel Islands", "Congo, Dem. Rep.", "Czech Republic", "Dominican Republic", "Faeroe Islands", "Falkland Islands", "Fm Yemen Dm", "Lao PDR", "Marshall Islands", "Philippines", "Seychelles", "Slovak Republic", "Syrian Arab Republic", "Turks and Caicos Islands", "United Arab Emirates", "United Kingdom", "Virgin Islands, U.S.", "United States"] if countries[0] and countries[0].name in list_countries_the: countries[0].name = "the "+countries[0].name - + prod_or_partner = "partner" # quick fix should be merged with item_type - + if not alert: if app_type == "casy": # raise Exception(app_name) @@ -1243,20 +1243,20 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 # Product elif app_type == "sapy": item_type = "countries" - if app_name == "stacked": + if app_name == "stacked": title = "Who %sed %s between %s and %s?" % (trade_flow.replace("_", " "), product.name_en, year_start, year_end) else: title = "Who %sed %s in %s?" % (trade_flow.replace("_", " "), product.name_en, year) - + prod_or_partner = "product" - + # Bilateral Country x Country elif app_type == "ccsy": # trade_flow_list = ["export", "import"] if _("net_export") in trade_flow_list: del trade_flow_list[trade_flow_list.index(_("net_export"))] if _("net_import") in trade_flow_list: del trade_flow_list[trade_flow_list.index(_("net_import"))] article = "to" if trade_flow == "export" else "from" - if app_name == "stacked": + if app_name == "stacked": title = "What did %s %s %s %s between %s and %s?" % (countries[0].name, trade_flow.replace("_", " "), article, countries[1].name, year_start, year_end) else: title = "What did %s %s %s %s in %s?" % (countries[0].name, trade_flow.replace("_", " "), article, countries[1].name, year) @@ -1265,13 +1265,13 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 elif app_type == "cspy": if "net_export" in trade_flow_list: del trade_flow_list[trade_flow_list.index("net_export")] if "net_import" in trade_flow_list: del trade_flow_list[trade_flow_list.index("net_import")] - item_type = "countries" + item_type = "countries" article = "to" if trade_flow == "export" else "from" if app_name == "stacked": title = "Where did %s %s %s %s between %s and %s?" % (countries[0].name, trade_flow.replace("_", " "), product.name_en, article, year_start, year_end) else: title = "Where did %s %s %s %s in %s?" % (countries[0].name, trade_flow.replace("_", " "), product.name_en, article, year) - + prod_or_partner = "product" # Return page without visualization data @@ -1329,6 +1329,19 @@ def explore(request, app_name, trade_flow, country1, country2, product, year="20 "previous_page": request.META.get('HTTP_REFERER', None), "item_type": item_type}, context_instance=RequestContext(request)) +def explore_random(request): + """Pick a random country and explore that, for the /explore link on the top + of the main template.""" + random_country = Country.objects.get_random().name_3char.lower() + return HttpResponseRedirect(reverse('observatory.views.explore', + args=('tree_map', + 'export', + random_country, + 'all', + 'show', + 2012 + ))) + '''attr_products / ''' def attr_products(request, prod_class): @@ -1340,20 +1353,20 @@ def attr_products(request, prod_class): ## Classification & Django Data Call name = "name_%s" % lang - # Get attribute information + # Get attribute information if prod_class == "sitc4": world_trade = list(Sitc4_py.objects.all().values('year','product_id','world_trade')) attr_list = list(Sitc4.objects.all().values('code',name,'id','color')) attr = {} - for i in attr_list: + for i in attr_list: attr[i['code']] = {'code':i['code'],'name':i[name],'color':i['color']} #.extra(where=['CHAR_LENGTH(code) = 2']) - + elif prod_class == "hs4": world_trade = list(Hs4_py.objects.all().values('year','product_id','world_trade')) attr_list = list(Hs4.objects.all().values('code',name,'id','community_id__color')) attr = {} - for i in attr_list: + for i in attr_list: attr[i['code']] = {'code':i['code'],'name':i[name],'item_id':i['id'],'color':i['community_id__color']} return HttpResponse(Sitc4.objects.get_all(lang) if prod_class == "sitc4" else Hs4.objects.get_all(lang)) @@ -1364,7 +1377,7 @@ def api_casy(request, trade_flow, country1, year): # start = time.time() # Setup the hash dictionary request_hash_dictionary = collections.OrderedDict() - + # Store the country code country_code = country1 '''Init variables''' @@ -1374,7 +1387,7 @@ def api_casy(request, trade_flow, country1, year): lang = request.GET.get("lang", lang) crawler = request.GET.get("_escaped_fragment_", False) country1 = Country.objects.get(name_3char=country1) - + '''Set query params with our changes''' query_params = request.GET.copy() query_params["lang"] = lang @@ -1394,7 +1407,7 @@ def api_casy(request, trade_flow, country1, year): request_hash_dictionary['app_name'] = app_name request_hash_dictionary['lang'] = lang request_hash_dictionary['prod_class'] = prod_class - + # Add the arguments to the request hash dictionary request_hash_dictionary['trade_flow'] = trade_flow request_hash_dictionary['country'] = country_code @@ -1410,7 +1423,7 @@ def api_casy(request, trade_flow, country1, year): # We are here, so let us store this data somewhere request_hash_string = "_".join( request_hash_dictionary.values() ) #base64.b64encode( request_unique_hash ) - + # Setup the store data store_data = request.build_absolute_uri().replace( "product_classification", "prod_class" ) + "||" store_page_url = request.build_absolute_uri().replace( "/api/", "/explore/" + app_name + "/" ) @@ -1418,7 +1431,7 @@ def api_casy(request, trade_flow, country1, year): store_page_url = store_page_url.replace( "product_classification", "prod_class" ) store_data = store_data + store_page_url + "||" store_data = store_data + request_hash_string - + # Write the store data to file store_file = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".store", "w+" ) store_file.write( store_data ) @@ -1426,13 +1439,13 @@ def api_casy(request, trade_flow, country1, year): # Set proper permissions since we want the cron to remove the file as well os.chmod( settings.DATA_FILES_PATH + "/" + request_hash_string + ".store", 0777 ) - + if ( os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".svg" ) is True ): # Check the request data type if ( request.GET.get( 'data_type', None ) is None ): # Let us get the data from the file response_json_data = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".svg", "r" ) - + # Set the return data returnData = response_json_data.read() @@ -1443,34 +1456,34 @@ def api_casy(request, trade_flow, country1, year): if ( os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json" ) is True ): # Let us get the data from the file response_json_data = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json", "r" ) - + # Set the return data returnData = response_json_data.read() #"""Return to browser as JSON for AJAX request""" return HttpResponse( returnData ) - - # Get attribute information + + # Get attribute information if prod_class == "sitc4": world_trade = list(Sitc4_py.objects.all().values('year','product_id','world_trade')) attr_list = list(Sitc4.objects.all().values('code',name,'id','color')) attr = {} - for i in attr_list: + for i in attr_list: attr[i['code']] = {'code':i['code'],'name':i[name],'color':i['color']} #.extra(where=['CHAR_LENGTH(code) = 2']) - + elif prod_class == "hs4": world_trade = list(Hs4_py.objects.all().values('year','product_id','world_trade')) attr_list = list(Hs4.objects.all().values('code',name,'id','community_id__color')) attr = {} - for i in attr_list: + for i in attr_list: attr[i['code']] = {'code':i['code'],'name':i[name],'item_id':i['id'],'color':i['community_id__color']} #.extra(where=['CHAR_LENGTH(code) = 2']) - + # get distince years from db, different for diff product classifications years_available = list(Sitc4_cpy.objects.values_list("year", flat=True).distinct()) if prod_class == "sitc4" else list(Hs4_cpy.objects.values_list("year", flat=True).distinct()) years_available.sort() - + magic = Cy.objects.filter(country=country1.id, year__range=(years_available[0], years_available[-1])).values('year', @@ -1478,12 +1491,12 @@ def api_casy(request, trade_flow, country1, year): 'pc_current', 'notpc_constant') magic_numbers = {} - for i in magic: - magic_numbers[i['year']] = {"pc_constant":i['pc_constant'], + for i in magic: + magic_numbers[i['year']] = {"pc_constant":i['pc_constant'], "pc_current":i['pc_current'], "notpc_constant":i["notpc_constant"]} - - + + '''Define parameters for query''' year_where = "AND year = %s" % (year,) if crawler == "" else " " rca_col = "null" @@ -1497,67 +1510,67 @@ def api_casy(request, trade_flow, country1, year): rca_col = "export_rca" else: val_col = "import_value as val" - + """Create query [year, id, abbrv, name_lang, val, export_rca]""" q = """ SELECT cpy.year, p.id, p.code, p.name_%s, p.community_id, c.color,c.name, %s, %s, distance, opp_gain, py.pci - FROM %sobservatory_%s_cpy as cpy, %sobservatory_%s as p, %sobservatory_%s_community as c, %sobservatory_%s_py as py + FROM %sobservatory_%s_cpy as cpy, %sobservatory_%s as p, %sobservatory_%s_community as c, %sobservatory_%s_py as py WHERE country_id=%s and cpy.product_id = p.id %s and p.community_id = c.id and py.product_id=p.id and cpy.year=py.year HAVING val > 0 ORDER BY val DESC """ % (lang, val_col, rca_col, DB_PREFIX, prod_class, DB_PREFIX, prod_class, DB_PREFIX, prod_class, DB_PREFIX, prod_class, country1.id, year_where) - + """Prepare JSON response""" json_response = {} - + """Check cache""" if settings.REDIS: #raw = get_redis_connection('default') raw = redis.Redis("localhost") - key = "%s:%s:%s:%s:%s" % (country1.name_3char, "all", "show", prod_class, trade_flow) + key = "%s:%s:%s:%s:%s" % (country1.name_3char, "all", "show", prod_class, trade_flow) # See if this key is already in the cache #cache_query = raw.hget(key, 'data') cache_query = raw.get(key) if (cache_query == None): - + rows = raw_q(query=q, params=None) total_val = sum([r[4] for r in rows]) """Add percentage value to return vals""" """Add percentage value to return vals""" # rows = [list(r) + [(r[4] / total_val)*100] for r in rows] - rows = [{"year":r[0], "item_id":r[1], "abbrv":r[2], "name":r[3], "value":r[7], "rca":r[8], + rows = [{"year":r[0], "item_id":r[1], "abbrv":r[2], "name":r[3], "value":r[7], "rca":r[8], "distance":r[9],"opp_gain":r[10], "pci": r[11], "share": (r[7] / total_val)*100, "community_id": r[4], "color": r[5], "community_name":r[6], "code":r[2], "id": r[2]} for r in rows] - + if crawler == "": - return [rows, total_val, ["#", "Year", "Abbrv", "Name", "Value", "RCA", "%"]] + return [rows, total_val, ["#", "Year", "Abbrv", "Name", "Value", "RCA", "%"]] # SAVE key in cache. - - json_response["data"] = rows - + + json_response["data"] = rows + raw.set(key, msgpack.dumps(rows))#, 'data', json.dumps(rows)) - + else: # If already cached, now simply retrieve encoded = cache_query decoded = msgpack.loads(encoded) - json_response["data"] = decoded - + json_response["data"] = decoded + else: rows = raw_q(query=q, params=None) total_val = sum([r[7] for r in rows]) # raise Exception(q,r,(r[7]/total_val)*100) """Add percentage value to return vals""" # rows = [list(r) + [(r[4] / total_val)*100] for r in rows] - rows = [{"year":r[0], "item_id":r[1], "abbrv":r[2], "name":r[3], "value":r[7], "rca":r[8], + rows = [{"year":r[0], "item_id":r[1], "abbrv":r[2], "name":r[3], "value":r[7], "rca":r[8], "distance":r[9],"opp_gain":r[10], "pci": r[11], "share": (r[7] / total_val)*100, "community_id": r[4], "color": r[5], "community_name":r[6], "code":r[2], "id": r[2]} for r in rows] - + if crawler == "": return [rows, total_val, ["#", "Year", "Abbrv", "Name", "Value", "RCA", "%"]] - - json_response["data"] = rows - + + json_response["data"] = rows + json_response["attr"] = attr json_response["attr_data"] = Sitc4.objects.get_all(lang) if prod_class == "sitc4" else Hs4.objects.get_all(lang) json_response["country1"] = country1.to_json() @@ -1617,8 +1630,8 @@ def api_sapy(request, trade_flow, product, year): request_hash_dictionary['product_dispaly'] = product_code request_hash_dictionary['year'] = year # We are here, so let us store this data somewhere - request_hash_string = "_".join(request_hash_dictionary.values()) - + request_hash_string = "_".join(request_hash_dictionary.values()) + # Setup the store data store_data = request.build_absolute_uri().replace( "product_classification", "prod_class" ) + "||" store_page_url = request.build_absolute_uri().replace( "/api/", "/explore/" + app_name + "/" ) @@ -1626,19 +1639,19 @@ def api_sapy(request, trade_flow, product, year): store_page_url = store_page_url.replace( "product_classification", "prod_class" ) store_data = store_data + store_page_url + "||" store_data = store_data + request_hash_string - + # Write the store data to file store_file = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".store", "w+" ) store_file.write( store_data ) store_file.close() - - + + if ( os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".svg" ) is True ): # Check the request data type if ( request.GET.get( 'data_type', None ) is None ): # Let us get the data from the file response_json_data = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".svg", "r" ) - + # Set the return data returnData = response_json_data.read() @@ -1649,17 +1662,17 @@ def api_sapy(request, trade_flow, product, year): if ( os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json" ) is True ): # Let us get the data from the file response_json_data = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json", "r" ) - + # Set the return data returnData = response_json_data.read() #"""Return to browser as JSON for AJAX request""" return HttpResponse( returnData ) - + '''Grab extraneous details''' ## Clasification & Django Data Call name = "name_%s" % lang - + '''Grab extraneous details''' if prod_class == "sitc4": # attr_list = list(Sitc4.objects.all().values('code','name','color')) @@ -1674,20 +1687,20 @@ def api_sapy(request, trade_flow, product, year): attr = {} for i in attr_list: attr[i['code']] = {'code':i['code'],'name':i[name],'item_id':i['id'],'color':i['community_id__color']} - - - # Create dictionary of region codes + + + # Create dictionary of region codes region_list = list(Country_region.objects.all().values()) #.extra(where=['CHAR_LENGTH(code) = 2']) region = {} for i in region_list: - region[i['id']] = i - + region[i['id']] = i + # Create dictinoary for continent groupings continent_list = list(Country.objects.all().distinct().values('continent')) continents = {} for i,k in enumerate(continent_list): continents[k['continent']] = i*1000 - + """Define parameters for query""" year_where = "AND year = %s" % (year,) if crawler == "" else " " rca_col = "null" @@ -1701,7 +1714,7 @@ def api_sapy(request, trade_flow, product, year): rca_col = "export_rca" else: val_col = "import_value as val" - + """Create query [year, id, abbrv, name_lang, val, export_rca]""" q = """ SELECT year, c.id, c.name_3char, c.name_%s, c.region_id, c.continent, %s, %s @@ -1710,10 +1723,10 @@ def api_sapy(request, trade_flow, product, year): HAVING val > 0 ORDER BY val DESC """ % (lang, val_col, rca_col, DB_PREFIX, prod_class, DB_PREFIX, product.id, year_where) - + """Prepare JSON response""" json_response = {} - + """Check cache""" if settings.REDIS: # raw = get_redis_connection('default') @@ -1729,21 +1742,21 @@ def api_sapy(request, trade_flow, product, year): # rows = [list(r) + [(r[4] / total_val)*100] for r in rows] rows = [{"year":r[0], "item_id":r[1], "abbrv":r[2], "name":r[3], "value":r[6], "rca":r[7], "share": (r[6] / total_val)*100, "id": r[1], "region_id":r[4],"continent":r[5]} for r in rows] - + if crawler == "": return [rows, total_val, ["#", "Year", "Abbrv", "Name", "Value", "RCA", "%"]] - + json_response["data"] = rows - + # SAVE key in cache. - raw.set(key, msgpack.dumps(rows)) - + raw.set(key, msgpack.dumps(rows)) + else: # If already cached, now simply retrieve encoded = cache_query decoded = msgpack.loads(encoded) json_response["data"] = decoded - + else: rows = raw_q(query=q, params=None) total_val = sum([r[6] for r in rows]) @@ -1752,12 +1765,12 @@ def api_sapy(request, trade_flow, product, year): # rows = [list(r) + [(r[4] / total_val)*100] for r in rows] rows = [{"year":r[0], "item_id":r[1], "abbrv":r[2], "name":r[3], "value":r[6], "rca":r[7], "share": (r[6] / total_val)*100, "id": r[1], "region_id":r[4],"continent":r[5]} for r in rows] - + if crawler == "": return [rows, total_val, ["#", "Year", "Abbrv", "Name", "Value", "RCA", "%"]] - + json_response["data"] = rows - + json_response["attr_data"] = Country.objects.get_all(lang) json_response["product"] = product.to_json() json_response["title"] = "Who %sed %s?" % (trade_flow.replace("_", " "), product.name_en) @@ -1767,13 +1780,13 @@ def api_sapy(request, trade_flow, product, year): json_response["attr"] = attr json_response["region"]= region json_response["continents"] = continents - json_response["other"] = query_params - + json_response["other"] = query_params + if not os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json" ): response_json_file = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json", "w+" ) response_json_file.write( json.dumps( json_response ) ) response_json_file.close() - + # raise Exception(time.time() - start) # Check the request data type if ( request.GET.get( 'data_type', None ) is None ): @@ -1782,8 +1795,8 @@ def api_sapy(request, trade_flow, product, year): elif ( request.GET.get( 'data_type', '' ) == 'json' ): """Return to browser as JSON for AJAX request""" return HttpResponse(json.dumps(json_response)) - - + + ''' / show / product / ''' def api_csay(request, trade_flow, country1, year): """Init variables""" @@ -1813,10 +1826,10 @@ def api_csay(request, trade_flow, country1, year): request_hash_dictionary['product_dispaly'] = 'show' request_hash_dictionary['country2'] = 'all' request_hash_dictionary['year'] = year - + # We are here, so let us store this data somewhere request_hash_string = "_".join(request_hash_dictionary.values()) - + # Setup the store data store_data = request.build_absolute_uri().replace( "product_classification", "prod_class" ) + "||" store_page_url = request.build_absolute_uri().replace( "/api/", "/explore/" + app_name + "/" ) @@ -1824,18 +1837,18 @@ def api_csay(request, trade_flow, country1, year): store_page_url = store_page_url.replace( "product_classification", "prod_class" ) store_data = store_data + store_page_url + "||" store_data = store_data + request_hash_string - + # Write the store data to file store_file = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".store", "w+" ) store_file.write( store_data ) store_file.close() - + if ( os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".svg" ) is True ): # Check the request data type if ( request.GET.get( 'data_type', None ) is None ): # Let us get the data from the file response_json_data = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".svg", "r" ) - + # Set the return data returnData = response_json_data.read() @@ -1846,7 +1859,7 @@ def api_csay(request, trade_flow, country1, year): if ( os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json" ) is True ): # Let us get the data from the file response_json_data = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json", "r" ) - + # Set the return data returnData = response_json_data.read() @@ -1855,14 +1868,14 @@ def api_csay(request, trade_flow, country1, year): '''Grab extraneous details''' region_list = list(Country_region.objects.all().values()) #.extra(where=['CHAR_LENGTH(code) = 2']) region = {} - for i in region_list: + for i in region_list: region[i['id']] = i - + continent_list = list(Country.objects.all().distinct().values('continent')) continents = {} - for i,k in enumerate(continent_list): + for i,k in enumerate(continent_list): continents[k['continent']] = i*1000 - + """Define parameters for query""" year_where = "AND year = %s" % (year,) if crawler == "" else " " rca_col = "null" @@ -1878,7 +1891,7 @@ def api_csay(request, trade_flow, country1, year): '''Create query [year, id, abbrv, name_lang, val, rca]''' q = """ SELECT year, c.id, c.name_3char, c.name_%s, c.region_id, c.continent, %s, %s - FROM %sobservatory_%s_ccpy as ccpy, %sobservatory_country as c + FROM %sobservatory_%s_ccpy as ccpy, %sobservatory_country as c WHERE origin_id=%s and ccpy.destination_id = c.id %s GROUP BY year, destination_id HAVING val > 0 @@ -1887,7 +1900,7 @@ def api_csay(request, trade_flow, country1, year): """Prepare JSON response""" json_response = {} - + """Check cache""" if settings.REDIS: raw = redis.Redis("localhost") @@ -1897,40 +1910,40 @@ def api_csay(request, trade_flow, country1, year): if (cache_query == None): rows = raw_q(query=q, params=None) total_val = sum([r[6] for r in rows]) - + """Add percentage value to return vals""" # rows = [list(r) + [(r[4] / total_val)*100] for r in rows] rows = [{"year":r[0], "item_id":r[1], "abbrv":r[2], "name":r[3], "value":r[6], "rca":r[7], "share": (r[6] / total_val)*100, "id":r[1], "region_id":r[4], "continent":r[5]} for r in rows] - + if crawler == "": return [rows, total_val, ["#", "Year", "Abbrv", "Name", "Value", "RCA", "%"]] - + json_response["data"] = rows - + # SAVE key in cache. raw.set(key, msgpack.dumps(rows)) - + else: # If already cached, now simply retrieve encoded = cache_query decoded = msgpack.loads(encoded) json_response["data"] = decoded - + else: rows = raw_q(query=q, params=None) total_val = sum([r[6] for r in rows]) - + """Add percentage value to return vals""" # rows = [list(r) + [(r[4] / total_val)*100] for r in rows] rows = [{"year":r[0], "item_id":r[1], "abbrv":r[2], "name":r[3], "value":r[6], "rca":r[7], "share": (r[6] / total_val)*100, "id":r[1], "region_id":r[4], "continent":r[5]} for r in rows] - + if crawler == "": return [rows, total_val, ["#", "Year", "Abbrv", "Name", "Value", "RCA", "%"]] - + json_response["data"] = rows - + # get distince years from db, different for diff product classifications years_available = list(Sitc4_cpy.objects.values_list("year", flat=True).distinct()) if prod_class == "sitc4" else list(Hs4_cpy.objects.values_list("year", flat=True).distinct()) years_available.sort() @@ -1942,14 +1955,14 @@ def api_csay(request, trade_flow, country1, year): 'pc_current', 'notpc_constant') magic_numbers = {} - for i in magic: - magic_numbers[i['year']] = {"pc_constant":i['pc_constant'], + for i in magic: + magic_numbers[i['year']] = {"pc_constant":i['pc_constant'], "pc_current":i['pc_current'], - "notpc_constant":i["notpc_constant"]} - + "notpc_constant":i["notpc_constant"]} + """Set article variable for question """ article = "to" if trade_flow == "export" else "from" - + json_response["attr_data"] = Country.objects.get_all(lang) json_response["country1"] = country1.to_json() json_response["title"] = "Where does %s %s %s?" % (country1.name, trade_flow, article) @@ -1961,7 +1974,7 @@ def api_csay(request, trade_flow, country1, year): json_response["prod_class"] = prod_class json_response["magic_numbers"] = magic_numbers json_response["other"] = query_params - + # raise Exception(time.time() - start) # Check the request data type if ( request.GET.get( 'data_type', None ) is None ): @@ -1970,8 +1983,8 @@ def api_csay(request, trade_flow, country1, year): elif ( request.GET.get( 'data_type', '' ) == 'json' ): """Return to browser as JSON for AJAX request""" return HttpResponse(json.dumps(json_response)) - - + + def api_ccsy(request, trade_flow, country1, country2, year): # import time # start = time.time() @@ -2001,7 +2014,7 @@ def api_ccsy(request, trade_flow, country1, country2, year): request_hash_dictionary['app_name'] = app_name request_hash_dictionary['lang'] = lang request_hash_dictionary['prod_class'] = prod_class - + #Set country_code to Country country_code1=Country.objects.get(name=country1) country_code2=Country.objects.get(name=country2) @@ -2013,10 +2026,10 @@ def api_ccsy(request, trade_flow, country1, country2, year): request_hash_dictionary['country2'] = country_code_two request_hash_dictionary['product_dispaly'] = 'show' request_hash_dictionary['year'] = year - + # We are here, so let us store this data somewhere - request_hash_string = "_".join(request_hash_dictionary.values()) - + request_hash_string = "_".join(request_hash_dictionary.values()) + # Setup the store data store_data = request.build_absolute_uri().replace( "product_classification", "prod_class" ) + "||" store_page_url = request.build_absolute_uri().replace( "/api/", "/explore/" + app_name + "/" ) @@ -2024,18 +2037,18 @@ def api_ccsy(request, trade_flow, country1, country2, year): store_page_url = store_page_url.replace( "product_classification", "prod_class" ) store_data = store_data + store_page_url + "||" store_data = store_data + request_hash_string - + # Write the store data to file store_file = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".store", "w+" ) store_file.write( store_data ) store_file.close() - + if ( os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".svg" ) is True ): # Check the request data type if ( request.GET.get( 'data_type', None ) is None ): # Let us get the data from the file response_json_data = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".svg", "r" ) - + # Set the return data returnData = response_json_data.read() @@ -2046,7 +2059,7 @@ def api_ccsy(request, trade_flow, country1, country2, year): if ( os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json" ) is True ): # Let us get the data from the file response_json_data = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json", "r" ) - + # Set the return data returnData = response_json_data.read() @@ -2057,18 +2070,18 @@ def api_ccsy(request, trade_flow, country1, country2, year): # attr_list = list(Sitc4.objects.all().values('code','name','color')) attr_list = list(Sitc4.objects.all().values('code',name,'id','color')) attr = {} - for i in attr_list: + for i in attr_list: attr[i['code']] = {'code':i['code'],'name':i[name],'color':i['color']} #.extra(where=['CHAR_LENGTH(code) = 2']) elif prod_class == "hs4": # attr_list = list(Hs4.objects.all().values('code','name')) #.extra(where=['CHAR_LENGTH(code) = 2']) attr_list = list(Hs4.objects.all().values('code',name,'id','community_id__color')) attr = {} - for i in attr_list: + for i in attr_list: attr[i['code']] = {'code':i['code'],'name':i[name],'item_id':i['id'],'color':i['community_id__color']} - - + + '''Define parameters for query''' year_where = "AND year = %s" % (year,) if crawler == "" else " " rca_col = "null" @@ -2080,19 +2093,19 @@ def api_ccsy(request, trade_flow, country1, country2, year): val_col = "export_value as val" else: val_col = "import_value as val" - + '''Create query''' q = """ SELECT year, p.id, p.code, p.name_%s, p.community_id, c.name, c.color, %s, %s - FROM %sobservatory_%s_ccpy as ccpy, %sobservatory_%s as p, %sobservatory_%s_community as c + FROM %sobservatory_%s_ccpy as ccpy, %sobservatory_%s as p, %sobservatory_%s_community as c WHERE origin_id=%s and destination_id=%s and ccpy.product_id = p.id and p.community_id = c.id %s HAVING val > 0 ORDER BY val DESC """ % (lang, val_col, rca_col, DB_PREFIX, prod_class, DB_PREFIX, prod_class, DB_PREFIX, prod_class, country1.id, country2.id, year_where) - + """Prepare JSON response""" json_response = {} - + """Check cache""" if settings.REDIS: #raw = get_redis_connection('default') @@ -2100,45 +2113,45 @@ def api_ccsy(request, trade_flow, country1, country2, year): key = "%s:%s:%s:%s:%s" % (country1.name_3char, country2.name_3char, "show", prod_class, trade_flow) # See if this key is already in the cache cache_query = raw.get(key) - if(cache_query == None): + if(cache_query == None): rows = raw_q(query=q, params=None) total_val = sum([r[7] for r in rows]) - + """Add percentage value to return vals""" # rows = [list(r) + [(r[4] / total_val)*100] for r in rows] - rows = [{"year":r[0], "item_id":r[1], "abbrv":r[2], "name":r[3], "value":r[7], "rca":r[5], + rows = [{"year":r[0], "item_id":r[1], "abbrv":r[2], "name":r[3], "value":r[7], "rca":r[5], "share": (r[7] / total_val)*100, "community_id":r[4],"community_name":r[5],"color":r[6], "code":r[2], "id": r[2]} for r in rows] - + if crawler == "": return [rows, total_val, ["#", "Year", "Abbrv", "Name", "Value", "RCA", "%"]] - + json_response["data"] = rows - + # SAVE key in cache. raw.set(key, msgpack.dumps(rows)) - - else: + + else: # If already cached, now simply retrieve encoded = cache_query decoded = msgpack.loads(encoded) json_response["data"] = decoded - + else: rows = raw_q(query=q, params=None) total_val = sum([r[7] for r in rows]) - + """Add percentage value to return vals""" # rows = [list(r) + [(r[4] / total_val)*100] for r in rows] - rows = [{"year":r[0], "item_id":r[1], "abbrv":r[2], "name":r[3], "value":r[7], "rca":r[5], + rows = [{"year":r[0], "item_id":r[1], "abbrv":r[2], "name":r[3], "value":r[7], "rca":r[5], "share": (r[7] / total_val)*100, "community_id":r[4],"community_name":r[5],"color":r[6], "code":r[2], "id": r[2]} for r in rows] - + if crawler == "": return [rows, total_val, ["#", "Year", "Abbrv", "Name", "Value", "RCA", "%"]] - + json_response["data"] = rows - + # get distince years from db, different for diff product classifications years_available = list(Sitc4_cpy.objects.values_list("year", flat=True).distinct()) if prod_class == "sitc4" else list(Hs4_cpy.objects.values_list("year", flat=True).distinct()) years_available.sort() @@ -2150,8 +2163,8 @@ def api_ccsy(request, trade_flow, country1, country2, year): 'pc_current', 'notpc_constant') magic_numbers = {} - for i in magic: - magic_numbers[i['year']] = {"pc_constant":i['pc_constant'], + for i in magic: + magic_numbers[i['year']] = {"pc_constant":i['pc_constant'], "pc_current":i['pc_current'], "notpc_constant":i["notpc_constant"]} @@ -2167,7 +2180,7 @@ def api_ccsy(request, trade_flow, country1, country2, year): json_response["attr"] = attr json_response["class"] = prod_class json_response["other"] = query_params - + if not os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json" ): response_json_file = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json", "w+" ) response_json_file.write( json.dumps( json_response ) ) @@ -2181,7 +2194,7 @@ def api_ccsy(request, trade_flow, country1, country2, year): elif ( request.GET.get( 'data_type', '' ) == 'json' ): """Return to browser as JSON for AJAX request""" return HttpResponse(json.dumps(json_response)) - + def api_cspy(request, trade_flow, country1, product, year): '''Init variables''' prod_class = request.session['product_classification'] if 'product_classification' in request.session else "hs4" @@ -2194,7 +2207,7 @@ def api_cspy(request, trade_flow, country1, product, year): app_name = request.session['app_name'] if 'app_name' in request.session else "" product = clean_product(product, prod_class) article = "to" if trade_flow == "export" else "from" - + '''Set query params with our changes''' query_params = request.GET.copy() query_params["lang"] = lang @@ -2215,7 +2228,7 @@ def api_cspy(request, trade_flow, country1, product, year): request_hash_dictionary['year'] = year # We are here, so let us store this data somewhere request_hash_string = "_".join(request_hash_dictionary.values()) #base64.b64encode( request_unique_hash ) - + # Setup the store data store_data = request.build_absolute_uri().replace( "product_classification", "prod_class" ) + "||" store_page_url = request.build_absolute_uri().replace( "/api/", "/explore/" + app_name + "/" ) @@ -2223,18 +2236,18 @@ def api_cspy(request, trade_flow, country1, product, year): store_page_url = store_page_url.replace( "product_classification", "prod_class" ) store_data = store_data + store_page_url + "||" store_data = store_data + request_hash_string - + # Write the store data to file store_file = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".store", "w+" ) store_file.write( store_data ) store_file.close() - + if ( os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".svg" ) is True ): # Check the request data type if ( request.GET.get( 'data_type', None ) is None ): # Let us get the data from the file response_json_data = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".svg", "r" ) - + # Set the return data returnData = response_json_data.read() @@ -2245,24 +2258,24 @@ def api_cspy(request, trade_flow, country1, product, year): if ( os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json" ) is True ): # Let us get the data from the file response_json_data = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json", "r" ) - + # Set the return data returnData = response_json_data.read() #"""Return to browser as JSON for AJAX request""" return HttpResponse( returnData ) - + '''Grab extraneous details''' region_list = list(Country_region.objects.all().values()) #.extra(where=['CHAR_LENGTH(code) = 2']) region = {} - for i in region_list: + for i in region_list: region[i['id']] = i - + continent_list = list(Country.objects.all().distinct().values('continent')) continents = {} - for i,k in enumerate(continent_list): + for i,k in enumerate(continent_list): continents[k['continent']] = i*1000 - + '''Define parameters for query''' year_where = "AND year = %s" % (year,) if crawler == "" else " " rca_col = "null" @@ -2274,23 +2287,23 @@ def api_cspy(request, trade_flow, country1, product, year): val_col = "export_value as val" else: val_col = "import_value as val" - + '''Create query''' q = """ - SELECT year, c.id, c.name_3char, c.name_%s, c.region_id, c.continent, %s, %s - FROM %sobservatory_%s_ccpy as ccpy, %sobservatory_country as c + SELECT year, c.id, c.name_3char, c.name_%s, c.region_id, c.continent, %s, %s + FROM %sobservatory_%s_ccpy as ccpy, %sobservatory_country as c WHERE origin_id=%s and ccpy.product_id=%s and ccpy.destination_id = c.id %s GROUP BY year, destination_id HAVING val > 0 ORDER BY val DESC """ % (lang, val_col, rca_col, DB_PREFIX, prod_class, DB_PREFIX, country1.id, product.id, year_where) - + """Prepare JSON response""" json_response = {} - + """Check cache""" if settings.REDIS: - raw = redis.Redis("localhost") + raw = redis.Redis("localhost") key = "%s:%s:%s:%s:%s" % (country1.name_3char, "show", product.id, prod_class, trade_flow) # See if this key is already in the cache cache_query = raw.get(key) @@ -2298,41 +2311,41 @@ def api_cspy(request, trade_flow, country1, product, year): if (cache_query == None): rows = raw_q(query=q, params=None) total_val = sum([r[6] for r in rows]) - + """Add percentage value to return vals""" # rows = [list(r) + [(r[4] / total_val)*100] for r in rows] rows = [{"year":r[0], "item_id":r[1], "abbrv":r[2], "name":r[3], "value":r[6], "rca":r[7], "share": (r[6] / total_val)*100, "region_id": r[4], "continent": r[5], "id":r[1]} for r in rows] - + if crawler == "": return [rows, total_val, ["#", "Year", "Abbrv", "Name", "Value", "RCA", "%"]] - + json_response["data"] = rows - + # SAVE key in cache. raw.set(key, msgpack.dumps(rows)) - + else: # If already cached, now simply retrieve encoded = cache_query decoded = msgpack.loads(encoded) json_response["data"] = decoded - + else: rows = raw_q(query=q, params=None) total_val = sum([r[6] for r in rows]) - + """Add percentage value to return vals""" # rows = [list(r) + [(r[4] / total_val)*100] for r in rows] rows = [{"year":r[0], "item_id":r[1], "abbrv":r[2], "name":r[3], "value":r[6], "rca":r[7], "share": (r[6] / total_val)*100, "region_id": r[4], "continent": r[5], "id":r[1]} for r in rows] - + if crawler == "": return [rows, total_val, ["#", "Year", "Abbrv", "Name", "Value", "RCA", "%"]] - + json_response["data"] = rows - - + + article = "to" if trade_flow == "export" else "from" # get distince years from db, different for diff product classifications @@ -2346,12 +2359,12 @@ def api_cspy(request, trade_flow, country1, product, year): 'pc_current', 'notpc_constant') magic_numbers = {} - for i in magic: - magic_numbers[i['year']] = {"pc_constant":i['pc_constant'], + for i in magic: + magic_numbers[i['year']] = {"pc_constant":i['pc_constant'], "pc_current":i['pc_current'], "notpc_constant":i["notpc_constant"]} - json_response["magic_numbers"] = magic_numbers + json_response["magic_numbers"] = magic_numbers json_response["attr_data"] = Country.objects.get_all(lang) json_response["title"] = "Where does %s %s %s %s?" % (country1.name, trade_flow, product.name_en, article) json_response["country1"] = country1.to_json() @@ -2362,8 +2375,8 @@ def api_cspy(request, trade_flow, country1, product, year): json_response["region"]= region json_response["app_type"] = "cspy" json_response["class"] = prod_class - json_response["other"] = query_params - + json_response["other"] = query_params + if not os.path.exists( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json" ): response_json_file = open( settings.DATA_FILES_PATH + "/" + request_hash_string + ".json", "w+" ) response_json_file.write( json.dumps( json_response ) ) @@ -2377,7 +2390,7 @@ def api_cspy(request, trade_flow, country1, product, year): elif ( request.GET.get( 'data_type', '' ) == 'json' ): """Return to browser as JSON for AJAX request""" return HttpResponse(json.dumps(json_response)) - + # Embed for iframe def embed(request, app_name, trade_flow, country1, country2, product, year): lang = request.GET.get("lang", "en") @@ -2388,7 +2401,7 @@ def embed(request, app_name, trade_flow, country1, country2, product, year): # get distince years from db, different for diff product classifications years_available = list(Sitc4_cpy.objects.values_list("year", flat=True).distinct()) if prod_class == "sitc4" else list(Hs4_cpy.objects.values_list("year", flat=True).distinct()) years_available.sort() - + return render_to_response("explore/embed.html", {"app":app_name, "trade_flow": trade_flow, "country1":country1, "country2":country2, "product":product, "year":year, "other":json.dumps(query_string),"years_available":json.dumps(years_available), "lang":lang}) @@ -2397,7 +2410,7 @@ def embed(request, app_name, trade_flow, country1, country2, product, year): ################### -## Abandoned function? Coresponding model/tables 'wdi' & 'wdi_cwy' +## Abandoned function? Coresponding model/tables 'wdi' & 'wdi_cwy' ################### def get_similar_productive(country, year): # correlation = request.GET.get("c", "pearson") @@ -2433,7 +2446,7 @@ def get_similar_productive(country, year): return render_to_response("explore/similar.html", {"cors": cors}) ################### -## Abandoned function? Coresponding model/tables 'wdi' & 'wdi_cwy' +## Abandoned function? Coresponding model/tables 'wdi' & 'wdi_cwy' ################### def similar_wdi(request, country, indicator, year): y = int(year) @@ -2502,19 +2515,19 @@ def get_app_type(country1, country2, product, year): # country / all / show / year if country2 == "all" and product == "show": return "casy" - + # country / show / all / year elif country2 == "show" and product == "all": return "csay" - + # show / all / product / year elif country1 == "show" and country2 == "all": return "sapy" - + # country / country / show / year elif product == "show": return "ccsy" - + # country / show / product / year else: return "cspy" diff --git a/html/template.html b/html/template.html index 042ef958..66044cb3 100644 --- a/html/template.html +++ b/html/template.html @@ -35,7 +35,7 @@

Explore +
  • Explore
  • Stories
  • Rankings
  • New Book
    • @@ -162,4 +162,4 @@