From bdc9e064739886026c558c221ef3b727eb04f409 Mon Sep 17 00:00:00 2001 From: Mali Akmanalp Date: Mon, 2 Nov 2015 18:57:23 -0500 Subject: [PATCH] Create Department-Country-Year data model and API: COL-434 --- colombia/api_schemas.py | 8 ++++++++ colombia/data/models.py | 17 ++++++++++++++++- colombia/data/views.py | 25 ++++++++++++++++++++++++- colombia/datasets.py | 12 ++++++++---- colombia/import.py | 4 ++++ colombia/models.py | 2 +- 6 files changed, 61 insertions(+), 7 deletions(-) diff --git a/colombia/api_schemas.py b/colombia/api_schemas.py index f69e7d8..c8fe85d 100644 --- a/colombia/api_schemas.py +++ b/colombia/api_schemas.py @@ -70,6 +70,12 @@ class Meta: fields = ("export_value", "country_id", "product_id", "year") +class CountryDepartmentYearSchema(ma.Schema): + + class Meta: + fields = ("export_value", "country_id", "location_id", "year") + + class XIndustryYearSchema(ma.Schema): location_id = ma.fields.Integer(default=0) @@ -138,6 +144,8 @@ class ColombiaMetadataSchema(MetadataSchema): country_municipality_product_year = CountryMunicipalityProductYearSchema(many=True) country_department_product_year = CountryDepartmentProductYearSchema(many=True) +country_department_year = CountryDepartmentYearSchema(many=True) + product_year = ProductYearSchema(many=True) industry_year = IndustryYearSchema(many=True) occupation_year = OccupationYearSchema(many=True) diff --git a/colombia/data/models.py b/colombia/data/models.py index 338cfe6..a2a0669 100644 --- a/colombia/data/models.py +++ b/colombia/data/models.py @@ -105,7 +105,7 @@ def product_id(cls): level = db.Column(product_enum) export_value = db.Column(db.BIGINT) - num_plants = db.Column(db.Integer) + export_num_plants = db.Column(db.Integer) class CountryDepartmentProductYear(CountryXProductYear): @@ -118,6 +118,21 @@ class CountryMunicipalityProductYear(CountryXProductYear): __tablename__ = "country_municipality_product_year" +class CountryDepartmentYear(BaseModel, IDMixin): + + __tablename__ = "country_department_year" + + country_id = db.Column(db.Integer, db.ForeignKey(Location.id)) + location_id = db.Column(db.Integer, db.ForeignKey(Location.id)) + year = db.Column(db.Integer) + + location = db.relationship(Location, foreign_keys=[location_id]) + country = db.relationship(Location, foreign_keys=[country_id]) + + export_value = db.Column(db.BIGINT) + export_num_plants = db.Column(db.Integer) + + class DepartmentYear(BaseModel, IDMixin): __tablename__ = "department_year" diff --git a/colombia/data/views.py b/colombia/data/views.py index b62443c..e345139 100644 --- a/colombia/data/views.py +++ b/colombia/data/views.py @@ -5,7 +5,7 @@ MunicipalityIndustryYear, ProductYear, IndustryYear, DepartmentYear, Location, CountryMunicipalityProductYear, CountryDepartmentProductYear, OccupationYear, - OccupationIndustryYear) + OccupationIndustryYear, CountryDepartmentYear) from ..api_schemas import marshal from .routing import lookup_classification_level from .. import api_schemas as schemas @@ -203,6 +203,26 @@ def eey_location_subregions_trade(entity_type, entity_id, buildingblock_level): return jsonify(data=[x._asdict() for x in q]) +def eey_location_partners(entity_type, entity_id, buildingblock_level): + + if buildingblock_level != "country": + msg = "Data doesn't exist at level {}. Try country.".format(buildingblock_level) + abort(400, body=msg) + + # Assert level of sub_id is same as entity_id + location_level = lookup_classification_level("location", entity_id) + + if location_level == "department": + q = CountryDepartmentYear.query\ + .filter_by(location_id=entity_id)\ + .all() + return marshal(schemas.country_department_year, q) + else: + msg = "Data doesn't exist at location level {}"\ + .format(location_level) + abort(400, body=msg) + + def eey_industry_occupations(entity_type, entity_id, buildingblock_level): if buildingblock_level != "minor_group": @@ -245,6 +265,9 @@ def eey_industry_occupations(entity_type, entity_id, buildingblock_level): }, "subregions_trade": { "func": eey_location_subregions_trade + }, + "partners": { + "func": eey_location_partners } } }, diff --git a/colombia/datasets.py b/colombia/datasets.py index 2b0c552..6f61d63 100644 --- a/colombia/datasets.py +++ b/colombia/datasets.py @@ -328,7 +328,7 @@ def load_trade4digit_municipality(): "p": "product", "yr": "year", "X_rcpy_d": "export_value", - "NP_rcpy": "num_plants" + "NP_rcpy": "export_num_plants" }, "classification_fields": { "location": { @@ -351,9 +351,13 @@ def load_trade4digit_municipality(): }, "facet_fields": ["location", "country", "product", "year"], "facets": { + ("country_id", "location_id", "year"): { + "export_value": first, + "export_num_plants": first + }, ("country_id", "location_id", "product_id", "year"): { "export_value": first, - "num_plants": first + "export_num_plants": first } } } @@ -367,7 +371,7 @@ def load_trade4digit_municipality(): "p": "product", "yr": "year", "X_rcpy_d": "export_value", - "NP_rcpy": "num_plants" + "NP_rcpy": "export_num_plants" }, "classification_fields": { "location": { @@ -392,7 +396,7 @@ def load_trade4digit_municipality(): "facets": { ("country_id", "location_id", "product_id", "year"): { "export_value": first, - "num_plants": first + "export_num_plants": first } } } diff --git a/colombia/import.py b/colombia/import.py index 066fc3e..63c77eb 100644 --- a/colombia/import.py +++ b/colombia/import.py @@ -141,6 +141,10 @@ df.to_sql("country_department_product_year", db.engine, index=False, chunksize=10000, if_exists="append") + df = ret[("country_id", "location_id", "year")].reset_index() + df.to_sql("country_department_year", db.engine, + index=False, chunksize=10000, if_exists="append") + # Country - industry- y ear ret = process_dataset(industry4digit_country) df = ret[('location_id', 'industry_id', 'year')].reset_index() diff --git a/colombia/models.py b/colombia/models.py index 903a0ba..10c44af 100644 --- a/colombia/models.py +++ b/colombia/models.py @@ -5,4 +5,4 @@ DepartmentIndustryYear, IndustryYear, MunicipalityIndustryYear, MSAProductYear, MSAIndustryYear, OccupationYear, - OccupationIndustryYear) + OccupationIndustryYear, CountryDepartmentYear)