ranking.py

import datacommons as dc
import json
import urllib
import sys
import threading
import os.path

threadedp = False

US_dcid = "country/USA"

statVarConfig = None
allStatVars = []
def readVars (file):
    f = open(file)
    jsstr = f.read()
    f.close()
    statVarConfig = json.loads(jsstr)
    for svc in statVarConfig:
        for sv in svc["statsVars"]:
            if (sv not in allStatVars):
                allStatVars.append(sv)
            if ("relatedChart" in svc and "denominator" in svc["relatedChart"]):
                denom = svc["relatedChart"]["denominator"]
                if (denom not in allStatVars):
                    allStatVars.append(denom)

    print("%s stat vars" % (len(allStatVars)))

readVars("../website/server/chart_config.json")

def loadDataFromDC (statsFile):
    items = RankStatVarPlace.pendingItems
    start = 0
    inc = 100

    while (start < len(items)) :
        if (start + inc < len(items)):
            next = items[start:start+inc]
        else :
            next = items[start:]
        if (threadedp) :
            th = threading.Thread(target=getSVInThread, args=[next])
        else:
            getSVInThread(next)
        start = start + inc


    def getSV ():
        items = RankStatVarPlace.pendingItems.copy()
        getSVInThread(items)
        #th = threading.Thread(target=getSVInThread, args=[items])
        RankStatVarPlace.pendingItems = []
     #   th.start()
        
        
def getSVInThread (items):
    childIds = []
    for ch in items:
        childIds.append(ch.dcid)
        # this can be made much faster ...
    vals = dc.get_stat_all(childIds, allStatVars)
    for pl in items:
        for statVar in allStatVars:
            try :
                plvals = vals[pl.dcid]
                svals = plvals[statVar]
                (val, date) = getLatest(svals)
                pl.statVars[statVar] = val
           #     print("%s %s %s %s" % (pl.dcid, statVar, val, date))
            except ValueError:
                continue
            except KeyError:
                continue
        print("%s %i" % (pl.dcid, len(pl.statVars)))

def getLatest(sval) :
    date = 0
    val = None
    for series in sval['sourceSeries']:
        for key in series['val'].keys():
            dd = int(key)
            if (dd > date) :
                date = dd
                val = series['val'][key]
    return (val, date)


class RankStatVarPlace:

    idPlaceMap = {}
    pendingItems = []

    @staticmethod
    def getPlace (type, dcid) :
        if (dcid in RankStatVarPlace.idPlaceMap):
            return RankStatVarPlace.idPlaceMap[dcid]
        else :
            return RankStatVarPlace(type, dcid)
        
    def __init__ (self, type, dcid) :
        self.dcid = dcid
        self.type = type
        self.children = []
        self.statVars = {}
        RankStatVarPlace.idPlaceMap[dcid] = self
        if ('wiki' not in dcid and self not in RankStatVarPlace.pendingItems):
            RankStatVarPlace.pendingItems.append(self)

    def addChild (self, child) :
        self.children.append(child)

        
    
            
def buildPlaceTree (placeFile):
    if (os.path.isfile(placeFile)):
        ff = open(placeFile)
        for line in ff:
            it = line.strip().split('\t')
            pl = RankStatVarPlace.getPlace(it[1], it[0])
            par = RankStatVarPlace.getPlace(it[3], it[2])
            par.addChild(pl)
    else :
        US = RankStatVarPlace.getPlace("Country", US_dcid)
        states = dc.get_places_in([US_dcid], "State")[US_dcid]
        for state in states:
            st = buildPlaceTreeState(state)
            US.addChild(st)
        ff = open(placeFile, "w")
        for c in US.children:
            dumpPlaces(ff, c, US)
        ff.close()

def dumpPlaces (ff, node, parent) :
    str = "%s\t%s\t%s\t%s\n" % (node.dcid, node.type, parent.dcid, parent.type)
    ff.write(str)
    for it in node.children:
        dumpPlaces(ff, it, node)

def buildPlaceTreeState(state):
    st = RankStatVarPlace.getPlace("State", state)
    counties = dc.get_places_in([state], "County")[state]
    for county in counties:
        cnt = RankStatVarPlace.getPlace("County", county)
        st.addChild(cnt)
        cities = dc.get_places_in([county], "City")[county]
        for city in cities:
            ct = RankStatVarPlace.getPlace("City", city)
            cnt.addChild(ct)
    return st



def dumpStatsInt (f, node):
    str = node.dcid +"\t" + node.type
    if (len(node.statVars) > 0):
        for stat in allStatVars:
            if (stat in node.statVars):
                str = str + "\t" + "%i" % (node.statVars[stat])
            else :
                str = str + "\t" + " "
        f.write(str + "\n")
    for child in node.children:
        dumpStatsInt(f, child)

def dumpStats (file):
    f = open(file, "w")
    str = "dcid\ttype"
    for stat in allStatVars:
        str = str + "\t" + stat
    f.write(str + "\n")
    US = RankStatVarPlace.getPlace("Country", US_dcid)
    for c in US.children:
        dumpStatsInt(f, c)
    f.close()

def loadDataFromFile (statsFile):
    f = open(statsFile)
    varLine = f.readLine()
    vars = varLine.strip().split('\t')
    for line in f:
        vals = line.strip().split('\t')
        dcid = vals[0]
        type = vals[1]
        node = RankStatVarPlace.getPlace(type, dcid)
        node.vars = {}
        i = 2
        while (i < len(vars)):
            node.vars[vars[i]] = vals[i]
            i = i + 1
                   
    
def loadStats (statsFile, reloadp) :
    if ((not os.path.isfile(statsFile)) or reloadp) :
        loadDataFromDC(statsFile)
        if (len(statsFile) > 0):
            dumpStats(statsFile)
    else:
        loadDataFromFile(statsFile)
    

buildPlaceTree("places.txt")
print("Finished Building Place Tree")
#loadStats("fullStats.tsv", 1)
loadStats("", 1)