Skip to content

Commit

Permalink
Merge pull request #208 from bacetiner/master
Browse files Browse the repository at this point in the history 
Miscellaneous Performance Improvements
  • Loading branch information
@bacetiner
bacetiner authored May 22, 2024
2 parents aa7d800 + 9257da3 commit f388926
Show file tree
Hide file tree
Showing 3 changed files with 114 additions and 45 deletions.
20 changes: 13 additions & 7 deletions brails/workflow/FootprintHandler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,13 +91,19 @@ def __fetch_roi(self,queryarea:str,outfile:str=''):
r = requests.get(nominatimquery, headers=headers)
datalist = r.json()

areafound = False
for data in datalist:
queryarea_osmid = data['osm_id']
queryarea_name = data['display_name']
if data['osm_type']=='relation':
areafound = True
break
if r.status_code==200:
areafound = False
for data in datalist:
queryarea_osmid = data['osm_id']
queryarea_name = data['display_name']
if data['osm_type']=='relation':
areafound = True
break
else:
sys.exit('Please try again. Server performing region search '
"returned the following error message: {datalist['title']}. " +
' This is a server-side error that does not require a' +
' change to BRAILS inputs.')

if areafound==True:
try:
Expand Down
17 changes: 12 additions & 5 deletions brails/workflow/NSIParser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,10 +154,13 @@ def GetRawDataROI(self, roi, outfile:str) -> None:

# If ROI is defined as a list of lists of footprint coordinates:
if type(roi) is list:
# Determine the coordinates of the bounding box including the footprints:
bbox = self.__get_bbox(roi)
footprints = roi.copy()
bpoly = None
if len(roi)>0:
# Determine the coordinates of the bounding box including the footprints:
bbox = self.__get_bbox(roi)
footprints = roi.copy()
bpoly = None
else:
sys.exit('No building footprint detected. Please try again for a larger region of interest')
else:
try:
roitype = roi.geom_type
Expand Down Expand Up @@ -284,7 +287,11 @@ def occ_parser(occtype):
return attributes

# Determine the coordinates of the bounding box including the footprints:
bbox = self.__get_bbox(footprints)
if len(footprints)>0:
bbox = self.__get_bbox(footprints)
else:
sys.exit('No building footprint detected. Please provide valid ' +
'footprint data to run this method.')

# Get the NBI data for computed bounding box:
datadict = self.__get_nbi_data(bbox)
Expand Down
122 changes: 89 additions & 33 deletions brails/workflow/TransportationElementHandler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
import requests
import copy
import json
from shapely.geometry import LineString, Point
from shapely.geometry import Polygon, LineString, Point, box
from shapely import wkt
from requests.adapters import HTTPAdapter, Retry
from brails.workflow.FootprintHandler import FootprintHandler
Expand All @@ -52,58 +52,80 @@ def __init__(self):
self.queryarea = ''
self.output_files = {'roads':'Roads.geojson'}

def fetch_transportation_elements(self,queryarea):
def fetch_transportation_elements(self, queryarea:str):

def query_generator(bpoly,eltype):
def query_generator(bpoly: Polygon, eltype:str) -> str:
# Get the bounds of the entered bounding polygon and lowercase the
# entered element type:
bbox = bpoly.bounds
eltype = eltype.lower()

# If element type is bridge, generate an NBI query for the bounds
# of bpoly:
if eltype=='bridge':
query = ('https://geo.dot.gov/server/rest/services/Hosted/' +
'National_Bridge_Inventory_DS/FeatureServer/0/query?'+
'where=1%3D1&outFields=*'+
f"&geometry={bbox[0]}%2C{bbox[1]}%2C{bbox[2]}%2C{bbox[3]}" +
'&geometryType=esriGeometryEnvelope&inSR=4326' +
'&spatialRel=esriSpatialRelIntersects&outSR=4326&f=json')

# If element type is tunnel, generate an NTI query for the bounds
# of bpoly:
elif eltype=='tunnel':
query = ('https://geo.dot.gov/server/rest/services/Hosted/' +
'National_Tunnel_Inventory_DS/FeatureServer/0/query?' +
'where=1%3D1&outFields=*'
f"&geometry={bbox[0]}%2C{bbox[1]}%2C{bbox[2]}%2C{bbox[3]}" +
'&geometryType=esriGeometryEnvelope&inSR=4326' +
'&spatialRel=esriSpatialRelIntersects&outSR=4326&f=json')

# If element type is railroad, generate an NARNL query for the
# bounds of bpoly:
elif eltype=='railroad':
query = ('https://geo.dot.gov/server/rest/services/Hosted/' +
'North_American_Rail_Network_Lines_DS/FeatureServer/0/query?' +
'where=1%3D1&outFields=*'+
f"&geometry={bbox[0]}%2C{bbox[1]}%2C{bbox[2]}%2C{bbox[3]}" +
'&geometryType=esriGeometryEnvelope&inSR=4326' +
'&spatialRel=esriSpatialRelIntersects&outSR=4326&f=json')

# If element type is primary_road, generate a TIGER query for the
# bounds of bpoly:
elif eltype=='primary_road':
query = ('https://tigerweb.geo.census.gov/arcgis/rest/services/' +
'TIGERweb/Transportation/MapServer/2/query?where=&text='+
'&outFields=OID,NAME,MTFCC'
f"&geometry={bbox[0]}%2C{bbox[1]}%2C{bbox[2]}%2C{bbox[3]}" +
'&geometryType=esriGeometryEnvelope&inSR=4326' +
'&spatialRel=esriSpatialRelIntersects&outSR=4326&f=json')

# If element type is secondary_road, generate a TIGER query for the
# bounds of bpoly:
elif eltype=='secondary_road':
query = ('https://tigerweb.geo.census.gov/arcgis/rest/services/' +
'TIGERweb/Transportation/MapServer/6/query?where=&text='+
'&outFields=OID,NAME,MTFCC'
f"&geometry={bbox[0]}%2C{bbox[1]}%2C{bbox[2]}%2C{bbox[3]}" +
'&geometryType=esriGeometryEnvelope&inSR=4326' +
'&spatialRel=esriSpatialRelIntersects&outSR=4326&f=json')

# If element type is local_road, generate a TIGER query for the
# bounds of bpoly:
elif eltype=='local_road':
query = ('https://tigerweb.geo.census.gov/arcgis/rest/services/' +
'TIGERweb/Transportation/MapServer/8/query?where=&text='+
'&outFields=OID,NAME,MTFCC'
f"&geometry={bbox[0]}%2C{bbox[1]}%2C{bbox[2]}%2C{bbox[3]}" +
'&geometryType=esriGeometryEnvelope&inSR=4326' +
'&spatialRel=esriSpatialRelIntersects&outSR=4326&f=json')

# Otherwise, raise a NotImplementedError:
else:
raise NotImplementedError('Element type not implemented')
return query

def get_el_counts(bpoly, eltype: str) -> int:
def create_pooling_session():
# Define a retry stratey for common error codes to use when
# downloading data:
s = requests.Session()
Expand All @@ -112,24 +134,54 @@ def get_el_counts(bpoly, eltype: str) -> int:
status_forcelist=[500, 502, 503, 504])
s.mount('https://', HTTPAdapter(max_retries=retries))

return s

def get_el_counts(bpoly:Polygon, eltype: str) -> int:
# Create a persistent requests session:
s = create_pooling_session()

# Create the query required to get the element counts:
query = query_generator(bpoly,eltype)
query.replace('outSR=4326','returnCountOnly=true')
query = query.replace('outSR=4326','returnCountOnly=true')

# Download element count using the defined retry strategy:
# Download the element count for the bounding polygon using the
# defined retry strategy:
r = s.get(query)
elcount = r.json()['count']

return elcount
return elcount

def get_max_el_count(eltype: str) -> int:
# Create a persistent requests session:
s = create_pooling_session()

# Create the query required to get the element counts:
query = query_generator(box(-1,1,-1,1),eltype)
query = query.split('/query?')
query = query[0] + '?f=pjson'

# Download the maximum element count for the bounding polygon using
# the defined retry strategy:
r = s.get(query)
maxelcount = r.json()['maxRecordCount']

return maxelcount

def conv2geojson(datalist,eltype,bpoly):
def list2geojson(datalist:list, eltype:str, bpoly:Polygon) -> dict:
# Lowercase the entered element type string:
eltype = eltype.lower()

# Populate the geojson header:
geojson = {'type':'FeatureCollection',
"crs": {"type": "name", "properties":
{"name": "urn:ogc:def:crs:OGC:1.3:CRS84"}},
'features':[]}

# Scroll through each item in the datalist:
for item in datalist:
# If the element is a bridge or tunnel, parse its geometry as a
# point and check if the extracted point is within the bounding
# polygon:
if eltype in ['bridge','tunnel']:
geometry = [item['geometry']['x'],item['geometry']['y']]
if bpoly.contains(Point(geometry)):
Expand All @@ -140,6 +192,9 @@ def conv2geojson(datalist,eltype,bpoly):
else:
continue
else:
# If the element is a road segment, parse it as a MultiLineString
# and check if the extracted segment is within the bounding
# polygon:
geometry = item['geometry']['paths']
if bpoly.intersects(LineString(geometry[0])):
feature = {'type':'Feature',
Expand All @@ -148,18 +203,34 @@ def conv2geojson(datalist,eltype,bpoly):
'coordinates':[]}}
else:
continue

# Copy the obtained geometry in a feature dictionary:
feature['geometry']['coordinates'] = geometry.copy()

# Read item attributes
properties = item['attributes']
for prop in properties:
if prop.count('_')>1:
propname = prop[:prop.rindex('_')]

# For each attribute:
for prop in properties.keys():
# Clean up the property name from redundant numeric text:
if '_' in prop:
strsegs = prop.split('_')
removestr = ''
for seg in strsegs:
if any(char.isdigit() for char in seg):
removestr = '_' + seg
propname = prop.replace(removestr,'')
else:
propname = prop

# Write the property in a feature dictionary:
feature['properties'][propname] = properties[prop]

# Add the feature in the geojson dictionary:
geojson['features'].append(feature)
return geojson

def print_el_counts(datalist,eltype):
def print_el_counts(datalist: list, eltype:str):
nel = len(datalist)
eltype_print = eltype.replace('_',' ')

Expand All @@ -175,14 +246,11 @@ def print_el_counts(datalist,eltype):

print(f'Found {nel} {eltype_print} {elntype}{suffix}')

def write2geojson(bpoly,eltype):
# Define a retry stratey for common error codes to use when
# downloading data:
s = requests.Session()
retries = Retry(total=10,
backoff_factor=0.1,
status_forcelist=[500, 502, 503, 504])
s.mount('https://', HTTPAdapter(max_retries=retries))
def write2geojson(bpoly:Polygon, eltype:str) -> dict:
nel = get_el_counts(bpoly,eltype)

# Create a persistent requests session:
s = create_pooling_session()

# Download element data using the defined retry strategy:
query = query_generator(bpoly,eltype)
Expand All @@ -197,7 +265,7 @@ def write2geojson(bpoly,eltype):
datalist = r.json()['features']

# If road data convert it into GeoJSON format:
jsonout = conv2geojson(datalist,eltype,bpoly)
jsonout = list2geojson(datalist,eltype,bpoly)

# If not road data convert it into GeoJSON format and write it into
# a file:
Expand Down Expand Up @@ -260,18 +328,6 @@ def combine_write_roadjsons(roadjsons,bpoly):
print_el_counts(roadjsons_combined['features'],'road')
with open('Roads.geojson', 'w') as output_file:
json.dump(roadjsons_combined, output_file, indent=2)

def mergeDataList(dataLists):
merged = dataLists[0]
for i in range(1,len(dataLists)):
data = dataLists[i]
for left, right in zip(merged,data):
if left['attributes']['OBJECTID'] != right['attributes']['OBJECTID']:
print(("Data downloaded from the National Bridge Inventory")+
("has miss matching OBJECTID between batches.")+
("This may be solved by running Brails again"))
left['attributes'].update(right['attributes'])
return merged

self.queryarea = queryarea

Expand Down

0 comments on commit f388926

Please sign in to comment.