Skip to content

Commit

Permalink
Merge pull request #209 from ibi-group/fix-mtc-sync
Browse files Browse the repository at this point in the history 
Fix feed sync with MTC RTD database
  • Loading branch information
@landonreed
landonreed authored Jun 24, 2019
2 parents a99a4f6 + 289ed0e commit eedf4e7
Showing 1 changed file with 48 additions and 56 deletions.
104 changes: 48 additions & 56 deletions src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,20 @@
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collection;

import static com.conveyal.datatools.manager.models.ExternalFeedSourceProperty.constructId;

/**
* This class implements the {@link ExternalFeedResource} interface for the MTC RTD database list of carriers (transit
* operators) and allows the Data Tools application to read and sync the list of carriers to a set of feed sources for a
* given project.
*
* This is generally intended as an initialization step to importing feed sources into a project; however, it should
* support subsequent sync requests (e.g., if new agencies are expected in the external feed resource, syncing should
* import those OR if feed properties are expected to have changed in the external feed resource, they should be updated
* accordingly in Data Tools).
*
* Created by demory on 3/30/16.
*/
public class MtcFeedResource implements ExternalFeedResource {
Expand All @@ -48,11 +58,15 @@ public String getResourceType() {
return RESOURCE_TYPE;
}

/**
* Fetch the list of feeds from the MTC endpoint, create any feed sources that do not match on agencyID, and update
* the external feed source properties.
*/
@Override
public void importFeedsForProject(Project project, String authHeader) throws IOException, IllegalAccessException {
URL url;
ObjectMapper mapper = new ObjectMapper();
// single list from MTC
// A single list of feeds is returned from the MTC Carrier endpoint.
try {
url = new URL(rtdApi + "/Carrier");
} catch(MalformedURLException ex) {
Expand All @@ -61,76 +75,54 @@ public void importFeedsForProject(Project project, String authHeader) throws IOE
}

try {
HttpURLConnection con = (HttpURLConnection) url.openConnection();

// optional default is GET
con.setRequestMethod("GET");

HttpURLConnection conn = (HttpURLConnection) url.openConnection();
//add request header
con.setRequestProperty("User-Agent", "User-Agent");

conn.setRequestProperty("User-Agent", "User-Agent");
// add auth header
LOG.info("authHeader="+authHeader);
con.setRequestProperty("Authorization", authHeader);

int responseCode = con.getResponseCode();
LOG.info("Sending 'GET' request to URL : " + url);
LOG.info("Response Code : " + responseCode);

BufferedReader in = new BufferedReader(
new InputStreamReader(con.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
conn.setRequestProperty("Authorization", authHeader);

while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();

String json = response.toString();
RtdCarrier[] results = mapper.readValue(json, RtdCarrier[].class);
for (int i = 0; i < results.length; i++) {
// String className = "RtdCarrier";
// Object car = Class.forName(className).newInstance();
RtdCarrier car = results[i];
//LOG.info("car id=" + car.AgencyId + " name=" + car.AgencyName);
LOG.info("Sending 'GET' request to URL : {}", url);
LOG.info("Response Code : {}", conn.getResponseCode());

RtdCarrier[] carriers = mapper.readValue(conn.getInputStream(), RtdCarrier[].class);
Collection<FeedSource> projectFeedSources = project.retrieveProjectFeedSources();
// Iterate over carriers found in response and update properties. Also, create a feed source for any carriers
// found in the response that do not correspond to an agency ID found in the external feed source properties.
for (int i = 0; i < carriers.length; i++) {
RtdCarrier carrier = carriers[i];
FeedSource source = null;

// check if a FeedSource with this AgencyId already exists
for (FeedSource existingSource : project.retrieveProjectFeedSources()) {
// Check if a FeedSource with this AgencyId already exists.
for (FeedSource existingSource : projectFeedSources) {
ExternalFeedSourceProperty agencyIdProp;
agencyIdProp = Persistence.externalFeedSourceProperties.getById(constructId(existingSource, this.getResourceType(),
AGENCY_ID_FIELDNAME));
if (agencyIdProp != null && agencyIdProp.value != null && agencyIdProp.value.equals(car.AgencyId)) {
//LOG.info("already exists: " + car.AgencyId);
String propertyId = constructId(existingSource, this.getResourceType(), AGENCY_ID_FIELDNAME);
agencyIdProp = Persistence.externalFeedSourceProperties.getById(propertyId);
if (agencyIdProp != null && agencyIdProp.value != null && agencyIdProp.value.equals(carrier.AgencyId)) {
source = existingSource;
}
}

String feedName;
if (car.AgencyName != null) {
feedName = car.AgencyName;
} else if (car.AgencyShortName != null) {
feedName = car.AgencyShortName;
} else {
feedName = car.AgencyId;
}

// Feed source does not exist. Create one using carrier properties.
if (source == null) {
// Derive the name from carrier properties found in response.
String feedName = carrier.AgencyName != null
? carrier.AgencyName
: carrier.AgencyShortName != null
? carrier.AgencyShortName
: carrier.AgencyId;
// Create new feed source to store in application database.
source = new FeedSource(feedName);
source.projectId = project.id;
LOG.info("Creating feed source {} from carrier response. (Did not previously exist.)", feedName);
// Store the feed source if it does not already exist.
Persistence.feedSources.create(source);
}
else source.name = feedName;

source.projectId = project.id;
// Store the feed source.
Persistence.feedSources.create(source);

// create / update the properties
// TODO: Does any property on the feed source need to be updated from the carrier (e.g., name).

for(Field carrierField : car.getClass().getDeclaredFields()) {
// Create / update the properties
LOG.info("Updating props for {}", source.name);
for(Field carrierField : carrier.getClass().getDeclaredFields()) {
String fieldName = carrierField.getName();
String fieldValue = carrierField.get(car) != null ? carrierField.get(car).toString() : null;
String fieldValue = carrierField.get(carrier) != null ? carrierField.get(carrier).toString() : null;
ExternalFeedSourceProperty prop = new ExternalFeedSourceProperty(source, this.getResourceType(), fieldName, fieldValue);
if (Persistence.externalFeedSourceProperties.getById(prop.id) == null) {
Persistence.externalFeedSourceProperties.create(prop);
Expand Down

0 comments on commit eedf4e7

Please sign in to comment.