diff --git a/src/main/java/com/conveyal/gtfs/error/NewGTFSErrorType.java b/src/main/java/com/conveyal/gtfs/error/NewGTFSErrorType.java index 80154ae83..e73b2e514 100644 --- a/src/main/java/com/conveyal/gtfs/error/NewGTFSErrorType.java +++ b/src/main/java/com/conveyal/gtfs/error/NewGTFSErrorType.java @@ -32,6 +32,7 @@ public enum NewGTFSErrorType { MISSING_ARRIVAL_OR_DEPARTURE(Priority.MEDIUM, "First and last stop times are required to have both an arrival and departure time."), MISSING_COLUMN(Priority.MEDIUM, "A required column was missing from a table."), MISSING_FIELD(Priority.MEDIUM, "A required field was missing or empty in a particular row."), + MISSING_FOREIGN_TABLE_REFERENCE(Priority.HIGH, "This line references an ID that must exist in a single foreign table."), MISSING_SHAPE(Priority.MEDIUM, "???"), MISSING_TABLE(Priority.MEDIUM, "This table is required by the GTFS specification but is missing."), MULTIPLE_SHAPES_FOR_PATTERN(Priority.MEDIUM, "Multiple shapes found for a single unique sequence of stops (i.e, trip pattern)."), diff --git a/src/main/java/com/conveyal/gtfs/loader/Field.java b/src/main/java/com/conveyal/gtfs/loader/Field.java index 52cda789c..3531fdaa5 100644 --- a/src/main/java/com/conveyal/gtfs/loader/Field.java +++ b/src/main/java/com/conveyal/gtfs/loader/Field.java @@ -8,6 +8,7 @@ import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.SQLType; +import java.util.LinkedHashSet; import java.util.Set; /** @@ -46,7 +47,7 @@ public abstract class Field { * Indicates that this field acts as a foreign key to this referenced table. This is used when checking referential * integrity when loading a feed. * */ - public Table referenceTable = null; + public Set referenceTables = new LinkedHashSet<>(); private boolean shouldBeIndexed; private boolean emptyValuePermitted; private boolean isConditionallyRequired; @@ -138,7 +139,7 @@ public boolean isRequired () { * a many-to-many reference. */ public boolean isForeignReference () { - return this.referenceTable != null; + return !this.referenceTables.isEmpty(); } /** @@ -181,7 +182,7 @@ public boolean shouldBeIndexed() { * @return this same Field instance */ public Field isReferenceTo(Table table) { - this.referenceTable = table; + referenceTables.add(table); return this; } diff --git a/src/main/java/com/conveyal/gtfs/loader/JDBCTableReader.java b/src/main/java/com/conveyal/gtfs/loader/JDBCTableReader.java index 6f4fa07c2..0eaa80331 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JDBCTableReader.java +++ b/src/main/java/com/conveyal/gtfs/loader/JDBCTableReader.java @@ -1,5 +1,6 @@ package com.conveyal.gtfs.loader; +import com.conveyal.gtfs.model.Calendar; import com.conveyal.gtfs.model.Entity; import com.conveyal.gtfs.storage.StorageException; import gnu.trove.map.TObjectIntMap; @@ -146,6 +147,18 @@ public int getRowCount() { } } + /** + * Provide reader for calendar table. + */ + public static JDBCTableReader getCalendarTableReader(DataSource dataSource, String tablePrefix) { + return new JDBCTableReader( + Table.CALENDAR, + dataSource, + tablePrefix + ".", + EntityPopulator.CALENDAR + ); + } + private class EntityIterator implements Iterator { private Connection connection; // Will remain open for the duration of the iteration. diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java index bf77ba7e1..78e6591d7 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java @@ -23,13 +23,16 @@ import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.sql.Connection; import java.sql.SQLException; import java.time.LocalDate; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -79,7 +82,7 @@ public JdbcGtfsExporter(String feedId, String outFile, DataSource dataSource, bo /** * Utility method to check if an exception uses a specific service. */ - public Boolean exceptionInvolvesService(ScheduleException ex, String serviceId) { + public boolean exceptionInvolvesService(ScheduleException ex, String serviceId) { return ( ex.addedService.contains(serviceId) || ex.removedService.contains(serviceId) || @@ -99,7 +102,7 @@ public FeedLoadResult exportTables() { FeedLoadResult result = new FeedLoadResult(); try { - zipOutputStream = new ZipOutputStream(new FileOutputStream(outFile)); + zipOutputStream = new ZipOutputStream(Files.newOutputStream(Paths.get(outFile))); long startTime = System.currentTimeMillis(); // We get a single connection object and share it across several different methods. // This ensures that actions taken in one method are visible to all subsequent SQL statements. @@ -132,40 +135,55 @@ public FeedLoadResult exportTables() { if (fromEditor) { // Export schedule exceptions in place of calendar dates if exporting a feed/schema that represents an editor snapshot. GTFSFeed feed = new GTFSFeed(); - // FIXME: The below table readers should probably just share a connection with the exporter. - JDBCTableReader exceptionsReader = - new JDBCTableReader(Table.SCHEDULE_EXCEPTIONS, dataSource, feedIdToExport + ".", - EntityPopulator.SCHEDULE_EXCEPTION); - JDBCTableReader calendarsReader = - new JDBCTableReader(Table.CALENDAR, dataSource, feedIdToExport + ".", - EntityPopulator.CALENDAR); - Iterable calendars = calendarsReader.getAll(); + JDBCTableReader exceptionsReader =new JDBCTableReader( + Table.SCHEDULE_EXCEPTIONS, + dataSource, + feedIdToExport + ".", + EntityPopulator.SCHEDULE_EXCEPTION + ); + JDBCTableReader calendarReader = JDBCTableReader.getCalendarTableReader(dataSource, feedIdToExport); + Iterable calendars = calendarReader.getAll(); Iterable exceptionsIterator = exceptionsReader.getAll(); - List exceptions = new ArrayList<>(); - // FIXME: Doing this causes the connection to stay open, but it is closed in the finalizer so it should - // not be a big problem. - for (ScheduleException exception : exceptionsIterator) { - exceptions.add(exception); + List calendarExceptions = new ArrayList<>(); + List calendarDateExceptions = new ArrayList<>(); + // Separate distinct calendar date exceptions from those associated with calendars. + for (ScheduleException ex : exceptionsIterator) { + if (ex.exemplar.equals(ScheduleException.ExemplarServiceDescriptor.CALENDAR_DATE_SERVICE)) { + calendarDateExceptions.add(ex); + } else { + calendarExceptions.add(ex); + } } + + int calendarDateCount = calendarDateExceptions.size(); + // Extract calendar date services, convert to calendar date and add to the feed. + for (ScheduleException ex : calendarDateExceptions) { + for (LocalDate date : ex.dates) { + String serviceId = ex.customSchedule.get(0); + CalendarDate calendarDate = new CalendarDate(); + calendarDate.date = date; + calendarDate.service_id = serviceId; + calendarDate.exception_type = 1; + Service service = new Service(serviceId); + service.calendar_dates.put(date, calendarDate); + // If the calendar dates provided contain duplicates (e.g. two or more identical service ids + // that are NOT associated with a calendar) only the first entry would persist export. To + // resolve this a unique key consisting of service id and date is used. + feed.services.put(String.format("%s-%s", calendarDate.service_id, calendarDate.date), service); + } + } + // check whether the feed is organized in a format with the calendars.txt file - if (calendarsReader.getRowCount() > 0) { + if (calendarReader.getRowCount() > 0) { // feed does have calendars.txt file, continue export with strategy of matching exceptions // to calendar to output calendar_dates.txt - int calendarDateCount = 0; for (Calendar cal : calendars) { Service service = new Service(cal.service_id); service.calendar = cal; - for (ScheduleException ex : exceptions.stream() + for (ScheduleException ex : calendarExceptions.stream() .filter(ex -> exceptionInvolvesService(ex, cal.service_id)) .collect(Collectors.toList()) ) { - if (ex.exemplar.equals(ScheduleException.ExemplarServiceDescriptor.SWAP) && - (!ex.addedService.contains(cal.service_id) && !ex.removedService.contains(cal.service_id))) { - // Skip swap exception if cal is not referenced by added or removed service. - // This is not technically necessary, but the output is cleaner/more intelligible. - continue; - } - for (LocalDate date : ex.dates) { if (date.isBefore(cal.start_date) || date.isAfter(cal.end_date)) { // No need to write dates that do not apply @@ -179,7 +197,7 @@ public FeedLoadResult exportTables() { LOG.info("Adding exception {} (type={}) for calendar {} on date {}", ex.name, calendarDate.exception_type, cal.service_id, date); if (service.calendar_dates.containsKey(date)) - throw new IllegalArgumentException("Duplicate schedule exceptions on " + date.toString()); + throw new IllegalArgumentException("Duplicate schedule exceptions on " + date); service.calendar_dates.put(date, calendarDate); calendarDateCount += 1; @@ -187,14 +205,16 @@ public FeedLoadResult exportTables() { } feed.services.put(cal.service_id, service); } - if (calendarDateCount == 0) { - LOG.info("No calendar dates found. Skipping table."); - } else { - LOG.info("Writing {} calendar dates from schedule exceptions", calendarDateCount); - new CalendarDate.Writer(feed).writeTable(zipOutputStream); - } + } + if (calendarDateCount == 0) { + LOG.info("No calendar dates found. Skipping table."); } else { - // No calendar records exist, export calendar_dates as is and hope for the best. + LOG.info("Writing {} calendar dates from schedule exceptions", calendarDateCount); + new CalendarDate.Writer(feed).writeTable(zipOutputStream); + } + + if (calendarReader.getRowCount() == 0 && calendarDateExceptions.isEmpty()) { + // No calendar or calendar date service records exist, export calendar_dates as is and hope for the best. // This situation will occur in at least 2 scenarios: // 1. A GTFS has been loaded into the editor that had only the calendar_dates.txt file // and no further edits were made before exporting to a snapshot diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsSnapshotter.java b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsSnapshotter.java index 3b36a9b8e..a4e05ecf4 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsSnapshotter.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsSnapshotter.java @@ -2,6 +2,7 @@ import com.conveyal.gtfs.model.Calendar; import com.conveyal.gtfs.model.CalendarDate; +import com.conveyal.gtfs.model.ScheduleException; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; @@ -210,15 +211,14 @@ private TableLoadResult createScheduleExceptionsTable() { tablePrefix.replace(".", ""), true ); - String sql = String.format( - "insert into %s (name, dates, exemplar, added_service, removed_service) values (?, ?, ?, ?, ?)", - scheduleExceptionsTableName - ); - PreparedStatement scheduleExceptionsStatement = connection.prepareStatement(sql); - final BatchTracker scheduleExceptionsTracker = new BatchTracker( - "schedule_exceptions", - scheduleExceptionsStatement - ); + + // Fetch all entries in the calendar table to generate set of serviceIds that exist in the calendar + // table. + JDBCTableReader calendarReader = JDBCTableReader.getCalendarTableReader(dataSource, feedIdToSnapshot); + Set calendarServiceIds = new HashSet<>(); + for (Calendar calendar : calendarReader.getAll()) { + calendarServiceIds.add(calendar.service_id); + } JDBCTableReader calendarDatesReader = new JDBCTableReader( Table.CALENDAR_DATES, @@ -231,65 +231,80 @@ private TableLoadResult createScheduleExceptionsTable() { // Keep track of calendars by service id in case we need to add dummy calendar entries. Map dummyCalendarsByServiceId = new HashMap<>(); - // Iterate through calendar dates to build up to get maps from exceptions to their dates. + // Iterate through calendar dates to build up appropriate service dates. Multimap removedServiceForDate = HashMultimap.create(); Multimap addedServiceForDate = HashMultimap.create(); + HashMap> calendarDateService = new HashMap<>(); for (CalendarDate calendarDate : calendarDates) { - // Skip any null dates - if (calendarDate.date == null) { - LOG.warn("Encountered calendar date record with null value for date field. Skipping."); + // Skip any null dates or service ids. + if (calendarDate.date == null || calendarDate.service_id == null) { + LOG.warn("Encountered calendar date record with null value for date/service_id field. Skipping."); continue; } String date = calendarDate.date.format(DateTimeFormatter.BASIC_ISO_DATE); - if (calendarDate.exception_type == 1) { - addedServiceForDate.put(date, calendarDate.service_id); - // create (if needed) and extend range of dummy calendar that would need to be created if we are - // copying from a feed that doesn't have the calendar.txt file - Calendar calendar = dummyCalendarsByServiceId.getOrDefault(calendarDate.service_id, new Calendar()); - calendar.service_id = calendarDate.service_id; - if (calendar.start_date == null || calendar.start_date.isAfter(calendarDate.date)) { - calendar.start_date = calendarDate.date; - } - if (calendar.end_date == null || calendar.end_date.isBefore(calendarDate.date)) { - calendar.end_date = calendarDate.date; + if (calendarServiceIds.contains(calendarDate.service_id)) { + // Calendar date is related to a calendar. + if (calendarDate.exception_type == 1) { + addedServiceForDate.put(date, calendarDate.service_id); + extendDummyCalendarRange(dummyCalendarsByServiceId, calendarDate); + } else { + removedServiceForDate.put(date, calendarDate.service_id); } - dummyCalendarsByServiceId.put(calendarDate.service_id, calendar); } else { - removedServiceForDate.put(date, calendarDate.service_id); + // Calendar date is not related to a calendar. Group calendar dates by service id. + if (calendarDateService.containsKey(calendarDate.service_id)) { + calendarDateService.get(calendarDate.service_id).add(date); + } else { + Set dates = new HashSet<>(); + dates.add(date); + calendarDateService.put(calendarDate.service_id, dates); + } + } } + + String sql = String.format( + "insert into %s (name, dates, exemplar, custom_schedule, added_service, removed_service) values (?, ?, ?, ?, ?, ?)", + scheduleExceptionsTableName + ); + PreparedStatement scheduleExceptionsStatement = connection.prepareStatement(sql); + final BatchTracker scheduleExceptionsTracker = new BatchTracker( + "schedule_exceptions", + scheduleExceptionsStatement + ); + // Iterate through dates with added or removed service and add to database. // For usability and simplicity of code, don't attempt to find all dates with similar // added and removed services, but simply create an entry for each found date. for (String date : Sets.union(removedServiceForDate.keySet(), addedServiceForDate.keySet())) { - scheduleExceptionsStatement.setString(1, date); - String[] dates = {date}; - scheduleExceptionsStatement.setArray(2, connection.createArrayOf("text", dates)); - scheduleExceptionsStatement.setInt(3, 9); // FIXME use better static type - scheduleExceptionsStatement.setArray( - 4, - connection.createArrayOf("text", addedServiceForDate.get(date).toArray()) + createScheduledExceptionStatement( + scheduleExceptionsStatement, + scheduleExceptionsTracker, + date, + new String[] {date}, + ScheduleException.ExemplarServiceDescriptor.SWAP, + new String[] {}, + addedServiceForDate.get(date).toArray(), + removedServiceForDate.get(date).toArray() ); - scheduleExceptionsStatement.setArray( - 5, - connection.createArrayOf("text", removedServiceForDate.get(date).toArray()) - ); - scheduleExceptionsTracker.addBatch(); } - scheduleExceptionsTracker.executeRemaining(); - // fetch all entries in the calendar table to generate set of serviceIds that exist in the calendar - // table. - JDBCTableReader calendarReader = new JDBCTableReader( - Table.CALENDAR, - dataSource, - feedIdToSnapshot + ".", - EntityPopulator.CALENDAR - ); - Set calendarServiceIds = new HashSet<>(); - for (Calendar calendar : calendarReader.getAll()) { - calendarServiceIds.add(calendar.service_id); + for (Map.Entry> entry : calendarDateService.entrySet()) { + String serviceId = entry.getKey(); + String[] dates = entry.getValue().toArray(new String[0]); + createScheduledExceptionStatement( + scheduleExceptionsStatement, + scheduleExceptionsTracker, + // Unique-ish schedule name that shouldn't conflict with existing service ids. + String.format("%s-%s", serviceId, dates[0]), + dates, + ScheduleException.ExemplarServiceDescriptor.CALENDAR_DATE_SERVICE, + new String[] {serviceId}, + new String[] {}, + new String[] {} + ); } + scheduleExceptionsTracker.executeRemaining(); // For service_ids that only existed in the calendar_dates table, insert auto-generated, "blank" // (no days of week specified) calendar entries. @@ -342,6 +357,53 @@ private TableLoadResult createScheduleExceptionsTable() { } } + /** + * Create (if needed) and extend range of dummy calendars that would need to be created if we are copying from a + * feed that doesn't have the calendar.txt file. + */ + private void extendDummyCalendarRange(Map dummyCalendarsByServiceId, CalendarDate calendarDate) { + Calendar calendar = dummyCalendarsByServiceId.getOrDefault(calendarDate.service_id, new Calendar()); + calendar.service_id = calendarDate.service_id; + if (calendar.start_date == null || calendar.start_date.isAfter(calendarDate.date)) { + calendar.start_date = calendarDate.date; + } + if (calendar.end_date == null || calendar.end_date.isBefore(calendarDate.date)) { + calendar.end_date = calendarDate.date; + } + dummyCalendarsByServiceId.put(calendarDate.service_id, calendar); + } + + /** + * Populate schedule exception statement and add to batch tracker. + */ + private void createScheduledExceptionStatement( + PreparedStatement scheduleExceptionsStatement, + BatchTracker scheduleExceptionsTracker, + String name, + String[] dates, + ScheduleException.ExemplarServiceDescriptor exemplarServiceDescriptor, + Object[] customSchedule, + Object[] addedServicesForDate, + Object[] removedServicesForDate + ) throws SQLException { + scheduleExceptionsStatement.setString(1, name); + scheduleExceptionsStatement.setArray(2, connection.createArrayOf("text", dates)); + scheduleExceptionsStatement.setInt(3, exemplarServiceDescriptor.getValue()); + scheduleExceptionsStatement.setArray( + 4, + connection.createArrayOf("text", customSchedule) + ); + scheduleExceptionsStatement.setArray( + 5, + connection.createArrayOf("text", addedServicesForDate) + ); + scheduleExceptionsStatement.setArray( + 6, + connection.createArrayOf("text", removedServicesForDate) + ); + scheduleExceptionsTracker.addBatch(); + } + /** * Helper method to determine if a table exists within a namespace. */ diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java index 86ee2c771..8484e80c3 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java @@ -2,6 +2,7 @@ import com.conveyal.gtfs.model.Entity; import com.conveyal.gtfs.model.PatternStop; +import com.conveyal.gtfs.model.ScheduleException.ExemplarServiceDescriptor; import com.conveyal.gtfs.model.Shape; import com.conveyal.gtfs.model.StopTime; import com.conveyal.gtfs.storage.StorageException; @@ -597,6 +598,7 @@ private String updateChildTable( boolean hasOrderField = orderFieldName != null; int previousOrder = -1; TIntSet orderValues = new TIntHashSet(); + Multimap> foreignReferencesPerTable = HashMultimap.create(); Multimap referencesPerTable = HashMultimap.create(); int cumulativeTravelTime = 0; for (JsonNode entityNode : subEntities) { @@ -605,19 +607,9 @@ private String updateChildTable( // Always override the key field (shape_id for shapes, pattern_id for patterns) regardless of the entity's // actual value. subEntity.put(keyField.name, keyValue); - // Check any references the sub entity might have. For example, this checks that stop_id values on - // pattern_stops refer to entities that actually exist in the stops table. NOTE: This skips the "specTable", - // i.e., for pattern stops it will not check pattern_id references. This is enforced above with the put key - // field statement above. - for (Field field : subTable.specFields()) { - if (field.referenceTable != null && !field.referenceTable.name.equals(specTable.name)) { - JsonNode refValueNode = subEntity.get(field.name); - // Skip over references that are null but not required (e.g., route_id in fare_rules). - if (refValueNode.isNull() && !field.isRequired()) continue; - String refValue = refValueNode.asText(); - referencesPerTable.put(field.referenceTable, refValue); - } - } + + checkTableReferences(foreignReferencesPerTable, referencesPerTable, specTable, subTable, subEntity); + // Insert new sub-entity. if (entityCount == 0) { // If handling first iteration, create the prepared statement (later iterations will add to batch). @@ -694,6 +686,40 @@ private String updateChildTable( return keyValue; } + /** + * Check any references the sub entity might have. For example, this checks that a service_id defined in a trip + * refers to a calendar or calendar date. NOTE: This skips the "specTable", i.e., for pattern stops it will not + * check pattern_id references. This is enforced above with the put key field statement above. + */ + private void checkTableReferences( + Multimap> foreignReferencesPerTable, + Multimap referencesPerTable, + Table specTable, + Table subTable, + ObjectNode subEntity + ) { + for (Field field : subTable.specFields()) { + if (field.referenceTables.isEmpty()) continue; + Multimap foreignReferences = HashMultimap.create(); + for (Table referenceTable : field.referenceTables) { + if (!referenceTable.name.equals(specTable.name)) { + JsonNode refValueNode = subEntity.get(field.name); + // Skip over references that are null but not required (e.g., route_id in fare_rules). + if (refValueNode.isNull() && !field.isRequired()) continue; + String refValue = refValueNode.asText(); + if (field.referenceTables.size() == 1) { + referencesPerTable.put(referenceTable, refValue); + } else { + foreignReferences.put(referenceTable, refValue); + } + } + } + if (!foreignReferences.isEmpty()) { + foreignReferencesPerTable.put(subTable, foreignReferences); + } + } + } + /** * Delete existing sub-entities for given key value for when an update to the parent entity is made (i.e., the parent * entity is not being newly created). Examples of sub-entities include stop times for trips, pattern stops for a @@ -1303,6 +1329,58 @@ private static long handleStatementExecution(PreparedStatement statement, boolea } } + private void checkUniqueIdsAndUpdateReferencingTables( + TIntSet uniqueIds, + Integer id, + String namespace, + Table table, + String keyValue, + Boolean isCreating, + Field keyField + ) throws SQLException { + int size = uniqueIds.size(); + if (size == 0 || (size == 1 && id != null && uniqueIds.contains(id))) { + // OK. + if (size == 0 && !isCreating) { + // FIXME: Need to update referencing tables because entity has changed ID. + // Entity key value is being changed to an entirely new one. If there are entities that + // reference this value, we need to update them. + updateReferencingTables(namespace, table, id, keyValue, keyField); + } + } else { + // Conflict. The different conflict conditions are outlined below. + if (size == 1) { + // There was one match found. + if (isCreating) { + // Under no circumstance should a new entity have a conflict with existing key field. + throw new SQLException( + String.format("New %s's %s value (%s) conflicts with an existing record in table.", + table.entityClass.getSimpleName(), + keyField.name, + keyValue) + ); + } + if (!uniqueIds.contains(id)) { + // There are two circumstances we could encounter here. + // 1. The key value for this entity has been updated to match some other entity's key value (conflict). + // 2. The int ID provided in the request parameter does not match any rows in the table. + throw new SQLException("Key field must be unique and request parameter ID must exist."); + } + } else if (size > 1) { + // FIXME: Handle edge case where original data set contains duplicate values for key field and this is an + // attempt to rectify bad data. + String message = String.format( + "%d %s entities shares the same key field (%s=%s)! Key field must be unique.", + size, + table.name, + keyField.name, + keyValue); + LOG.error(message); + throw new SQLException(message); + } + } + } + /** * Checks for modification of GTFS key field (e.g., stop_id, route_id) in supplied JSON object and ensures * both uniqueness and that referencing tables are appropriately updated. @@ -1344,46 +1422,34 @@ private void ensureReferentialIntegrity( String keyValue = jsonObject.get(keyField).asText(); // If updating key field, check that there is no ID conflict on value (e.g., stop_id or route_id) TIntSet uniqueIds = getIdsForCondition(tableName, keyField, keyValue, connection); - int size = uniqueIds.size(); - if (size == 0 || (size == 1 && id != null && uniqueIds.contains(id))) { - // OK. - if (size == 0 && !isCreating) { - // FIXME: Need to update referencing tables because entity has changed ID. - // Entity key value is being changed to an entirely new one. If there are entities that - // reference this value, we need to update them. - updateReferencingTables(namespace, table, id, keyValue); - } - } else { - // Conflict. The different conflict conditions are outlined below. - if (size == 1) { - // There was one match found. - if (isCreating) { - // Under no circumstance should a new entity have a conflict with existing key field. - throw new SQLException( - String.format("New %s's %s value (%s) conflicts with an existing record in table.", - table.entityClass.getSimpleName(), - keyField, - keyValue) - ); - } - if (!uniqueIds.contains(id)) { - // There are two circumstances we could encounter here. - // 1. The key value for this entity has been updated to match some other entity's key value (conflict). - // 2. The int ID provided in the request parameter does not match any rows in the table. - throw new SQLException("Key field must be unique and request parameter ID must exist."); - } - } else if (size > 1) { - // FIXME: Handle edge case where original data set contains duplicate values for key field and this is an - // attempt to rectify bad data. - String message = String.format( - "%d %s entities shares the same key field (%s=%s)! Key field must be unique.", - size, - table.name, - keyField, - keyValue); - LOG.error(message); - throw new SQLException(message); - } + checkUniqueIdsAndUpdateReferencingTables( + uniqueIds, + id, + namespace, + table, + keyValue, + isCreating, + table.getFieldForName(table.getKeyFieldName()) + ); + + if (table.name.equals("schedule_exceptions") && + jsonObject.has("exemplar") && + jsonObject.get("exemplar").asInt() == ExemplarServiceDescriptor.CALENDAR_DATE_SERVICE.getValue() + ) { + // Special case for schedule_exceptions where for exception type 10 and service_id is also a key. + String calendarDateServiceKey = "custom_schedule"; + Field calendarDateServiceKeyField = table.getFieldForName(calendarDateServiceKey); + String calendarDateServiceKeyVal = jsonObject.get(calendarDateServiceKey).asText(); + TIntSet calendarDateServiceUniqueIds = getIdsForCondition(tableName, calendarDateServiceKey, calendarDateServiceKeyVal, connection); + checkUniqueIdsAndUpdateReferencingTables( + calendarDateServiceUniqueIds, + id, + namespace, + table, + calendarDateServiceKeyVal, + isCreating, + calendarDateServiceKeyField + ); } } @@ -1409,7 +1475,13 @@ private static TIntSet getIdsForCondition( String keyValue, Connection connection ) throws SQLException { - String idCheckSql = String.format("select id from %s where %s = ?", tableName, keyField); + String idCheckSql; + if (keyField.equals("custom_schedule")) { + // The custom_schedule field of an exception based service contains an array and requires an "any" query. + idCheckSql = String.format("select id from %s where ? = any (%s)", tableName, keyField); + } else { + idCheckSql = String.format("select id from %s where %s = ?", tableName, keyField); + } // Create statement for counting rows selected PreparedStatement statement = connection.prepareStatement(idCheckSql); statement.setString(1, keyValue); @@ -1442,9 +1514,12 @@ private static Set
getReferencingTables(Table table) { // which could have unexpected behaviour. referencingTables.add(gtfsTable); } - if (field.isForeignReference() && field.referenceTable.name.equals(table.name)) { - // If any of the table's fields are foreign references to the specified table, add to the return set. - referencingTables.add(gtfsTable); + if (field.isForeignReference()) { + for (Table refTable : field.referenceTables) { + if (refTable.name.equals(table.name)) { + referencingTables.add(gtfsTable); + } + } } } } @@ -1535,16 +1610,18 @@ private void updateReferencingTables( String namespace, Table table, int id, - String newKeyValue + String newKeyValue, + Field keyField ) throws SQLException { - Field keyField = table.getFieldForName(table.getKeyFieldName()); Class entityClass = table.getEntityClass(); // Determine method (update vs. delete) depending on presence of newKeyValue field. SqlMethod sqlMethod = newKeyValue != null ? SqlMethod.UPDATE : SqlMethod.DELETE; Set
referencingTables = getReferencingTables(table); // If there are no referencing tables, there is no need to update any values (e.g., . if (referencingTables.size() == 0) return; - String keyValue = getValueForId(id, keyField.name, namespace, table, connection); + // Exception based service contains a single service ID in custom_schedule + String sqlKeyFieldName = keyField.name == "custom_schedule" ? "custom_schedule[1]" : keyField.name; + String keyValue = getValueForId(id, sqlKeyFieldName, namespace, table, connection); if (keyValue == null) { // FIXME: should we still check referencing tables for null value? LOG.warn("Entity {} to {} has null value for {}. Skipping references check.", id, sqlMethod, keyField); @@ -1568,76 +1645,80 @@ private void updateReferencingTables( } else { // General deletion for (Field field : referencingTable.editorFields()) { - if (field.isForeignReference() && field.referenceTable.name.equals(table.name)) { - // Get statement to update or delete entities that reference the key value. - PreparedStatement updateStatement = getUpdateReferencesStatement(sqlMethod, refTableName, field, keyValue, newKeyValue); - LOG.info(updateStatement.toString()); - result = updateStatement.executeUpdate(); - if (result > 0) { - // FIXME: is this where a delete hook should go? (E.g., CalendarController subclass would override - // deleteEntityHook). - if (sqlMethod.equals(SqlMethod.DELETE)) { - ArrayList patternAndRouteIds = new ArrayList<>(); - // Check for restrictions on delete. - if (table.isCascadeDeleteRestricted()) { - // The entity must not have any referencing entities in order to delete it. - connection.rollback(); - if (entityClass.getSimpleName().equals("Stop")) { - String patternStopLookup = String.format( - "select distinct p.id, r.id, r.route_short_name, r.route_id " + - "from %s.pattern_stops ps " + - "inner join " + - "%s.patterns p " + - "on p.pattern_id = ps.pattern_id " + - "inner join " + - "%s.routes r " + - "on p.route_id = r.route_id " + - "where %s = '%s'", - namespace, - namespace, - namespace, - keyField.name, - keyValue - ); - PreparedStatement patternStopSelectStatement = connection.prepareStatement(patternStopLookup); - if (patternStopSelectStatement.execute()) { - ResultSet resultSet = patternStopSelectStatement.getResultSet(); - while (resultSet.next()) { - patternAndRouteIds.add( - String.format("{%s-%s-%s-%s}", - getResultSetString(1, resultSet), - getResultSetString(2, resultSet), - getResultSetString(3, resultSet), - getResultSetString(4, resultSet) - ) + if (field.isForeignReference()) { + for (Table refTable : field.referenceTables) { + if (refTable.name.equals(table.name)) { + // Get statement to update or delete entities that reference the key value. + PreparedStatement updateStatement = getUpdateReferencesStatement(sqlMethod, refTableName, field, keyValue, newKeyValue); + LOG.info(updateStatement.toString()); + result = updateStatement.executeUpdate(); + if (result > 0) { + // FIXME: is this where a delete hook should go? (E.g., CalendarController subclass would override + // deleteEntityHook). + if (sqlMethod.equals(SqlMethod.DELETE)) { + ArrayList patternAndRouteIds = new ArrayList<>(); + // Check for restrictions on delete. + if (table.isCascadeDeleteRestricted()) { + // The entity must not have any referencing entities in order to delete it. + connection.rollback(); + if (entityClass.getSimpleName().equals("Stop")) { + String patternStopLookup = String.format( + "select distinct p.id, r.id " + + "from %s.pattern_stops ps " + + "inner join " + + "%s.patterns p " + + "on p.pattern_id = ps.pattern_id " + + "inner join " + + "%s.routes r " + + "on p.route_id = r.route_id " + + "where %s = '%s'", + namespace, + namespace, + namespace, + keyField.name, + keyValue + ); + PreparedStatement patternStopSelectStatement = connection.prepareStatement(patternStopLookup); + if (patternStopSelectStatement.execute()) { + ResultSet resultSet = patternStopSelectStatement.getResultSet(); + while (resultSet.next()) { + patternAndRouteIds.add( + String.format("{%s-%s-%s-%s}", + getResultSetString(1, resultSet), + getResultSetString(2, resultSet), + getResultSetString(3, resultSet), + getResultSetString(4, resultSet) + ) + ); + } + } + } + String message = String.format( + "Cannot delete %s %s=%s. %d %s reference this %s.", + entityClass.getSimpleName(), + keyField.name, + keyValue, + result, + referencingTable.name, + entityClass.getSimpleName() + ); + if (patternAndRouteIds.size() > 0) { + // Append referenced patterns data to the end of the error. + message = String.format( + "%s%nReferenced patterns: [%s]", + message, + StringUtils.join(patternAndRouteIds, ",") ); } + LOG.warn(message); + throw new SQLException(message); } } - String message = String.format( - "Cannot delete %s %s=%s. %d %s reference this %s.", - entityClass.getSimpleName(), - keyField.name, - keyValue, - result, - referencingTable.name, - entityClass.getSimpleName() - ); - if (patternAndRouteIds.size() > 0) { - // Append referenced patterns data to the end of the error. - message = String.format( - "%s\nReferenced patterns: [%s]", - message, - StringUtils.join(patternAndRouteIds, ",") - ); - } - LOG.warn(message); - throw new SQLException(message); + LOG.info("{} reference(s) in {} {}D!", result, refTableName, sqlMethod); + } else { + LOG.info("No references in {} found!", refTableName); } } - LOG.info("{} reference(s) in {} {}D!", result, refTableName, sqlMethod); - } else { - LOG.info("No references in {} found!", refTableName); } } } @@ -1645,6 +1726,25 @@ private void updateReferencingTables( } } + /** + * Traditional method signature for updateReferencingTables, updating exception based service requires + * passing the keyField. + * @param namespace + * @param table + * @param id + * @param newKeyValue + * @throws SQLException + */ + private void updateReferencingTables( + String namespace, + Table table, + int id, + String newKeyValue + ) throws SQLException { + Field keyField = table.getFieldForName(table.getKeyFieldName()); + updateReferencingTables(namespace, table, id, newKeyValue, keyField); + } + /** * To prevent orphaned descendants, delete them before joining references are deleted. For the relationship * route -> pattern -> pattern stop, delete pattern stop before deleting the joining pattern. diff --git a/src/main/java/com/conveyal/gtfs/loader/ReferenceTracker.java b/src/main/java/com/conveyal/gtfs/loader/ReferenceTracker.java index acd1178c7..32b3d57c7 100644 --- a/src/main/java/com/conveyal/gtfs/loader/ReferenceTracker.java +++ b/src/main/java/com/conveyal/gtfs/loader/ReferenceTracker.java @@ -1,6 +1,7 @@ package com.conveyal.gtfs.loader; import com.conveyal.gtfs.error.NewGTFSError; +import com.conveyal.gtfs.error.NewGTFSErrorType; import com.conveyal.gtfs.loader.conditions.ConditionalRequirement; import com.google.common.collect.HashMultimap; @@ -8,8 +9,10 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import static com.conveyal.gtfs.error.NewGTFSErrorType.DUPLICATE_ID; +import static com.conveyal.gtfs.error.NewGTFSErrorType.MISSING_FOREIGN_TABLE_REFERENCE; import static com.conveyal.gtfs.error.NewGTFSErrorType.REFERENTIAL_INTEGRITY; /** @@ -79,15 +82,19 @@ public Set checkReferencesAndUniqueness(String keyValue, int lineN // First, handle referential integrity check. boolean isOrderField = field.name.equals(orderField); if (field.isForeignReference()) { - // Check referential integrity if the field is a foreign reference. Note: the - // reference table must be loaded before the table/value being currently checked. - String referenceField = field.referenceTable.getKeyFieldName(); - String referenceTransitId = String.join(":", referenceField, value); - - if (!this.transitIds.contains(referenceTransitId)) { - // If the reference tracker does not contain + TreeSet badValues = new TreeSet<>(); + if (!hasMatchingReference(field, value, badValues)) { + // If the reference tracker does not contain a match. + NewGTFSErrorType errorType = (field.referenceTables.size() > 1) + ? MISSING_FOREIGN_TABLE_REFERENCE + : REFERENTIAL_INTEGRITY; NewGTFSError referentialIntegrityError = NewGTFSError - .forLine(table, lineNumber, REFERENTIAL_INTEGRITY, referenceTransitId) + .forLine( + table, + lineNumber, + errorType, + String.join(", ", badValues) + ) .setEntityId(keyValue); // If the field is an order field, set the sequence for the new error. if (isOrderField) referentialIntegrityError.setSequence(value); @@ -148,6 +155,32 @@ public Set checkReferencesAndUniqueness(String keyValue, int lineN return errors; } + /** + * Check foreign references. If the foreign reference is present in one of the tables, there is no + * need to check the remainder. If no matching foreign reference is found, flag integrity error. + * Note: The reference table must be loaded before the table/value being currently checked. + */ + private boolean hasMatchingReference(Field field, String value, TreeSet badValues) { + for (Table referenceTable : field.referenceTables) { + if (checkReference(referenceTable.getKeyFieldName(), value, badValues)) { + return true; + } + } + return false; + } + + /** + * Check that a reference is valid. + */ + private boolean checkReference(String referenceField, String reference, TreeSet badValues) { + String referenceTransitId = String.join(":", referenceField, reference); + if (this.transitIds.contains(referenceTransitId)) { + return true; + } else { + badValues.add(referenceTransitId); + } + return false; + } /** * Work through each conditionally required check assigned to fields within a table. First check the reference field diff --git a/src/main/java/com/conveyal/gtfs/loader/Table.java b/src/main/java/com/conveyal/gtfs/loader/Table.java index d13615452..1ac7a0030 100644 --- a/src/main/java/com/conveyal/gtfs/loader/Table.java +++ b/src/main/java/com/conveyal/gtfs/loader/Table.java @@ -153,7 +153,6 @@ public Table (String name, Class entityClass, Requirement requ public static final Table SCHEDULE_EXCEPTIONS = new Table("schedule_exceptions", ScheduleException.class, EDITOR, new StringField("name", REQUIRED), // FIXME: This makes name the key field... - // FIXME: Change to DateListField new DateListField("dates", REQUIRED), new ShortField("exemplar", REQUIRED, 9), new StringListField("custom_schedule", OPTIONAL).isReferenceTo(CALENDAR), @@ -162,7 +161,7 @@ public Table (String name, Class entityClass, Requirement requ ); public static final Table CALENDAR_DATES = new Table("calendar_dates", CalendarDate.class, OPTIONAL, - new StringField("service_id", REQUIRED).isReferenceTo(CALENDAR), + new StringField("service_id", REQUIRED), new DateField("date", REQUIRED), new IntegerField("exception_type", REQUIRED, 1, 2) ).keyFieldIsNotUnique() @@ -329,9 +328,8 @@ public Table (String name, Class entityClass, Requirement requ public static final Table TRIPS = new Table("trips", Trip.class, REQUIRED, new StringField("trip_id", REQUIRED), new StringField("route_id", REQUIRED).isReferenceTo(ROUTES).indexThisColumn(), - // FIXME: Should this also optionally reference CALENDAR_DATES? // FIXME: Do we need an index on service_id - new StringField("service_id", REQUIRED).isReferenceTo(CALENDAR), + new StringField("service_id", REQUIRED).isReferenceTo(CALENDAR).isReferenceTo(CALENDAR_DATES).isReferenceTo(SCHEDULE_EXCEPTIONS), new StringField("trip_headsign", OPTIONAL), new StringField("trip_short_name", OPTIONAL), new ShortField("direction_id", OPTIONAL, 1), diff --git a/src/main/java/com/conveyal/gtfs/model/Calendar.java b/src/main/java/com/conveyal/gtfs/model/Calendar.java index 22c6029b9..097ca0eaf 100644 --- a/src/main/java/com/conveyal/gtfs/model/Calendar.java +++ b/src/main/java/com/conveyal/gtfs/model/Calendar.java @@ -143,12 +143,7 @@ protected void writeOneRow(Calendar c) throws IOException { @Override protected Iterator iterator() { // wrap an iterator over services - Iterator calIt = Iterators.transform(feed.services.values().iterator(), new Function () { - @Override - public Calendar apply (Service s) { - return s.calendar; - } - }); + Iterator calIt = Iterators.transform(feed.services.values().iterator(), s -> s.calendar); // not every service has a calendar (e.g. TriMet has no calendars, just calendar dates). // This is legal GTFS, so skip services with no calendar diff --git a/src/main/java/com/conveyal/gtfs/model/CalendarDate.java b/src/main/java/com/conveyal/gtfs/model/CalendarDate.java index b737e62ff..8153e7544 100644 --- a/src/main/java/com/conveyal/gtfs/model/CalendarDate.java +++ b/src/main/java/com/conveyal/gtfs/model/CalendarDate.java @@ -109,12 +109,7 @@ protected void writeOneRow(CalendarDate d) throws IOException { @Override protected Iterator iterator() { Iterator serviceIterator = feed.services.values().iterator(); - return Iterators.concat(Iterators.transform(serviceIterator, new Function> () { - @Override - public Iterator apply(Service service) { - return service.calendar_dates.values().iterator(); - } - })); + return Iterators.concat(Iterators.transform(serviceIterator, service -> service.calendar_dates.values().iterator())); } } } diff --git a/src/main/java/com/conveyal/gtfs/model/ScheduleException.java b/src/main/java/com/conveyal/gtfs/model/ScheduleException.java index 932875ccc..e88c04658 100644 --- a/src/main/java/com/conveyal/gtfs/model/ScheduleException.java +++ b/src/main/java/com/conveyal/gtfs/model/ScheduleException.java @@ -73,6 +73,8 @@ public boolean serviceRunsOn(Calendar calendar) { if (removedService != null && removedService.contains(calendar.service_id)) { return false; } + case CALENDAR_DATE_SERVICE: + return false; default: // can't actually happen, but java requires a default with a return here return false; @@ -84,7 +86,7 @@ public boolean serviceRunsOn(Calendar calendar) { * For example, run Sunday service on Presidents' Day, or no service on New Year's Day. */ public enum ExemplarServiceDescriptor { - MONDAY(0), TUESDAY(1), WEDNESDAY(2), THURSDAY(3), FRIDAY(4), SATURDAY(5), SUNDAY(6), NO_SERVICE(7), CUSTOM(8), SWAP(9), MISSING(-1); + MONDAY(0), TUESDAY(1), WEDNESDAY(2), THURSDAY(3), FRIDAY(4), SATURDAY(5), SUNDAY(6), NO_SERVICE(7), CUSTOM(8), SWAP(9), CALENDAR_DATE_SERVICE(10), MISSING(-1); private final int value; @@ -119,6 +121,8 @@ public static ExemplarServiceDescriptor exemplarFromInt (int value) { return ExemplarServiceDescriptor.CUSTOM; case 9: return ExemplarServiceDescriptor.SWAP; + case 10: + return ExemplarServiceDescriptor.CALENDAR_DATE_SERVICE; default: return ExemplarServiceDescriptor.MISSING; } diff --git a/src/main/java/com/conveyal/gtfs/validator/ServiceValidator.java b/src/main/java/com/conveyal/gtfs/validator/ServiceValidator.java index c2ede5888..7e64b7af3 100644 --- a/src/main/java/com/conveyal/gtfs/validator/ServiceValidator.java +++ b/src/main/java/com/conveyal/gtfs/validator/ServiceValidator.java @@ -183,8 +183,8 @@ select durations.service_id, duration_seconds, days_active from ( for (String tripId : serviceInfo.tripIds) { registerError( NewGTFSError.forTable(Table.TRIPS, NewGTFSErrorType.TRIP_NEVER_ACTIVE) - .setEntityId(tripId) - .setBadValue(tripId)); + .setEntityId(tripId) + .setBadValue(tripId)); } } if (serviceInfo.tripIds.isEmpty()) { @@ -247,7 +247,7 @@ select durations.service_id, duration_seconds, days_active from ( // Check for low or zero service, which seems to happen even when services are defined. // This will also catch cases where dateInfo was null and the new instance contains no service. registerError(NewGTFSError.forFeed(NewGTFSErrorType.DATE_NO_SERVICE, - DateField.GTFS_DATE_FORMATTER.format(date))); + DateField.GTFS_DATE_FORMATTER.format(date))); } } } @@ -320,7 +320,7 @@ select durations.service_id, duration_seconds, days_active from ( String serviceDurationsTableName = feed.getTableNameWithSchemaPrefix("service_durations"); sql = String.format("create table %s (service_id varchar, route_type integer, " + - "duration_seconds integer, primary key (service_id, route_type))", serviceDurationsTableName); + "duration_seconds integer, primary key (service_id, route_type))", serviceDurationsTableName); LOG.info(sql); statement.execute(sql); sql = String.format("insert into %s values (?, ?, ?)", serviceDurationsTableName); diff --git a/src/test/java/com/conveyal/gtfs/GTFSFeedTest.java b/src/test/java/com/conveyal/gtfs/GTFSFeedTest.java index e92c54167..7afd7189b 100644 --- a/src/test/java/com/conveyal/gtfs/GTFSFeedTest.java +++ b/src/test/java/com/conveyal/gtfs/GTFSFeedTest.java @@ -62,10 +62,10 @@ public static void setUpClass() { } /** - * Make sure a roundtrip of loading a GTFS zip file and then writing another zip file can be performed. + * Make sure a round-trip of loading a GTFS zip file and then writing another zip file can be performed. */ @Test - public void canDoRoundtripLoadAndWriteToZipFile() throws IOException { + public void canDoRoundTripLoadAndWriteToZipFile() throws IOException { // create a temp file for this test File outZip = File.createTempFile("fake-agency-output", ".zip"); @@ -97,6 +97,14 @@ public void canDoRoundtripLoadAndWriteToZipFile() throws IOException { new DataExpectation("end_date", "20170917") } ), + new FileTestCase( + "calendar_dates.txt", + new DataExpectation[]{ + new DataExpectation("service_id", "calendar-date-service"), + new DataExpectation("date", "20170917"), + new DataExpectation("exception_type", "1") + } + ), new FileTestCase( "routes.txt", new DataExpectation[]{ diff --git a/src/test/java/com/conveyal/gtfs/GTFSTest.java b/src/test/java/com/conveyal/gtfs/GTFSTest.java index e14050f56..5eacb02a9 100644 --- a/src/test/java/com/conveyal/gtfs/GTFSTest.java +++ b/src/test/java/com/conveyal/gtfs/GTFSTest.java @@ -130,13 +130,21 @@ public void canLoadAndExportSimpleAgency() { * Tests that a GTFS feed with bad date values in calendars.txt and calendar_dates.txt can pass the integration test. */ @Test - public void canLoadFeedWithBadDates () { + void canLoadFeedWithBadDates () { PersistenceExpectation[] expectations = PersistenceExpectation.list( new PersistenceExpectation( "calendar", new RecordExpectation[]{ new RecordExpectation("start_date", null) } + ), + new PersistenceExpectation( + "calendar_dates", + new RecordExpectation[]{ + new RecordExpectation("service_id", "123_ID_NOT_EXISTS"), + new RecordExpectation("date", "20190301"), + new RecordExpectation("exception_type", "1") + } ) ); ErrorExpectation[] errorExpectations = ErrorExpectation.list( @@ -144,7 +152,6 @@ public void canLoadFeedWithBadDates () { new ErrorExpectation(NewGTFSErrorType.DATE_FORMAT), new ErrorExpectation(NewGTFSErrorType.DATE_FORMAT), new ErrorExpectation(NewGTFSErrorType.DATE_FORMAT), - new ErrorExpectation(NewGTFSErrorType.REFERENTIAL_INTEGRITY), new ErrorExpectation(NewGTFSErrorType.DATE_FORMAT), new ErrorExpectation(NewGTFSErrorType.DATE_FORMAT), // The below "wrong number of fields" errors are for empty new lines @@ -154,7 +161,7 @@ public void canLoadFeedWithBadDates () { new ErrorExpectation(NewGTFSErrorType.WRONG_NUMBER_OF_FIELDS), new ErrorExpectation(NewGTFSErrorType.WRONG_NUMBER_OF_FIELDS), new ErrorExpectation(NewGTFSErrorType.WRONG_NUMBER_OF_FIELDS), - new ErrorExpectation(NewGTFSErrorType.REFERENTIAL_INTEGRITY), + new ErrorExpectation(NewGTFSErrorType.MISSING_FOREIGN_TABLE_REFERENCE), new ErrorExpectation(NewGTFSErrorType.ROUTE_LONG_NAME_CONTAINS_SHORT_NAME), new ErrorExpectation(NewGTFSErrorType.FEED_TRAVEL_TIMES_ROUNDED), new ErrorExpectation(NewGTFSErrorType.SERVICE_NEVER_ACTIVE), @@ -416,25 +423,53 @@ public void canLoadAndExportSimpleAgencyWithMixtureOfCalendarDefinitions() { new RecordExpectation("exception_type", 2) } ), - // calendar-dates.txt-only expectation new PersistenceExpectation( - "calendar", + "calendar_dates", new RecordExpectation[]{ new RecordExpectation( "service_id", "only-in-calendar-dates-txt" ), - new RecordExpectation("start_date", 20170916), - new RecordExpectation("end_date", 20170916) - }, - true + new RecordExpectation("date", 20170916), + new RecordExpectation("exception_type", 1) + } ), new PersistenceExpectation( "calendar_dates", new RecordExpectation[]{ new RecordExpectation( - "service_id", "only-in-calendar-dates-txt" + "service_id", "calendar-dates-txt-service-one" ), - new RecordExpectation("date", 20170916), + new RecordExpectation("date", 20170917), + new RecordExpectation("exception_type", 1) + } + ), + new PersistenceExpectation( + "calendar_dates", + new RecordExpectation[]{ + new RecordExpectation( + "service_id", "calendar-dates-txt-service-two" + ), + new RecordExpectation("date", 20170918), + new RecordExpectation("exception_type", 1) + } + ), + new PersistenceExpectation( + "calendar_dates", + new RecordExpectation[]{ + new RecordExpectation( + "service_id", "calendar-dates-txt-service-three" + ), + new RecordExpectation("date", 20170917), + new RecordExpectation("exception_type", 1) + } + ), + new PersistenceExpectation( + "calendar_dates", + new RecordExpectation[]{ + new RecordExpectation( + "service_id", "calendar-dates-txt-service-three" + ), + new RecordExpectation("date", 20170918), new RecordExpectation("exception_type", 1) } ), @@ -514,7 +549,9 @@ public void canLoadAndExportSimpleAgencyWithMixtureOfCalendarDefinitions() { ErrorExpectation[] errorExpectations = ErrorExpectation.list( new ErrorExpectation(NewGTFSErrorType.MISSING_FIELD), new ErrorExpectation(NewGTFSErrorType.ROUTE_LONG_NAME_CONTAINS_SHORT_NAME), - new ErrorExpectation(NewGTFSErrorType.FEED_TRAVEL_TIMES_ROUNDED) + new ErrorExpectation(NewGTFSErrorType.ROUTE_LONG_NAME_CONTAINS_SHORT_NAME), + new ErrorExpectation(NewGTFSErrorType.FEED_TRAVEL_TIMES_ROUNDED), + new ErrorExpectation(NewGTFSErrorType.SERVICE_UNUSED) ); assertThat( runIntegrationTestOnFolder( @@ -1244,16 +1281,6 @@ private void assertThatPersistenceExpectationRecordWasFound( new RecordExpectation("end_date", "20170917") } ), - new PersistenceExpectation( - "calendar_dates", - new RecordExpectation[]{ - new RecordExpectation( - "service_id", "04100312-8fe1-46a5-a9f2-556f39478f57" - ), - new RecordExpectation("date", 20170916), - new RecordExpectation("exception_type", 2) - } - ), new PersistenceExpectation( "fare_attributes", new RecordExpectation[]{ diff --git a/src/test/resources/fake-agency-mixture-of-calendar-definitions/calendar_dates.txt b/src/test/resources/fake-agency-mixture-of-calendar-definitions/calendar_dates.txt index ea060f019..6e4f013d1 100755 --- a/src/test/resources/fake-agency-mixture-of-calendar-definitions/calendar_dates.txt +++ b/src/test/resources/fake-agency-mixture-of-calendar-definitions/calendar_dates.txt @@ -1,3 +1,7 @@ service_id,date,exception_type in-both-calendar-txt-and-calendar-dates,20170920,2 -only-in-calendar-dates-txt,20170916,1 \ No newline at end of file +only-in-calendar-dates-txt,20170916,1 +calendar-dates-txt-service-one,20170917,1 +calendar-dates-txt-service-two,20170918,1 +calendar-dates-txt-service-three,20170917,1 +calendar-dates-txt-service-three,20170918,1 \ No newline at end of file diff --git a/src/test/resources/fake-agency-mixture-of-calendar-definitions/routes.txt b/src/test/resources/fake-agency-mixture-of-calendar-definitions/routes.txt index 35ea7aa67..b13480efa 100755 --- a/src/test/resources/fake-agency-mixture-of-calendar-definitions/routes.txt +++ b/src/test/resources/fake-agency-mixture-of-calendar-definitions/routes.txt @@ -1,2 +1,3 @@ agency_id,route_id,route_short_name,route_long_name,route_desc,route_type,route_url,route_color,route_text_color,route_branding_url 1,1,1,Route 1,,3,,7CE6E7,FFFFFF, +1,2,2,Route 2,,3,,7CE6E7,FFFFFF, diff --git a/src/test/resources/fake-agency-mixture-of-calendar-definitions/stop_times.txt b/src/test/resources/fake-agency-mixture-of-calendar-definitions/stop_times.txt index fe0a9ad12..12ad079e6 100755 --- a/src/test/resources/fake-agency-mixture-of-calendar-definitions/stop_times.txt +++ b/src/test/resources/fake-agency-mixture-of-calendar-definitions/stop_times.txt @@ -5,3 +5,7 @@ non-frequency-trip-2,08:00:00,08:00:00,4u6g,1,,0,0,0.0000000, non-frequency-trip-2,08:01:00,08:01:00,johv,2,,0,0,341.4491961, frequency-trip,09:00:00,09:00:00,4u6g,1,,0,0,0.0000000, frequency-trip,09:01:00,09:01:00,johv,2,,0,0,341.4491961, +exception-trip-1,09:00:00,09:00:00,4u6g,1,,0,0,0.0000000, +exception-trip-1,09:01:00,09:01:00,johv,2,,0,0,341.4491961, +exception-trip-2,09:00:00,09:00:00,4u6g,1,,0,0,0.0000000, +exception-trip-2,09:01:00,09:01:00,johv,2,,0,0,341.4491961, diff --git a/src/test/resources/fake-agency-mixture-of-calendar-definitions/trips.txt b/src/test/resources/fake-agency-mixture-of-calendar-definitions/trips.txt index 077253974..221642959 100755 --- a/src/test/resources/fake-agency-mixture-of-calendar-definitions/trips.txt +++ b/src/test/resources/fake-agency-mixture-of-calendar-definitions/trips.txt @@ -1,4 +1,6 @@ route_id,trip_id,trip_headsign,trip_short_name,direction_id,block_id,shape_id,bikes_allowed,wheelchair_accessible,service_id 1,non-frequency-trip,,,0,,5820f377-f947-4728-ac29-ac0102cbc34e,0,0,only-in-calendar-dates-txt 1,non-frequency-trip-2,,,0,,5820f377-f947-4728-ac29-ac0102cbc34e,0,0,only-in-calendar-txt -1,frequency-trip,,,0,,5820f377-f947-4728-ac29-ac0102cbc34e,0,0,in-both-calendar-txt-and-calendar-dates \ No newline at end of file +1,frequency-trip,,,0,,5820f377-f947-4728-ac29-ac0102cbc34e,0,0,in-both-calendar-txt-and-calendar-dates +2,exception-trip-1,,,0,,5820f377-f947-4728-ac29-ac0102cbc34e,0,0,calendar-dates-txt-service-one +2,exception-trip-2,,,0,,5820f377-f947-4728-ac29-ac0102cbc34e,0,0,calendar-dates-txt-service-two \ No newline at end of file diff --git a/src/test/resources/fake-agency/calendar_dates.txt b/src/test/resources/fake-agency/calendar_dates.txt index 403ee2bbe..5d0a31806 100755 --- a/src/test/resources/fake-agency/calendar_dates.txt +++ b/src/test/resources/fake-agency/calendar_dates.txt @@ -1,2 +1,3 @@ service_id,date,exception_type -04100312-8fe1-46a5-a9f2-556f39478f57,20170916,2 \ No newline at end of file +04100312-8fe1-46a5-a9f2-556f39478f57,20170916,2 +calendar-date-service,20170917,1 \ No newline at end of file diff --git a/src/test/resources/fake-agency/stop_times.txt b/src/test/resources/fake-agency/stop_times.txt index 5d8793689..1cfefaa49 100755 --- a/src/test/resources/fake-agency/stop_times.txt +++ b/src/test/resources/fake-agency/stop_times.txt @@ -3,3 +3,5 @@ a30277f8-e50a-4a85-9141-b1e0da9d429d,07:00:00,07:00:00,4u6g,1,Test stop headsign a30277f8-e50a-4a85-9141-b1e0da9d429d,07:01:00,07:01:00,johv,2,Test stop headsign 2,0,0,341.4491961, frequency-trip,08:00:00,08:00:00,4u6g,1,Test stop headsign frequency trip,0,0,0.0000000, frequency-trip,08:29:00,08:29:00,1234,2,Test stop headsign frequency trip 2,0,0,341.4491961, +calendar-date-trip,08:00:00,08:00:00,4u6g,1,Test stop headsign calendar date trip,0,0,0.0000000, +calendar-date-trip,08:29:00,08:29:00,1234,2,Test stop headsign calendar date trip 2,0,0,341.4491961, \ No newline at end of file diff --git a/src/test/resources/fake-agency/trips.txt b/src/test/resources/fake-agency/trips.txt index eab14b86d..982c01e0f 100755 --- a/src/test/resources/fake-agency/trips.txt +++ b/src/test/resources/fake-agency/trips.txt @@ -1,3 +1,4 @@ route_id,trip_id,trip_headsign,trip_short_name,direction_id,block_id,shape_id,bikes_allowed,wheelchair_accessible,service_id 1,a30277f8-e50a-4a85-9141-b1e0da9d429d,,,0,,5820f377-f947-4728-ac29-ac0102cbc34e,0,0,04100312-8fe1-46a5-a9f2-556f39478f57 -1,frequency-trip,,,0,,5820f377-f947-4728-ac29-ac0102cbc34e,0,0,04100312-8fe1-46a5-a9f2-556f39478f57 \ No newline at end of file +1,frequency-trip,,,0,,5820f377-f947-4728-ac29-ac0102cbc34e,0,0,04100312-8fe1-46a5-a9f2-556f39478f57 +1,calendar-date-trip,,,0,,5820f377-f947-4728-ac29-ac0102cbc34e,0,0,calendar-date-service \ No newline at end of file diff --git a/src/test/resources/graphql/feedRowCounts.txt b/src/test/resources/graphql/feedRowCounts.txt index 116395f6f..74ec31965 100644 --- a/src/test/resources/graphql/feedRowCounts.txt +++ b/src/test/resources/graphql/feedRowCounts.txt @@ -5,7 +5,6 @@ query($namespace: String) { row_counts { agency calendar - calendar_dates errors routes stops diff --git a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchFeedRowCounts-0.json b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchFeedRowCounts-0.json index 4f2b230be..a8f58d527 100644 --- a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchFeedRowCounts-0.json +++ b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchFeedRowCounts-0.json @@ -5,12 +5,11 @@ "row_counts" : { "agency" : 1, "calendar" : 1, - "calendar_dates" : 1, "errors" : 6, "routes" : 1, - "stop_times" : 4, + "stop_times" : 6, "stops" : 5, - "trips" : 2 + "trips" : 3 }, "snapshot_of" : null } diff --git a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchPatterns-0.json b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchPatterns-0.json index 3732092fe..5c45497fd 100644 --- a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchPatterns-0.json +++ b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchPatterns-0.json @@ -14,11 +14,11 @@ "id" : 1, "pattern_id" : "1", "pickup_type" : 0, - "stop_headsign" : "Test stop headsign", "shape_dist_traveled" : 0.0, "stop" : [ { "stop_id" : "4u6g" } ], + "stop_headsign" : "Test stop headsign", "stop_id" : "4u6g", "stop_sequence" : 0, "timepoint" : null @@ -29,11 +29,11 @@ "id" : 2, "pattern_id" : "1", "pickup_type" : 0, - "stop_headsign" : "Test stop headsign 2", "shape_dist_traveled" : 341.4491961, "stop" : [ { "stop_id" : "johv" } ], + "stop_headsign" : "Test stop headsign 2", "stop_id" : "johv", "stop_sequence" : 1, "timepoint" : null @@ -106,7 +106,7 @@ }, { "direction_id" : 0, "id" : 2, - "name" : "2 stops from Butler Ln to Child Stop (1 trips)", + "name" : "2 stops from Butler Ln to Child Stop (2 trips)", "pattern_id" : "2", "pattern_stops" : [ { "default_dwell_time" : 0, @@ -115,11 +115,11 @@ "id" : 3, "pattern_id" : "2", "pickup_type" : 0, - "stop_headsign" : "Test stop headsign frequency trip", "shape_dist_traveled" : 0.0, "stop" : [ { "stop_id" : "4u6g" } ], + "stop_headsign" : "Test stop headsign calendar date trip", "stop_id" : "4u6g", "stop_sequence" : 0, "timepoint" : null @@ -130,11 +130,11 @@ "id" : 4, "pattern_id" : "2", "pickup_type" : 0, - "stop_headsign" : "Test stop headsign frequency trip 2", "shape_dist_traveled" : 341.4491961, "stop" : [ { "stop_id" : "1234" } ], + "stop_headsign" : "Test stop headsign calendar date trip 2", "stop_id" : "1234", "stop_sequence" : 1, "timepoint" : null @@ -199,9 +199,11 @@ }, { "stop_id" : "1234" } ], - "trip_count" : 1, + "trip_count" : 2, "trips" : [ { "trip_id" : "frequency-trip" + }, { + "trip_id" : "calendar-date-trip" } ], "use_frequency" : null } ] diff --git a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchRoutes-0.json b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchRoutes-0.json index 4792aa782..1c1a10c26 100644 --- a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchRoutes-0.json +++ b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchRoutes-0.json @@ -30,11 +30,13 @@ }, { "stop_id" : "1234" } ], - "trip_count" : 2, + "trip_count" : 3, "trips" : [ { "trip_id" : "a30277f8-e50a-4a85-9141-b1e0da9d429d" }, { "trip_id" : "frequency-trip" + }, { + "trip_id" : "calendar-date-trip" } ], "wheelchair_accessible" : null } ] diff --git a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchServices-0.json b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchServices-0.json index 26fc242ed..9911140b9 100644 --- a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchServices-0.json +++ b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchServices-0.json @@ -16,6 +16,18 @@ }, { "trip_id" : "frequency-trip" } ] + }, { + "dates" : [ "20170917" ], + "duration_seconds" : "1740", + "durations" : [ { + "duration_seconds" : 1740, + "route_type" : 3 + } ], + "n_days_active" : "1", + "service_id" : "calendar-date-service", + "trips" : [ { + "trip_id" : "calendar-date-trip" + } ] } ] } } diff --git a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchStopTimes-0.json b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchStopTimes-0.json index 370176de9..08f52fc97 100644 --- a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchStopTimes-0.json +++ b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchStopTimes-0.json @@ -41,11 +41,33 @@ "drop_off_type" : 0, "pickup_type" : 0, "shape_dist_traveled" : 341.4491961, - "stop_headsign" : "Test stop headsign frequency trip", + "stop_headsign" : "Test stop headsign frequency trip 2", "stop_id" : "1234", "stop_sequence" : 2, "timepoint" : null, "trip_id" : "frequency-trip" + }, { + "arrival_time" : 28800, + "departure_time" : 28800, + "drop_off_type" : 0, + "pickup_type" : 0, + "shape_dist_traveled" : 0.0, + "stop_headsign" : "Test stop headsign calendar date trip", + "stop_id" : "4u6g", + "stop_sequence" : 1, + "timepoint" : null, + "trip_id" : "calendar-date-trip" + }, { + "arrival_time" : 30540, + "departure_time" : 30540, + "drop_off_type" : 0, + "pickup_type" : 0, + "shape_dist_traveled" : 341.4491961, + "stop_headsign" : "Test stop headsign calendar date trip 2", + "stop_id" : "1234", + "stop_sequence" : 2, + "timepoint" : null, + "trip_id" : "calendar-date-trip" } ] } } diff --git a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchStops-0.json b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchStops-0.json index 0594014cd..15a993f2d 100644 --- a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchStops-0.json +++ b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchStops-0.json @@ -20,7 +20,7 @@ "stop_lat" : 37.0612132, "stop_lon" : -122.0074332, "stop_name" : "Butler Ln", - "stop_time_count" : 2, + "stop_time_count" : 3, "stop_times" : [ { "stop_id" : "4u6g", "stop_sequence" : 1, @@ -29,6 +29,10 @@ "stop_id" : "4u6g", "stop_sequence" : 1, "trip_id" : "frequency-trip" + }, { + "stop_id" : "4u6g", + "stop_sequence" : 1, + "trip_id" : "calendar-date-trip" } ], "stop_timezone" : null, "stop_url" : null, @@ -94,11 +98,15 @@ "stop_lat" : 37.06662, "stop_lon" : -122.07772, "stop_name" : "Child Stop", - "stop_time_count" : 1, + "stop_time_count" : 2, "stop_times" : [ { "stop_id" : "1234", "stop_sequence" : 2, "trip_id" : "frequency-trip" + }, { + "stop_id" : "1234", + "stop_sequence" : 2, + "trip_id" : "calendar-date-trip" } ], "stop_timezone" : null, "stop_url" : null, diff --git a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchTrips-0.json b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchTrips-0.json index deb41d06b..6e31418a8 100644 --- a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchTrips-0.json +++ b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchTrips-0.json @@ -150,6 +150,79 @@ "trip_id" : "frequency-trip", "trip_short_name" : null, "wheelchair_accessible" : 0 + }, { + "bikes_allowed" : 0, + "block_id" : null, + "direction_id" : 0, + "frequencies" : [ ], + "id" : 4, + "pattern_id" : "2", + "route_id" : "1", + "service_id" : "calendar-date-service", + "shape" : [ { + "point_type" : null, + "shape_dist_traveled" : 0.0, + "shape_id" : "5820f377-f947-4728-ac29-ac0102cbc34e", + "shape_pt_lat" : 37.0612132, + "shape_pt_lon" : -122.0074332, + "shape_pt_sequence" : 1 + }, { + "point_type" : null, + "shape_dist_traveled" : 7.4997067, + "shape_id" : "5820f377-f947-4728-ac29-ac0102cbc34e", + "shape_pt_lat" : 37.061172, + "shape_pt_lon" : -122.0075, + "shape_pt_sequence" : 2 + }, { + "point_type" : null, + "shape_dist_traveled" : 33.8739075, + "shape_id" : "5820f377-f947-4728-ac29-ac0102cbc34e", + "shape_pt_lat" : 37.061359, + "shape_pt_lon" : -122.007683, + "shape_pt_sequence" : 3 + }, { + "point_type" : null, + "shape_dist_traveled" : 109.0402932, + "shape_id" : "5820f377-f947-4728-ac29-ac0102cbc34e", + "shape_pt_lat" : 37.060878, + "shape_pt_lon" : -122.008278, + "shape_pt_sequence" : 4 + }, { + "point_type" : null, + "shape_dist_traveled" : 184.6078298, + "shape_id" : "5820f377-f947-4728-ac29-ac0102cbc34e", + "shape_pt_lat" : 37.060359, + "shape_pt_lon" : -122.008828, + "shape_pt_sequence" : 5 + }, { + "point_type" : null, + "shape_dist_traveled" : 265.8053023, + "shape_id" : "5820f377-f947-4728-ac29-ac0102cbc34e", + "shape_pt_lat" : 37.059761, + "shape_pt_lon" : -122.009354, + "shape_pt_sequence" : 6 + }, { + "point_type" : null, + "shape_dist_traveled" : 357.8617018, + "shape_id" : "5820f377-f947-4728-ac29-ac0102cbc34e", + "shape_pt_lat" : 37.059066, + "shape_pt_lon" : -122.009919, + "shape_pt_sequence" : 7 + } ], + "shape_id" : "5820f377-f947-4728-ac29-ac0102cbc34e", + "stop_times" : [ { + "stop_id" : "4u6g", + "stop_sequence" : 1, + "trip_id" : "calendar-date-trip" + }, { + "stop_id" : "1234", + "stop_sequence" : 2, + "trip_id" : "calendar-date-trip" + } ], + "trip_headsign" : null, + "trip_id" : "calendar-date-trip", + "trip_short_name" : null, + "wheelchair_accessible" : 0 } ] } } diff --git a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canSanitizeSQLInjectionSentAsKeyValue-0.json b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canSanitizeSQLInjectionSentAsKeyValue-0.json index 9b1e74973..04c98ecdc 100644 --- a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canSanitizeSQLInjectionSentAsKeyValue-0.json +++ b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canSanitizeSQLInjectionSentAsKeyValue-0.json @@ -30,11 +30,13 @@ }, { "stop_id" : "1234" } ], - "trip_count" : 2, + "trip_count" : 3, "trips" : [ { "trip_id" : "a30277f8-e50a-4a85-9141-b1e0da9d429d" }, { "trip_id" : "frequency-trip" + }, { + "trip_id" : "calendar-date-trip" } ], "wheelchair_accessible" : null } ]