diff --git a/src/main/java/com/conveyal/gtfs/loader/Field.java b/src/main/java/com/conveyal/gtfs/loader/Field.java index 13c2c74a9..3531fdaa5 100644 --- a/src/main/java/com/conveyal/gtfs/loader/Field.java +++ b/src/main/java/com/conveyal/gtfs/loader/Field.java @@ -8,7 +8,7 @@ import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.SQLType; -import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.Set; /** @@ -47,7 +47,7 @@ public abstract class Field { * Indicates that this field acts as a foreign key to this referenced table. This is used when checking referential * integrity when loading a feed. * */ - public HashSet referenceTables = new HashSet<>(); + public Set
referenceTables = new LinkedHashSet<>(); private boolean shouldBeIndexed; private boolean emptyValuePermitted; private boolean isConditionallyRequired; diff --git a/src/main/java/com/conveyal/gtfs/loader/JDBCTableReader.java b/src/main/java/com/conveyal/gtfs/loader/JDBCTableReader.java index 6f4fa07c2..0eaa80331 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JDBCTableReader.java +++ b/src/main/java/com/conveyal/gtfs/loader/JDBCTableReader.java @@ -1,5 +1,6 @@ package com.conveyal.gtfs.loader; +import com.conveyal.gtfs.model.Calendar; import com.conveyal.gtfs.model.Entity; import com.conveyal.gtfs.storage.StorageException; import gnu.trove.map.TObjectIntMap; @@ -146,6 +147,18 @@ public int getRowCount() { } } + /** + * Provide reader for calendar table. + */ + public static JDBCTableReader getCalendarTableReader(DataSource dataSource, String tablePrefix) { + return new JDBCTableReader( + Table.CALENDAR, + dataSource, + tablePrefix + ".", + EntityPopulator.CALENDAR + ); + } + private class EntityIterator implements Iterator { private Connection connection; // Will remain open for the duration of the iteration. diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java index 4c1919439..430f6434c 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java @@ -28,13 +28,16 @@ import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.sql.Connection; import java.sql.SQLException; import java.time.LocalDate; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -86,7 +89,7 @@ public JdbcGtfsExporter(String feedId, String outFile, DataSource dataSource, bo /** * Utility method to check if an exception uses a specific service. */ - public Boolean exceptionInvolvesService(ScheduleException ex, String serviceId) { + public boolean exceptionInvolvesService(ScheduleException ex, String serviceId) { return ( ex.addedService.contains(serviceId) || ex.removedService.contains(serviceId) || @@ -106,7 +109,7 @@ public FeedLoadResult exportTables() { FeedLoadResult result = new FeedLoadResult(); try { - zipOutputStream = new ZipOutputStream(new FileOutputStream(outFile)); + zipOutputStream = new ZipOutputStream(Files.newOutputStream(Paths.get(outFile))); long startTime = System.currentTimeMillis(); // We get a single connection object and share it across several different methods. // This ensures that actions taken in one method are visible to all subsequent SQL statements. @@ -142,40 +145,55 @@ public FeedLoadResult exportTables() { if (fromEditor) { // Export schedule exceptions in place of calendar dates if exporting a feed/schema that represents an editor snapshot. GTFSFeed feed = new GTFSFeed(); - // FIXME: The below table readers should probably just share a connection with the exporter. - JDBCTableReader exceptionsReader = - new JDBCTableReader(Table.SCHEDULE_EXCEPTIONS, dataSource, feedIdToExport + ".", - EntityPopulator.SCHEDULE_EXCEPTION); - JDBCTableReader calendarsReader = - new JDBCTableReader(Table.CALENDAR, dataSource, feedIdToExport + ".", - EntityPopulator.CALENDAR); - Iterable calendars = calendarsReader.getAll(); + JDBCTableReader exceptionsReader =new JDBCTableReader( + Table.SCHEDULE_EXCEPTIONS, + dataSource, + feedIdToExport + ".", + EntityPopulator.SCHEDULE_EXCEPTION + ); + JDBCTableReader calendarReader = JDBCTableReader.getCalendarTableReader(dataSource, feedIdToExport); + Iterable calendars = calendarReader.getAll(); Iterable exceptionsIterator = exceptionsReader.getAll(); - List exceptions = new ArrayList<>(); - // FIXME: Doing this causes the connection to stay open, but it is closed in the finalizer so it should - // not be a big problem. - for (ScheduleException exception : exceptionsIterator) { - exceptions.add(exception); + List calendarExceptions = new ArrayList<>(); + List calendarDateExceptions = new ArrayList<>(); + // Separate distinct calendar date exceptions from those associated with calendars. + for (ScheduleException ex : exceptionsIterator) { + if (ex.exemplar.equals(ScheduleException.ExemplarServiceDescriptor.CALENDAR_DATE_SERVICE)) { + calendarDateExceptions.add(ex); + } else { + calendarExceptions.add(ex); + } } + + int calendarDateCount = calendarDateExceptions.size(); + // Extract calendar date services, convert to calendar date and add to the feed. + for (ScheduleException ex : calendarDateExceptions) { + for (LocalDate date : ex.dates) { + String serviceId = ex.customSchedule.get(0); + CalendarDate calendarDate = new CalendarDate(); + calendarDate.date = date; + calendarDate.service_id = serviceId; + calendarDate.exception_type = 1; + Service service = new Service(serviceId); + service.calendar_dates.put(date, calendarDate); + // If the calendar dates provided contain duplicates (e.g. two or more identical service ids + // that are NOT associated with a calendar) only the first entry would persist export. To + // resolve this a unique key consisting of service id and date is used. + feed.services.put(String.format("%s-%s", calendarDate.service_id, calendarDate.date), service); + } + } + // check whether the feed is organized in a format with the calendars.txt file - if (calendarsReader.getRowCount() > 0) { + if (calendarReader.getRowCount() > 0) { // feed does have calendars.txt file, continue export with strategy of matching exceptions // to calendar to output calendar_dates.txt - int calendarDateCount = 0; for (Calendar cal : calendars) { Service service = new Service(cal.service_id); service.calendar = cal; - for (ScheduleException ex : exceptions.stream() + for (ScheduleException ex : calendarExceptions.stream() .filter(ex -> exceptionInvolvesService(ex, cal.service_id)) .collect(Collectors.toList()) ) { - if (ex.exemplar.equals(ScheduleException.ExemplarServiceDescriptor.SWAP) && - (!ex.addedService.contains(cal.service_id) && !ex.removedService.contains(cal.service_id))) { - // Skip swap exception if cal is not referenced by added or removed service. - // This is not technically necessary, but the output is cleaner/more intelligible. - continue; - } - for (LocalDate date : ex.dates) { if (date.isBefore(cal.start_date) || date.isAfter(cal.end_date)) { // No need to write dates that do not apply @@ -189,7 +207,7 @@ public FeedLoadResult exportTables() { LOG.info("Adding exception {} (type={}) for calendar {} on date {}", ex.name, calendarDate.exception_type, cal.service_id, date); if (service.calendar_dates.containsKey(date)) - throw new IllegalArgumentException("Duplicate schedule exceptions on " + date.toString()); + throw new IllegalArgumentException("Duplicate schedule exceptions on " + date); service.calendar_dates.put(date, calendarDate); calendarDateCount += 1; @@ -197,14 +215,16 @@ public FeedLoadResult exportTables() { } feed.services.put(cal.service_id, service); } - if (calendarDateCount == 0) { - LOG.info("No calendar dates found. Skipping table."); - } else { - LOG.info("Writing {} calendar dates from schedule exceptions", calendarDateCount); - new CalendarDate.Writer(feed).writeTable(zipOutputStream); - } + } + if (calendarDateCount == 0) { + LOG.info("No calendar dates found. Skipping table."); } else { - // No calendar records exist, export calendar_dates as is and hope for the best. + LOG.info("Writing {} calendar dates from schedule exceptions", calendarDateCount); + new CalendarDate.Writer(feed).writeTable(zipOutputStream); + } + + if (calendarReader.getRowCount() == 0 && calendarDateExceptions.isEmpty()) { + // No calendar or calendar date service records exist, export calendar_dates as is and hope for the best. // This situation will occur in at least 2 scenarios: // 1. A GTFS has been loaded into the editor that had only the calendar_dates.txt file // and no further edits were made before exporting to a snapshot diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsSnapshotter.java b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsSnapshotter.java index 74f182b65..b310924af 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsSnapshotter.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsSnapshotter.java @@ -2,6 +2,7 @@ import com.conveyal.gtfs.model.Calendar; import com.conveyal.gtfs.model.CalendarDate; +import com.conveyal.gtfs.model.ScheduleException; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; @@ -217,15 +218,14 @@ private TableLoadResult createScheduleExceptionsTable() { tablePrefix.replace(".", ""), true ); - String sql = String.format( - "insert into %s (name, dates, exemplar, added_service, removed_service) values (?, ?, ?, ?, ?)", - scheduleExceptionsTableName - ); - PreparedStatement scheduleExceptionsStatement = connection.prepareStatement(sql); - final BatchTracker scheduleExceptionsTracker = new BatchTracker( - "schedule_exceptions", - scheduleExceptionsStatement - ); + + // Fetch all entries in the calendar table to generate set of serviceIds that exist in the calendar + // table. + JDBCTableReader calendarReader = JDBCTableReader.getCalendarTableReader(dataSource, feedIdToSnapshot); + Set calendarServiceIds = new HashSet<>(); + for (Calendar calendar : calendarReader.getAll()) { + calendarServiceIds.add(calendar.service_id); + } JDBCTableReader calendarDatesReader = new JDBCTableReader( Table.CALENDAR_DATES, @@ -238,65 +238,80 @@ private TableLoadResult createScheduleExceptionsTable() { // Keep track of calendars by service id in case we need to add dummy calendar entries. Map dummyCalendarsByServiceId = new HashMap<>(); - // Iterate through calendar dates to build up to get maps from exceptions to their dates. + // Iterate through calendar dates to build up appropriate service dates. Multimap removedServiceForDate = HashMultimap.create(); Multimap addedServiceForDate = HashMultimap.create(); + HashMap> calendarDateService = new HashMap<>(); for (CalendarDate calendarDate : calendarDates) { - // Skip any null dates - if (calendarDate.date == null) { - LOG.warn("Encountered calendar date record with null value for date field. Skipping."); + // Skip any null dates or service ids. + if (calendarDate.date == null || calendarDate.service_id == null) { + LOG.warn("Encountered calendar date record with null value for date/service_id field. Skipping."); continue; } String date = calendarDate.date.format(DateTimeFormatter.BASIC_ISO_DATE); - if (calendarDate.exception_type == 1) { - addedServiceForDate.put(date, calendarDate.service_id); - // create (if needed) and extend range of dummy calendar that would need to be created if we are - // copying from a feed that doesn't have the calendar.txt file - Calendar calendar = dummyCalendarsByServiceId.getOrDefault(calendarDate.service_id, new Calendar()); - calendar.service_id = calendarDate.service_id; - if (calendar.start_date == null || calendar.start_date.isAfter(calendarDate.date)) { - calendar.start_date = calendarDate.date; - } - if (calendar.end_date == null || calendar.end_date.isBefore(calendarDate.date)) { - calendar.end_date = calendarDate.date; + if (calendarServiceIds.contains(calendarDate.service_id)) { + // Calendar date is related to a calendar. + if (calendarDate.exception_type == 1) { + addedServiceForDate.put(date, calendarDate.service_id); + extendDummyCalendarRange(dummyCalendarsByServiceId, calendarDate); + } else { + removedServiceForDate.put(date, calendarDate.service_id); } - dummyCalendarsByServiceId.put(calendarDate.service_id, calendar); } else { - removedServiceForDate.put(date, calendarDate.service_id); + // Calendar date is not related to a calendar. Group calendar dates by service id. + if (calendarDateService.containsKey(calendarDate.service_id)) { + calendarDateService.get(calendarDate.service_id).add(date); + } else { + Set dates = new HashSet<>(); + dates.add(date); + calendarDateService.put(calendarDate.service_id, dates); + } + } } + + String sql = String.format( + "insert into %s (name, dates, exemplar, custom_schedule, added_service, removed_service) values (?, ?, ?, ?, ?, ?)", + scheduleExceptionsTableName + ); + PreparedStatement scheduleExceptionsStatement = connection.prepareStatement(sql); + final BatchTracker scheduleExceptionsTracker = new BatchTracker( + "schedule_exceptions", + scheduleExceptionsStatement + ); + // Iterate through dates with added or removed service and add to database. // For usability and simplicity of code, don't attempt to find all dates with similar // added and removed services, but simply create an entry for each found date. for (String date : Sets.union(removedServiceForDate.keySet(), addedServiceForDate.keySet())) { - scheduleExceptionsStatement.setString(1, date); - String[] dates = {date}; - scheduleExceptionsStatement.setArray(2, connection.createArrayOf("text", dates)); - scheduleExceptionsStatement.setInt(3, 9); // FIXME use better static type - scheduleExceptionsStatement.setArray( - 4, - connection.createArrayOf("text", addedServiceForDate.get(date).toArray()) + createScheduledExceptionStatement( + scheduleExceptionsStatement, + scheduleExceptionsTracker, + date, + new String[] {date}, + ScheduleException.ExemplarServiceDescriptor.SWAP, + new String[] {}, + addedServiceForDate.get(date).toArray(), + removedServiceForDate.get(date).toArray() ); - scheduleExceptionsStatement.setArray( - 5, - connection.createArrayOf("text", removedServiceForDate.get(date).toArray()) - ); - scheduleExceptionsTracker.addBatch(); } - scheduleExceptionsTracker.executeRemaining(); - // fetch all entries in the calendar table to generate set of serviceIds that exist in the calendar - // table. - JDBCTableReader calendarReader = new JDBCTableReader( - Table.CALENDAR, - dataSource, - feedIdToSnapshot + ".", - EntityPopulator.CALENDAR - ); - Set calendarServiceIds = new HashSet<>(); - for (Calendar calendar : calendarReader.getAll()) { - calendarServiceIds.add(calendar.service_id); + for (Map.Entry> entry : calendarDateService.entrySet()) { + String serviceId = entry.getKey(); + String[] dates = entry.getValue().toArray(new String[0]); + createScheduledExceptionStatement( + scheduleExceptionsStatement, + scheduleExceptionsTracker, + // Unique-ish schedule name that shouldn't conflict with existing service ids. + String.format("%s-%s", serviceId, dates[0]), + dates, + ScheduleException.ExemplarServiceDescriptor.CALENDAR_DATE_SERVICE, + new String[] {serviceId}, + new String[] {}, + new String[] {} + ); } + scheduleExceptionsTracker.executeRemaining(); // For service_ids that only existed in the calendar_dates table, insert auto-generated, "blank" // (no days of week specified) calendar entries. @@ -349,6 +364,53 @@ private TableLoadResult createScheduleExceptionsTable() { } } + /** + * Create (if needed) and extend range of dummy calendars that would need to be created if we are copying from a + * feed that doesn't have the calendar.txt file. + */ + private void extendDummyCalendarRange(Map dummyCalendarsByServiceId, CalendarDate calendarDate) { + Calendar calendar = dummyCalendarsByServiceId.getOrDefault(calendarDate.service_id, new Calendar()); + calendar.service_id = calendarDate.service_id; + if (calendar.start_date == null || calendar.start_date.isAfter(calendarDate.date)) { + calendar.start_date = calendarDate.date; + } + if (calendar.end_date == null || calendar.end_date.isBefore(calendarDate.date)) { + calendar.end_date = calendarDate.date; + } + dummyCalendarsByServiceId.put(calendarDate.service_id, calendar); + } + + /** + * Populate schedule exception statement and add to batch tracker. + */ + private void createScheduledExceptionStatement( + PreparedStatement scheduleExceptionsStatement, + BatchTracker scheduleExceptionsTracker, + String name, + String[] dates, + ScheduleException.ExemplarServiceDescriptor exemplarServiceDescriptor, + Object[] customSchedule, + Object[] addedServicesForDate, + Object[] removedServicesForDate + ) throws SQLException { + scheduleExceptionsStatement.setString(1, name); + scheduleExceptionsStatement.setArray(2, connection.createArrayOf("text", dates)); + scheduleExceptionsStatement.setInt(3, exemplarServiceDescriptor.getValue()); + scheduleExceptionsStatement.setArray( + 4, + connection.createArrayOf("text", customSchedule) + ); + scheduleExceptionsStatement.setArray( + 5, + connection.createArrayOf("text", addedServicesForDate) + ); + scheduleExceptionsStatement.setArray( + 6, + connection.createArrayOf("text", removedServicesForDate) + ); + scheduleExceptionsTracker.addBatch(); + } + /** * Helper method to determine if a table exists within a namespace. */ diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java index 81ebceb85..5dd43be65 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java @@ -4,6 +4,7 @@ import com.conveyal.gtfs.model.Location; import com.conveyal.gtfs.model.LocationShape; import com.conveyal.gtfs.model.Stop; +import com.conveyal.gtfs.model.ScheduleException.ExemplarServiceDescriptor; import com.conveyal.gtfs.model.Shape; import com.conveyal.gtfs.model.StopTime; import com.conveyal.gtfs.storage.StorageException; @@ -20,6 +21,7 @@ import gnu.trove.set.TIntSet; import gnu.trove.set.hash.TIntHashSet; import org.apache.commons.dbutils.DbUtils; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -726,10 +728,16 @@ private String updateChildTable( } /** +<<<<<<< HEAD * Check any references the sub entity might have. For example, this checks that stop_id values on * pattern_stops refer to entities that actually exist in the stops table. NOTE: This skips the "specTable", * i.e., for pattern stops it will not check pattern_id references. This is enforced above with the put key * field statement above. +======= + * Check any references the sub entity might have. For example, this checks that a service_id defined in a trip + * refers to a calendar or calendar date. NOTE: This skips the "specTable", i.e., for pattern stops it will not + * check pattern_id references. This is enforced above with the put key field statement above. +>>>>>>> dev */ private void checkTableReferences( Multimap> foreignReferencesPerTable, @@ -1031,6 +1039,58 @@ private static long handleStatementExecution(PreparedStatement statement, boolea } } + private void checkUniqueIdsAndUpdateReferencingTables( + TIntSet uniqueIds, + Integer id, + String namespace, + Table table, + String keyValue, + Boolean isCreating, + Field keyField + ) throws SQLException { + int size = uniqueIds.size(); + if (size == 0 || (size == 1 && id != null && uniqueIds.contains(id))) { + // OK. + if (size == 0 && !isCreating) { + // FIXME: Need to update referencing tables because entity has changed ID. + // Entity key value is being changed to an entirely new one. If there are entities that + // reference this value, we need to update them. + updateReferencingTables(namespace, table, id, keyValue, keyField); + } + } else { + // Conflict. The different conflict conditions are outlined below. + if (size == 1) { + // There was one match found. + if (isCreating) { + // Under no circumstance should a new entity have a conflict with existing key field. + throw new SQLException( + String.format("New %s's %s value (%s) conflicts with an existing record in table.", + table.entityClass.getSimpleName(), + keyField.name, + keyValue) + ); + } + if (!uniqueIds.contains(id)) { + // There are two circumstances we could encounter here. + // 1. The key value for this entity has been updated to match some other entity's key value (conflict). + // 2. The int ID provided in the request parameter does not match any rows in the table. + throw new SQLException("Key field must be unique and request parameter ID must exist."); + } + } else if (size > 1) { + // FIXME: Handle edge case where original data set contains duplicate values for key field and this is an + // attempt to rectify bad data. + String message = String.format( + "%d %s entities shares the same key field (%s=%s)! Key field must be unique.", + size, + table.name, + keyField.name, + keyValue); + LOG.error(message); + throw new SQLException(message); + } + } + } + /** * Checks for modification of GTFS key field (e.g., stop_id, route_id) in supplied JSON object and ensures * both uniqueness and that referencing tables are appropriately updated. @@ -1072,46 +1132,34 @@ private void ensureReferentialIntegrity( String keyValue = jsonObject.get(keyField).asText(); // If updating key field, check that there is no ID conflict on value (e.g., stop_id or route_id) TIntSet uniqueIds = getIdsForCondition(tableName, keyField, keyValue, connection); - int size = uniqueIds.size(); - if (size == 0 || (size == 1 && id != null && uniqueIds.contains(id))) { - // OK. - if (size == 0 && !isCreating) { - // FIXME: Need to update referencing tables because entity has changed ID. - // Entity key value is being changed to an entirely new one. If there are entities that - // reference this value, we need to update them. - updateReferencingTables(namespace, table, id, keyValue); - } - } else { - // Conflict. The different conflict conditions are outlined below. - if (size == 1) { - // There was one match found. - if (isCreating) { - // Under no circumstance should a new entity have a conflict with existing key field. - throw new SQLException( - String.format("New %s's %s value (%s) conflicts with an existing record in table.", - table.entityClass.getSimpleName(), - keyField, - keyValue) - ); - } - if (!uniqueIds.contains(id)) { - // There are two circumstances we could encounter here. - // 1. The key value for this entity has been updated to match some other entity's key value (conflict). - // 2. The int ID provided in the request parameter does not match any rows in the table. - throw new SQLException("Key field must be unique and request parameter ID must exist."); - } - } else if (size > 1) { - // FIXME: Handle edge case where original data set contains duplicate values for key field and this is an - // attempt to rectify bad data. - String message = String.format( - "%d %s entities shares the same key field (%s=%s)! Key field must be unique.", - size, - table.name, - keyField, - keyValue); - LOG.error(message); - throw new SQLException(message); - } + checkUniqueIdsAndUpdateReferencingTables( + uniqueIds, + id, + namespace, + table, + keyValue, + isCreating, + table.getFieldForName(table.getKeyFieldName()) + ); + + if (table.name.equals("schedule_exceptions") && + jsonObject.has("exemplar") && + jsonObject.get("exemplar").asInt() == ExemplarServiceDescriptor.CALENDAR_DATE_SERVICE.getValue() + ) { + // Special case for schedule_exceptions where for exception type 10 and service_id is also a key. + String calendarDateServiceKey = "custom_schedule"; + Field calendarDateServiceKeyField = table.getFieldForName(calendarDateServiceKey); + String calendarDateServiceKeyVal = jsonObject.get(calendarDateServiceKey).asText(); + TIntSet calendarDateServiceUniqueIds = getIdsForCondition(tableName, calendarDateServiceKey, calendarDateServiceKeyVal, connection); + checkUniqueIdsAndUpdateReferencingTables( + calendarDateServiceUniqueIds, + id, + namespace, + table, + calendarDateServiceKeyVal, + isCreating, + calendarDateServiceKeyField + ); } } @@ -1137,7 +1185,13 @@ private static TIntSet getIdsForCondition( String keyValue, Connection connection ) throws SQLException { - String idCheckSql = String.format("select id from %s where %s = ?", tableName, keyField); + String idCheckSql; + if (keyField.equals("custom_schedule")) { + // The custom_schedule field of an exception based service contains an array and requires an "any" query. + idCheckSql = String.format("select id from %s where ? = any (%s)", tableName, keyField); + } else { + idCheckSql = String.format("select id from %s where %s = ?", tableName, keyField); + } // Create statement for counting rows selected PreparedStatement statement = connection.prepareStatement(idCheckSql); statement.setString(1, keyValue); @@ -1266,16 +1320,18 @@ private void updateReferencingTables( String namespace, Table table, int id, - String newKeyValue + String newKeyValue, + Field keyField ) throws SQLException { - Field keyField = table.getFieldForName(table.getKeyFieldName()); Class entityClass = table.getEntityClass(); // Determine method (update vs. delete) depending on presence of newKeyValue field. SqlMethod sqlMethod = newKeyValue != null ? SqlMethod.UPDATE : SqlMethod.DELETE; Set
referencingTables = getReferencingTables(table); // If there are no referencing tables, there is no need to update any values (e.g., . if (referencingTables.isEmpty()) return; - String keyValue = getValueForId(id, keyField.name, namespace, table, connection); + // Exception based service contains a single service ID in custom_schedule + String sqlKeyFieldName = keyField.name == "custom_schedule" ? "custom_schedule[1]" : keyField.name; + String keyValue = getValueForId(id, sqlKeyFieldName, namespace, table, connection); if (keyValue == null) { // FIXME: should we still check referencing tables for null value? LOG.warn("Entity {} to {} has null value for {}. Skipping references check.", id, sqlMethod, keyField); @@ -1291,9 +1347,10 @@ private void updateReferencingTables( for (Table referencingTable : referencingTables) { // Update/delete foreign references that match the key value. String refTableName = String.join(".", namespace, referencingTable.name); + int result; if (table.name.equals("schedule_exceptions") && referencingTable.name.equals("calendar_dates")) { // Custom logic for calendar dates because schedule_exceptions does not reference calendar dates right now. - int result = deleteCalendarDatesForException(id, namespace, table, refTableName); + result = deleteCalendarDatesForException(id, namespace, table, refTableName); LOG.info("Deleted {} entries in calendar dates associated with schedule exception {}", result, id); } else { for (Field field : referencingTable.editorFields()) { @@ -1302,26 +1359,69 @@ private void updateReferencingTables( if (refTable.name.equals(table.name)) { // Get statement to update or delete entities that reference the key value. PreparedStatement updateStatement = getUpdateReferencesStatement(sqlMethod, refTableName, field, keyValue, newKeyValue); - LOG.info("{}", updateStatement); - int result = updateStatement.executeUpdate(); + LOG.info(updateStatement.toString()); + result = updateStatement.executeUpdate(); if (result > 0) { // FIXME: is this where a delete hook should go? (E.g., CalendarController subclass would override // deleteEntityHook). - if (sqlMethod.equals(SqlMethod.DELETE) && table.isCascadeDeleteRestricted()) { - // Check for restrictions on delete. The entity must not have any referencing - // entities in order to delete it. - connection.rollback(); - String message = String.format( - "Cannot delete %s %s=%s. %d %s reference this %s.", - entityClass.getSimpleName(), - keyField.name, - keyValue, - result, - referencingTable.name, - entityClass.getSimpleName() - ); - LOG.warn(message); - throw new SQLException(message); + if (sqlMethod.equals(SqlMethod.DELETE)) { + ArrayList patternAndRouteIds = new ArrayList<>(); + // Check for restrictions on delete. + if (table.isCascadeDeleteRestricted()) { + // The entity must not have any referencing entities in order to delete it. + connection.rollback(); + if (entityClass.getSimpleName().equals("Stop")) { + String patternStopLookup = String.format( + "select distinct p.id, r.id " + + "from %s.pattern_stops ps " + + "inner join " + + "%s.patterns p " + + "on p.pattern_id = ps.pattern_id " + + "inner join " + + "%s.routes r " + + "on p.route_id = r.route_id " + + "where %s = '%s'", + namespace, + namespace, + namespace, + keyField.name, + keyValue + ); + PreparedStatement patternStopSelectStatement = connection.prepareStatement(patternStopLookup); + if (patternStopSelectStatement.execute()) { + ResultSet resultSet = patternStopSelectStatement.getResultSet(); + while (resultSet.next()) { + patternAndRouteIds.add( + String.format("{%s-%s-%s-%s}", + getResultSetString(1, resultSet), + getResultSetString(2, resultSet), + getResultSetString(3, resultSet), + getResultSetString(4, resultSet) + ) + ); + } + } + } + String message = String.format( + "Cannot delete %s %s=%s. %d %s reference this %s.", + entityClass.getSimpleName(), + keyField.name, + keyValue, + result, + referencingTable.name, + entityClass.getSimpleName() + ); + if (patternAndRouteIds.size() > 0) { + // Append referenced patterns data to the end of the error. + message = String.format( + "%s%nReferenced patterns: [%s]", + message, + StringUtils.join(patternAndRouteIds, ",") + ); + } + LOG.warn(message); + throw new SQLException(message); + } } LOG.info("{} reference(s) in {} {}D!", result, refTableName, sqlMethod); } else { @@ -1338,6 +1438,25 @@ private void updateReferencingTables( } } + /** + * Traditional method signature for updateReferencingTables, updating exception based service requires + * passing the keyField. + * @param namespace + * @param table + * @param id + * @param newKeyValue + * @throws SQLException + */ + private void updateReferencingTables( + String namespace, + Table table, + int id, + String newKeyValue + ) throws SQLException { + Field keyField = table.getFieldForName(table.getKeyFieldName()); + updateReferencingTables(namespace, table, id, newKeyValue, keyField); + } + /** * To prevent orphaned descendants, delete them before joining references are deleted. For the relationship * route -> pattern -> pattern stop, delete pattern stop before deleting the joining pattern. diff --git a/src/main/java/com/conveyal/gtfs/loader/ReferenceTracker.java b/src/main/java/com/conveyal/gtfs/loader/ReferenceTracker.java index f428ee79a..094e6e867 100644 --- a/src/main/java/com/conveyal/gtfs/loader/ReferenceTracker.java +++ b/src/main/java/com/conveyal/gtfs/loader/ReferenceTracker.java @@ -176,6 +176,22 @@ public Set checkReferencesAndUniqueness(String keyValue, int lineN return errors; } + /** +<<<<<<< HEAD +======= + * Check foreign references. If the foreign reference is present in one of the tables, there is no + * need to check the remainder. If no matching foreign reference is found, flag integrity error. + * Note: The reference table must be loaded before the table/value being currently checked. + */ + private boolean hasMatchingReference(Field field, String value, TreeSet badValues) { + for (Table referenceTable : field.referenceTables) { + if (checkReference(referenceTable.getKeyFieldName(), value, badValues)) { + return true; + } + } + return false; + } + /** * Check that a reference is valid. */ @@ -189,7 +205,6 @@ private boolean checkReference(String referenceField, String reference, TreeSet< return false; } - /** * Work through each conditionally required check assigned to fields within a table. First check the reference field * to confirm if it meets the conditions whereby the conditional field is required. If the conditional field is diff --git a/src/main/java/com/conveyal/gtfs/loader/Table.java b/src/main/java/com/conveyal/gtfs/loader/Table.java index 70426c838..4a5b6f110 100644 --- a/src/main/java/com/conveyal/gtfs/loader/Table.java +++ b/src/main/java/com/conveyal/gtfs/loader/Table.java @@ -176,7 +176,7 @@ public Table (String name, Class entityClass, Requirement requ ); public static final Table CALENDAR_DATES = new Table("calendar_dates", CalendarDate.class, OPTIONAL, - new StringField("service_id", REQUIRED).isReferenceTo(CALENDAR), + new StringField("service_id", REQUIRED), new DateField("date", REQUIRED), new IntegerField("exception_type", REQUIRED, 1, 2) ).keyFieldIsNotUnique() @@ -345,9 +345,8 @@ public Table (String name, Class entityClass, Requirement requ public static final Table TRIPS = new Table("trips", Trip.class, REQUIRED, new StringField("trip_id", REQUIRED), new StringField("route_id", REQUIRED).isReferenceTo(ROUTES).indexThisColumn(), - // FIXME: Should this also optionally reference CALENDAR_DATES? // FIXME: Do we need an index on service_id - new StringField("service_id", REQUIRED).isReferenceTo(CALENDAR), + new StringField("service_id", REQUIRED).isReferenceTo(CALENDAR).isReferenceTo(CALENDAR_DATES).isReferenceTo(SCHEDULE_EXCEPTIONS), new StringField("trip_headsign", OPTIONAL), new StringField("trip_short_name", OPTIONAL), new ShortField("direction_id", OPTIONAL, 1), diff --git a/src/main/java/com/conveyal/gtfs/model/Calendar.java b/src/main/java/com/conveyal/gtfs/model/Calendar.java index 4be105396..18c17bedb 100644 --- a/src/main/java/com/conveyal/gtfs/model/Calendar.java +++ b/src/main/java/com/conveyal/gtfs/model/Calendar.java @@ -144,12 +144,7 @@ protected void writeOneRow(Calendar c) throws IOException { @Override protected Iterator iterator() { // wrap an iterator over services - Iterator calIt = Iterators.transform(feed.services.values().iterator(), new Function () { - @Override - public Calendar apply (Service s) { - return s.calendar; - } - }); + Iterator calIt = Iterators.transform(feed.services.values().iterator(), s -> s.calendar); // not every service has a calendar (e.g. TriMet has no calendars, just calendar dates). // This is legal GTFS, so skip services with no calendar diff --git a/src/main/java/com/conveyal/gtfs/model/CalendarDate.java b/src/main/java/com/conveyal/gtfs/model/CalendarDate.java index b737e62ff..8153e7544 100644 --- a/src/main/java/com/conveyal/gtfs/model/CalendarDate.java +++ b/src/main/java/com/conveyal/gtfs/model/CalendarDate.java @@ -109,12 +109,7 @@ protected void writeOneRow(CalendarDate d) throws IOException { @Override protected Iterator iterator() { Iterator serviceIterator = feed.services.values().iterator(); - return Iterators.concat(Iterators.transform(serviceIterator, new Function> () { - @Override - public Iterator apply(Service service) { - return service.calendar_dates.values().iterator(); - } - })); + return Iterators.concat(Iterators.transform(serviceIterator, service -> service.calendar_dates.values().iterator())); } } } diff --git a/src/main/java/com/conveyal/gtfs/model/ScheduleException.java b/src/main/java/com/conveyal/gtfs/model/ScheduleException.java index 932875ccc..e88c04658 100644 --- a/src/main/java/com/conveyal/gtfs/model/ScheduleException.java +++ b/src/main/java/com/conveyal/gtfs/model/ScheduleException.java @@ -73,6 +73,8 @@ public boolean serviceRunsOn(Calendar calendar) { if (removedService != null && removedService.contains(calendar.service_id)) { return false; } + case CALENDAR_DATE_SERVICE: + return false; default: // can't actually happen, but java requires a default with a return here return false; @@ -84,7 +86,7 @@ public boolean serviceRunsOn(Calendar calendar) { * For example, run Sunday service on Presidents' Day, or no service on New Year's Day. */ public enum ExemplarServiceDescriptor { - MONDAY(0), TUESDAY(1), WEDNESDAY(2), THURSDAY(3), FRIDAY(4), SATURDAY(5), SUNDAY(6), NO_SERVICE(7), CUSTOM(8), SWAP(9), MISSING(-1); + MONDAY(0), TUESDAY(1), WEDNESDAY(2), THURSDAY(3), FRIDAY(4), SATURDAY(5), SUNDAY(6), NO_SERVICE(7), CUSTOM(8), SWAP(9), CALENDAR_DATE_SERVICE(10), MISSING(-1); private final int value; @@ -119,6 +121,8 @@ public static ExemplarServiceDescriptor exemplarFromInt (int value) { return ExemplarServiceDescriptor.CUSTOM; case 9: return ExemplarServiceDescriptor.SWAP; + case 10: + return ExemplarServiceDescriptor.CALENDAR_DATE_SERVICE; default: return ExemplarServiceDescriptor.MISSING; } diff --git a/src/main/java/com/conveyal/gtfs/validator/ServiceValidator.java b/src/main/java/com/conveyal/gtfs/validator/ServiceValidator.java index aad3da12c..74d1e5867 100644 --- a/src/main/java/com/conveyal/gtfs/validator/ServiceValidator.java +++ b/src/main/java/com/conveyal/gtfs/validator/ServiceValidator.java @@ -192,8 +192,8 @@ select durations.service_id, duration_seconds, days_active from ( for (String tripId : serviceInfo.tripIds) { registerError( NewGTFSError.forTable(Table.TRIPS, NewGTFSErrorType.TRIP_NEVER_ACTIVE) - .setEntityId(tripId) - .setBadValue(tripId)); + .setEntityId(tripId) + .setBadValue(tripId)); } } if (serviceInfo.tripIds.isEmpty()) { @@ -256,7 +256,7 @@ select durations.service_id, duration_seconds, days_active from ( // Check for low or zero service, which seems to happen even when services are defined. // This will also catch cases where dateInfo was null and the new instance contains no service. registerError(NewGTFSError.forFeed(NewGTFSErrorType.DATE_NO_SERVICE, - DateField.GTFS_DATE_FORMATTER.format(date))); + DateField.GTFS_DATE_FORMATTER.format(date))); } } } @@ -329,7 +329,7 @@ select durations.service_id, duration_seconds, days_active from ( String serviceDurationsTableName = feed.getTableNameWithSchemaPrefix("service_durations"); sql = String.format("create table %s (service_id varchar, route_type integer, " + - "duration_seconds integer, primary key (service_id, route_type))", serviceDurationsTableName); + "duration_seconds integer, primary key (service_id, route_type))", serviceDurationsTableName); LOG.info(sql); statement.execute(sql); sql = String.format("insert into %s values (?, ?, ?)", serviceDurationsTableName); diff --git a/src/test/java/com/conveyal/gtfs/GTFSFeedTest.java b/src/test/java/com/conveyal/gtfs/GTFSFeedTest.java index b705f882c..8ac38f8ce 100644 --- a/src/test/java/com/conveyal/gtfs/GTFSFeedTest.java +++ b/src/test/java/com/conveyal/gtfs/GTFSFeedTest.java @@ -45,7 +45,21 @@ public static void setUpClass() { * Make sure a round-trip of loading a GTFS zip file and then writing another zip file can be performed. */ @Test - public void canDoRoundtripLoadAndWriteToZipFile() throws IOException { + public void canDoRoundTripLoadAndWriteToZipFile() throws IOException { + // create a temp file for this test + File outZip = File.createTempFile("fake-agency-output", ".zip"); + + // delete file to make sure we can assert that this program created the file + outZip.delete(); + + GTFSFeed feed = GTFSFeed.fromFile(simpleGtfsZipFileName); + feed.toFile(outZip.getAbsolutePath()); + feed.close(); + assertThat(outZip.exists(), is(true)); + + // assert that rows of data were written to files within the zipfile + ZipFile zip = new ZipFile(outZip); + FileTestCase[] fileTestCases = { // agency.txt new FileTestCase( @@ -63,6 +77,14 @@ public void canDoRoundtripLoadAndWriteToZipFile() throws IOException { new DataExpectation("end_date", "20170917") } ), + new FileTestCase( + "calendar_dates.txt", + new DataExpectation[]{ + new DataExpectation("service_id", "calendar-date-service"), + new DataExpectation("date", "20170917"), + new DataExpectation("exception_type", "1") + } + ), new FileTestCase( "routes.txt", new DataExpectation[]{ diff --git a/src/test/java/com/conveyal/gtfs/GTFSTest.java b/src/test/java/com/conveyal/gtfs/GTFSTest.java index d9c6a2641..412a73e7e 100644 --- a/src/test/java/com/conveyal/gtfs/GTFSTest.java +++ b/src/test/java/com/conveyal/gtfs/GTFSTest.java @@ -137,13 +137,21 @@ public void canLoadAndExportSimpleAgency() { * Tests that a GTFS feed with bad date values in calendars.txt and calendar_dates.txt can pass the integration test. */ @Test - public void canLoadFeedWithBadDates () { + void canLoadFeedWithBadDates () { PersistenceExpectation[] expectations = PersistenceExpectation.list( new PersistenceExpectation( "calendar", new RecordExpectation[]{ new RecordExpectation("start_date", null) } + ), + new PersistenceExpectation( + "calendar_dates", + new RecordExpectation[]{ + new RecordExpectation("service_id", "123_ID_NOT_EXISTS"), + new RecordExpectation("date", "20190301"), + new RecordExpectation("exception_type", "1") + } ) ); ErrorExpectation[] errorExpectations = ErrorExpectation.list( @@ -151,7 +159,6 @@ public void canLoadFeedWithBadDates () { new ErrorExpectation(NewGTFSErrorType.DATE_FORMAT), new ErrorExpectation(NewGTFSErrorType.DATE_FORMAT), new ErrorExpectation(NewGTFSErrorType.DATE_FORMAT), - new ErrorExpectation(NewGTFSErrorType.REFERENTIAL_INTEGRITY), new ErrorExpectation(NewGTFSErrorType.DATE_FORMAT), new ErrorExpectation(NewGTFSErrorType.DATE_FORMAT), // The below "wrong number of fields" errors are for empty new lines @@ -161,7 +168,7 @@ public void canLoadFeedWithBadDates () { new ErrorExpectation(NewGTFSErrorType.WRONG_NUMBER_OF_FIELDS), new ErrorExpectation(NewGTFSErrorType.WRONG_NUMBER_OF_FIELDS), new ErrorExpectation(NewGTFSErrorType.WRONG_NUMBER_OF_FIELDS), - new ErrorExpectation(NewGTFSErrorType.REFERENTIAL_INTEGRITY), + new ErrorExpectation(NewGTFSErrorType.MISSING_FOREIGN_TABLE_REFERENCE), new ErrorExpectation(NewGTFSErrorType.ROUTE_LONG_NAME_CONTAINS_SHORT_NAME), new ErrorExpectation(NewGTFSErrorType.SERVICE_NEVER_ACTIVE), new ErrorExpectation(NewGTFSErrorType.TRIP_NEVER_ACTIVE), @@ -435,25 +442,53 @@ public void canLoadAndExportSimpleAgencyWithMixtureOfCalendarDefinitions() { new RecordExpectation("exception_type", 2) } ), - // calendar-dates.txt-only expectation new PersistenceExpectation( - "calendar", + "calendar_dates", new RecordExpectation[]{ new RecordExpectation( "service_id", "only-in-calendar-dates-txt" ), - new RecordExpectation("start_date", 20170916), - new RecordExpectation("end_date", 20170916) - }, - true + new RecordExpectation("date", 20170916), + new RecordExpectation("exception_type", 1) + } ), new PersistenceExpectation( "calendar_dates", new RecordExpectation[]{ new RecordExpectation( - "service_id", "only-in-calendar-dates-txt" + "service_id", "calendar-dates-txt-service-one" ), - new RecordExpectation("date", 20170916), + new RecordExpectation("date", 20170917), + new RecordExpectation("exception_type", 1) + } + ), + new PersistenceExpectation( + "calendar_dates", + new RecordExpectation[]{ + new RecordExpectation( + "service_id", "calendar-dates-txt-service-two" + ), + new RecordExpectation("date", 20170918), + new RecordExpectation("exception_type", 1) + } + ), + new PersistenceExpectation( + "calendar_dates", + new RecordExpectation[]{ + new RecordExpectation( + "service_id", "calendar-dates-txt-service-three" + ), + new RecordExpectation("date", 20170917), + new RecordExpectation("exception_type", 1) + } + ), + new PersistenceExpectation( + "calendar_dates", + new RecordExpectation[]{ + new RecordExpectation( + "service_id", "calendar-dates-txt-service-three" + ), + new RecordExpectation("date", 20170918), new RecordExpectation("exception_type", 1) } ), @@ -533,6 +568,8 @@ public void canLoadAndExportSimpleAgencyWithMixtureOfCalendarDefinitions() { ErrorExpectation[] errorExpectations = ErrorExpectation.list( new ErrorExpectation(NewGTFSErrorType.MISSING_FIELD), new ErrorExpectation(NewGTFSErrorType.ROUTE_LONG_NAME_CONTAINS_SHORT_NAME), + new ErrorExpectation(NewGTFSErrorType.ROUTE_LONG_NAME_CONTAINS_SHORT_NAME), + new ErrorExpectation(NewGTFSErrorType.SERVICE_UNUSED), new ErrorExpectation(NewGTFSErrorType.FEED_TRAVEL_TIMES_ROUNDED) ); assertThat( @@ -1278,16 +1315,6 @@ private void assertThatPersistenceExpectationRecordWasFound( new RecordExpectation("end_date", "20170917") } ), - new PersistenceExpectation( - "calendar_dates", - new RecordExpectation[]{ - new RecordExpectation( - "service_id", "04100312-8fe1-46a5-a9f2-556f39478f57" - ), - new RecordExpectation("date", 20170916), - new RecordExpectation("exception_type", 2) - } - ), new PersistenceExpectation( "fare_attributes", new RecordExpectation[]{ diff --git a/src/test/resources/fake-agency-mixture-of-calendar-definitions/calendar_dates.txt b/src/test/resources/fake-agency-mixture-of-calendar-definitions/calendar_dates.txt index ea060f019..6e4f013d1 100755 --- a/src/test/resources/fake-agency-mixture-of-calendar-definitions/calendar_dates.txt +++ b/src/test/resources/fake-agency-mixture-of-calendar-definitions/calendar_dates.txt @@ -1,3 +1,7 @@ service_id,date,exception_type in-both-calendar-txt-and-calendar-dates,20170920,2 -only-in-calendar-dates-txt,20170916,1 \ No newline at end of file +only-in-calendar-dates-txt,20170916,1 +calendar-dates-txt-service-one,20170917,1 +calendar-dates-txt-service-two,20170918,1 +calendar-dates-txt-service-three,20170917,1 +calendar-dates-txt-service-three,20170918,1 \ No newline at end of file diff --git a/src/test/resources/fake-agency-mixture-of-calendar-definitions/routes.txt b/src/test/resources/fake-agency-mixture-of-calendar-definitions/routes.txt index 35ea7aa67..b13480efa 100755 --- a/src/test/resources/fake-agency-mixture-of-calendar-definitions/routes.txt +++ b/src/test/resources/fake-agency-mixture-of-calendar-definitions/routes.txt @@ -1,2 +1,3 @@ agency_id,route_id,route_short_name,route_long_name,route_desc,route_type,route_url,route_color,route_text_color,route_branding_url 1,1,1,Route 1,,3,,7CE6E7,FFFFFF, +1,2,2,Route 2,,3,,7CE6E7,FFFFFF, diff --git a/src/test/resources/fake-agency-mixture-of-calendar-definitions/stop_times.txt b/src/test/resources/fake-agency-mixture-of-calendar-definitions/stop_times.txt index fe0a9ad12..12ad079e6 100755 --- a/src/test/resources/fake-agency-mixture-of-calendar-definitions/stop_times.txt +++ b/src/test/resources/fake-agency-mixture-of-calendar-definitions/stop_times.txt @@ -5,3 +5,7 @@ non-frequency-trip-2,08:00:00,08:00:00,4u6g,1,,0,0,0.0000000, non-frequency-trip-2,08:01:00,08:01:00,johv,2,,0,0,341.4491961, frequency-trip,09:00:00,09:00:00,4u6g,1,,0,0,0.0000000, frequency-trip,09:01:00,09:01:00,johv,2,,0,0,341.4491961, +exception-trip-1,09:00:00,09:00:00,4u6g,1,,0,0,0.0000000, +exception-trip-1,09:01:00,09:01:00,johv,2,,0,0,341.4491961, +exception-trip-2,09:00:00,09:00:00,4u6g,1,,0,0,0.0000000, +exception-trip-2,09:01:00,09:01:00,johv,2,,0,0,341.4491961, diff --git a/src/test/resources/fake-agency-mixture-of-calendar-definitions/trips.txt b/src/test/resources/fake-agency-mixture-of-calendar-definitions/trips.txt index 077253974..221642959 100755 --- a/src/test/resources/fake-agency-mixture-of-calendar-definitions/trips.txt +++ b/src/test/resources/fake-agency-mixture-of-calendar-definitions/trips.txt @@ -1,4 +1,6 @@ route_id,trip_id,trip_headsign,trip_short_name,direction_id,block_id,shape_id,bikes_allowed,wheelchair_accessible,service_id 1,non-frequency-trip,,,0,,5820f377-f947-4728-ac29-ac0102cbc34e,0,0,only-in-calendar-dates-txt 1,non-frequency-trip-2,,,0,,5820f377-f947-4728-ac29-ac0102cbc34e,0,0,only-in-calendar-txt -1,frequency-trip,,,0,,5820f377-f947-4728-ac29-ac0102cbc34e,0,0,in-both-calendar-txt-and-calendar-dates \ No newline at end of file +1,frequency-trip,,,0,,5820f377-f947-4728-ac29-ac0102cbc34e,0,0,in-both-calendar-txt-and-calendar-dates +2,exception-trip-1,,,0,,5820f377-f947-4728-ac29-ac0102cbc34e,0,0,calendar-dates-txt-service-one +2,exception-trip-2,,,0,,5820f377-f947-4728-ac29-ac0102cbc34e,0,0,calendar-dates-txt-service-two \ No newline at end of file diff --git a/src/test/resources/fake-agency/calendar_dates.txt b/src/test/resources/fake-agency/calendar_dates.txt index 403ee2bbe..5d0a31806 100755 --- a/src/test/resources/fake-agency/calendar_dates.txt +++ b/src/test/resources/fake-agency/calendar_dates.txt @@ -1,2 +1,3 @@ service_id,date,exception_type -04100312-8fe1-46a5-a9f2-556f39478f57,20170916,2 \ No newline at end of file +04100312-8fe1-46a5-a9f2-556f39478f57,20170916,2 +calendar-date-service,20170917,1 \ No newline at end of file diff --git a/src/test/resources/fake-agency/stop_times.txt b/src/test/resources/fake-agency/stop_times.txt index 61b66b857..88358ba62 100755 --- a/src/test/resources/fake-agency/stop_times.txt +++ b/src/test/resources/fake-agency/stop_times.txt @@ -4,3 +4,5 @@ a30277f8-e50a-4a85-9141-b1e0da9d429d,07:01:00,07:01:00,johv,2,Test stop headsign frequency-trip,08:00:00,08:00:00,4u6g,1,Test stop headsign frequency trip,0,0,0.0000000, frequency-trip,08:29:00,08:29:00,1234,2,Test stop headsign frequency trip 1,0,0,0.0000000, frequency-trip,08:30:00,08:30:00,area1,3,Test stop headsign frequency trip 2,0,0,0.0000000, +calendar-date-trip,08:00:00,08:00:00,4u6g,1,Test stop headsign calendar date trip,0,0,0.0000000, +calendar-date-trip,08:29:00,08:29:00,1234,2,Test stop headsign calendar date trip 2,0,0,341.4491961, diff --git a/src/test/resources/fake-agency/trips.txt b/src/test/resources/fake-agency/trips.txt index eab14b86d..982c01e0f 100755 --- a/src/test/resources/fake-agency/trips.txt +++ b/src/test/resources/fake-agency/trips.txt @@ -1,3 +1,4 @@ route_id,trip_id,trip_headsign,trip_short_name,direction_id,block_id,shape_id,bikes_allowed,wheelchair_accessible,service_id 1,a30277f8-e50a-4a85-9141-b1e0da9d429d,,,0,,5820f377-f947-4728-ac29-ac0102cbc34e,0,0,04100312-8fe1-46a5-a9f2-556f39478f57 -1,frequency-trip,,,0,,5820f377-f947-4728-ac29-ac0102cbc34e,0,0,04100312-8fe1-46a5-a9f2-556f39478f57 \ No newline at end of file +1,frequency-trip,,,0,,5820f377-f947-4728-ac29-ac0102cbc34e,0,0,04100312-8fe1-46a5-a9f2-556f39478f57 +1,calendar-date-trip,,,0,,5820f377-f947-4728-ac29-ac0102cbc34e,0,0,calendar-date-service \ No newline at end of file diff --git a/src/test/resources/graphql/feedRowCounts.txt b/src/test/resources/graphql/feedRowCounts.txt index 116395f6f..74ec31965 100644 --- a/src/test/resources/graphql/feedRowCounts.txt +++ b/src/test/resources/graphql/feedRowCounts.txt @@ -5,7 +5,6 @@ query($namespace: String) { row_counts { agency calendar - calendar_dates errors routes stops diff --git a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchFeedRowCounts-0.json b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchFeedRowCounts-0.json index 08028c6db..e9fc696e3 100644 --- a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchFeedRowCounts-0.json +++ b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchFeedRowCounts-0.json @@ -10,7 +10,7 @@ "routes" : 1, "stop_times" : 6, "stops" : 5, - "trips" : 2 + "trips" : 3 }, "snapshot_of" : null } diff --git a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchPatterns-0.json b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchPatterns-0.json index 16f1b6980..8415a9b8f 100644 --- a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchPatterns-0.json +++ b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchPatterns-0.json @@ -237,9 +237,11 @@ }, { "stop_id" : "1234567" } ], - "trip_count" : 1, + "trip_count" : 2, "trips" : [ { "trip_id" : "frequency-trip" + }, { + "trip_id" : "calendar-date-trip" } ], "use_frequency" : null } ] diff --git a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchRoutes-0.json b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchRoutes-0.json index 4acbbb20f..cf54cefd3 100644 --- a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchRoutes-0.json +++ b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchRoutes-0.json @@ -34,11 +34,13 @@ }, { "stop_id" : "1234567" } ], - "trip_count" : 2, + "trip_count" : 3, "trips" : [ { "trip_id" : "a30277f8-e50a-4a85-9141-b1e0da9d429d" }, { "trip_id" : "frequency-trip" + }, { + "trip_id" : "calendar-date-trip" } ], "wheelchair_accessible" : null } ] diff --git a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchServices-0.json b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchServices-0.json index 787d58aed..e8430e435 100644 --- a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchServices-0.json +++ b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchServices-0.json @@ -16,6 +16,18 @@ }, { "trip_id" : "frequency-trip" } ] + }, { + "dates" : [ "20170917" ], + "duration_seconds" : "1740", + "durations" : [ { + "duration_seconds" : 1740, + "route_type" : 3 + } ], + "n_days_active" : "1", + "service_id" : "calendar-date-service", + "trips" : [ { + "trip_id" : "calendar-date-trip" + } ] } ] } } diff --git a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchStopTimes-0.json b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchStopTimes-0.json index 7d3c4467d..d1277752f 100644 --- a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchStopTimes-0.json +++ b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchStopTimes-0.json @@ -41,7 +41,7 @@ "drop_off_type" : 0, "pickup_type" : 0, "shape_dist_traveled" : 341.4491961, - "stop_headsign" : "Test stop headsign frequency trip", + "stop_headsign" : "Test stop headsign frequency trip 2", "stop_id" : "1234", "stop_sequence" : 2, "timepoint" : 1, @@ -68,6 +68,28 @@ "stop_sequence" : 4, "timepoint" : 1, "trip_id" : "frequency-trip" + }, { + "arrival_time" : 28800, + "departure_time" : 28800, + "drop_off_type" : 0, + "pickup_type" : 0, + "shape_dist_traveled" : 0.0, + "stop_headsign" : "Test stop headsign calendar date trip", + "stop_id" : "4u6g", + "stop_sequence" : 1, + "timepoint" : null, + "trip_id" : "calendar-date-trip" + }, { + "arrival_time" : 30540, + "departure_time" : 30540, + "drop_off_type" : 0, + "pickup_type" : 0, + "shape_dist_traveled" : 341.4491961, + "stop_headsign" : "Test stop headsign calendar date trip 2", + "stop_id" : "1234", + "stop_sequence" : 2, + "timepoint" : null, + "trip_id" : "calendar-date-trip" } ] } } diff --git a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchStops-0.json b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchStops-0.json index 149bdedd7..ace2093f0 100644 --- a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchStops-0.json +++ b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchStops-0.json @@ -18,7 +18,7 @@ "stop_lat" : 37.0612132, "stop_lon" : -122.0074332, "stop_name" : "Butler Ln", - "stop_time_count" : 2, + "stop_time_count" : 3, "stop_times" : [ { "stop_id" : "4u6g", "stop_sequence" : 1, @@ -27,6 +27,10 @@ "stop_id" : "4u6g", "stop_sequence" : 1, "trip_id" : "frequency-trip" + }, { + "stop_id" : "4u6g", + "stop_sequence" : 1, + "trip_id" : "calendar-date-trip" } ], "stop_timezone" : null, "stop_url" : null, @@ -92,11 +96,15 @@ "stop_lat" : 37.06662, "stop_lon" : -122.07772, "stop_name" : "Child Stop", - "stop_time_count" : 1, + "stop_time_count" : 2, "stop_times" : [ { "stop_id" : "1234", "stop_sequence" : 2, "trip_id" : "frequency-trip" + }, { + "stop_id" : "1234", + "stop_sequence" : 2, + "trip_id" : "calendar-date-trip" } ], "stop_timezone" : null, "stop_url" : null, diff --git a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchTrips-0.json b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchTrips-0.json index a3485c9c9..4e0e1a37b 100644 --- a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchTrips-0.json +++ b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchTrips-0.json @@ -158,6 +158,79 @@ "trip_id" : "frequency-trip", "trip_short_name" : null, "wheelchair_accessible" : 0 + }, { + "bikes_allowed" : 0, + "block_id" : null, + "direction_id" : 0, + "frequencies" : [ ], + "id" : 4, + "pattern_id" : "2", + "route_id" : "1", + "service_id" : "calendar-date-service", + "shape" : [ { + "point_type" : null, + "shape_dist_traveled" : 0.0, + "shape_id" : "5820f377-f947-4728-ac29-ac0102cbc34e", + "shape_pt_lat" : 37.0612132, + "shape_pt_lon" : -122.0074332, + "shape_pt_sequence" : 1 + }, { + "point_type" : null, + "shape_dist_traveled" : 7.4997067, + "shape_id" : "5820f377-f947-4728-ac29-ac0102cbc34e", + "shape_pt_lat" : 37.061172, + "shape_pt_lon" : -122.0075, + "shape_pt_sequence" : 2 + }, { + "point_type" : null, + "shape_dist_traveled" : 33.8739075, + "shape_id" : "5820f377-f947-4728-ac29-ac0102cbc34e", + "shape_pt_lat" : 37.061359, + "shape_pt_lon" : -122.007683, + "shape_pt_sequence" : 3 + }, { + "point_type" : null, + "shape_dist_traveled" : 109.0402932, + "shape_id" : "5820f377-f947-4728-ac29-ac0102cbc34e", + "shape_pt_lat" : 37.060878, + "shape_pt_lon" : -122.008278, + "shape_pt_sequence" : 4 + }, { + "point_type" : null, + "shape_dist_traveled" : 184.6078298, + "shape_id" : "5820f377-f947-4728-ac29-ac0102cbc34e", + "shape_pt_lat" : 37.060359, + "shape_pt_lon" : -122.008828, + "shape_pt_sequence" : 5 + }, { + "point_type" : null, + "shape_dist_traveled" : 265.8053023, + "shape_id" : "5820f377-f947-4728-ac29-ac0102cbc34e", + "shape_pt_lat" : 37.059761, + "shape_pt_lon" : -122.009354, + "shape_pt_sequence" : 6 + }, { + "point_type" : null, + "shape_dist_traveled" : 357.8617018, + "shape_id" : "5820f377-f947-4728-ac29-ac0102cbc34e", + "shape_pt_lat" : 37.059066, + "shape_pt_lon" : -122.009919, + "shape_pt_sequence" : 7 + } ], + "shape_id" : "5820f377-f947-4728-ac29-ac0102cbc34e", + "stop_times" : [ { + "stop_id" : "4u6g", + "stop_sequence" : 1, + "trip_id" : "calendar-date-trip" + }, { + "stop_id" : "1234", + "stop_sequence" : 2, + "trip_id" : "calendar-date-trip" + } ], + "trip_headsign" : null, + "trip_id" : "calendar-date-trip", + "trip_short_name" : null, + "wheelchair_accessible" : 0 } ] } } diff --git a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canSanitizeSQLInjectionSentAsKeyValue-0.json b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canSanitizeSQLInjectionSentAsKeyValue-0.json index 0c5811306..2f21dba08 100644 --- a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canSanitizeSQLInjectionSentAsKeyValue-0.json +++ b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canSanitizeSQLInjectionSentAsKeyValue-0.json @@ -34,11 +34,13 @@ }, { "stop_id" : "1234567" } ], - "trip_count" : 2, + "trip_count" : 3, "trips" : [ { "trip_id" : "a30277f8-e50a-4a85-9141-b1e0da9d429d" }, { "trip_id" : "frequency-trip" + }, { + "trip_id" : "calendar-date-trip" } ], "wheelchair_accessible" : null } ]