Merge pull request #385 from conveyal/feature/DT-183-Exception-based-…

…scheduling

Exception based scheduling

src/main/java/com/conveyal/gtfs/error/NewGTFSErrorType.java

@@ -32,6 +32,7 @@ public enum NewGTFSErrorType {
     MISSING_ARRIVAL_OR_DEPARTURE(Priority.MEDIUM, "First and last stop times are required to have both an arrival and departure time."),
     MISSING_COLUMN(Priority.MEDIUM, "A required column was missing from a table."),
     MISSING_FIELD(Priority.MEDIUM, "A required field was missing or empty in a particular row."),
+    MISSING_FOREIGN_TABLE_REFERENCE(Priority.HIGH, "This line references an ID that must exist in a single foreign table."),
     MISSING_SHAPE(Priority.MEDIUM, "???"),
     MISSING_TABLE(Priority.MEDIUM, "This table is required by the GTFS specification but is missing."),
     MULTIPLE_SHAPES_FOR_PATTERN(Priority.MEDIUM, "Multiple shapes found for a single unique sequence of stops (i.e, trip pattern)."),

src/main/java/com/conveyal/gtfs/loader/Field.java

@@ -8,6 +8,7 @@
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
 import java.sql.SQLType;
+import java.util.LinkedHashSet;
 import java.util.Set;
 
 /**
@@ -46,7 +47,7 @@ public abstract class Field {
      * Indicates that this field acts as a foreign key to this referenced table. This is used when checking referential
      * integrity when loading a feed.
      * */
-    public Table referenceTable = null;
+    public Set<Table> referenceTables = new LinkedHashSet<>();
     private boolean shouldBeIndexed;
     private boolean emptyValuePermitted;
     private boolean isConditionallyRequired;
@@ -138,7 +139,7 @@ public boolean isRequired () {
      * a many-to-many reference.
      */
     public boolean isForeignReference () {
-        return this.referenceTable != null;
+        return !this.referenceTables.isEmpty();
     }
 
     /**
@@ -181,7 +182,7 @@ public boolean shouldBeIndexed() {
      * @return this same Field instance
      */
     public Field isReferenceTo(Table table) {
-        this.referenceTable = table;
+        referenceTables.add(table);
         return this;
     }
 

src/main/java/com/conveyal/gtfs/loader/JDBCTableReader.java

@@ -1,5 +1,6 @@
 package com.conveyal.gtfs.loader;
 
+import com.conveyal.gtfs.model.Calendar;
 import com.conveyal.gtfs.model.Entity;
 import com.conveyal.gtfs.storage.StorageException;
 import gnu.trove.map.TObjectIntMap;
@@ -146,6 +147,18 @@ public int getRowCount() {
         }
     }
 
+    /**
+     * Provide reader for calendar table.
+     */
+    public static JDBCTableReader<Calendar> getCalendarTableReader(DataSource dataSource, String tablePrefix) {
+        return new JDBCTableReader(
+            Table.CALENDAR,
+            dataSource,
+            tablePrefix + ".",
+            EntityPopulator.CALENDAR
+        );
+    }
+
     private class EntityIterator implements Iterator<T> {
 
         private Connection connection; // Will remain open for the duration of the iteration.

src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java

@@ -23,13 +23,16 @@
 import java.nio.file.FileSystems;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.sql.Connection;
 import java.sql.SQLException;
 import java.time.LocalDate;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
@@ -79,7 +82,7 @@ public JdbcGtfsExporter(String feedId, String outFile, DataSource dataSource, bo
     /**
      * Utility method to check if an exception uses a specific service.
      */
-    public Boolean exceptionInvolvesService(ScheduleException ex, String serviceId) {
+    public boolean exceptionInvolvesService(ScheduleException ex, String serviceId) {
         return (
             ex.addedService.contains(serviceId) ||
             ex.removedService.contains(serviceId) ||
@@ -99,7 +102,7 @@ public FeedLoadResult exportTables() {
         FeedLoadResult result = new FeedLoadResult();
 
         try {
-            zipOutputStream = new ZipOutputStream(new FileOutputStream(outFile));
+            zipOutputStream = new ZipOutputStream(Files.newOutputStream(Paths.get(outFile)));
             long startTime = System.currentTimeMillis();
             // We get a single connection object and share it across several different methods.
             // This ensures that actions taken in one method are visible to all subsequent SQL statements.
@@ -132,40 +135,55 @@ public FeedLoadResult exportTables() {
             if (fromEditor) {
                 // Export schedule exceptions in place of calendar dates if exporting a feed/schema that represents an editor snapshot.
                 GTFSFeed feed = new GTFSFeed();
-                // FIXME: The below table readers should probably just share a connection with the exporter.
-                JDBCTableReader<ScheduleException> exceptionsReader =
-                        new JDBCTableReader(Table.SCHEDULE_EXCEPTIONS, dataSource, feedIdToExport + ".",
-                                EntityPopulator.SCHEDULE_EXCEPTION);
-                JDBCTableReader<Calendar> calendarsReader =
-                        new JDBCTableReader(Table.CALENDAR, dataSource, feedIdToExport + ".",
-                                EntityPopulator.CALENDAR);
-                Iterable<Calendar> calendars = calendarsReader.getAll();
+                JDBCTableReader<ScheduleException> exceptionsReader =new JDBCTableReader(
+                    Table.SCHEDULE_EXCEPTIONS,
+                    dataSource,
+                    feedIdToExport + ".",
+                    EntityPopulator.SCHEDULE_EXCEPTION
+                );
+                JDBCTableReader<Calendar> calendarReader = JDBCTableReader.getCalendarTableReader(dataSource, feedIdToExport);
+                Iterable<Calendar> calendars = calendarReader.getAll();
                 Iterable<ScheduleException> exceptionsIterator = exceptionsReader.getAll();
-                List<ScheduleException> exceptions = new ArrayList<>();
-                // FIXME: Doing this causes the connection to stay open, but it is closed in the finalizer so it should
-                // not be a big problem.
-                for (ScheduleException exception : exceptionsIterator) {
-                    exceptions.add(exception);
+                List<ScheduleException> calendarExceptions = new ArrayList<>();
+                List<ScheduleException> calendarDateExceptions = new ArrayList<>();
+                // Separate distinct calendar date exceptions from those associated with calendars.
+                for (ScheduleException ex : exceptionsIterator) {
+                    if (ex.exemplar.equals(ScheduleException.ExemplarServiceDescriptor.CALENDAR_DATE_SERVICE)) {
+                        calendarDateExceptions.add(ex);
+                    } else {
+                        calendarExceptions.add(ex);
+                    }
                 }
+
+                int calendarDateCount = calendarDateExceptions.size();
+                // Extract calendar date services, convert to calendar date and add to the feed.
+                for (ScheduleException ex : calendarDateExceptions) {
+                    for (LocalDate date : ex.dates) {
+                        String serviceId = ex.customSchedule.get(0);
+                        CalendarDate calendarDate = new CalendarDate();
+                        calendarDate.date = date;
+                        calendarDate.service_id = serviceId;
+                        calendarDate.exception_type = 1;
+                        Service service = new Service(serviceId);
+                        service.calendar_dates.put(date, calendarDate);
+                        // If the calendar dates provided contain duplicates (e.g. two or more identical service ids
+                        // that are NOT associated with a calendar) only the first entry would persist export. To
+                        // resolve this a unique key consisting of service id and date is used.
+                        feed.services.put(String.format("%s-%s", calendarDate.service_id, calendarDate.date), service);
+                    }
+                }
+
                 // check whether the feed is organized in a format with the calendars.txt file
-                if (calendarsReader.getRowCount() > 0) {
+                if (calendarReader.getRowCount() > 0) {
                     // feed does have calendars.txt file, continue export with strategy of matching exceptions
                     // to calendar to output calendar_dates.txt
-                    int calendarDateCount = 0;
                     for (Calendar cal : calendars) {
                         Service service = new Service(cal.service_id);
                         service.calendar = cal;
-                        for (ScheduleException ex : exceptions.stream()
+                        for (ScheduleException ex : calendarExceptions.stream()
                             .filter(ex -> exceptionInvolvesService(ex, cal.service_id))
                             .collect(Collectors.toList())
                         ) {
-                            if (ex.exemplar.equals(ScheduleException.ExemplarServiceDescriptor.SWAP) &&
-                                (!ex.addedService.contains(cal.service_id) && !ex.removedService.contains(cal.service_id))) {
-                                // Skip swap exception if cal is not referenced by added or removed service.
-                                // This is not technically necessary, but the output is cleaner/more intelligible.
-                                continue;
-                            }
-
                             for (LocalDate date : ex.dates) {
                                 if (date.isBefore(cal.start_date) || date.isAfter(cal.end_date)) {
                                     // No need to write dates that do not apply
@@ -179,22 +197,24 @@ public FeedLoadResult exportTables() {
                                 LOG.info("Adding exception {} (type={}) for calendar {} on date {}", ex.name, calendarDate.exception_type, cal.service_id, date);
 
                                 if (service.calendar_dates.containsKey(date))
-                                    throw new IllegalArgumentException("Duplicate schedule exceptions on " + date.toString());
+                                    throw new IllegalArgumentException("Duplicate schedule exceptions on " + date);
 
                                 service.calendar_dates.put(date, calendarDate);
                                 calendarDateCount += 1;
                             }
                         }
                         feed.services.put(cal.service_id, service);
                     }
-                    if (calendarDateCount == 0) {
-                        LOG.info("No calendar dates found. Skipping table.");
-                    } else {
-                        LOG.info("Writing {} calendar dates from schedule exceptions", calendarDateCount);
-                        new CalendarDate.Writer(feed).writeTable(zipOutputStream);
-                    }
+                }
+                if (calendarDateCount == 0) {
+                    LOG.info("No calendar dates found. Skipping table.");
                 } else {
-                    // No calendar records exist, export calendar_dates as is and hope for the best.
+                    LOG.info("Writing {} calendar dates from schedule exceptions", calendarDateCount);
+                    new CalendarDate.Writer(feed).writeTable(zipOutputStream);
+                }
+
+                if (calendarReader.getRowCount() == 0 && calendarDateExceptions.isEmpty()) {
+                    // No calendar or calendar date service records exist, export calendar_dates as is and hope for the best.
                     // This situation will occur in at least 2 scenarios:
                     // 1.  A GTFS has been loaded into the editor that had only the calendar_dates.txt file
                     //     and no further edits were made before exporting to a snapshot