Merge pull request #567 from ibi-group/add-option-for-publishing-patt…

…erns

Add option to export patterns

pom.xml

@@ -272,7 +272,7 @@
             <groupId>com.github.conveyal</groupId>
             <artifactId>gtfs-lib</artifactId>
             <!-- Latest dev build on jitpack.io -->
-            <version>80a968cd3a071aa46a994ce2632535303a6e4384</version>
+            <version>10307d43c6</version>
             <!-- Exclusions added in order to silence SLF4J warnings about multiple bindings:
                 http://www.slf4j.org/codes.html#multiple_bindings
             -->

src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java

@@ -63,6 +63,12 @@ private static Snapshot getSnapshotFromRequest(Request req) {
         return Persistence.snapshots.getById(id);
     }
 
+    private static boolean getPublishProprietaryFiles(Request req) {
+        return Boolean.parseBoolean(
+            req.queryParamOrDefault("publishProprietaryFiles", Boolean.FALSE.toString())
+        );
+    }
+
     /**
      * HTTP endpoint that returns the list of snapshots for a given feed source.
      */
@@ -84,6 +90,7 @@ private static String createSnapshot (Request req, Response res) throws IOExcept
         boolean publishNewVersion = Boolean.parseBoolean(
             req.queryParamOrDefault("publishNewVersion", Boolean.FALSE.toString())
         );
+        boolean publishProprietaryFiles = getPublishProprietaryFiles(req);
         FeedSource feedSource = FeedVersionController.requestFeedSourceById(req, Actions.EDIT, "feedId");
         // Take fields from request body for creating snapshot (i.e., feedId/feedSourceId, name, comment).
         Snapshot snapshot = json.read(req.body());
@@ -99,7 +106,7 @@ private static String createSnapshot (Request req, Response res) throws IOExcept
                 new CreateSnapshotJob(userProfile, snapshot, bufferIsEmpty, !bufferIsEmpty, false);
         // Add publish feed version job if specified by request.
         if (publishNewVersion) {
-            createSnapshotJob.addNextJob(new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, userProfile));
+            createSnapshotJob.addNextJob(new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, userProfile, publishProprietaryFiles));
         }
         // Begin asynchronous execution.
         JobUtils.heavyExecutor.execute(createSnapshotJob);
@@ -173,9 +180,10 @@ private static String restoreSnapshot (Request req, Response res) {
     private static String downloadSnapshotAsGTFS(Request req, Response res) {
         Auth0UserProfile userProfile = req.attribute("user");
         Snapshot snapshot = getSnapshotFromRequest(req);
+        boolean publishProprietaryFiles = getPublishProprietaryFiles(req);
         // Create and kick off export job.
         // FIXME: what if a snapshot is already written to S3?
-        ExportSnapshotToGTFSJob exportSnapshotToGTFSJob = new ExportSnapshotToGTFSJob(userProfile,  snapshot);
+        ExportSnapshotToGTFSJob exportSnapshotToGTFSJob = new ExportSnapshotToGTFSJob(userProfile, snapshot, publishProprietaryFiles);
         JobUtils.heavyExecutor.execute(exportSnapshotToGTFSJob);
         return formatJobMessage(exportSnapshotToGTFSJob.jobId, "Exporting snapshot to GTFS.");
     }

src/main/java/com/conveyal/datatools/editor/jobs/ExportSnapshotToGTFSJob.java

@@ -28,16 +28,18 @@ public class ExportSnapshotToGTFSJob extends MonitorableJob {
     private final Snapshot snapshot;
     private final FeedVersion feedVersion;
     private File tempFile;
+    private final boolean publishProprietaryFiles;
 
-    public ExportSnapshotToGTFSJob(Auth0UserProfile owner, Snapshot snapshot, FeedVersion feedVersion) {
+    public ExportSnapshotToGTFSJob(Auth0UserProfile owner, Snapshot snapshot, FeedVersion feedVersion, boolean publishProprietaryFiles) {
         super(owner, "Exporting snapshot " + snapshot.name, JobType.EXPORT_SNAPSHOT_TO_GTFS);
         this.snapshot = snapshot;
         this.feedVersion = feedVersion;
+        this.publishProprietaryFiles = publishProprietaryFiles;
         status.update("Starting database snapshot...", 10);
     }
 
-    public ExportSnapshotToGTFSJob(Auth0UserProfile owner, Snapshot snapshot) {
-        this(owner, snapshot, null);
+    public ExportSnapshotToGTFSJob(Auth0UserProfile owner, Snapshot snapshot, boolean publishProprietaryFiles) {
+        this(owner, snapshot, null, publishProprietaryFiles);
     }
 
     @JsonProperty
@@ -57,7 +59,7 @@ public void jobLogic() {
             status.fail("Error creating local file for snapshot.", e);
             return;
         }
-        JdbcGtfsExporter exporter = new JdbcGtfsExporter(snapshot.namespace, tempFile.getAbsolutePath(), DataManager.GTFS_DATA_SOURCE, true);
+        JdbcGtfsExporter exporter = new JdbcGtfsExporter(snapshot.namespace, tempFile.getAbsolutePath(), DataManager.GTFS_DATA_SOURCE, true, publishProprietaryFiles);
         FeedLoadResult result = exporter.exportTables();
         if (result.fatalException != null) {
             status.fail(String.format("Error (%s) encountered while exporting database tables.", result.fatalException));

src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java

@@ -172,14 +172,16 @@ protected static FeedVersion cleanFeedVersionForNonAdmins(FeedVersion feedVersio
     private static boolean createFeedVersionFromSnapshot (Request req, Response res) {
 
         Auth0UserProfile userProfile = req.attribute("user");
+        Boolean publishProprietaryFiles = Boolean.parseBoolean(req.queryParams("publishProprietaryFiles"));
         // TODO: Should the ability to create a feedVersion from snapshot be controlled by the 'edit-gtfs' privilege?
         FeedSource feedSource = requestFeedSourceById(req, Actions.MANAGE);
         Snapshot snapshot = Persistence.snapshots.getById(req.queryParams("snapshotId"));
         if (snapshot == null) {
             logMessageAndHalt(req, 400, "Must provide valid snapshot ID");
         }
+        // Publishing the proprietary files will preserve the pattern names in the newly published feed version.
         CreateFeedVersionFromSnapshotJob createFromSnapshotJob =
-            new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, userProfile);
+            new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, userProfile, publishProprietaryFiles);
         JobUtils.heavyExecutor.execute(createFromSnapshotJob);
 
         return true;

src/main/java/com/conveyal/datatools/manager/jobs/CreateFeedVersionFromSnapshotJob.java

@@ -22,11 +22,13 @@ public class CreateFeedVersionFromSnapshotJob extends FeedSourceJob {
 
     private final FeedVersion feedVersion;
     private final Snapshot snapshot;
+    private final boolean publishProprietaryFiles;
 
-    public CreateFeedVersionFromSnapshotJob(FeedSource feedSource, Snapshot snapshot, Auth0UserProfile owner) {
+    public CreateFeedVersionFromSnapshotJob(FeedSource feedSource, Snapshot snapshot, Auth0UserProfile owner, boolean publishProprietaryFiles) {
         super(owner, "Creating Feed Version from Snapshot for " + feedSource.name, JobType.CREATE_FEEDVERSION_FROM_SNAPSHOT);
         this.feedVersion = new FeedVersion(feedSource, snapshot);
         this.snapshot = snapshot;
+        this.publishProprietaryFiles = publishProprietaryFiles;
     }
 
     @Override
@@ -35,7 +37,7 @@ public void jobLogic() {
         // Add the jobs to handle this operation in order.
         addNextJob(
             // First export the snapshot to GTFS.
-            new ExportSnapshotToGTFSJob(owner, snapshot, feedVersion),
+            new ExportSnapshotToGTFSJob(owner, snapshot, feedVersion, publishProprietaryFiles),
             // Then, process feed version once GTFS file written.
             new ProcessSingleFeedJob(feedVersion, owner, true)
         );

src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java

@@ -130,7 +130,8 @@ public void jobLogic() {
             snapshot.feedTransformResult = dbTarget.feedTransformResult;
             // If the user has selected to create a new version from the resulting snapshot, do so here.
             if (rules.createNewVersion) {
-                addNextJob(new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, owner));
+                // Publishing the proprietary files will preserve the pattern names in the newly created feed version.
+                addNextJob(new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, owner, true));
             }
         }
 

src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java

@@ -160,7 +160,7 @@ void canDeleteTrips() throws IOException {
         );
         // Fetch snapshot where modifications were made and create new version from it.
         Snapshot snapshotWithModifications = feedSource.retrieveSnapshots().iterator().next();
-        CreateFeedVersionFromSnapshotJob newVersionJob = new CreateFeedVersionFromSnapshotJob(feedSource, snapshotWithModifications, user);
+        CreateFeedVersionFromSnapshotJob newVersionJob = new CreateFeedVersionFromSnapshotJob(feedSource, snapshotWithModifications, user, false);
         newVersionJob.run();
         // Grab the modified version and check that the trips count matches expectation.
         FeedVersion newVersion = feedSource.retrieveLatest();