From 023f1943318fffd5dd710be4a155f9e697a37656 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Thu, 5 Oct 2023 15:53:47 -0400 Subject: [PATCH 1/8] feat(SnapshotExport): initial work to add option to export patterns --- pom.xml | 4 ++-- .../editor/controllers/api/SnapshotController.java | 5 ++++- .../editor/jobs/ExportSnapshotToGTFSJob.java | 12 +++++++++++- .../controllers/api/FeedVersionController.java | 2 +- .../jobs/CreateFeedVersionFromSnapshotJob.java | 6 ++++-- .../datatools/manager/jobs/ProcessSingleFeedJob.java | 2 +- .../manager/jobs/ArbitraryTransformJobTest.java | 2 +- 7 files changed, 24 insertions(+), 9 deletions(-) diff --git a/pom.xml b/pom.xml index 5f5a80fe2..60e1a74b8 100644 --- a/pom.xml +++ b/pom.xml @@ -265,10 +265,10 @@ - AWS S3 SDK - putting/getting objects into/out of S3. --> - com.github.conveyal + com.conveyal gtfs-lib - 80a968cd3a071aa46a994ce2632535303a6e4384 + 7.0.2 diff --git a/src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java b/src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java index bce416e93..6ed68bcc7 100644 --- a/src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java +++ b/src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java @@ -84,6 +84,9 @@ private static String createSnapshot (Request req, Response res) throws IOExcept boolean publishNewVersion = Boolean.parseBoolean( req.queryParamOrDefault("publishNewVersion", Boolean.FALSE.toString()) ); + boolean publishProprietaryFiles = Boolean.parseBoolean( + req.queryParamOrDefault("publishProprietaryFiles", Boolean.FALSE.toString()) + ); FeedSource feedSource = FeedVersionController.requestFeedSourceById(req, Actions.EDIT, "feedId"); // Take fields from request body for creating snapshot (i.e., feedId/feedSourceId, name, comment). Snapshot snapshot = json.read(req.body()); @@ -99,7 +102,7 @@ private static String createSnapshot (Request req, Response res) throws IOExcept new CreateSnapshotJob(userProfile, snapshot, bufferIsEmpty, !bufferIsEmpty, false); // Add publish feed version job if specified by request. if (publishNewVersion) { - createSnapshotJob.addNextJob(new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, userProfile)); + createSnapshotJob.addNextJob(new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, userProfile, publishProprietaryFiles)); } // Begin asynchronous execution. JobUtils.heavyExecutor.execute(createSnapshotJob); diff --git a/src/main/java/com/conveyal/datatools/editor/jobs/ExportSnapshotToGTFSJob.java b/src/main/java/com/conveyal/datatools/editor/jobs/ExportSnapshotToGTFSJob.java index 3798c2ee8..53c56b746 100644 --- a/src/main/java/com/conveyal/datatools/editor/jobs/ExportSnapshotToGTFSJob.java +++ b/src/main/java/com/conveyal/datatools/editor/jobs/ExportSnapshotToGTFSJob.java @@ -28,11 +28,21 @@ public class ExportSnapshotToGTFSJob extends MonitorableJob { private final Snapshot snapshot; private final FeedVersion feedVersion; private File tempFile; + private final boolean publishProprietaryFiles; public ExportSnapshotToGTFSJob(Auth0UserProfile owner, Snapshot snapshot, FeedVersion feedVersion) { super(owner, "Exporting snapshot " + snapshot.name, JobType.EXPORT_SNAPSHOT_TO_GTFS); this.snapshot = snapshot; this.feedVersion = feedVersion; + this.publishProprietaryFiles = false; // DEFAULT. TODO: don't hide this here. + status.update("Starting database snapshot...", 10); + } + + public ExportSnapshotToGTFSJob(Auth0UserProfile owner, Snapshot snapshot, FeedVersion feedVersion, boolean publishProprietaryFiles) { + super(owner, "Exporting snapshot " + snapshot.name, JobType.EXPORT_SNAPSHOT_TO_GTFS); + this.snapshot = snapshot; + this.feedVersion = feedVersion; + this.publishProprietaryFiles = publishProprietaryFiles; status.update("Starting database snapshot...", 10); } @@ -57,7 +67,7 @@ public void jobLogic() { status.fail("Error creating local file for snapshot.", e); return; } - JdbcGtfsExporter exporter = new JdbcGtfsExporter(snapshot.namespace, tempFile.getAbsolutePath(), DataManager.GTFS_DATA_SOURCE, true); + JdbcGtfsExporter exporter = new JdbcGtfsExporter(snapshot.namespace, tempFile.getAbsolutePath(), DataManager.GTFS_DATA_SOURCE, true, publishProprietaryFiles); FeedLoadResult result = exporter.exportTables(); if (result.fatalException != null) { status.fail(String.format("Error (%s) encountered while exporting database tables.", result.fatalException)); diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java index 035269e66..a33a64881 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java @@ -179,7 +179,7 @@ private static boolean createFeedVersionFromSnapshot (Request req, Response res) logMessageAndHalt(req, 400, "Must provide valid snapshot ID"); } CreateFeedVersionFromSnapshotJob createFromSnapshotJob = - new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, userProfile); + new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, userProfile, false); JobUtils.heavyExecutor.execute(createFromSnapshotJob); return true; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/CreateFeedVersionFromSnapshotJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/CreateFeedVersionFromSnapshotJob.java index 6c711c74e..c6929a93b 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/CreateFeedVersionFromSnapshotJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/CreateFeedVersionFromSnapshotJob.java @@ -22,11 +22,13 @@ public class CreateFeedVersionFromSnapshotJob extends FeedSourceJob { private final FeedVersion feedVersion; private final Snapshot snapshot; + private final boolean publishProprietaryFiles; - public CreateFeedVersionFromSnapshotJob(FeedSource feedSource, Snapshot snapshot, Auth0UserProfile owner) { + public CreateFeedVersionFromSnapshotJob(FeedSource feedSource, Snapshot snapshot, Auth0UserProfile owner, boolean publishProprietaryFiles) { super(owner, "Creating Feed Version from Snapshot for " + feedSource.name, JobType.CREATE_FEEDVERSION_FROM_SNAPSHOT); this.feedVersion = new FeedVersion(feedSource, snapshot); this.snapshot = snapshot; + this.publishProprietaryFiles = publishProprietaryFiles; } @Override @@ -35,7 +37,7 @@ public void jobLogic() { // Add the jobs to handle this operation in order. addNextJob( // First export the snapshot to GTFS. - new ExportSnapshotToGTFSJob(owner, snapshot, feedVersion), + new ExportSnapshotToGTFSJob(owner, snapshot, feedVersion, publishProprietaryFiles), // Then, process feed version once GTFS file written. new ProcessSingleFeedJob(feedVersion, owner, true) ); diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java index f26eba655..27585d946 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java @@ -130,7 +130,7 @@ public void jobLogic() { snapshot.feedTransformResult = dbTarget.feedTransformResult; // If the user has selected to create a new version from the resulting snapshot, do so here. if (rules.createNewVersion) { - addNextJob(new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, owner)); + addNextJob(new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, owner, false)); } } diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java index a8924973e..9303bf821 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java @@ -160,7 +160,7 @@ void canDeleteTrips() throws IOException { ); // Fetch snapshot where modifications were made and create new version from it. Snapshot snapshotWithModifications = feedSource.retrieveSnapshots().iterator().next(); - CreateFeedVersionFromSnapshotJob newVersionJob = new CreateFeedVersionFromSnapshotJob(feedSource, snapshotWithModifications, user); + CreateFeedVersionFromSnapshotJob newVersionJob = new CreateFeedVersionFromSnapshotJob(feedSource, snapshotWithModifications, user, false); newVersionJob.run(); // Grab the modified version and check that the trips count matches expectation. FeedVersion newVersion = feedSource.retrieveLatest(); From d4e780430a0969d08bd484a32e251db5f2b0d446 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Fri, 6 Oct 2023 15:53:10 -0400 Subject: [PATCH 2/8] refactor(SnapshotExport): update gtfs-lib, add comment --- pom.xml | 4 ++-- .../manager/controllers/api/FeedVersionController.java | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 60e1a74b8..5f5a80fe2 100644 --- a/pom.xml +++ b/pom.xml @@ -265,10 +265,10 @@ - AWS S3 SDK - putting/getting objects into/out of S3. --> - com.conveyal + com.github.conveyal gtfs-lib - 7.0.2 + 80a968cd3a071aa46a994ce2632535303a6e4384 diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java index a33a64881..f4fe5c4d6 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java @@ -178,6 +178,7 @@ private static boolean createFeedVersionFromSnapshot (Request req, Response res) if (snapshot == null) { logMessageAndHalt(req, 400, "Must provide valid snapshot ID"); } + // TODO: Allow publishing with proprietary files from this endpoint? CreateFeedVersionFromSnapshotJob createFromSnapshotJob = new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, userProfile, false); JobUtils.heavyExecutor.execute(createFromSnapshotJob); From 1a92792ce9fa09737b94d87e507add8e5acfcd3b Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Fri, 6 Oct 2023 15:56:30 -0400 Subject: [PATCH 3/8] refactor(SnapshotExport): update gtfs-lib again --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5f5a80fe2..0c3d529d1 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - 80a968cd3a071aa46a994ce2632535303a6e4384 + a5f60417e38043d35148aa6873c93056e69e9c87 From 87e54a46fbe4bc60cecf12c297de19c57b28dab5 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 10 Oct 2023 09:55:18 +0100 Subject: [PATCH 4/8] refactor(Upstream changes to export call): Updated to export proprietary files with comments --- pom.xml | 2 +- .../editor/controllers/api/SnapshotController.java | 13 +++++++++---- .../editor/jobs/ExportSnapshotToGTFSJob.java | 12 ++---------- .../controllers/api/FeedVersionController.java | 4 ++-- .../manager/jobs/ProcessSingleFeedJob.java | 3 ++- 5 files changed, 16 insertions(+), 18 deletions(-) diff --git a/pom.xml b/pom.xml index 0c3d529d1..8affc0ccb 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - a5f60417e38043d35148aa6873c93056e69e9c87 + 026599ba4907f4c42b24e883d154e63b0777dd31 diff --git a/src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java b/src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java index 6ed68bcc7..ebe45b692 100644 --- a/src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java +++ b/src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java @@ -63,6 +63,12 @@ private static Snapshot getSnapshotFromRequest(Request req) { return Persistence.snapshots.getById(id); } + private static boolean getPublishProprietaryFiles(Request req) { + return Boolean.parseBoolean( + req.queryParamOrDefault("publishProprietaryFiles",Boolean.FALSE.toString()) + ); + } + /** * HTTP endpoint that returns the list of snapshots for a given feed source. */ @@ -84,9 +90,7 @@ private static String createSnapshot (Request req, Response res) throws IOExcept boolean publishNewVersion = Boolean.parseBoolean( req.queryParamOrDefault("publishNewVersion", Boolean.FALSE.toString()) ); - boolean publishProprietaryFiles = Boolean.parseBoolean( - req.queryParamOrDefault("publishProprietaryFiles", Boolean.FALSE.toString()) - ); + boolean publishProprietaryFiles = getPublishProprietaryFiles(req); FeedSource feedSource = FeedVersionController.requestFeedSourceById(req, Actions.EDIT, "feedId"); // Take fields from request body for creating snapshot (i.e., feedId/feedSourceId, name, comment). Snapshot snapshot = json.read(req.body()); @@ -176,9 +180,10 @@ private static String restoreSnapshot (Request req, Response res) { private static String downloadSnapshotAsGTFS(Request req, Response res) { Auth0UserProfile userProfile = req.attribute("user"); Snapshot snapshot = getSnapshotFromRequest(req); + boolean publishProprietaryFiles = getPublishProprietaryFiles(req); // Create and kick off export job. // FIXME: what if a snapshot is already written to S3? - ExportSnapshotToGTFSJob exportSnapshotToGTFSJob = new ExportSnapshotToGTFSJob(userProfile, snapshot); + ExportSnapshotToGTFSJob exportSnapshotToGTFSJob = new ExportSnapshotToGTFSJob(userProfile, snapshot, publishProprietaryFiles); JobUtils.heavyExecutor.execute(exportSnapshotToGTFSJob); return formatJobMessage(exportSnapshotToGTFSJob.jobId, "Exporting snapshot to GTFS."); } diff --git a/src/main/java/com/conveyal/datatools/editor/jobs/ExportSnapshotToGTFSJob.java b/src/main/java/com/conveyal/datatools/editor/jobs/ExportSnapshotToGTFSJob.java index 53c56b746..fc76890b5 100644 --- a/src/main/java/com/conveyal/datatools/editor/jobs/ExportSnapshotToGTFSJob.java +++ b/src/main/java/com/conveyal/datatools/editor/jobs/ExportSnapshotToGTFSJob.java @@ -30,14 +30,6 @@ public class ExportSnapshotToGTFSJob extends MonitorableJob { private File tempFile; private final boolean publishProprietaryFiles; - public ExportSnapshotToGTFSJob(Auth0UserProfile owner, Snapshot snapshot, FeedVersion feedVersion) { - super(owner, "Exporting snapshot " + snapshot.name, JobType.EXPORT_SNAPSHOT_TO_GTFS); - this.snapshot = snapshot; - this.feedVersion = feedVersion; - this.publishProprietaryFiles = false; // DEFAULT. TODO: don't hide this here. - status.update("Starting database snapshot...", 10); - } - public ExportSnapshotToGTFSJob(Auth0UserProfile owner, Snapshot snapshot, FeedVersion feedVersion, boolean publishProprietaryFiles) { super(owner, "Exporting snapshot " + snapshot.name, JobType.EXPORT_SNAPSHOT_TO_GTFS); this.snapshot = snapshot; @@ -46,8 +38,8 @@ public ExportSnapshotToGTFSJob(Auth0UserProfile owner, Snapshot snapshot, FeedVe status.update("Starting database snapshot...", 10); } - public ExportSnapshotToGTFSJob(Auth0UserProfile owner, Snapshot snapshot) { - this(owner, snapshot, null); + public ExportSnapshotToGTFSJob(Auth0UserProfile owner, Snapshot snapshot, boolean publishProprietaryFiles) { + this(owner, snapshot, null, publishProprietaryFiles); } @JsonProperty diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java index f4fe5c4d6..8f855efea 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java @@ -178,9 +178,9 @@ private static boolean createFeedVersionFromSnapshot (Request req, Response res) if (snapshot == null) { logMessageAndHalt(req, 400, "Must provide valid snapshot ID"); } - // TODO: Allow publishing with proprietary files from this endpoint? + // Publishing the proprietary files will preserve the pattern names in the newly published feed version. CreateFeedVersionFromSnapshotJob createFromSnapshotJob = - new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, userProfile, false); + new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, userProfile, true); JobUtils.heavyExecutor.execute(createFromSnapshotJob); return true; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java index 27585d946..c267fdd9a 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java @@ -130,7 +130,8 @@ public void jobLogic() { snapshot.feedTransformResult = dbTarget.feedTransformResult; // If the user has selected to create a new version from the resulting snapshot, do so here. if (rules.createNewVersion) { - addNextJob(new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, owner, false)); + // Publishing the proprietary files will preserve the pattern names in the newly created feed version. + addNextJob(new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, owner, true)); } } From 7f64621960eb162b7fe162db790d45e93c03ed04 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 10 Oct 2023 12:33:43 +0100 Subject: [PATCH 5/8] refactor(pom.xml): Bumped GTFS-lib version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8affc0ccb..5cb786373 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - 026599ba4907f4c42b24e883d154e63b0777dd31 + 10307d43c6610d5dd1cc07d30091596816772727 From 4b641c05be2f1188a2a7ebcd0d5d89b8f43eef4d Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Mon, 6 Nov 2023 17:00:35 -0500 Subject: [PATCH 6/8] refactor(FeedVersionController): parse publishProprietaryFiles for snapshot req --- .../manager/controllers/api/FeedVersionController.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java index 8f855efea..9dffd6adc 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java @@ -172,6 +172,7 @@ protected static FeedVersion cleanFeedVersionForNonAdmins(FeedVersion feedVersio private static boolean createFeedVersionFromSnapshot (Request req, Response res) { Auth0UserProfile userProfile = req.attribute("user"); + Boolean publishProprietaryFiles = Boolean.parseBoolean(req.queryParams("publishProprietaryFiles")); // TODO: Should the ability to create a feedVersion from snapshot be controlled by the 'edit-gtfs' privilege? FeedSource feedSource = requestFeedSourceById(req, Actions.MANAGE); Snapshot snapshot = Persistence.snapshots.getById(req.queryParams("snapshotId")); @@ -180,7 +181,7 @@ private static boolean createFeedVersionFromSnapshot (Request req, Response res) } // Publishing the proprietary files will preserve the pattern names in the newly published feed version. CreateFeedVersionFromSnapshotJob createFromSnapshotJob = - new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, userProfile, true); + new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, userProfile, publishProprietaryFiles); JobUtils.heavyExecutor.execute(createFromSnapshotJob); return true; From 1c03a7571ba64169656ea51ac1e897e858106753 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Wed, 15 Nov 2023 13:50:16 -0500 Subject: [PATCH 7/8] refactor(SnapshotController): update formatting --- .../datatools/editor/controllers/api/SnapshotController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java b/src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java index ebe45b692..58d93fee9 100644 --- a/src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java +++ b/src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java @@ -65,7 +65,7 @@ private static Snapshot getSnapshotFromRequest(Request req) { private static boolean getPublishProprietaryFiles(Request req) { return Boolean.parseBoolean( - req.queryParamOrDefault("publishProprietaryFiles",Boolean.FALSE.toString()) + req.queryParamOrDefault("publishProprietaryFiles", Boolean.FALSE.toString()) ); } From b2a3116e10b2f7bd95554ad220a2568344b39d66 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Wed, 15 Nov 2023 13:57:31 -0500 Subject: [PATCH 8/8] refactor(deps): update gtfs lib commit --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5cb786373..3d44b0dd6 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - 10307d43c6610d5dd1cc07d30091596816772727 + 10307d43c6