Merge pull request #324 from conveyal/dev

gtfs-lib new major version release

.github/workflows/maven.yml

@@ -48,10 +48,6 @@ jobs:
         key: maven-local-repo
     - name: Build with Maven
       run: mvn --no-transfer-progress package
-    - name: Codecov
-      # this first codecov run will upload a report associated with the commit set through CI environment variables
-      uses: codecov/[email protected]
-      continue-on-error: true
     - name: Clear contents of the target directory
       # Avoids issues where maven-semantic-release attempts to upload
       # multiple versions/builds (and fails due to the pre-existence of the version on maven central).
@@ -71,7 +67,3 @@ jobs:
         GH_TOKEN: ${{ secrets.GH_TOKEN }}
       run: |
        semantic-release --prepare @conveyal/maven-semantic-release --publish @semantic-release/github,@conveyal/maven-semantic-release --verify-conditions @semantic-release/github,@conveyal/maven-semantic-release --verify-release @conveyal/maven-semantic-release --use-conveyal-workflow --dev-branch=dev --skip-maven-deploy
-       if [[ "$GITHUB_REF_SLUG" = "master" ]]; then
-         bash <(curl -s https://codecov.io/bash) -C "$(git rev-parse HEAD)"
-         bash <(curl -s https://codecov.io/bash) -C "$(git rev-parse HEAD^)"
-       fi

.gitignore

@@ -8,3 +8,5 @@
 *.iml
 .idea/
 target/
+
+lambda$*.json

README.md

@@ -19,15 +19,24 @@ A gtfs-lib GTFSFeed object should faithfully represent the contents of a single
 gtfs-lib can be used as a Java library or run via the command line.  If using this library with PostgreSQL for persistence, you must use at least version 9.6 of PostgreSQL.
 
 ### Library (maven)
-
+Include gtfs-lib as a library in your project with the following dependency in your `pom.xml`.
 ```xml
 <dependency>
-  <groupId>com.conveyal</groupId>
+  <!-- Note: to use a local version development version, the groupId should be changed to com.conveyal -->
+  <groupId>com.github.conveyal</groupId>
   <artifactId>gtfs-lib</artifactId>
   <version>${choose-a-version}</version>
 </dependency>
 ```
 
+#### Jitpack
+
+gtfs-lib builds are hosted on [jitpack](https://jitpack.io/#conveyal/gtfs-lib). 
+
+[Release versions](https://github.com/conveyal/gtfs-lib/releases) are available by default.
+
+Branch- (e.g. `dev-SNAPSHOT`) or commit-specific (using a 10 character commit ID like `a04294e420`) snapshot builds can be triggered by clicking `Get` for the build of your choice on jitpack's website or visiting https://jitpack.io/#conveyal/gtfs-lib/YOUR_VERSION.
+
 ### Command line
 
 ```bash

pom.xml

@@ -123,22 +123,6 @@
           </execution>
         </executions>
       </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-gpg-plugin</artifactId>
-        <version>1.6</version>
-        <executions>
-          <execution>
-            <id>sign-artifacts</id>
-            <!-- We sign in the verify phase, which means it will happen for install and deploy but not package. -->
-            <phase>verify</phase>
-            <goals>
-              <goal>sign</goal>
-            </goals>
-          </execution>
-        </executions>
-      </plugin>
-
       <plugin>
         <!-- Attach a version of the JAR that includes all dependencies -->
         <groupId>org.apache.maven.plugins</groupId>
@@ -234,6 +218,13 @@
       <version>5.7.0</version>
       <scope>test</scope>
     </dependency>
+    <!-- Used for parameterized tests -->
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-params</artifactId>
+      <version>5.6.2</version>
+      <scope>test</scope>
+    </dependency>
     <!-- Parses command line arguments to main methods. -->
     <dependency>
     	<groupId>com.beust</groupId>
@@ -292,7 +283,7 @@
     <dependency>
     	<groupId>commons-io</groupId>
     	<artifactId>commons-io</artifactId>
-    	<version>2.4</version>
+    	<version>2.7</version>
     </dependency>
     <!-- Commons DBUtils is a collection of utility methods that help avoid boilerplate when working with SQL. -->
     <dependency>

src/main/java/com/conveyal/gtfs/GTFSFeed.java

@@ -14,7 +14,6 @@
 import org.locationtech.jts.algorithm.ConvexHull;
 import org.locationtech.jts.geom.*;
 import org.locationtech.jts.index.strtree.STRtree;
-import org.locationtech.jts.simplify.DouglasPeuckerSimplifier;
 import org.mapdb.BTreeMap;
 import org.mapdb.DB;
 import org.mapdb.DBMaker;
@@ -66,6 +65,8 @@ public class GTFSFeed implements Cloneable, Closeable {
     public final Map<String, Stop> stops;
     public final Map<String, Transfer> transfers;
     public final BTreeMap<String, Trip> trips;
+    public final Map<String, Translation> translations;
+    public final Map<String, Attribution> attributions;
 
     public final Set<String> transitIds = new HashSet<>();
     /** CRC32 of the GTFS file this was loaded from */
@@ -636,6 +637,8 @@ private GTFSFeed (DB db) {
         fares = db.getTreeMap("fares");
         services = db.getTreeMap("services");
         shape_points = db.getTreeMap("shape_points");
+        translations = db.getTreeMap("translations");
+        attributions = db.getTreeMap("attributions");
 
         feedId = db.getAtomicString("feed_id").get();
         checksum = db.getAtomicLong("checksum").get();

src/main/java/com/conveyal/gtfs/TripPatternKey.java

@@ -24,6 +24,8 @@ public class TripPatternKey {
     public TIntList arrivalTimes = new TIntArrayList();
     public TIntList departureTimes = new TIntArrayList();
     public TIntList timepoints = new TIntArrayList();
+    public TIntList continuous_pickup = new TIntArrayList();
+    public TIntList continuous_drop_off = new TIntArrayList();
     public TDoubleList shapeDistances = new TDoubleArrayList();
 
     public TripPatternKey (String routeId) {
@@ -39,6 +41,8 @@ public void addStopTime (StopTime st) {
         departureTimes.add(st.departure_time);
         timepoints.add(st.timepoint);
         shapeDistances.add(st.shape_dist_traveled);
+        continuous_pickup.add(st.continuous_pickup);
+        continuous_drop_off.add(st.continuous_drop_off);
     }
 
     @Override

src/main/java/com/conveyal/gtfs/error/NewGTFSError.java

@@ -1,5 +1,6 @@
 package com.conveyal.gtfs.error;
 
+import com.conveyal.gtfs.loader.LineContext;
 import com.conveyal.gtfs.loader.Table;
 import com.conveyal.gtfs.model.Entity;
 import org.slf4j.Logger;
@@ -82,6 +83,14 @@ public static NewGTFSError forLine (Table table, int lineNumber, NewGTFSErrorTyp
         return error;
     }
 
+    // Factory Builder for cases where an entity has not yet been constructed, but we know the line number.
+    public static NewGTFSError forLine(LineContext lineContext, NewGTFSErrorType errorType, String badValue) {
+        NewGTFSError error = new NewGTFSError(lineContext.table.getEntityClass(), errorType);
+        error.lineNumber = lineContext.lineNumber;
+        error.badValue = badValue;
+        return error;
+    }
+
     // Factory Builder for cases where the entity has already been decoded and an error is discovered during validation
     public static NewGTFSError forEntity(Entity entity, NewGTFSErrorType errorType) {
         NewGTFSError error = new NewGTFSError(entity.getClass(), errorType);

src/main/java/com/conveyal/gtfs/error/NewGTFSErrorType.java

@@ -8,9 +8,11 @@
  */
 public enum NewGTFSErrorType {
     // Standard errors.
+    AGENCY_ID_REQUIRED_FOR_MULTI_AGENCY_FEEDS(Priority.HIGH, "For GTFS feeds with more than one agency, agency_id is required."),
     BOOLEAN_FORMAT(Priority.MEDIUM, "A GTFS boolean field must contain the value 1 or 0."),
     COLOR_FORMAT(Priority.MEDIUM, "A color should be specified with six-characters (three two-digit hexadecimal numbers)."),
     COLUMN_NAME_UNSAFE(Priority.HIGH, "Column header contains characters not safe in SQL, it was renamed."),
+    CONDITIONALLY_REQUIRED(Priority.HIGH, "A conditionally required field was missing in a particular row."),
     CURRENCY_UNKNOWN(Priority.MEDIUM, "The currency code was not recognized."),
     DATE_FORMAT(Priority.MEDIUM, "Date format should be YYYYMMDD."),
     DATE_NO_SERVICE(Priority.MEDIUM, "No service_ids were active on a date within the range of dates with defined service."),

src/main/java/com/conveyal/gtfs/graphql/GraphQLGtfsSchema.java

@@ -144,9 +144,12 @@ public class GraphQLGtfsSchema {
             .description("A GTFS feed_info object")
             .field(MapFetcher.field("id", GraphQLInt))
             .field(MapFetcher.field("feed_id"))
+            .field(MapFetcher.field("feed_contact_email"))
+            .field(MapFetcher.field("feed_contact_url"))
             .field(MapFetcher.field("feed_publisher_name"))
             .field(MapFetcher.field("feed_publisher_url"))
             .field(MapFetcher.field("feed_lang"))
+            .field(MapFetcher.field("default_lang"))
             .field(MapFetcher.field("feed_start_date"))
             .field(MapFetcher.field("feed_end_date"))
             .field(MapFetcher.field("feed_version"))
@@ -257,10 +260,37 @@ public class GraphQLGtfsSchema {
             .field(MapFetcher.field("timepoint", GraphQLInt))
             .field(MapFetcher.field("drop_off_type", GraphQLInt))
             .field(MapFetcher.field("pickup_type", GraphQLInt))
-            // Editor-specific fields
+            .field(MapFetcher.field("continuous_drop_off", GraphQLInt))
+            .field(MapFetcher.field("continuous_pickup", GraphQLInt))
             .field(MapFetcher.field("shape_dist_traveled", GraphQLFloat))
             .build();
 
+    // Represents rows from attributions.txt
+    public static final GraphQLObjectType attributionsType = newObject().name("attributions")
+        .field(MapFetcher.field("attribution_id"))
+        .field(MapFetcher.field("agency_id"))
+        .field(MapFetcher.field("route_id"))
+        .field(MapFetcher.field("trip_id"))
+        .field(MapFetcher.field("organization_name"))
+        .field(MapFetcher.field("is_producer", GraphQLInt))
+        .field(MapFetcher.field("is_operator", GraphQLInt))
+        .field(MapFetcher.field("is_authority", GraphQLInt))
+        .field(MapFetcher.field("attribution_url"))
+        .field(MapFetcher.field("attribution_email"))
+        .field(MapFetcher.field("attribution_phone"))
+        .build();
+
+    // Represents rows from translations.txt
+    public static final GraphQLObjectType translationsType = newObject().name("translations")
+        .field(MapFetcher.field("table_name"))
+        .field(MapFetcher.field("field_name"))
+        .field(MapFetcher.field("language"))
+        .field(MapFetcher.field("translation"))
+        .field(MapFetcher.field("record_id"))
+        .field(MapFetcher.field("record_sub_id"))
+        .field(MapFetcher.field("field_value"))
+        .build();
+
     // Represents rows from routes.txt
     public static final GraphQLObjectType routeType = newObject().name("route")
             .description("A line from a GTFS routes.txt table")
@@ -272,7 +302,8 @@ public class GraphQLGtfsSchema {
             .field(MapFetcher.field("route_desc"))
             .field(MapFetcher.field("route_url"))
             .field(MapFetcher.field("route_branding_url"))
-            // TODO route_type as enum or int
+            .field(MapFetcher.field("continuous_drop_off", GraphQLInt))
+            .field(MapFetcher.field("continuous_pickup", GraphQLInt))
             .field(MapFetcher.field("route_type", GraphQLInt))
             .field(MapFetcher.field("route_color"))
             .field(MapFetcher.field("route_text_color"))
@@ -341,6 +372,7 @@ public class GraphQLGtfsSchema {
             .field(MapFetcher.field("stop_url"))
             .field(MapFetcher.field("stop_timezone"))
             .field(MapFetcher.field("parent_station"))
+            .field(MapFetcher.field("platform_code"))
             .field(MapFetcher.field("location_type", GraphQLInt))
             .field(MapFetcher.field("wheelchair_boarding", GraphQLInt))
             // Returns all stops that reference parent stop's stop_id
@@ -403,6 +435,8 @@ public class GraphQLGtfsSchema {
             .field(MapFetcher.field("shape_dist_traveled", GraphQLFloat))
             .field(MapFetcher.field("drop_off_type", GraphQLInt))
             .field(MapFetcher.field("pickup_type", GraphQLInt))
+            .field(MapFetcher.field("continuous_drop_off", GraphQLInt))
+            .field(MapFetcher.field("continuous_pickup", GraphQLInt))
             .field(MapFetcher.field("stop_sequence", GraphQLInt))
             .field(MapFetcher.field("timepoint", GraphQLInt))
             // FIXME: This will only returns a list with one stop entity (unless there is a referential integrity issue)
@@ -763,6 +797,26 @@ public class GraphQLGtfsSchema {
                     .dataFetcher(new JDBCFetcher("services"))
                     .build()
             )
+            .field(newFieldDefinition()
+                .name("attributions")
+                .type(new GraphQLList(GraphQLGtfsSchema.attributionsType))
+                .argument(stringArg("namespace")) // FIXME maybe these nested namespace arguments are not doing anything.
+                .argument(intArg(ID_ARG))
+                .argument(intArg(LIMIT_ARG))
+                .argument(intArg(OFFSET_ARG))
+                .dataFetcher(new JDBCFetcher("attributions"))
+                .build()
+            )
+            .field(newFieldDefinition()
+                .name("translations")
+                .type(new GraphQLList(GraphQLGtfsSchema.translationsType))
+                .argument(stringArg("namespace")) // FIXME maybe these nested namespace arguments are not doing anything.
+                .argument(intArg(ID_ARG))
+                .argument(intArg(LIMIT_ARG))
+                .argument(intArg(OFFSET_ARG))
+                .dataFetcher(new JDBCFetcher("translations"))
+                .build()
+            )
             .build();
 
     /**

src/main/java/com/conveyal/gtfs/loader/EntityPopulator.java

@@ -64,6 +64,8 @@ public interface EntityPopulator<T> {
         patternStop.stop_sequence = getIntIfPresent(result, "stop_sequence", columnForName);
         patternStop.timepoint = getIntIfPresent(result, "timepoint", columnForName);
         patternStop.shape_dist_traveled = getDoubleIfPresent(result, "shape_dist_traveled", columnForName);
+        patternStop.continuous_pickup   = getIntIfPresent   (result, "continuous_pickup",   columnForName);
+        patternStop.continuous_drop_off = getIntIfPresent   (result, "continuous_drop_off", columnForName);
         return patternStop;
     };
 
@@ -139,17 +141,19 @@ public interface EntityPopulator<T> {
     };
 
     EntityPopulator<Route> ROUTE = (result, columnForName) -> {
-        Route route              = new Route();
-        route.route_id           = getStringIfPresent(result, "route_id",           columnForName);
-        route.agency_id          = getStringIfPresent(result, "agency_id",          columnForName);
-        route.route_short_name   = getStringIfPresent(result, "route_short_name",   columnForName);
-        route.route_long_name    = getStringIfPresent(result, "route_long_name",    columnForName);
-        route.route_desc         = getStringIfPresent(result, "route_desc",         columnForName);
-        route.route_type         = getIntIfPresent   (result, "route_type",         columnForName);
-        route.route_color        = getStringIfPresent(result, "route_color",        columnForName);
-        route.route_text_color   = getStringIfPresent(result, "route_text_color",   columnForName);
-        route.route_url          = getUrlIfPresent   (result, "route_url",          columnForName);
-        route.route_branding_url = getUrlIfPresent   (result, "route_branding_url", columnForName);
+        Route route               = new Route();
+        route.route_id            = getStringIfPresent(result, "route_id",            columnForName);
+        route.agency_id           = getStringIfPresent(result, "agency_id",           columnForName);
+        route.route_short_name    = getStringIfPresent(result, "route_short_name",    columnForName);
+        route.route_long_name     = getStringIfPresent(result, "route_long_name",     columnForName);
+        route.route_desc          = getStringIfPresent(result, "route_desc",          columnForName);
+        route.route_type          = getIntIfPresent   (result, "route_type",          columnForName);
+        route.route_color         = getStringIfPresent(result, "route_color",         columnForName);
+        route.route_text_color    = getStringIfPresent(result, "route_text_color",    columnForName);
+        route.route_url           = getUrlIfPresent   (result, "route_url",           columnForName);
+        route.route_branding_url  = getUrlIfPresent   (result, "route_branding_url",  columnForName);
+        route.continuous_pickup   = getIntIfPresent   (result, "continuous_pickup",   columnForName);
+        route.continuous_drop_off = getIntIfPresent   (result, "continuous_drop_off", columnForName);
         return route;
     };
 
@@ -166,7 +170,8 @@ public interface EntityPopulator<T> {
         stop.stop_timezone  = getStringIfPresent(result, "stop_timezone",  columnForName);
         stop.stop_url       = getUrlIfPresent   (result, "stop_url",       columnForName);
         stop.location_type  = getIntIfPresent   (result, "location_type",  columnForName);
-        stop.wheelchair_boarding = Integer.toString(getIntIfPresent(result, "wheelchair_boarding", columnForName));
+        stop.wheelchair_boarding = getIntIfPresent(result, "wheelchair_boarding", columnForName);
+        stop.platform_code  = getStringIfPresent(result, "platform_code",  columnForName);
         return stop;
     };
 
@@ -205,6 +210,8 @@ public interface EntityPopulator<T> {
         stopTime.stop_headsign       = getStringIfPresent(result, "stop_headsign", columnForName);
         stopTime.pickup_type         = getIntIfPresent   (result, "pickup_type", columnForName);
         stopTime.drop_off_type       = getIntIfPresent   (result, "drop_off_type", columnForName);
+        stopTime.continuous_pickup   = getIntIfPresent   (result, "continuous_pickup", columnForName);
+        stopTime.continuous_drop_off = getIntIfPresent   (result, "continuous_drop_off", columnForName);
         stopTime.timepoint           = getIntIfPresent   (result, "timepoint", columnForName);
         stopTime.shape_dist_traveled = getDoubleIfPresent(result, "shape_dist_traveled", columnForName);
         return stopTime;

src/main/java/com/conveyal/gtfs/loader/FeedLoadResult.java

@@ -34,6 +34,8 @@ public class FeedLoadResult implements Serializable {
     public TableLoadResult stopTimes;
     public TableLoadResult transfers;
     public TableLoadResult trips;
+    public TableLoadResult translations;
+    public TableLoadResult attributions;
 
     public long loadTimeMillis;
     public long completionTime;
@@ -59,5 +61,7 @@ public FeedLoadResult (boolean constructTableResults) {
         stopTimes = new TableLoadResult();
         transfers = new TableLoadResult();
         trips = new TableLoadResult();
+        translations = new TableLoadResult();
+        attributions = new TableLoadResult();
     }
 }