diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
index 695b467c4..d30bb6689 100644
--- a/.github/workflows/maven.yml
+++ b/.github/workflows/maven.yml
@@ -48,10 +48,6 @@ jobs:
key: maven-local-repo
- name: Build with Maven
run: mvn --no-transfer-progress package
- - name: Codecov
- # this first codecov run will upload a report associated with the commit set through CI environment variables
- uses: codecov/codecov-action@v1.2.0
- continue-on-error: true
- name: Clear contents of the target directory
# Avoids issues where maven-semantic-release attempts to upload
# multiple versions/builds (and fails due to the pre-existence of the version on maven central).
@@ -71,7 +67,3 @@ jobs:
GH_TOKEN: ${{ secrets.GH_TOKEN }}
run: |
semantic-release --prepare @conveyal/maven-semantic-release --publish @semantic-release/github,@conveyal/maven-semantic-release --verify-conditions @semantic-release/github,@conveyal/maven-semantic-release --verify-release @conveyal/maven-semantic-release --use-conveyal-workflow --dev-branch=dev --skip-maven-deploy
- if [[ "$GITHUB_REF_SLUG" = "master" ]]; then
- bash <(curl -s https://codecov.io/bash) -C "$(git rev-parse HEAD)"
- bash <(curl -s https://codecov.io/bash) -C "$(git rev-parse HEAD^)"
- fi
diff --git a/.gitignore b/.gitignore
index 928ad1b18..dc693fe79 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,3 +8,5 @@
*.iml
.idea/
target/
+
+lambda$*.json
\ No newline at end of file
diff --git a/README.md b/README.md
index 1daff3de6..8dc117d4b 100644
--- a/README.md
+++ b/README.md
@@ -19,15 +19,24 @@ A gtfs-lib GTFSFeed object should faithfully represent the contents of a single
gtfs-lib can be used as a Java library or run via the command line. If using this library with PostgreSQL for persistence, you must use at least version 9.6 of PostgreSQL.
### Library (maven)
-
+Include gtfs-lib as a library in your project with the following dependency in your `pom.xml`.
```xml
- com.conveyal
+
+ com.github.conveyal
gtfs-lib
${choose-a-version}
```
+#### Jitpack
+
+gtfs-lib builds are hosted on [jitpack](https://jitpack.io/#conveyal/gtfs-lib).
+
+[Release versions](https://github.com/conveyal/gtfs-lib/releases) are available by default.
+
+Branch- (e.g. `dev-SNAPSHOT`) or commit-specific (using a 10 character commit ID like `a04294e420`) snapshot builds can be triggered by clicking `Get` for the build of your choice on jitpack's website or visiting https://jitpack.io/#conveyal/gtfs-lib/YOUR_VERSION.
+
### Command line
```bash
diff --git a/maven-artifact-signing-key.asc.enc b/maven-artifact-signing-key.asc.enc
deleted file mode 100644
index c611046ba..000000000
Binary files a/maven-artifact-signing-key.asc.enc and /dev/null differ
diff --git a/maven-settings.xml b/maven-settings.xml
deleted file mode 100644
index eac060f03..000000000
--- a/maven-settings.xml
+++ /dev/null
@@ -1,26 +0,0 @@
-
-
-
-
- ossrh
- ${env.OSSRH_JIRA_USERNAME}
- ${env.OSSRH_JIRA_PASSWORD}
-
-
-
-
- ossrh
-
- true
-
-
- gpg
- ${env.GPG_KEY_NAME}
- ${env.GPG_PASSPHRASE}
-
-
-
-
-
diff --git a/pom.xml b/pom.xml
index d51c95e50..2bb3afa99 100644
--- a/pom.xml
+++ b/pom.xml
@@ -123,22 +123,6 @@
-
- org.apache.maven.plugins
- maven-gpg-plugin
- 1.6
-
-
- sign-artifacts
-
- verify
-
- sign
-
-
-
-
-
org.apache.maven.plugins
@@ -234,6 +218,13 @@
5.7.0
test
+
+
+ org.junit.jupiter
+ junit-jupiter-params
+ 5.6.2
+ test
+
com.beust
@@ -292,7 +283,7 @@
commons-io
commons-io
- 2.4
+ 2.7
diff --git a/src/main/java/com/conveyal/gtfs/GTFSFeed.java b/src/main/java/com/conveyal/gtfs/GTFSFeed.java
index c07c11a5f..8e6de45e2 100644
--- a/src/main/java/com/conveyal/gtfs/GTFSFeed.java
+++ b/src/main/java/com/conveyal/gtfs/GTFSFeed.java
@@ -14,7 +14,6 @@
import org.locationtech.jts.algorithm.ConvexHull;
import org.locationtech.jts.geom.*;
import org.locationtech.jts.index.strtree.STRtree;
-import org.locationtech.jts.simplify.DouglasPeuckerSimplifier;
import org.mapdb.BTreeMap;
import org.mapdb.DB;
import org.mapdb.DBMaker;
@@ -66,6 +65,8 @@ public class GTFSFeed implements Cloneable, Closeable {
public final Map stops;
public final Map transfers;
public final BTreeMap trips;
+ public final Map translations;
+ public final Map attributions;
public final Set transitIds = new HashSet<>();
/** CRC32 of the GTFS file this was loaded from */
@@ -636,6 +637,8 @@ private GTFSFeed (DB db) {
fares = db.getTreeMap("fares");
services = db.getTreeMap("services");
shape_points = db.getTreeMap("shape_points");
+ translations = db.getTreeMap("translations");
+ attributions = db.getTreeMap("attributions");
feedId = db.getAtomicString("feed_id").get();
checksum = db.getAtomicLong("checksum").get();
diff --git a/src/main/java/com/conveyal/gtfs/TripPatternKey.java b/src/main/java/com/conveyal/gtfs/TripPatternKey.java
index e06a5f6ab..9331483e0 100644
--- a/src/main/java/com/conveyal/gtfs/TripPatternKey.java
+++ b/src/main/java/com/conveyal/gtfs/TripPatternKey.java
@@ -24,6 +24,8 @@ public class TripPatternKey {
public TIntList arrivalTimes = new TIntArrayList();
public TIntList departureTimes = new TIntArrayList();
public TIntList timepoints = new TIntArrayList();
+ public TIntList continuous_pickup = new TIntArrayList();
+ public TIntList continuous_drop_off = new TIntArrayList();
public TDoubleList shapeDistances = new TDoubleArrayList();
public TripPatternKey (String routeId) {
@@ -39,6 +41,8 @@ public void addStopTime (StopTime st) {
departureTimes.add(st.departure_time);
timepoints.add(st.timepoint);
shapeDistances.add(st.shape_dist_traveled);
+ continuous_pickup.add(st.continuous_pickup);
+ continuous_drop_off.add(st.continuous_drop_off);
}
@Override
diff --git a/src/main/java/com/conveyal/gtfs/error/NewGTFSError.java b/src/main/java/com/conveyal/gtfs/error/NewGTFSError.java
index e350fe671..98e61895f 100644
--- a/src/main/java/com/conveyal/gtfs/error/NewGTFSError.java
+++ b/src/main/java/com/conveyal/gtfs/error/NewGTFSError.java
@@ -1,5 +1,6 @@
package com.conveyal.gtfs.error;
+import com.conveyal.gtfs.loader.LineContext;
import com.conveyal.gtfs.loader.Table;
import com.conveyal.gtfs.model.Entity;
import org.slf4j.Logger;
@@ -82,6 +83,14 @@ public static NewGTFSError forLine (Table table, int lineNumber, NewGTFSErrorTyp
return error;
}
+ // Factory Builder for cases where an entity has not yet been constructed, but we know the line number.
+ public static NewGTFSError forLine(LineContext lineContext, NewGTFSErrorType errorType, String badValue) {
+ NewGTFSError error = new NewGTFSError(lineContext.table.getEntityClass(), errorType);
+ error.lineNumber = lineContext.lineNumber;
+ error.badValue = badValue;
+ return error;
+ }
+
// Factory Builder for cases where the entity has already been decoded and an error is discovered during validation
public static NewGTFSError forEntity(Entity entity, NewGTFSErrorType errorType) {
NewGTFSError error = new NewGTFSError(entity.getClass(), errorType);
diff --git a/src/main/java/com/conveyal/gtfs/error/NewGTFSErrorType.java b/src/main/java/com/conveyal/gtfs/error/NewGTFSErrorType.java
index dd8f4be99..49f44fa3f 100644
--- a/src/main/java/com/conveyal/gtfs/error/NewGTFSErrorType.java
+++ b/src/main/java/com/conveyal/gtfs/error/NewGTFSErrorType.java
@@ -8,9 +8,11 @@
*/
public enum NewGTFSErrorType {
// Standard errors.
+ AGENCY_ID_REQUIRED_FOR_MULTI_AGENCY_FEEDS(Priority.HIGH, "For GTFS feeds with more than one agency, agency_id is required."),
BOOLEAN_FORMAT(Priority.MEDIUM, "A GTFS boolean field must contain the value 1 or 0."),
COLOR_FORMAT(Priority.MEDIUM, "A color should be specified with six-characters (three two-digit hexadecimal numbers)."),
COLUMN_NAME_UNSAFE(Priority.HIGH, "Column header contains characters not safe in SQL, it was renamed."),
+ CONDITIONALLY_REQUIRED(Priority.HIGH, "A conditionally required field was missing in a particular row."),
CURRENCY_UNKNOWN(Priority.MEDIUM, "The currency code was not recognized."),
DATE_FORMAT(Priority.MEDIUM, "Date format should be YYYYMMDD."),
DATE_NO_SERVICE(Priority.MEDIUM, "No service_ids were active on a date within the range of dates with defined service."),
diff --git a/src/main/java/com/conveyal/gtfs/graphql/GraphQLGtfsSchema.java b/src/main/java/com/conveyal/gtfs/graphql/GraphQLGtfsSchema.java
index e752660cb..acd077d3d 100644
--- a/src/main/java/com/conveyal/gtfs/graphql/GraphQLGtfsSchema.java
+++ b/src/main/java/com/conveyal/gtfs/graphql/GraphQLGtfsSchema.java
@@ -144,9 +144,12 @@ public class GraphQLGtfsSchema {
.description("A GTFS feed_info object")
.field(MapFetcher.field("id", GraphQLInt))
.field(MapFetcher.field("feed_id"))
+ .field(MapFetcher.field("feed_contact_email"))
+ .field(MapFetcher.field("feed_contact_url"))
.field(MapFetcher.field("feed_publisher_name"))
.field(MapFetcher.field("feed_publisher_url"))
.field(MapFetcher.field("feed_lang"))
+ .field(MapFetcher.field("default_lang"))
.field(MapFetcher.field("feed_start_date"))
.field(MapFetcher.field("feed_end_date"))
.field(MapFetcher.field("feed_version"))
@@ -257,10 +260,37 @@ public class GraphQLGtfsSchema {
.field(MapFetcher.field("timepoint", GraphQLInt))
.field(MapFetcher.field("drop_off_type", GraphQLInt))
.field(MapFetcher.field("pickup_type", GraphQLInt))
- // Editor-specific fields
+ .field(MapFetcher.field("continuous_drop_off", GraphQLInt))
+ .field(MapFetcher.field("continuous_pickup", GraphQLInt))
.field(MapFetcher.field("shape_dist_traveled", GraphQLFloat))
.build();
+ // Represents rows from attributions.txt
+ public static final GraphQLObjectType attributionsType = newObject().name("attributions")
+ .field(MapFetcher.field("attribution_id"))
+ .field(MapFetcher.field("agency_id"))
+ .field(MapFetcher.field("route_id"))
+ .field(MapFetcher.field("trip_id"))
+ .field(MapFetcher.field("organization_name"))
+ .field(MapFetcher.field("is_producer", GraphQLInt))
+ .field(MapFetcher.field("is_operator", GraphQLInt))
+ .field(MapFetcher.field("is_authority", GraphQLInt))
+ .field(MapFetcher.field("attribution_url"))
+ .field(MapFetcher.field("attribution_email"))
+ .field(MapFetcher.field("attribution_phone"))
+ .build();
+
+ // Represents rows from translations.txt
+ public static final GraphQLObjectType translationsType = newObject().name("translations")
+ .field(MapFetcher.field("table_name"))
+ .field(MapFetcher.field("field_name"))
+ .field(MapFetcher.field("language"))
+ .field(MapFetcher.field("translation"))
+ .field(MapFetcher.field("record_id"))
+ .field(MapFetcher.field("record_sub_id"))
+ .field(MapFetcher.field("field_value"))
+ .build();
+
// Represents rows from routes.txt
public static final GraphQLObjectType routeType = newObject().name("route")
.description("A line from a GTFS routes.txt table")
@@ -272,7 +302,8 @@ public class GraphQLGtfsSchema {
.field(MapFetcher.field("route_desc"))
.field(MapFetcher.field("route_url"))
.field(MapFetcher.field("route_branding_url"))
- // TODO route_type as enum or int
+ .field(MapFetcher.field("continuous_drop_off", GraphQLInt))
+ .field(MapFetcher.field("continuous_pickup", GraphQLInt))
.field(MapFetcher.field("route_type", GraphQLInt))
.field(MapFetcher.field("route_color"))
.field(MapFetcher.field("route_text_color"))
@@ -341,6 +372,7 @@ public class GraphQLGtfsSchema {
.field(MapFetcher.field("stop_url"))
.field(MapFetcher.field("stop_timezone"))
.field(MapFetcher.field("parent_station"))
+ .field(MapFetcher.field("platform_code"))
.field(MapFetcher.field("location_type", GraphQLInt))
.field(MapFetcher.field("wheelchair_boarding", GraphQLInt))
// Returns all stops that reference parent stop's stop_id
@@ -403,6 +435,8 @@ public class GraphQLGtfsSchema {
.field(MapFetcher.field("shape_dist_traveled", GraphQLFloat))
.field(MapFetcher.field("drop_off_type", GraphQLInt))
.field(MapFetcher.field("pickup_type", GraphQLInt))
+ .field(MapFetcher.field("continuous_drop_off", GraphQLInt))
+ .field(MapFetcher.field("continuous_pickup", GraphQLInt))
.field(MapFetcher.field("stop_sequence", GraphQLInt))
.field(MapFetcher.field("timepoint", GraphQLInt))
// FIXME: This will only returns a list with one stop entity (unless there is a referential integrity issue)
@@ -763,6 +797,26 @@ public class GraphQLGtfsSchema {
.dataFetcher(new JDBCFetcher("services"))
.build()
)
+ .field(newFieldDefinition()
+ .name("attributions")
+ .type(new GraphQLList(GraphQLGtfsSchema.attributionsType))
+ .argument(stringArg("namespace")) // FIXME maybe these nested namespace arguments are not doing anything.
+ .argument(intArg(ID_ARG))
+ .argument(intArg(LIMIT_ARG))
+ .argument(intArg(OFFSET_ARG))
+ .dataFetcher(new JDBCFetcher("attributions"))
+ .build()
+ )
+ .field(newFieldDefinition()
+ .name("translations")
+ .type(new GraphQLList(GraphQLGtfsSchema.translationsType))
+ .argument(stringArg("namespace")) // FIXME maybe these nested namespace arguments are not doing anything.
+ .argument(intArg(ID_ARG))
+ .argument(intArg(LIMIT_ARG))
+ .argument(intArg(OFFSET_ARG))
+ .dataFetcher(new JDBCFetcher("translations"))
+ .build()
+ )
.build();
/**
diff --git a/src/main/java/com/conveyal/gtfs/loader/EntityPopulator.java b/src/main/java/com/conveyal/gtfs/loader/EntityPopulator.java
index 762176262..7105c6c71 100644
--- a/src/main/java/com/conveyal/gtfs/loader/EntityPopulator.java
+++ b/src/main/java/com/conveyal/gtfs/loader/EntityPopulator.java
@@ -64,6 +64,8 @@ public interface EntityPopulator {
patternStop.stop_sequence = getIntIfPresent(result, "stop_sequence", columnForName);
patternStop.timepoint = getIntIfPresent(result, "timepoint", columnForName);
patternStop.shape_dist_traveled = getDoubleIfPresent(result, "shape_dist_traveled", columnForName);
+ patternStop.continuous_pickup = getIntIfPresent (result, "continuous_pickup", columnForName);
+ patternStop.continuous_drop_off = getIntIfPresent (result, "continuous_drop_off", columnForName);
return patternStop;
};
@@ -139,17 +141,19 @@ public interface EntityPopulator {
};
EntityPopulator ROUTE = (result, columnForName) -> {
- Route route = new Route();
- route.route_id = getStringIfPresent(result, "route_id", columnForName);
- route.agency_id = getStringIfPresent(result, "agency_id", columnForName);
- route.route_short_name = getStringIfPresent(result, "route_short_name", columnForName);
- route.route_long_name = getStringIfPresent(result, "route_long_name", columnForName);
- route.route_desc = getStringIfPresent(result, "route_desc", columnForName);
- route.route_type = getIntIfPresent (result, "route_type", columnForName);
- route.route_color = getStringIfPresent(result, "route_color", columnForName);
- route.route_text_color = getStringIfPresent(result, "route_text_color", columnForName);
- route.route_url = getUrlIfPresent (result, "route_url", columnForName);
- route.route_branding_url = getUrlIfPresent (result, "route_branding_url", columnForName);
+ Route route = new Route();
+ route.route_id = getStringIfPresent(result, "route_id", columnForName);
+ route.agency_id = getStringIfPresent(result, "agency_id", columnForName);
+ route.route_short_name = getStringIfPresent(result, "route_short_name", columnForName);
+ route.route_long_name = getStringIfPresent(result, "route_long_name", columnForName);
+ route.route_desc = getStringIfPresent(result, "route_desc", columnForName);
+ route.route_type = getIntIfPresent (result, "route_type", columnForName);
+ route.route_color = getStringIfPresent(result, "route_color", columnForName);
+ route.route_text_color = getStringIfPresent(result, "route_text_color", columnForName);
+ route.route_url = getUrlIfPresent (result, "route_url", columnForName);
+ route.route_branding_url = getUrlIfPresent (result, "route_branding_url", columnForName);
+ route.continuous_pickup = getIntIfPresent (result, "continuous_pickup", columnForName);
+ route.continuous_drop_off = getIntIfPresent (result, "continuous_drop_off", columnForName);
return route;
};
@@ -166,7 +170,8 @@ public interface EntityPopulator {
stop.stop_timezone = getStringIfPresent(result, "stop_timezone", columnForName);
stop.stop_url = getUrlIfPresent (result, "stop_url", columnForName);
stop.location_type = getIntIfPresent (result, "location_type", columnForName);
- stop.wheelchair_boarding = Integer.toString(getIntIfPresent(result, "wheelchair_boarding", columnForName));
+ stop.wheelchair_boarding = getIntIfPresent(result, "wheelchair_boarding", columnForName);
+ stop.platform_code = getStringIfPresent(result, "platform_code", columnForName);
return stop;
};
@@ -205,6 +210,8 @@ public interface EntityPopulator {
stopTime.stop_headsign = getStringIfPresent(result, "stop_headsign", columnForName);
stopTime.pickup_type = getIntIfPresent (result, "pickup_type", columnForName);
stopTime.drop_off_type = getIntIfPresent (result, "drop_off_type", columnForName);
+ stopTime.continuous_pickup = getIntIfPresent (result, "continuous_pickup", columnForName);
+ stopTime.continuous_drop_off = getIntIfPresent (result, "continuous_drop_off", columnForName);
stopTime.timepoint = getIntIfPresent (result, "timepoint", columnForName);
stopTime.shape_dist_traveled = getDoubleIfPresent(result, "shape_dist_traveled", columnForName);
return stopTime;
diff --git a/src/main/java/com/conveyal/gtfs/loader/FeedLoadResult.java b/src/main/java/com/conveyal/gtfs/loader/FeedLoadResult.java
index 69d62d9f7..3a61fdd50 100644
--- a/src/main/java/com/conveyal/gtfs/loader/FeedLoadResult.java
+++ b/src/main/java/com/conveyal/gtfs/loader/FeedLoadResult.java
@@ -34,6 +34,8 @@ public class FeedLoadResult implements Serializable {
public TableLoadResult stopTimes;
public TableLoadResult transfers;
public TableLoadResult trips;
+ public TableLoadResult translations;
+ public TableLoadResult attributions;
public long loadTimeMillis;
public long completionTime;
@@ -59,5 +61,7 @@ public FeedLoadResult (boolean constructTableResults) {
stopTimes = new TableLoadResult();
transfers = new TableLoadResult();
trips = new TableLoadResult();
+ translations = new TableLoadResult();
+ attributions = new TableLoadResult();
}
}
diff --git a/src/main/java/com/conveyal/gtfs/loader/Field.java b/src/main/java/com/conveyal/gtfs/loader/Field.java
index 2440c4624..52cda789c 100644
--- a/src/main/java/com/conveyal/gtfs/loader/Field.java
+++ b/src/main/java/com/conveyal/gtfs/loader/Field.java
@@ -2,6 +2,7 @@
import com.conveyal.gtfs.error.NewGTFSError;
import com.conveyal.gtfs.error.NewGTFSErrorType;
+import com.conveyal.gtfs.loader.conditions.ConditionalRequirement;
import com.google.common.collect.ImmutableSet;
import java.sql.PreparedStatement;
@@ -48,6 +49,9 @@ public abstract class Field {
public Table referenceTable = null;
private boolean shouldBeIndexed;
private boolean emptyValuePermitted;
+ private boolean isConditionallyRequired;
+ private boolean isForeign;
+ public ConditionalRequirement[] conditions;
public Field(String name, Requirement requirement) {
this.name = name;
@@ -137,6 +141,25 @@ public boolean isForeignReference () {
return this.referenceTable != null;
}
+ /**
+ * Fluent method to mark a field as having foreign references. If flagged, the field value is added to
+ * {@link ReferenceTracker#uniqueValuesForFields} to be used as a look-up for reference matches. Note: this is intended only for
+ * special cases (e.g., zone_id) where the field being referenced does not exist as the primary key of a table.
+ */
+ Field hasForeignReferences() {
+ isForeign = true;
+ return this;
+ }
+
+ /**
+ * Indicates that this field has foreign references. Note: this is intentionally distinct from
+ * {@link #isForeignReference()} and is intended only for special cases (e.g., zone_id) where the field being
+ * referenced does not exist as the primary key of a table.
+ */
+ public boolean isForeign() {
+ return isForeign;
+ }
+
/**
* Fluent method that indicates that a newly constructed field should be indexed after the table is loaded.
* FIXME: should shouldBeIndexed be determined based on presence of referenceTable?
@@ -152,7 +175,8 @@ public boolean shouldBeIndexed() {
}
/**
- * Fluent method indicates that this field is a reference to an entry in the table provided as an argument.
+ * Fluent method that indicates that this field is a reference to an entry in the table provided as an argument
+ * (i.e. it is a foreign reference).
* @param table
* @return this same Field instance
*/
@@ -180,10 +204,26 @@ public boolean isEmptyValuePermitted() {
/**
* Get the expression used to select this column from the database based on the prefix. The csvOutput parameter is
- * needed in overriden method implementations that have special ways of outputting certain fields. The prefix
+ * needed in overridden method implementations that have special ways of outputting certain fields. The prefix
* parameter is assumed to be either null or a string in the format: `schema.`
*/
public String getColumnExpression(String prefix, boolean csvOutput) {
return prefix != null ? String.format("%s%s", prefix, name) : name;
}
+
+ /**
+ * Mark this field as conditionally required. If needed an optional list of conditions can be provided.
+ */
+ public Field requireConditions(ConditionalRequirement...conditions) {
+ this.isConditionallyRequired = true;
+ this.conditions = conditions;
+ return this;
+ }
+
+ /**
+ * Indicates that this field is conditionally required.
+ */
+ public boolean isConditionallyRequired() {
+ return isConditionallyRequired;
+ }
}
diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsLoader.java b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsLoader.java
index 37036657d..b17a812c0 100644
--- a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsLoader.java
+++ b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsLoader.java
@@ -67,7 +67,7 @@ public class JdbcGtfsLoader {
public static final long INSERT_BATCH_SIZE = 500;
// Represents null in Postgres text format
- private static final String POSTGRES_NULL_TEXT = "\\N";
+ public static final String POSTGRES_NULL_TEXT = "\\N";
private static final Logger LOG = LoggerFactory.getLogger(JdbcGtfsLoader.class);
private String gtfsFilePath;
@@ -92,7 +92,9 @@ public JdbcGtfsLoader(String gtfsFilePath, DataSource dataSource) {
this.dataSource = dataSource;
}
- /** Get SQL string for creating the feed registry table (AKA, the "feeds" table). */
+ /**
+ * Get SQL string for creating the feed registry table (AKA, the "feeds" table).
+ */
public static String getCreateFeedRegistrySQL() {
return "create table if not exists feeds (namespace varchar primary key, md5 varchar, " +
"sha1 varchar, feed_id varchar, feed_version varchar, filename varchar, loaded_date timestamp, " +
@@ -110,7 +112,7 @@ public static String getCreateFeedRegistrySQL() {
// SHA1 took 1072 msec, 9fb356af4be2750f20955203787ec6f95d32ef22
// There appears to be no advantage to loading tables in parallel, as the whole loading process is I/O bound.
- public FeedLoadResult loadTables () {
+ public FeedLoadResult loadTables() {
// This result object will be returned to the caller to summarize the feed and report any critical errors.
FeedLoadResult result = new FeedLoadResult();
@@ -135,7 +137,7 @@ public FeedLoadResult loadTables () {
this.tablePrefix = randomIdString();
result.filename = gtfsFilePath;
result.uniqueIdentifier = tablePrefix;
-
+
// The order of the following four lines should not be changed because the schema needs to be in place
// before the error storage can be constructed, which in turn needs to exist in case any errors are
// encountered during the loading process.
@@ -155,14 +157,16 @@ public FeedLoadResult loadTables () {
result.calendarDates = load(Table.CALENDAR_DATES);
result.routes = load(Table.ROUTES);
result.fareAttributes = load(Table.FARE_ATTRIBUTES);
- result.fareRules = load(Table.FARE_RULES);
result.feedInfo = load(Table.FEED_INFO);
result.shapes = load(Table.SHAPES);
result.stops = load(Table.STOPS);
+ result.fareRules = load(Table.FARE_RULES);
result.transfers = load(Table.TRANSFERS);
result.trips = load(Table.TRIPS); // refs routes
result.frequencies = load(Table.FREQUENCIES); // refs trips
result.stopTimes = load(Table.STOP_TIMES);
+ result.translations = load(Table.TRANSLATIONS);
+ result.attributions = load(Table.ATTRIBUTIONS);
result.errorCount = errorStorage.getErrorCount();
// This will commit and close the single connection that has been shared between all preceding load steps.
errorStorage.commitAndClose();
@@ -180,15 +184,15 @@ public FeedLoadResult loadTables () {
}
return result;
}
-
+
/**
* Creates a schema/namespace in the database WITHOUT committing the changes.
* This does *not* setup any other tables or enter the schema name in a registry (@see #registerFeed).
- *
+ *
* @param connection Connection to the database to create the schema on.
* @param schemaName Name of the schema (i.e. table prefix). Should not include the dot suffix.
*/
- static void createSchema (Connection connection, String schemaName) {
+ static void createSchema(Connection connection, String schemaName) {
try {
Statement statement = connection.createStatement();
// FIXME do the following only on databases that support schemas.
@@ -205,13 +209,13 @@ static void createSchema (Connection connection, String schemaName) {
* Add a line to the list of loaded feeds showing that this feed has been loaded.
* We used to inspect feed_info here so we could make our table prefix based on feed ID and version.
* Now we just load feed_info like any other table.
- * // Create a row in the table of loaded feeds for this feed
+ * // Create a row in the table of loaded feeds for this feed
* Really this is not just making the table prefix - it's loading the feed_info and should also calculate hashes.
*
* Originally we were flattening all feed_info files into one root-level table, but that forces us to drop any
* custom fields in feed_info.
*/
- private void registerFeed (File gtfsFile) {
+ private void registerFeed(File gtfsFile) {
// FIXME is this extra CSV reader used anymore? Check comment below.
// First, inspect feed_info.txt to extract the ID and version.
@@ -243,7 +247,7 @@ private void registerFeed (File gtfsFile) {
// current_timestamp seems to be the only standard way to get the current time across all common databases.
// Record total load processing time?
PreparedStatement insertStatement = connection.prepareStatement(
- "insert into feeds values (?, ?, ?, ?, ?, ?, current_timestamp, null, false)");
+ "insert into feeds values (?, ?, ?, ?, ?, ?, current_timestamp, null, false)");
insertStatement.setString(1, tablePrefix);
insertStatement.setString(2, md5Hex);
insertStatement.setString(3, shaHex);
@@ -271,7 +275,7 @@ static void createFeedRegistryIfNotExists(Connection connection) throws SQLExcep
/**
* This wraps the main internal table loader method to catch exceptions and figure out how many errors happened.
*/
- private TableLoadResult load (Table table) {
+ private TableLoadResult load(Table table) {
// This object will be returned to the caller to summarize the contents of the table and any errors.
TableLoadResult tableLoadResult = new TableLoadResult();
int initialErrorCount = errorStorage.getErrorCount();
@@ -311,9 +315,10 @@ private int getTableSize(Table table) {
/**
* This function will throw any exception that occurs. Those exceptions will be handled by the outer load method.
+ *
* @return number of rows that were loaded.
*/
- private int loadInternal (Table table) throws Exception {
+ private int loadInternal(Table table) throws Exception {
CsvReader csvReader = table.getCsvReader(zip, errorStorage);
if (csvReader == null) {
LOG.info(String.format("file %s.txt not found in gtfs zipfile", table.name));
@@ -363,7 +368,7 @@ private int loadInternal (Table table) throws Exception {
// When outputting text, accumulate transformed strings to allow skipping rows when errors are encountered.
// One extra position in the array for the CSV line number.
String[] transformedStrings = new String[cleanFields.length + 1];
-
+ boolean tableHasConditionalRequirements = table.hasConditionalRequirements();
// Iterate over each record and prepare the record for storage in the table either through batch insert
// statements or postgres text copy operation.
while (csvReader.readRecord()) {
@@ -373,6 +378,7 @@ private int loadInternal (Table table) throws Exception {
errorStorage.storeError(NewGTFSError.forTable(table, TABLE_TOO_LONG));
break;
}
+ // Line 1 is considered the header row, so the first actual row of data will be line 2.
int lineNumber = ((int) csvReader.getCurrentRecord()) + 2;
if (lineNumber % 500_000 == 0) LOG.info("Processed {}", human(lineNumber));
if (csvReader.getColumnCount() != fields.length) {
@@ -403,10 +409,10 @@ private int loadInternal (Table table) throws Exception {
// error.
if (
table.name.equals("calendar_dates") &&
- "service_id".equals(field.name) &&
- "1".equals(csvReader.get(Field.getFieldIndex(fields, "exception_type")))
+ "service_id".equals(field.name) &&
+ "1".equals(csvReader.get(Field.getFieldIndex(fields, "exception_type")))
- ){
+ ) {
for (NewGTFSError error : errors) {
if (NewGTFSErrorType.REFERENTIAL_INTEGRITY.equals(error.errorType)) {
// Do not record bad service_id reference errors for calendar date entries that add service
@@ -430,6 +436,12 @@ private int loadInternal (Table table) throws Exception {
// Increment column index.
columnIndex += 1;
}
+ if (tableHasConditionalRequirements) {
+ LineContext lineContext = new LineContext(table, fields, transformedStrings, lineNumber);
+ errorStorage.storeErrors(
+ referenceTracker.checkConditionallyRequiredFields(lineContext)
+ );
+ }
if (postgresText) {
// Print a new line in the standard postgres text format:
// https://www.postgresql.org/docs/9.1/static/sql-copy.html#AEN64380
@@ -481,7 +493,7 @@ public static void copyFromFile(Connection connection, File file, String targetT
InputStream stream = new BufferedInputStream(new FileInputStream(file.getAbsolutePath()));
// Our connection pool wraps the Connection objects, so we need to unwrap the Postgres connection interface.
CopyManager copyManager = new CopyManager(connection.unwrap(BaseConnection.class));
- copyManager.copyIn(copySql, stream, 1024*1024);
+ copyManager.copyIn(copySql, stream, 1024 * 1024);
stream.close();
// It is also possible to load from local file if this code is running on the database server.
// statement.execute(String.format("copy %s from '%s'", table.name, tempTextFile.getAbsolutePath()));
@@ -515,9 +527,12 @@ public void setValueForField(Table table, int fieldIndex, int lineNumber, Field
if (postgresText) {
ValidateFieldResult result = field.validateAndConvert(string);
// If the result is null, use the null-setting method.
- if (result.clean == null) setFieldToNull(postgresText, transformedStrings, fieldIndex, field);
- // Otherwise, set the cleaned field according to its index.
- else transformedStrings[fieldIndex + 1] = result.clean;
+ if (result.clean == null) {
+ setFieldToNull(postgresText, transformedStrings, fieldIndex, field);
+ } else {
+ // Otherwise, set the cleaned field according to its index.
+ transformedStrings[fieldIndex + 1] = result.clean;
+ }
errors = result.errors;
} else {
errors = field.setParameter(insertStatement, fieldIndex + 2, string);
@@ -543,15 +558,18 @@ public void setValueForField(Table table, int fieldIndex, int lineNumber, Field
* Sets field to null in statement or string array depending on whether postgres is being used.
*/
private void setFieldToNull(boolean postgresText, String[] transformedStrings, int fieldIndex, Field field) {
- if (postgresText) transformedStrings[fieldIndex + 1] = POSTGRES_NULL_TEXT;
- // Adjust parameter index by two: indexes are one-based and the first one is the CSV line number.
- else try {
- // LOG.info("setting {} index to null", fieldIndex + 2);
- field.setNull(insertStatement, fieldIndex + 2);
- } catch (SQLException e) {
- e.printStackTrace();
- // FIXME: store error here? It appears that an exception should only be thrown if the type value is invalid,
- // the connection is closed, or the index is out of bounds. So storing an error may be unnecessary.
+ if (postgresText) {
+ transformedStrings[fieldIndex + 1] = POSTGRES_NULL_TEXT;
+ } else {
+ // Adjust parameter index by two: indexes are one-based and the first one is the CSV line number.
+ try {
+ // LOG.info("setting {} index to null", fieldIndex + 2);
+ field.setNull(insertStatement, fieldIndex + 2);
+ } catch (SQLException e) {
+ e.printStackTrace();
+ // FIXME: store error here? It appears that an exception should only be thrown if the type value is invalid,
+ // the connection is closed, or the index is out of bounds. So storing an error may be unnecessary.
+ }
}
}
@@ -563,7 +581,7 @@ private void setFieldToNull(boolean postgresText, String[] transformedStrings, i
*
* TODO add a test including SQL injection text (quote and semicolon)
*/
- public static String sanitize (String string, SQLErrorStorage errorStorage) {
+ public static String sanitize(String string, SQLErrorStorage errorStorage) {
String clean = string.replaceAll("[^\\p{Alnum}_]", "");
if (!clean.equals(string)) {
LOG.warn("SQL identifier '{}' was sanitized to '{}'", string, clean);
diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsSnapshotter.java b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsSnapshotter.java
index 610a4f8ac..3b36a9b8e 100644
--- a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsSnapshotter.java
+++ b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsSnapshotter.java
@@ -112,6 +112,8 @@ public SnapshotResult copyTables() {
result.stopTimes = copy(Table.STOP_TIMES, true);
result.transfers = copy(Table.TRANSFERS, true);
result.trips = copy(Table.TRIPS, true);
+ result.attributions = copy(Table.ATTRIBUTIONS, true);
+ result.translations = copy(Table.TRANSLATIONS, true);
result.completionTime = System.currentTimeMillis();
result.loadTimeMillis = result.completionTime - startTime;
LOG.info("Copying tables took {} sec", (result.loadTimeMillis) / 1000);
diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java
index 14665a7c0..666ba77bf 100644
--- a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java
+++ b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java
@@ -634,6 +634,8 @@ private String updateChildTable(
"timepoint",
"drop_off_type",
"pickup_type",
+ "continuous_pickup",
+ "continuous_drop_off",
"shape_dist_traveled"
);
}
@@ -1146,6 +1148,8 @@ private void insertBlankStopTimes(
stopTime.pickup_type = patternStop.pickup_type;
stopTime.timepoint = patternStop.timepoint;
stopTime.shape_dist_traveled = patternStop.shape_dist_traveled;
+ stopTime.continuous_drop_off = patternStop.continuous_drop_off;
+ stopTime.continuous_pickup = patternStop.continuous_pickup;
stopTime.stop_sequence = i;
// Update stop time with each trip ID and add to batch.
for (String tripId : tripIds) {
diff --git a/src/main/java/com/conveyal/gtfs/loader/LineContext.java b/src/main/java/com/conveyal/gtfs/loader/LineContext.java
new file mode 100644
index 000000000..2ef0f1900
--- /dev/null
+++ b/src/main/java/com/conveyal/gtfs/loader/LineContext.java
@@ -0,0 +1,46 @@
+package com.conveyal.gtfs.loader;
+
+/**
+ * Wrapper class that provides access to row values and line context (e.g., line number) for a particular row of GTFS
+ * data.
+ */
+public class LineContext {
+ public final Table table;
+ private final Field[] fields;
+ /**
+ * The row data has one extra value at the beginning of the array that represents the line number.
+ */
+ private final String[] rowDataWithLineNumber;
+ public final int lineNumber;
+
+ public LineContext(Table table, Field[] fields, String[] rowDataWithLineNumber, int lineNumber) {
+ this.table = table;
+ this.fields = fields;
+ this.rowDataWithLineNumber = rowDataWithLineNumber;
+ this.lineNumber = lineNumber;
+ }
+
+ /**
+ * Get value for a particular column index from a set of row data. Note: the row data here has one extra value at
+ * the beginning of the array that represents the line number (hence the +1). This is because the data is formatted
+ * for batch insertion into a postgres table.
+ */
+ public String getValueForRow(int columnIndex) {
+ return rowDataWithLineNumber[columnIndex + 1];
+ }
+
+ /**
+ * Overloaded method to provide value for the current line for a particular field.
+ */
+ public String getValueForRow(String fieldName) {
+ int fieldIndex = Field.getFieldIndex(fields, fieldName);
+ return rowDataWithLineNumber[fieldIndex + 1];
+ }
+
+ /**
+ * Overloaded method to provide value for the current line for the key field.
+ */
+ public String getEntityId() {
+ return getValueForRow(table.getKeyFieldIndex(fields));
+ }
+}
diff --git a/src/main/java/com/conveyal/gtfs/loader/ReferenceTracker.java b/src/main/java/com/conveyal/gtfs/loader/ReferenceTracker.java
index 7a01e25dd..acd1178c7 100644
--- a/src/main/java/com/conveyal/gtfs/loader/ReferenceTracker.java
+++ b/src/main/java/com/conveyal/gtfs/loader/ReferenceTracker.java
@@ -1,9 +1,12 @@
package com.conveyal.gtfs.loader;
import com.conveyal.gtfs.error.NewGTFSError;
+import com.conveyal.gtfs.loader.conditions.ConditionalRequirement;
+import com.google.common.collect.HashMultimap;
import java.util.Collections;
import java.util.HashSet;
+import java.util.Map;
import java.util.Set;
import static com.conveyal.gtfs.error.NewGTFSErrorType.DUPLICATE_ID;
@@ -21,6 +24,7 @@
*/
public class ReferenceTracker {
public final Set transitIds = new HashSet<>();
+ public final HashMultimap uniqueValuesForFields = HashMultimap.create();
public final Set transitIdsWithSequence = new HashSet<>();
/**
@@ -62,6 +66,13 @@ public Set checkReferencesAndUniqueness(String keyValue, int lineN
: !table.hasUniqueKeyField ? null : keyField;
String transitId = String.join(":", keyField, keyValue);
+ // Unique key values are needed for referential integrity checks as part of checks for fields that have
+ // conditional requirements. This also tracks "special" foreign keys like stop#zone_id that are not primary keys
+ // of the table they exist in.
+ if ((field.name.equals(keyField) && keyField.equals(uniqueKeyField)) || field.isForeign()) {
+ uniqueValuesForFields.put(field.name, value);
+ }
+
// If the field is optional and there is no value present, skip check.
if (!field.isRequired() && "".equals(value)) return Collections.emptySet();
@@ -136,4 +147,28 @@ public Set checkReferencesAndUniqueness(String keyValue, int lineN
}
return errors;
}
+
+
+ /**
+ * Work through each conditionally required check assigned to fields within a table. First check the reference field
+ * to confirm if it meets the conditions whereby the conditional field is required. If the conditional field is
+ * required confirm that a value has been provided, if not, log an error.
+ */
+ public Set checkConditionallyRequiredFields(LineContext lineContext) {
+ Set errors = new HashSet<>();
+ Map fieldsToCheck = lineContext.table.getConditionalRequirements();
+
+ // Work through each field that has been assigned a conditional requirement.
+ for (Map.Entry entry : fieldsToCheck.entrySet()) {
+ Field referenceField = entry.getKey();
+ ConditionalRequirement[] conditionalRequirements = entry.getValue();
+ // Work through each field's conditional requirements.
+ for (ConditionalRequirement conditionalRequirement : conditionalRequirements) {
+ errors.addAll(
+ conditionalRequirement.check(lineContext, referenceField, uniqueValuesForFields)
+ );
+ }
+ }
+ return errors;
+ }
}
diff --git a/src/main/java/com/conveyal/gtfs/loader/Table.java b/src/main/java/com/conveyal/gtfs/loader/Table.java
index 6ea3c2bcf..b397d9958 100644
--- a/src/main/java/com/conveyal/gtfs/loader/Table.java
+++ b/src/main/java/com/conveyal/gtfs/loader/Table.java
@@ -2,7 +2,15 @@
import com.conveyal.gtfs.error.NewGTFSError;
import com.conveyal.gtfs.error.SQLErrorStorage;
+import com.conveyal.gtfs.loader.conditions.AgencyHasMultipleRowsCheck;
+import com.conveyal.gtfs.loader.conditions.ConditionalRequirement;
+import com.conveyal.gtfs.loader.conditions.FieldInRangeCheck;
+import com.conveyal.gtfs.loader.conditions.FieldIsEmptyCheck;
+import com.conveyal.gtfs.loader.conditions.FieldNotEmptyAndMatchesValueCheck;
+import com.conveyal.gtfs.loader.conditions.ForeignRefExistsCheck;
+import com.conveyal.gtfs.loader.conditions.ReferenceFieldShouldBeProvidedCheck;
import com.conveyal.gtfs.model.Agency;
+import com.conveyal.gtfs.model.Attribution;
import com.conveyal.gtfs.model.Calendar;
import com.conveyal.gtfs.model.CalendarDate;
import com.conveyal.gtfs.model.Entity;
@@ -18,6 +26,7 @@
import com.conveyal.gtfs.model.Stop;
import com.conveyal.gtfs.model.StopTime;
import com.conveyal.gtfs.model.Transfer;
+import com.conveyal.gtfs.model.Translation;
import com.conveyal.gtfs.model.Trip;
import com.conveyal.gtfs.storage.StorageException;
import com.csvreader.CsvReader;
@@ -38,16 +47,16 @@
import java.util.Arrays;
import java.util.Collections;
import java.util.Enumeration;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import static com.conveyal.gtfs.error.NewGTFSErrorType.DUPLICATE_HEADER;
-import static com.conveyal.gtfs.error.NewGTFSErrorType.DUPLICATE_ID;
-import static com.conveyal.gtfs.error.NewGTFSErrorType.REFERENTIAL_INTEGRITY;
import static com.conveyal.gtfs.error.NewGTFSErrorType.TABLE_IN_SUBDIRECTORY;
import static com.conveyal.gtfs.loader.JdbcGtfsLoader.sanitize;
import static com.conveyal.gtfs.loader.Requirement.EDITOR;
@@ -98,15 +107,19 @@ public Table (String name, Class extends Entity> entityClass, Requirement requ
}
public static final Table AGENCY = new Table("agency", Agency.class, REQUIRED,
- new StringField("agency_id", OPTIONAL), // FIXME? only required if there are more than one
- new StringField("agency_name", REQUIRED),
- new URLField("agency_url", REQUIRED),
- new StringField("agency_timezone", REQUIRED), // FIXME new field type for time zones?
+ new StringField("agency_id", OPTIONAL).requireConditions(
+ // If there is more than one agency, the agency_id must be provided
+ // https://developers.google.com/transit/gtfs/reference#agencytxt
+ new AgencyHasMultipleRowsCheck()
+ ).hasForeignReferences(),
+ new StringField("agency_name", REQUIRED),
+ new URLField("agency_url", REQUIRED),
+ new StringField("agency_timezone", REQUIRED), // FIXME new field type for time zones?
new StringField("agency_lang", OPTIONAL), // FIXME new field type for languages?
- new StringField("agency_phone", OPTIONAL),
- new URLField("agency_branding_url", OPTIONAL),
- new URLField("agency_fare_url", OPTIONAL),
- new StringField("agency_email", OPTIONAL) // FIXME new field type for emails?
+ new StringField("agency_phone", OPTIONAL),
+ new URLField("agency_branding_url", OPTIONAL),
+ new URLField("agency_fare_url", OPTIONAL),
+ new StringField("agency_email", OPTIONAL) // FIXME new field type for emails?
).restrictDelete().addPrimaryKey();
// The GTFS spec says this table is required, but in practice it is not required if calendar_dates is present.
@@ -147,7 +160,11 @@ public Table (String name, Class extends Entity> entityClass, Requirement requ
new CurrencyField("currency_type", REQUIRED),
new ShortField("payment_method", REQUIRED, 1),
new ShortField("transfers", REQUIRED, 2).permitEmptyValue(),
- new StringField("agency_id", OPTIONAL), // FIXME? only required if there are more than one
+ new StringField("agency_id", OPTIONAL).requireConditions(
+ // If there is more than one agency, this agency_id is required.
+ // https://developers.google.com/transit/gtfs/reference#fare_attributestxt
+ new ReferenceFieldShouldBeProvidedCheck("agency_id")
+ ),
new IntegerField("transfer_duration", OPTIONAL)
).addPrimaryKey();
@@ -156,7 +173,7 @@ public Table (String name, Class extends Entity> entityClass, Requirement requ
public static final Table FEED_INFO = new Table("feed_info", FeedInfo.class, OPTIONAL,
new StringField("feed_publisher_name", REQUIRED),
// feed_id is not the first field because that would label it as the key field, which we do not want because the
- // key field cannot be optional.
+ // key field cannot be optional. feed_id is not part of the GTFS spec, but is required by OTP to associate static GTFS with GTFS-rt feeds.
new StringField("feed_id", OPTIONAL),
new URLField("feed_publisher_url", REQUIRED),
new LanguageField("feed_lang", REQUIRED),
@@ -166,41 +183,40 @@ public Table (String name, Class extends Entity> entityClass, Requirement requ
// Editor-specific field that represents default route values for use in editing.
new ColorField("default_route_color", EDITOR),
// FIXME: Should the route type max value be equivalent to GTFS spec's max?
- new IntegerField("default_route_type", EDITOR, 999)
+ new IntegerField("default_route_type", EDITOR, 999),
+ new LanguageField("default_lang", OPTIONAL),
+ new StringField("feed_contact_email", OPTIONAL),
+ new URLField("feed_contact_url", OPTIONAL)
).keyFieldIsNotUnique();
public static final Table ROUTES = new Table("routes", Route.class, REQUIRED,
new StringField("route_id", REQUIRED),
- new StringField("agency_id", OPTIONAL).isReferenceTo(AGENCY),
- new StringField("route_short_name", OPTIONAL), // one of short or long must be provided
- new StringField("route_long_name", OPTIONAL),
- new StringField("route_desc", OPTIONAL),
+ new StringField("agency_id", OPTIONAL).isReferenceTo(AGENCY).requireConditions(
+ // If there is more than one agency, this agency_id is required.
+ // https://developers.google.com/transit/gtfs/reference#routestxt
+ new ReferenceFieldShouldBeProvidedCheck("agency_id")
+ ),
+ new StringField("route_short_name", OPTIONAL), // one of short or long must be provided
+ new StringField("route_long_name", OPTIONAL),
+ new StringField("route_desc", OPTIONAL),
// Max route type according to the GTFS spec is 7; however, there is a GTFS proposal that could see this
// max value grow to around 1800: https://groups.google.com/forum/#!msg/gtfs-changes/keT5rTPS7Y0/71uMz2l6ke0J
new IntegerField("route_type", REQUIRED, 1800),
- new URLField("route_url", OPTIONAL),
- new URLField("route_branding_url", OPTIONAL),
- new ColorField("route_color", OPTIONAL), // really this is an int in hex notation
- new ColorField("route_text_color", OPTIONAL),
+ new URLField("route_url", OPTIONAL),
+ new URLField("route_branding_url", OPTIONAL),
+ new ColorField("route_color", OPTIONAL), // really this is an int in hex notation
+ new ColorField("route_text_color", OPTIONAL),
// Editor fields below.
new ShortField("publicly_visible", EDITOR, 1),
// wheelchair_accessible is an exemplar field applied to all trips on a route.
new ShortField("wheelchair_accessible", EDITOR, 2).permitEmptyValue(),
new IntegerField("route_sort_order", OPTIONAL, 0, Integer.MAX_VALUE),
// Status values are In progress (0), Pending approval (1), and Approved (2).
- new ShortField("status", EDITOR, 2)
+ new ShortField("status", EDITOR, 2),
+ new ShortField("continuous_pickup", OPTIONAL,3),
+ new ShortField("continuous_drop_off", OPTIONAL,3)
).addPrimaryKey();
- public static final Table FARE_RULES = new Table("fare_rules", FareRule.class, OPTIONAL,
- new StringField("fare_id", REQUIRED).isReferenceTo(FARE_ATTRIBUTES),
- new StringField("route_id", OPTIONAL).isReferenceTo(ROUTES),
- // FIXME: referential integrity check for zone_id for below three fields?
- new StringField("origin_id", OPTIONAL),
- new StringField("destination_id", OPTIONAL),
- new StringField("contains_id", OPTIONAL))
- .withParentTable(FARE_ATTRIBUTES)
- .addPrimaryKey().keyFieldIsNotUnique();
-
public static final Table SHAPES = new Table("shapes", ShapePoint.class, OPTIONAL,
new StringField("shape_id", REQUIRED),
new IntegerField("shape_pt_sequence", REQUIRED),
@@ -226,20 +242,53 @@ public Table (String name, Class extends Entity> entityClass, Requirement requ
).addPrimaryKey();
public static final Table STOPS = new Table("stops", Stop.class, REQUIRED,
- new StringField("stop_id", REQUIRED),
- new StringField("stop_code", OPTIONAL),
- new StringField("stop_name", REQUIRED),
- new StringField("stop_desc", OPTIONAL),
- new DoubleField("stop_lat", REQUIRED, -80, 80, 6),
- new DoubleField("stop_lon", REQUIRED, -180, 180, 6),
- new StringField("zone_id", OPTIONAL),
- new URLField("stop_url", OPTIONAL),
- new ShortField("location_type", OPTIONAL, 2),
- // FIXME: Need self-reference check during referential integrity check
- new StringField("parent_station", OPTIONAL), //.isReferenceToSelf()
- new StringField("stop_timezone", OPTIONAL),
- new ShortField("wheelchair_boarding", OPTIONAL, 2)
- ).restrictDelete().addPrimaryKey();
+ new StringField("stop_id", REQUIRED),
+ new StringField("stop_code", OPTIONAL),
+ // The actual conditions that will be acted upon are within the location_type field.
+ new StringField("stop_name", OPTIONAL).requireConditions(),
+ new StringField("stop_desc", OPTIONAL),
+ // The actual conditions that will be acted upon are within the location_type field.
+ new DoubleField("stop_lat", OPTIONAL, -80, 80, 6).requireConditions(),
+ // The actual conditions that will be acted upon are within the location_type field.
+ new DoubleField("stop_lon", OPTIONAL, -180, 180, 6).requireConditions(),
+ new StringField("zone_id", OPTIONAL).hasForeignReferences(),
+ new URLField("stop_url", OPTIONAL),
+ new ShortField("location_type", OPTIONAL, 4).requireConditions(
+ // If the location type is defined and within range, the dependent fields are required.
+ // https://developers.google.com/transit/gtfs/reference#stopstxt
+ new FieldInRangeCheck(0, 2, "stop_name"),
+ new FieldInRangeCheck(0, 2, "stop_lat"),
+ new FieldInRangeCheck(0, 2, "stop_lon"),
+ new FieldInRangeCheck(2, 4, "parent_station")
+ ),
+ // The actual conditions that will be acted upon are within the location_type field.
+ new StringField("parent_station", OPTIONAL).requireConditions(),
+ new StringField("stop_timezone", OPTIONAL),
+ new ShortField("wheelchair_boarding", OPTIONAL, 2),
+ new StringField("platform_code", OPTIONAL)
+ )
+ .restrictDelete()
+ .addPrimaryKey();
+
+ // GTFS reference: https://developers.google.com/transit/gtfs/reference#fare_rulestxt
+ public static final Table FARE_RULES = new Table("fare_rules", FareRule.class, OPTIONAL,
+ new StringField("fare_id", REQUIRED).isReferenceTo(FARE_ATTRIBUTES),
+ new StringField("route_id", OPTIONAL).isReferenceTo(ROUTES),
+ new StringField("origin_id", OPTIONAL).requireConditions(
+ // If the origin_id is defined, its value must exist as a zone_id in stops.txt.
+ new ForeignRefExistsCheck("zone_id", "fare_rules")
+ ),
+ new StringField("destination_id", OPTIONAL).requireConditions(
+ // If the destination_id is defined, its value must exist as a zone_id in stops.txt.
+ new ForeignRefExistsCheck("zone_id", "fare_rules")
+ ),
+ new StringField("contains_id", OPTIONAL).requireConditions(
+ // If the contains_id is defined, its value must exist as a zone_id in stops.txt.
+ new ForeignRefExistsCheck("zone_id", "fare_rules")
+ )
+ )
+ .withParentTable(FARE_ATTRIBUTES)
+ .addPrimaryKey().keyFieldIsNotUnique();
public static final Table PATTERN_STOP = new Table("pattern_stops", PatternStop.class, OPTIONAL,
new StringField("pattern_id", REQUIRED).isReferenceTo(PATTERNS),
@@ -252,7 +301,9 @@ public Table (String name, Class extends Entity> entityClass, Requirement requ
new IntegerField("drop_off_type", EDITOR, 2),
new IntegerField("pickup_type", EDITOR, 2),
new DoubleField("shape_dist_traveled", EDITOR, 0, Double.POSITIVE_INFINITY, -1),
- new ShortField("timepoint", EDITOR, 1)
+ new ShortField("timepoint", EDITOR, 1),
+ new ShortField("continuous_pickup", OPTIONAL,3),
+ new ShortField("continuous_drop_off", OPTIONAL,3)
).withParentTable(PATTERNS);
public static final Table TRANSFERS = new Table("transfers", Transfer.class, OPTIONAL,
@@ -266,16 +317,16 @@ public Table (String name, Class extends Entity> entityClass, Requirement requ
.hasCompoundKey();
public static final Table TRIPS = new Table("trips", Trip.class, REQUIRED,
- new StringField("trip_id", REQUIRED),
- new StringField("route_id", REQUIRED).isReferenceTo(ROUTES).indexThisColumn(),
+ new StringField("trip_id", REQUIRED),
+ new StringField("route_id", REQUIRED).isReferenceTo(ROUTES).indexThisColumn(),
// FIXME: Should this also optionally reference CALENDAR_DATES?
// FIXME: Do we need an index on service_id
- new StringField("service_id", REQUIRED).isReferenceTo(CALENDAR),
- new StringField("trip_headsign", OPTIONAL),
- new StringField("trip_short_name", OPTIONAL),
+ new StringField("service_id", REQUIRED).isReferenceTo(CALENDAR),
+ new StringField("trip_headsign", OPTIONAL),
+ new StringField("trip_short_name", OPTIONAL),
new ShortField("direction_id", OPTIONAL, 1),
- new StringField("block_id", OPTIONAL),
- new StringField("shape_id", OPTIONAL).isReferenceTo(SHAPES),
+ new StringField("block_id", OPTIONAL),
+ new StringField("shape_id", OPTIONAL).isReferenceTo(SHAPES),
new ShortField("wheelchair_accessible", OPTIONAL, 2),
new ShortField("bikes_allowed", OPTIONAL, 2),
// Editor-specific fields below.
@@ -295,6 +346,8 @@ public Table (String name, Class extends Entity> entityClass, Requirement requ
new StringField("stop_headsign", OPTIONAL),
new ShortField("pickup_type", OPTIONAL, 3),
new ShortField("drop_off_type", OPTIONAL, 3),
+ new ShortField("continuous_pickup", OPTIONAL, 3),
+ new ShortField("continuous_drop_off", OPTIONAL, 3),
new DoubleField("shape_dist_traveled", OPTIONAL, 0, Double.POSITIVE_INFINITY, 2),
new ShortField("timepoint", OPTIONAL, 1),
new IntegerField("fare_units_traveled", EXTENSION) // OpenOV NL extension
@@ -313,24 +366,59 @@ public Table (String name, Class extends Entity> entityClass, Requirement requ
.withParentTable(TRIPS)
.keyFieldIsNotUnique();
+ // GTFS reference: https://developers.google.com/transit/gtfs/reference#attributionstxt
+ public static final Table TRANSLATIONS = new Table("translations", Translation.class, OPTIONAL,
+ new StringField("table_name", REQUIRED),
+ new StringField("field_name", REQUIRED),
+ new LanguageField("language", REQUIRED),
+ new StringField("translation", REQUIRED),
+ new StringField("record_id", OPTIONAL).requireConditions(
+ // If the field_value is empty the record_id is required.
+ new FieldIsEmptyCheck("field_value")
+ ),
+ new StringField("record_sub_id", OPTIONAL).requireConditions(
+ // If the record_id is not empty and the value is stop_times the record_sub_id is required.
+ new FieldNotEmptyAndMatchesValueCheck("record_id", "stop_times")
+ ),
+ new StringField("field_value", OPTIONAL).requireConditions(
+ // If the record_id is empty the field_value is required.
+ new FieldIsEmptyCheck("record_id")
+ ))
+ .keyFieldIsNotUnique();
+
+ public static final Table ATTRIBUTIONS = new Table("attributions", Attribution.class, OPTIONAL,
+ new StringField("attribution_id", OPTIONAL),
+ new StringField("agency_id", OPTIONAL).isReferenceTo(AGENCY),
+ new LanguageField("route_id", OPTIONAL).isReferenceTo(ROUTES),
+ new StringField("trip_id", OPTIONAL).isReferenceTo(TRIPS),
+ new StringField("organization_name", REQUIRED),
+ new ShortField("is_producer", OPTIONAL, 1),
+ new ShortField("is_operator", OPTIONAL, 1),
+ new ShortField("is_authority", OPTIONAL, 1),
+ new URLField("attribution_url", OPTIONAL),
+ new StringField("attribution_email", OPTIONAL),
+ new StringField("attribution_phone", OPTIONAL));
+
/** List of tables in order needed for checking referential integrity during load stage. */
public static final Table[] tablesInOrder = {
- AGENCY,
- CALENDAR,
- SCHEDULE_EXCEPTIONS,
- CALENDAR_DATES,
- FARE_ATTRIBUTES,
- FEED_INFO,
- ROUTES,
- FARE_RULES,
- PATTERNS,
- SHAPES,
- STOPS,
- PATTERN_STOP,
- TRANSFERS,
- TRIPS,
- STOP_TIMES,
- FREQUENCIES
+ AGENCY,
+ CALENDAR,
+ SCHEDULE_EXCEPTIONS,
+ CALENDAR_DATES,
+ FARE_ATTRIBUTES,
+ FEED_INFO,
+ ROUTES,
+ PATTERNS,
+ SHAPES,
+ STOPS,
+ FARE_RULES,
+ PATTERN_STOP,
+ TRANSFERS,
+ TRIPS,
+ STOP_TIMES,
+ FREQUENCIES,
+ TRANSLATIONS,
+ ATTRIBUTIONS
};
/**
@@ -996,4 +1084,19 @@ public int getKeyFieldIndex(Field[] fields) {
String keyField = getKeyFieldName();
return Field.getFieldIndex(fields, keyField);
}
+
+ public boolean hasConditionalRequirements() {
+ return !getConditionalRequirements().isEmpty();
+ }
+
+ public Map getConditionalRequirements() {
+ Map fieldsWithConditions = new HashMap<>();
+ for (Field field : fields) {
+ if (field.isConditionallyRequired()) {
+ fieldsWithConditions.put(field, field.conditions);
+ }
+ }
+ return fieldsWithConditions;
+ }
+
}
diff --git a/src/main/java/com/conveyal/gtfs/loader/conditions/AgencyHasMultipleRowsCheck.java b/src/main/java/com/conveyal/gtfs/loader/conditions/AgencyHasMultipleRowsCheck.java
new file mode 100644
index 000000000..3b3b6ca9b
--- /dev/null
+++ b/src/main/java/com/conveyal/gtfs/loader/conditions/AgencyHasMultipleRowsCheck.java
@@ -0,0 +1,62 @@
+package com.conveyal.gtfs.loader.conditions;
+
+import com.conveyal.gtfs.error.NewGTFSError;
+import com.conveyal.gtfs.loader.Field;
+import com.conveyal.gtfs.loader.LineContext;
+import com.google.common.collect.HashMultimap;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import static com.conveyal.gtfs.error.NewGTFSErrorType.AGENCY_ID_REQUIRED_FOR_MULTI_AGENCY_FEEDS;
+import static com.conveyal.gtfs.loader.JdbcGtfsLoader.POSTGRES_NULL_TEXT;
+
+/**
+ * Conditional requirement to check that an agency_id has been provided if more than one row exists in agency.txt.
+ */
+public class AgencyHasMultipleRowsCheck extends ConditionalRequirement {
+
+ private final int FIRST_ROW = 2;
+ private final int SECOND_ROW = 3;
+
+ public AgencyHasMultipleRowsCheck() {
+ this.dependentFieldName = "agency_id";
+ }
+
+ /**
+ * Flag an error if there are multiple rows in agency.txt and the agency_id is missing for any rows.
+ */
+ public Set check(
+ LineContext lineContext,
+ Field referenceField,
+ HashMultimap uniqueValuesForFields
+ ) {
+ String dependentFieldValue = lineContext.getValueForRow(dependentFieldName);
+ Set errors = new HashSet<>();
+ Set agencyIdValues = uniqueValuesForFields.get(dependentFieldName);
+ // Do some awkward checks to determine if the first or second row (or another) is missing the agency_id.
+ boolean firstOrSecondMissingId = lineContext.lineNumber == SECOND_ROW && agencyIdValues.contains("");
+ boolean currentRowMissingId = POSTGRES_NULL_TEXT.equals(dependentFieldValue);
+ boolean secondRowMissingId = firstOrSecondMissingId && currentRowMissingId;
+ if (firstOrSecondMissingId || (lineContext.lineNumber > SECOND_ROW && currentRowMissingId)) {
+ // The check on the agency table is carried out whilst the agency table is being loaded so it
+ // is possible to compare the number of agencyIdValues added against the number of rows loaded to
+ // accurately determine missing agency_id values.
+ int lineNumber = secondRowMissingId
+ ? SECOND_ROW
+ : firstOrSecondMissingId
+ ? FIRST_ROW
+ : lineContext.lineNumber;
+ errors.add(
+ NewGTFSError.forLine(
+ lineContext.table,
+ lineNumber,
+ AGENCY_ID_REQUIRED_FOR_MULTI_AGENCY_FEEDS,
+ dependentFieldName
+ )
+ );
+ }
+ return errors;
+ }
+
+}
diff --git a/src/main/java/com/conveyal/gtfs/loader/conditions/ConditionalRequirement.java b/src/main/java/com/conveyal/gtfs/loader/conditions/ConditionalRequirement.java
new file mode 100644
index 000000000..25fe7087d
--- /dev/null
+++ b/src/main/java/com/conveyal/gtfs/loader/conditions/ConditionalRequirement.java
@@ -0,0 +1,28 @@
+package com.conveyal.gtfs.loader.conditions;
+
+import com.conveyal.gtfs.error.NewGTFSError;
+import com.conveyal.gtfs.loader.Field;
+import com.conveyal.gtfs.loader.LineContext;
+import com.google.common.collect.HashMultimap;
+
+import java.util.Set;
+
+/**
+ * An abstract class which primarily defines a method used by implementing classes to define specific conditional
+ * requirement checks.
+ */
+public abstract class ConditionalRequirement {
+ /** The name of the dependent field, which is a field that requires a specific value if the reference and
+ * (in some cases) dependent field checks meet certain conditions.
+ */
+ protected String dependentFieldName;
+
+ /**
+ * All sub classes must implement this method and provide related conditional checks.
+ */
+ public abstract Set check(
+ LineContext lineContext,
+ Field referenceField,
+ HashMultimap uniqueValuesForFields
+ );
+}
diff --git a/src/main/java/com/conveyal/gtfs/loader/conditions/FieldInRangeCheck.java b/src/main/java/com/conveyal/gtfs/loader/conditions/FieldInRangeCheck.java
new file mode 100644
index 000000000..47f752c45
--- /dev/null
+++ b/src/main/java/com/conveyal/gtfs/loader/conditions/FieldInRangeCheck.java
@@ -0,0 +1,88 @@
+package com.conveyal.gtfs.loader.conditions;
+
+import com.conveyal.gtfs.error.NewGTFSError;
+import com.conveyal.gtfs.loader.Field;
+import com.conveyal.gtfs.loader.LineContext;
+import com.google.common.collect.HashMultimap;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import static com.conveyal.gtfs.error.NewGTFSErrorType.CONDITIONALLY_REQUIRED;
+import static com.conveyal.gtfs.loader.JdbcGtfsLoader.POSTGRES_NULL_TEXT;
+
+/**
+ * Conditional requirement to check that a reference field value is within a defined range and the conditional field
+ * value has not be defined.
+ */
+public class FieldInRangeCheck extends ConditionalRequirement {
+ /** The minimum reference field value if a range check is being performed. */
+ protected int minReferenceValue;
+ /** The maximum reference field value if a range check is being performed. */
+ protected int maxReferenceValue;
+
+ public FieldInRangeCheck(
+ int minReferenceValue,
+ int maxReferenceValue,
+ String dependentFieldName
+ ) {
+ this.minReferenceValue = minReferenceValue;
+ this.maxReferenceValue = maxReferenceValue;
+ this.dependentFieldName = dependentFieldName;
+ }
+
+ /**
+ * If the reference field value is within a defined range and the conditional field value has not been defined, flag
+ * an error.
+ */
+ public Set check(
+ LineContext lineContext,
+ Field referenceField,
+ HashMultimap uniqueValuesForFields
+ ) {
+ Set errors = new HashSet<>();
+ String referenceFieldValue = lineContext.getValueForRow(referenceField.name);
+ String conditionalFieldValue = lineContext.getValueForRow(dependentFieldName);
+ boolean referenceValueMeetsRangeCondition =
+ !POSTGRES_NULL_TEXT.equals(referenceFieldValue) &&
+ // TODO use pre-existing method in ShortField?
+ isValueInRange(referenceFieldValue, minReferenceValue, maxReferenceValue);
+
+ if (!referenceValueMeetsRangeCondition) {
+ // If ref value does not meet the range condition, there is no need to check this conditional
+ // value for (e.g.) an empty value. Continue to the next check.
+ return errors;
+ }
+
+ if (POSTGRES_NULL_TEXT.equals(conditionalFieldValue)) {
+ // Reference value in range and conditionally required field is empty.
+ String message = String.format(
+ "%s is required when %s value is between %d and %d.",
+ dependentFieldName,
+ referenceField.name,
+ minReferenceValue,
+ maxReferenceValue
+ );
+ errors.add(
+ NewGTFSError.forLine(
+ lineContext,
+ CONDITIONALLY_REQUIRED,
+ message).setEntityId(lineContext.getEntityId())
+ );
+ }
+ return errors;
+ }
+
+ /**
+ * Check if the provided value is within the min and max values. If the field value can not be converted
+ * to a number it is assumed that the value is not a number and will therefore never be within the min/max range.
+ */
+ private boolean isValueInRange(String referenceFieldValue, int min, int max) {
+ try {
+ int fieldValue = Integer.parseInt(referenceFieldValue);
+ return fieldValue >= min && fieldValue <= max;
+ } catch (NumberFormatException e) {
+ return false;
+ }
+ }
+}
diff --git a/src/main/java/com/conveyal/gtfs/loader/conditions/FieldIsEmptyCheck.java b/src/main/java/com/conveyal/gtfs/loader/conditions/FieldIsEmptyCheck.java
new file mode 100644
index 000000000..3ba3c291d
--- /dev/null
+++ b/src/main/java/com/conveyal/gtfs/loader/conditions/FieldIsEmptyCheck.java
@@ -0,0 +1,55 @@
+package com.conveyal.gtfs.loader.conditions;
+
+import com.conveyal.gtfs.error.NewGTFSError;
+import com.conveyal.gtfs.loader.Field;
+import com.conveyal.gtfs.loader.LineContext;
+import com.google.common.collect.HashMultimap;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import static com.conveyal.gtfs.error.NewGTFSErrorType.CONDITIONALLY_REQUIRED;
+import static com.conveyal.gtfs.loader.JdbcGtfsLoader.POSTGRES_NULL_TEXT;
+
+/**
+ * Conditional requirement to check that if a dependent field value is empty then the reference field value is provided.
+ */
+public class FieldIsEmptyCheck extends ConditionalRequirement {
+
+ public FieldIsEmptyCheck(String dependentFieldName) {
+ this.dependentFieldName = dependentFieldName;
+ }
+
+ /**
+ * Check the dependent field value. If it is empty, the reference field value must be provided.
+ */
+ public Set check(
+ LineContext lineContext,
+ Field referenceField,
+ HashMultimap uniqueValuesForFields
+ ) {
+ Set errors = new HashSet<>();
+ String dependentFieldValue = lineContext.getValueForRow(dependentFieldName);
+ String referenceFieldValue = lineContext.getValueForRow(referenceField.name);
+ if (
+ POSTGRES_NULL_TEXT.equals(dependentFieldValue) &&
+ POSTGRES_NULL_TEXT.equals(referenceFieldValue)
+ ) {
+ // The reference field is required when the dependent field is empty.
+ String message = String.format(
+ "%s is required when %s is empty.",
+ referenceField.name,
+ dependentFieldName
+ );
+ errors.add(
+ NewGTFSError.forLine(
+ lineContext,
+ CONDITIONALLY_REQUIRED,
+ message).setEntityId(lineContext.getEntityId())
+ );
+
+ }
+ return errors;
+ }
+
+}
diff --git a/src/main/java/com/conveyal/gtfs/loader/conditions/FieldNotEmptyAndMatchesValueCheck.java b/src/main/java/com/conveyal/gtfs/loader/conditions/FieldNotEmptyAndMatchesValueCheck.java
new file mode 100644
index 000000000..c693120fe
--- /dev/null
+++ b/src/main/java/com/conveyal/gtfs/loader/conditions/FieldNotEmptyAndMatchesValueCheck.java
@@ -0,0 +1,62 @@
+package com.conveyal.gtfs.loader.conditions;
+
+import com.conveyal.gtfs.error.NewGTFSError;
+import com.conveyal.gtfs.loader.Field;
+import com.conveyal.gtfs.loader.LineContext;
+import com.google.common.collect.HashMultimap;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import static com.conveyal.gtfs.error.NewGTFSErrorType.CONDITIONALLY_REQUIRED;
+import static com.conveyal.gtfs.loader.JdbcGtfsLoader.POSTGRES_NULL_TEXT;
+
+/**
+ * Conditional requirement to check that a dependent field value is not empty and matches an expected value.
+ */
+public class FieldNotEmptyAndMatchesValueCheck extends ConditionalRequirement {
+ /** The expected dependent field value. */
+ private String requiredDependentFieldValue;
+
+ public FieldNotEmptyAndMatchesValueCheck(
+ String dependentFieldName,
+ String requiredDependentFieldValue
+ ) {
+ this.dependentFieldName = dependentFieldName;
+ this.requiredDependentFieldValue = requiredDependentFieldValue;
+ }
+
+ /**
+ * Check the dependent field value is not empty and matches the expected value.
+ */
+ public Set check(
+ LineContext lineContext,
+ Field referenceField,
+ HashMultimap uniqueValuesForFields
+ ) {
+ Set errors = new HashSet<>();
+ String dependentFieldValue = lineContext.getValueForRow(dependentFieldName);
+ String referenceFieldValue = lineContext.getValueForRow(referenceField.name);
+ if (
+ !POSTGRES_NULL_TEXT.equals(dependentFieldValue) &&
+ dependentFieldValue.equals(requiredDependentFieldValue) &&
+ POSTGRES_NULL_TEXT.equals(referenceFieldValue)
+ ) {
+ String message = String.format(
+ "%s is required and must match %s when %s is provided.",
+ referenceField.name,
+ requiredDependentFieldValue,
+ dependentFieldName
+ );
+ errors.add(
+ NewGTFSError.forLine(
+ lineContext,
+ CONDITIONALLY_REQUIRED,
+ message).setEntityId(lineContext.getEntityId())
+ );
+
+ }
+ return errors;
+ }
+
+}
diff --git a/src/main/java/com/conveyal/gtfs/loader/conditions/ForeignRefExistsCheck.java b/src/main/java/com/conveyal/gtfs/loader/conditions/ForeignRefExistsCheck.java
new file mode 100644
index 000000000..23c9d5e5d
--- /dev/null
+++ b/src/main/java/com/conveyal/gtfs/loader/conditions/ForeignRefExistsCheck.java
@@ -0,0 +1,62 @@
+package com.conveyal.gtfs.loader.conditions;
+
+import com.conveyal.gtfs.error.NewGTFSError;
+import com.conveyal.gtfs.loader.Field;
+import com.conveyal.gtfs.loader.LineContext;
+import com.conveyal.gtfs.loader.ReferenceTracker;
+import com.google.common.collect.HashMultimap;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import static com.conveyal.gtfs.error.NewGTFSErrorType.REFERENTIAL_INTEGRITY;
+import static com.conveyal.gtfs.loader.JdbcGtfsLoader.POSTGRES_NULL_TEXT;
+
+/**
+ * Conditional requirement to check that an expected foreign field value matches a conditional field value.
+ */
+public class ForeignRefExistsCheck extends ConditionalRequirement {
+ /** The reference table name. */
+ private String referenceTableName;
+
+ public ForeignRefExistsCheck(String dependentFieldName, String referenceTableName) {
+ this.dependentFieldName = dependentFieldName;
+ this.referenceTableName = referenceTableName;
+ }
+
+ /**
+ * Check that an expected foreign field value matches a conditional field value. Selected foreign field values are
+ * added to {@link ReferenceTracker#uniqueValuesForFields} as part of the load process and are used here to check
+ * conditional fields which have a dependency on them. e.g. stop#zone_id does not exist in stops table, but is
+ * required by fare_rules records (e.g. origin_id).
+ */
+ public Set check(
+ LineContext lineContext,
+ Field referenceField,
+ HashMultimap uniqueValuesForFields
+ ) {
+ Set errors = new HashSet<>();
+ String referenceFieldValue = lineContext.getValueForRow(referenceField.name);
+ // Expected reference in foreign field id list.
+ String foreignFieldReference =
+ String.join(
+ ":",
+ dependentFieldName,
+ referenceFieldValue
+ );
+ if (lineContext.table.name.equals(referenceTableName) &&
+ !POSTGRES_NULL_TEXT.equals(referenceFieldValue) &&
+ !uniqueValuesForFields.get(dependentFieldName).contains(foreignFieldReference)
+ ) {
+ errors.add(
+ NewGTFSError.forLine(
+ lineContext,
+ REFERENTIAL_INTEGRITY,
+ String.join(":", referenceField.name, foreignFieldReference)
+ ).setEntityId(lineContext.getEntityId())
+ );
+ }
+ return errors;
+ }
+
+}
diff --git a/src/main/java/com/conveyal/gtfs/loader/conditions/ReferenceFieldShouldBeProvidedCheck.java b/src/main/java/com/conveyal/gtfs/loader/conditions/ReferenceFieldShouldBeProvidedCheck.java
new file mode 100644
index 000000000..33fd55716
--- /dev/null
+++ b/src/main/java/com/conveyal/gtfs/loader/conditions/ReferenceFieldShouldBeProvidedCheck.java
@@ -0,0 +1,53 @@
+package com.conveyal.gtfs.loader.conditions;
+
+import com.conveyal.gtfs.error.NewGTFSError;
+import com.conveyal.gtfs.loader.Field;
+import com.conveyal.gtfs.loader.LineContext;
+import com.google.common.collect.HashMultimap;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import static com.conveyal.gtfs.error.NewGTFSErrorType.AGENCY_ID_REQUIRED_FOR_MULTI_AGENCY_FEEDS;
+import static com.conveyal.gtfs.loader.JdbcGtfsLoader.POSTGRES_NULL_TEXT;
+
+/**
+ * Conditional requirement to check that the reference field is not empty when the dependent field/table has multiple
+ * rows.
+ */
+public class ReferenceFieldShouldBeProvidedCheck extends ConditionalRequirement {
+
+ public ReferenceFieldShouldBeProvidedCheck(String dependentFieldName) {
+ this.dependentFieldName = dependentFieldName;
+ }
+
+ /**
+ * Checks that the reference field is not empty when the dependent field/table has multiple rows. This is
+ * principally designed for checking that routes#agency_id is filled when multiple agencies exist.
+ */
+ public Set check(
+ LineContext lineContext,
+ Field referenceField,
+ HashMultimap uniqueValuesForFields
+ ) {
+ String referenceFieldValue = lineContext.getValueForRow(referenceField.name);
+ Set errors = new HashSet<>();
+ int dependentFieldCount = uniqueValuesForFields.get(dependentFieldName).size();
+ if (dependentFieldCount > 1) {
+ // If there are multiple entries for the dependent field (including empty strings to account for any
+ // potentially missing values), the reference field must not be empty.
+ boolean referenceFieldIsEmpty = POSTGRES_NULL_TEXT.equals(referenceFieldValue);
+ if (referenceFieldIsEmpty) {
+ errors.add(
+ NewGTFSError.forLine(
+ lineContext,
+ AGENCY_ID_REQUIRED_FOR_MULTI_AGENCY_FEEDS,
+ null
+ ).setEntityId(lineContext.getEntityId())
+ );
+ }
+ }
+ return errors;
+ }
+
+}
diff --git a/src/main/java/com/conveyal/gtfs/model/Attribution.java b/src/main/java/com/conveyal/gtfs/model/Attribution.java
new file mode 100644
index 000000000..deaa30be4
--- /dev/null
+++ b/src/main/java/com/conveyal/gtfs/model/Attribution.java
@@ -0,0 +1,119 @@
+package com.conveyal.gtfs.model;
+
+import com.conveyal.gtfs.GTFSFeed;
+
+import java.io.IOException;
+import java.net.URL;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.Iterator;
+
+public class Attribution extends Entity {
+
+ String attribution_id;
+ String agency_id;
+ String route_id;
+ String trip_id;
+ String organization_name;
+ int is_producer;
+ int is_operator;
+ int is_authority;
+ URL attribution_url;
+ String attribution_email;
+ String attribution_phone;
+
+ @Override
+ public String getId () {
+ return attribution_id;
+ }
+
+ /**
+ * Sets the parameters for a prepared statement following the parameter order defined in
+ * {@link com.conveyal.gtfs.loader.Table#ATTRIBUTIONS}. JDBC prepared statement parameters use a one-based index.
+ */
+ @Override
+ public void setStatementParameters(PreparedStatement statement, boolean setDefaultId) throws SQLException {
+ int oneBasedIndex = 1;
+ if (!setDefaultId) statement.setInt(oneBasedIndex++, id);
+ statement.setString(oneBasedIndex++, attribution_id);
+ statement.setString(oneBasedIndex++, agency_id);
+ statement.setString(oneBasedIndex++, route_id);
+ statement.setString(oneBasedIndex++, trip_id);
+ statement.setString(oneBasedIndex++, organization_name);
+ setIntParameter(statement, oneBasedIndex++, is_producer);
+ setIntParameter(statement, oneBasedIndex++, is_operator);
+ setIntParameter(statement, oneBasedIndex++, is_authority);
+ statement.setString(oneBasedIndex++, attribution_url != null ? attribution_url .toString() : null);
+ statement.setString(oneBasedIndex++, attribution_email);
+ statement.setString(oneBasedIndex++, attribution_phone);
+ }
+
+ public static class Loader extends Entity.Loader {
+
+ public Loader(GTFSFeed feed) {
+ super(feed, "attributions");
+ }
+
+ @Override
+ protected boolean isRequired() {
+ return false;
+ }
+
+ @Override
+ public void loadOneRow() throws IOException {
+ Attribution a = new Attribution();
+ a.id = row + 1; // offset line number by 1 to account for 0-based row index
+ a.attribution_id = getStringField("attribution_id", false);
+ a.agency_id = getStringField("agency_id", false);
+ a.route_id = getStringField("route_id", true);
+ a.trip_id = getStringField("trip_id", false);
+ a.organization_name = getStringField("organization_name", true);
+ a.is_producer = getIntField("is_producer", false, 0, 1);
+ a.is_operator = getIntField("is_operator", false, 0, 1);
+ a.is_authority = getIntField("is_authority", false, 0, 1);
+ a.attribution_url = getUrlField("attribution_url", false);
+ a.attribution_email = getStringField("attribution_email", false);
+ a.attribution_phone = getStringField("attribution_phone", false);
+
+ // TODO clooge due to not being able to have null keys in mapdb
+ if (a.attribution_id == null) a.attribution_id = "NONE";
+
+ feed.attributions.put(a.attribution_id, a);
+ }
+ }
+
+ public static class Writer extends Entity.Writer {
+ public Writer (GTFSFeed feed) {
+ super(feed, "attribution");
+ }
+
+ @Override
+ protected void writeHeaders() throws IOException {
+ writer.writeRecord(new String[] {"attribution_id", "agency_id", "route_id", "trip_id", "organization_name",
+ "is_producer", "is_operator", "is_authority", "attribution_url", "attribution_email", "attribution_phone"});
+ }
+
+ @Override
+ protected void writeOneRow(Attribution a) throws IOException {
+ writeStringField(a.attribution_id);
+ writeStringField(a.agency_id);
+ writeStringField(a.route_id);
+ writeStringField(a.trip_id);
+ writeStringField(a.organization_name);
+ writeIntField(a.is_producer);
+ writeIntField(a.is_operator);
+ writeIntField(a.is_authority);
+ writeUrlField(a.attribution_url);
+ writeStringField(a.attribution_email);
+ writeStringField(a.attribution_phone);
+ endRecord();
+ }
+
+ @Override
+ protected Iterator iterator() {
+ return feed.attributions.values().iterator();
+ }
+ }
+
+
+}
diff --git a/src/main/java/com/conveyal/gtfs/model/FeedInfo.java b/src/main/java/com/conveyal/gtfs/model/FeedInfo.java
index 42fc61cbf..1cc94008d 100644
--- a/src/main/java/com/conveyal/gtfs/model/FeedInfo.java
+++ b/src/main/java/com/conveyal/gtfs/model/FeedInfo.java
@@ -23,6 +23,9 @@ public class FeedInfo extends Entity implements Cloneable {
public LocalDate feed_start_date;
public LocalDate feed_end_date;
public String feed_version;
+ public String default_lang;
+ public String feed_contact_email;
+ public URL feed_contact_url;
public FeedInfo clone () {
try {
@@ -50,6 +53,11 @@ public void setStatementParameters(PreparedStatement statement, boolean setDefau
feedStartDateField.setParameter(statement, oneBasedIndex++, feed_start_date);
feedEndDateField.setParameter(statement, oneBasedIndex++, feed_end_date);
statement.setString(oneBasedIndex++, feed_version);
+ statement.setString(oneBasedIndex++, default_lang);
+ statement.setString(oneBasedIndex++, feed_contact_email);
+ String feedContactUrl = feed_contact_url != null ? feed_contact_url.toString() : null;
+ statement.setString(oneBasedIndex++, feedContactUrl);
+
}
public static class Loader extends Entity.Loader {
@@ -74,6 +82,9 @@ public void loadOneRow() throws IOException {
fi.feed_start_date = getDateField("feed_start_date", false);
fi.feed_end_date = getDateField("feed_end_date", false);
fi.feed_version = getStringField("feed_version", false);
+ fi.default_lang = getStringField("default_lang", false);
+ fi.feed_contact_email = getStringField("feed_contact_email", false);
+ fi.feed_contact_url = getUrlField("feed_contact_url", false);
fi.feed = feed;
if (feed.feedInfo.isEmpty()) {
feed.feedInfo.put("NONE", fi);
@@ -93,7 +104,7 @@ public Writer(GTFSFeed feed) {
@Override
public void writeHeaders() throws IOException {
writer.writeRecord(new String[] {"feed_id", "feed_publisher_name", "feed_publisher_url", "feed_lang",
- "feed_start_date", "feed_end_date", "feed_version"});
+ "feed_start_date", "feed_end_date", "feed_version", "default_lang", "feed_contact_email", "feed_contact_url"});
}
@Override
@@ -110,6 +121,9 @@ public void writeOneRow(FeedInfo i) throws IOException {
else writeStringField("");
writeStringField(i.feed_version);
+ writeStringField(i.default_lang);
+ writeStringField(i.feed_contact_email);
+ writeUrlField(i.feed_contact_url);
endRecord();
}
diff --git a/src/main/java/com/conveyal/gtfs/model/PatternStop.java b/src/main/java/com/conveyal/gtfs/model/PatternStop.java
index 547280c23..62f343c66 100644
--- a/src/main/java/com/conveyal/gtfs/model/PatternStop.java
+++ b/src/main/java/com/conveyal/gtfs/model/PatternStop.java
@@ -23,6 +23,8 @@ public class PatternStop extends Entity {
public int pickup_type;
public int drop_off_type;
public int timepoint;
+ public int continuous_pickup = INT_MISSING;
+ public int continuous_drop_off = INT_MISSING;
public PatternStop () {}
diff --git a/src/main/java/com/conveyal/gtfs/model/Route.java b/src/main/java/com/conveyal/gtfs/model/Route.java
index c5eb05fbb..3c669e8e8 100644
--- a/src/main/java/com/conveyal/gtfs/model/Route.java
+++ b/src/main/java/com/conveyal/gtfs/model/Route.java
@@ -34,6 +34,8 @@ public class Route extends Entity { // implements Entity.Factory
public String route_text_color;
public URL route_branding_url;
public String feed_id;
+ public int continuous_pickup = INT_MISSING;
+ public int continuous_drop_off = INT_MISSING;
@Override
public String getId () {
@@ -64,6 +66,8 @@ public void setStatementParameters(PreparedStatement statement, boolean setDefau
// route_sort_order
setIntParameter(statement, oneBasedIndex++, route_sort_order);
setIntParameter(statement, oneBasedIndex++, 0);
+ setIntParameter(statement, oneBasedIndex++, continuous_pickup);
+ setIntParameter(statement, oneBasedIndex++, continuous_drop_off);
}
public static class Loader extends Entity.Loader {
@@ -104,6 +108,8 @@ public void loadOneRow() throws IOException {
r.route_color = getStringField("route_color", false);
r.route_text_color = getStringField("route_text_color", false);
r.route_branding_url = getUrlField("route_branding_url", false);
+ r.continuous_pickup = getIntField("continuous_pickup", true, 0, 3);
+ r.continuous_pickup = getIntField("continuous_drop_off", true, 0, 3);
r.feed = feed;
r.feed_id = feed.feedId;
// Attempting to put a null key or value will cause an NPE in BTreeMap
@@ -130,6 +136,8 @@ public void writeHeaders() throws IOException {
writeStringField("route_text_color");
writeStringField("route_branding_url");
writeStringField("route_sort_order");
+ writeStringField("continuous_pickup");
+ writeStringField("continuous_drop_off");
endRecord();
}
@@ -146,6 +154,8 @@ public void writeOneRow(Route r) throws IOException {
writeStringField(r.route_text_color);
writeUrlField(r.route_branding_url);
writeIntField(r.route_sort_order);
+ writeIntField(r.continuous_pickup);
+ writeIntField(r.continuous_drop_off);
endRecord();
}
diff --git a/src/main/java/com/conveyal/gtfs/model/Stop.java b/src/main/java/com/conveyal/gtfs/model/Stop.java
index 25a3ce82a..24923380e 100644
--- a/src/main/java/com/conveyal/gtfs/model/Stop.java
+++ b/src/main/java/com/conveyal/gtfs/model/Stop.java
@@ -22,9 +22,9 @@ public class Stop extends Entity {
public int location_type;
public String parent_station;
public String stop_timezone;
- // TODO should be int
- public String wheelchair_boarding;
+ public int wheelchair_boarding;
public String feed_id;
+ public String platform_code;
@Override
public String getId () {
@@ -39,12 +39,6 @@ public String getId () {
public void setStatementParameters(PreparedStatement statement, boolean setDefaultId) throws SQLException {
int oneBasedIndex = 1;
if (!setDefaultId) statement.setInt(oneBasedIndex++, id);
- int wheelchairBoarding = 0;
- try {
- wheelchairBoarding = Integer.parseInt(wheelchair_boarding);
- } catch (NumberFormatException e) {
- // Do nothing, wheelchairBoarding will remain zero.
- }
statement.setString(oneBasedIndex++, stop_id);
statement.setString(oneBasedIndex++, stop_code);
statement.setString(oneBasedIndex++, stop_name);
@@ -56,8 +50,8 @@ public void setStatementParameters(PreparedStatement statement, boolean setDefau
setIntParameter(statement, oneBasedIndex++, location_type);
statement.setString(oneBasedIndex++, parent_station);
statement.setString(oneBasedIndex++, stop_timezone);
- // FIXME: For some reason wheelchair boarding type is String
- setIntParameter(statement, oneBasedIndex++, wheelchairBoarding);
+ setIntParameter(statement, oneBasedIndex++, wheelchair_boarding);
+ statement.setString(oneBasedIndex++, platform_code);
}
public static class Loader extends Entity.Loader {
@@ -86,9 +80,10 @@ public void loadOneRow() throws IOException {
s.location_type = getIntField("location_type", false, 0, 1);
s.parent_station = getStringField("parent_station", false);
s.stop_timezone = getStringField("stop_timezone", false);
- s.wheelchair_boarding = getStringField("wheelchair_boarding", false);
+ s.wheelchair_boarding = getIntField("wheelchair_boarding", false, 0, 2);
s.feed = feed;
s.feed_id = feed.feedId;
+ s.platform_code = getStringField("platform_code", false);
/* TODO check ref integrity later, this table self-references via parent_station */
// Attempting to put a null key or value will cause an NPE in BTreeMap
if (s.stop_id != null) feed.stops.put(s.stop_id, s);
@@ -104,7 +99,7 @@ public Writer (GTFSFeed feed) {
@Override
public void writeHeaders() throws IOException {
writer.writeRecord(new String[] {"stop_id", "stop_code", "stop_name", "stop_desc", "stop_lat", "stop_lon", "zone_id",
- "stop_url", "location_type", "parent_station", "stop_timezone", "wheelchair_boarding"});
+ "stop_url", "location_type", "parent_station", "stop_timezone", "wheelchair_boarding", "platform_code"});
}
@Override
@@ -120,7 +115,8 @@ public void writeOneRow(Stop s) throws IOException {
writeIntField(s.location_type);
writeStringField(s.parent_station);
writeStringField(s.stop_timezone);
- writeStringField(s.wheelchair_boarding);
+ writeIntField(s.wheelchair_boarding);
+ writeStringField(s.platform_code);
endRecord();
}
diff --git a/src/main/java/com/conveyal/gtfs/model/StopTime.java b/src/main/java/com/conveyal/gtfs/model/StopTime.java
index 02445e84f..1a227a393 100644
--- a/src/main/java/com/conveyal/gtfs/model/StopTime.java
+++ b/src/main/java/com/conveyal/gtfs/model/StopTime.java
@@ -27,6 +27,8 @@ public class StopTime extends Entity implements Cloneable, Serializable {
public String stop_headsign;
public int pickup_type;
public int drop_off_type;
+ public int continuous_pickup = INT_MISSING;
+ public int continuous_drop_off = INT_MISSING;
public double shape_dist_traveled = DOUBLE_MISSING;
public int timepoint = INT_MISSING;
@@ -56,6 +58,8 @@ public void setStatementParameters(PreparedStatement statement, boolean setDefau
statement.setString(oneBasedIndex++, stop_headsign);
setIntParameter(statement, oneBasedIndex++, pickup_type);
setIntParameter(statement, oneBasedIndex++, drop_off_type);
+ setIntParameter(statement, oneBasedIndex++, continuous_pickup);
+ setIntParameter(statement, oneBasedIndex++, continuous_drop_off);
statement.setDouble(oneBasedIndex++, shape_dist_traveled);
setIntParameter(statement, oneBasedIndex++, timepoint);
}
@@ -85,6 +89,8 @@ public void loadOneRow() throws IOException {
st.stop_headsign = getStringField("stop_headsign", false);
st.pickup_type = getIntField("pickup_type", false, 0, 3); // TODO add ranges as parameters
st.drop_off_type = getIntField("drop_off_type", false, 0, 3);
+ st.continuous_pickup = getIntField("continuous_pickup", true, 0, 3);
+ st.continuous_pickup = getIntField("continuous_drop_off", true, 0, 3);
st.shape_dist_traveled = getDoubleField("shape_dist_traveled", false, 0D, Double.MAX_VALUE); // FIXME using both 0 and NaN for "missing", define DOUBLE_MISSING
st.timepoint = getIntField("timepoint", false, 0, 1, INT_MISSING);
st.feed = null; // this could circular-serialize the whole feed
@@ -108,7 +114,7 @@ public Writer (GTFSFeed feed) {
@Override
protected void writeHeaders() throws IOException {
writer.writeRecord(new String[] {"trip_id", "arrival_time", "departure_time", "stop_id", "stop_sequence", "stop_headsign",
- "pickup_type", "drop_off_type", "shape_dist_traveled", "timepoint"});
+ "pickup_type", "drop_off_type", "continuous_pickup", "continuous_drop_off", "shape_dist_traveled", "timepoint"});
}
@Override
@@ -121,6 +127,8 @@ protected void writeOneRow(StopTime st) throws IOException {
writeStringField(st.stop_headsign);
writeIntField(st.pickup_type);
writeIntField(st.drop_off_type);
+ writeIntField(st.continuous_pickup);
+ writeIntField(st.continuous_drop_off);
writeDoubleField(st.shape_dist_traveled);
writeIntField(st.timepoint);
endRecord();
diff --git a/src/main/java/com/conveyal/gtfs/model/Translation.java b/src/main/java/com/conveyal/gtfs/model/Translation.java
new file mode 100644
index 000000000..13d9ad981
--- /dev/null
+++ b/src/main/java/com/conveyal/gtfs/model/Translation.java
@@ -0,0 +1,107 @@
+package com.conveyal.gtfs.model;
+
+import com.conveyal.gtfs.GTFSFeed;
+
+import java.io.IOException;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.Iterator;
+
+public class Translation extends Entity {
+
+ public String table_name;
+ public String field_name;
+ public String language;
+ public String translation;
+ public String record_id;
+ public String record_sub_id;
+ public String field_value;
+
+ @Override
+ public String getId() {
+ return createId(table_name, field_name, language);
+ }
+
+ /**
+ * Sets the parameters for a prepared statement following the parameter order defined in
+ * {@link com.conveyal.gtfs.loader.Table#TRANSLATIONS}. JDBC prepared statement parameters use a one-based index.
+ */
+ @Override
+ public void setStatementParameters(PreparedStatement statement, boolean setDefaultId) throws SQLException {
+ int oneBasedIndex = 1;
+ if (!setDefaultId) statement.setInt(oneBasedIndex++, id);
+ statement.setString(oneBasedIndex++, table_name);
+ statement.setString(oneBasedIndex++, field_name);
+ statement.setString(oneBasedIndex++, language);
+ statement.setString(oneBasedIndex++, translation);
+ statement.setString(oneBasedIndex++, record_id);
+ statement.setString(oneBasedIndex++, record_sub_id);
+ statement.setString(oneBasedIndex++, field_value);
+ }
+
+ public static class Loader extends Entity.Loader {
+
+ public Loader(GTFSFeed feed) {
+ super(feed, "translation");
+ }
+
+ @Override
+ protected boolean isRequired() {
+ return false;
+ }
+
+ @Override
+ public void loadOneRow() throws IOException {
+ Translation t = new Translation();
+ t.id = row + 1; // offset line number by 1 to account for 0-based row index
+ t.table_name = getStringField("table_name", true);
+ t.field_name = getStringField("field_name", true);
+ t.field_name = getStringField("language", true);
+ t.translation = getStringField("translation", true);
+ t.record_id = getStringField("record_id", false);
+ t.record_sub_id = getStringField("record_sub_id", false);
+ t.field_value = getStringField("field_value", false);
+ feed.translations.put(
+ createId(t.table_name, t.field_name, t.language),
+ t
+ );
+ }
+ }
+
+ public static class Writer extends Entity.Writer {
+ public Writer (GTFSFeed feed) {
+ super(feed, "translation");
+ }
+
+ @Override
+ protected void writeHeaders() throws IOException {
+ writer.writeRecord(new String[] {"table_name", "field_name", "language", "translation", "record_id",
+ "record_sub_id", "field_value"});
+ }
+
+ @Override
+ protected void writeOneRow(Translation t) throws IOException {
+ writeStringField(t.table_name);
+ writeStringField(t.field_name);
+ writeStringField(t.language);
+ writeStringField(t.translation);
+ writeStringField(t.record_id);
+ writeStringField(t.record_sub_id);
+ writeStringField(t.field_value);
+ endRecord();
+ }
+
+ @Override
+ protected Iterator iterator() {
+ return feed.translations.values().iterator();
+ }
+ }
+
+ /**
+ * Translation entries have no ID in GTFS so we define one based on the fields in the translation entry.
+ */
+ private static String createId(String table_name, String field_name, String language) {
+ return String.format("%s_%s_%s", table_name, field_name, language);
+ }
+
+}
diff --git a/src/main/java/com/conveyal/gtfs/validator/NewTripTimesValidator.java b/src/main/java/com/conveyal/gtfs/validator/NewTripTimesValidator.java
index cf395e674..dffdc2c44 100644
--- a/src/main/java/com/conveyal/gtfs/validator/NewTripTimesValidator.java
+++ b/src/main/java/com/conveyal/gtfs/validator/NewTripTimesValidator.java
@@ -4,18 +4,21 @@
import com.conveyal.gtfs.loader.Feed;
import com.conveyal.gtfs.model.Entity;
import com.conveyal.gtfs.model.Route;
-import com.conveyal.gtfs.model.ShapePoint;
import com.conveyal.gtfs.model.Stop;
import com.conveyal.gtfs.model.StopTime;
import com.conveyal.gtfs.model.Trip;
-import com.google.common.collect.ListMultimap;
-import com.google.common.collect.MultimapBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
-import static com.conveyal.gtfs.error.NewGTFSErrorType.*;
+import static com.conveyal.gtfs.error.NewGTFSErrorType.CONDITIONALLY_REQUIRED;
+import static com.conveyal.gtfs.error.NewGTFSErrorType.MISSING_ARRIVAL_OR_DEPARTURE;
+import static com.conveyal.gtfs.error.NewGTFSErrorType.TRIP_TOO_FEW_STOP_TIMES;
/**
* Check that the travel times between adjacent stops in trips are reasonable.
@@ -114,6 +117,8 @@ private boolean fixInitialFinal (StopTime stopTime) {
registerError(stopTime, MISSING_ARRIVAL_OR_DEPARTURE);
fixMissingTimes(stopTime);
if (missingEitherTime(stopTime)) {
+ //TODO: Is this even needed? Already covered by MISSING_ARRIVAL_OR_DEPARTURE.
+ registerError(stopTime, CONDITIONALLY_REQUIRED, "First and last stop times are required to have both an arrival and departure time.");
return true;
}
}
@@ -135,17 +140,22 @@ private void processTrip (List stopTimes) {
// This error should already have been caught TODO verify.
return;
}
+
// Our code should only call this method with non-null stopTimes.
if (stopTimes.size() < 2) {
registerError(trip, TRIP_TOO_FEW_STOP_TIMES);
return;
}
+ boolean hasContinuousBehavior = false;
// Make a parallel list of stops based on the stop_times for this trip.
// We will remove any stop_times for stops that don't exist in the feed.
// We could ask the SQL server to do the join between stop_times and stops, but we want to check references.
List stops = new ArrayList<>();
for (Iterator it = stopTimes.iterator(); it.hasNext(); ) {
StopTime stopTime = it.next();
+ if (hasContinuousBehavior(stopTime.continuous_drop_off, stopTime.continuous_pickup)) {
+ hasContinuousBehavior = true;
+ }
Stop stop = stopById.get(stopTime.stop_id);
if (stop == null) {
// All bad references should have been recorded at import, we can just remove them from the trips.
@@ -166,6 +176,18 @@ private void processTrip (List stopTimes) {
// All bad references should have been recorded at import and null trip check is handled above, we can just
// ignore nulls.
Route route = routeById.get(trip.route_id);
+ if (route != null &&
+ hasContinuousBehavior(route.continuous_drop_off, route.continuous_pickup)) {
+ hasContinuousBehavior = true;
+ }
+
+ if (trip.shape_id == null && hasContinuousBehavior) {
+ registerError(
+ trip,
+ CONDITIONALLY_REQUIRED,
+ "shape_id is required when a trip has continuous behavior defined."
+ );
+ }
// Pass these same cleaned lists of stop_times and stops into each trip validator in turn.
for (TripValidator tripValidator : tripValidators) tripValidator.validateTrip(trip, route, stopTimes, stops);
}
@@ -181,4 +203,17 @@ public void complete (ValidationResult validationResult) {
}
}
+ /**
+ * Determine if a trip has continuous behaviour by checking the values that have been defined for continuous drop
+ * off and pickup.
+ */
+ private boolean hasContinuousBehavior(int continuousDropOff, int continuousPickup) {
+ return
+ continuousDropOff == 0 ||
+ continuousDropOff == 2 ||
+ continuousDropOff == 3 ||
+ continuousPickup == 0 ||
+ continuousPickup == 2 ||
+ continuousPickup == 3;
+ }
}
diff --git a/src/main/java/com/conveyal/gtfs/validator/PatternFinderValidator.java b/src/main/java/com/conveyal/gtfs/validator/PatternFinderValidator.java
index 9584b9e61..f24ac1c44 100644
--- a/src/main/java/com/conveyal/gtfs/validator/PatternFinderValidator.java
+++ b/src/main/java/com/conveyal/gtfs/validator/PatternFinderValidator.java
@@ -158,6 +158,8 @@ public void complete(ValidationResult validationResult) {
setIntParameter(insertPatternStopStatement,7, key.pickupTypes.get(i));
setDoubleParameter(insertPatternStopStatement, 8, key.shapeDistances.get(i));
setIntParameter(insertPatternStopStatement,9, key.timepoints.get(i));
+ setIntParameter(insertPatternStopStatement,10, key.continuous_pickup.get(i));
+ setIntParameter(insertPatternStopStatement,11, key.continuous_drop_off.get(i));
patternStopTracker.addBatch();
}
// Finally, update all trips on this pattern to reference this pattern's ID.
diff --git a/src/test/java/com/conveyal/gtfs/GTFSTest.java b/src/test/java/com/conveyal/gtfs/GTFSTest.java
index 87a2a60d1..dca6a72bc 100644
--- a/src/test/java/com/conveyal/gtfs/GTFSTest.java
+++ b/src/test/java/com/conveyal/gtfs/GTFSTest.java
@@ -224,6 +224,8 @@ public void canLoadAndExportSimpleAgencyInSubDirectory() {
new ErrorExpectation(NewGTFSErrorType.TABLE_IN_SUBDIRECTORY),
new ErrorExpectation(NewGTFSErrorType.TABLE_IN_SUBDIRECTORY),
new ErrorExpectation(NewGTFSErrorType.TABLE_IN_SUBDIRECTORY),
+ new ErrorExpectation(NewGTFSErrorType.TABLE_IN_SUBDIRECTORY),
+ new ErrorExpectation(NewGTFSErrorType.TABLE_IN_SUBDIRECTORY),
new ErrorExpectation(NewGTFSErrorType.ROUTE_LONG_NAME_CONTAINS_SHORT_NAME),
new ErrorExpectation(NewGTFSErrorType.FEED_TRAVEL_TIMES_ROUNDED),
new ErrorExpectation(NewGTFSErrorType.STOP_UNUSED),
@@ -560,8 +562,8 @@ private boolean runIntegrationTestOnZipFile(
ErrorExpectation[] errorExpectations,
FeedValidatorCreator... customValidators
) {
- String newDBName = TestUtils.generateNewDB();
- String dbConnectionUrl = String.join("/", JDBC_URL, newDBName);
+ String testDBName = TestUtils.generateNewDB();
+ String dbConnectionUrl = String.join("/", JDBC_URL, testDBName);
DataSource dataSource = TestUtils.createTestDataSource(dbConnectionUrl);
String namespace;
diff --git a/src/test/java/com/conveyal/gtfs/dto/FeedInfoDTO.java b/src/test/java/com/conveyal/gtfs/dto/FeedInfoDTO.java
index fb3feac2b..52ede95c3 100644
--- a/src/test/java/com/conveyal/gtfs/dto/FeedInfoDTO.java
+++ b/src/test/java/com/conveyal/gtfs/dto/FeedInfoDTO.java
@@ -14,4 +14,7 @@ public class FeedInfoDTO {
public String feed_version;
public String default_route_color;
public String default_route_type;
+ public String default_lang;
+ public String feed_contact_email;
+ public String feed_contact_url;
}
diff --git a/src/test/java/com/conveyal/gtfs/dto/PatternStopDTO.java b/src/test/java/com/conveyal/gtfs/dto/PatternStopDTO.java
index 12888f4d6..f61b68cb5 100644
--- a/src/test/java/com/conveyal/gtfs/dto/PatternStopDTO.java
+++ b/src/test/java/com/conveyal/gtfs/dto/PatternStopDTO.java
@@ -11,6 +11,8 @@ public class PatternStopDTO {
public Integer pickup_type;
public Integer stop_sequence;
public Integer timepoint;
+ public Integer continuous_pickup;
+ public Integer continuous_drop_off;
/** Empty constructor for deserialization */
public PatternStopDTO() {}
diff --git a/src/test/java/com/conveyal/gtfs/dto/RouteDTO.java b/src/test/java/com/conveyal/gtfs/dto/RouteDTO.java
index 0ea408213..ec15aca45 100644
--- a/src/test/java/com/conveyal/gtfs/dto/RouteDTO.java
+++ b/src/test/java/com/conveyal/gtfs/dto/RouteDTO.java
@@ -24,4 +24,6 @@ public class RouteDTO {
/** This field is incorrectly set to String in order to test how empty string literals are persisted to the database. */
public String route_sort_order;
public Integer status;
+ public int continuous_pickup;
+ public int continuous_drop_off;
}
diff --git a/src/test/java/com/conveyal/gtfs/dto/StopDTO.java b/src/test/java/com/conveyal/gtfs/dto/StopDTO.java
index 21ec9e4fa..66a333863 100644
--- a/src/test/java/com/conveyal/gtfs/dto/StopDTO.java
+++ b/src/test/java/com/conveyal/gtfs/dto/StopDTO.java
@@ -14,4 +14,5 @@ public class StopDTO {
public String parent_station;
public Integer location_type;
public Integer wheelchair_boarding;
+ public String platform_code;
}
diff --git a/src/test/java/com/conveyal/gtfs/dto/StopTimeDTO.java b/src/test/java/com/conveyal/gtfs/dto/StopTimeDTO.java
index 9e385e3ca..7941c68ee 100644
--- a/src/test/java/com/conveyal/gtfs/dto/StopTimeDTO.java
+++ b/src/test/java/com/conveyal/gtfs/dto/StopTimeDTO.java
@@ -11,6 +11,8 @@ public class StopTimeDTO {
public Integer drop_off_type;
public Integer pickup_type;
public Double shape_dist_traveled;
+ public int continuous_pickup;
+ public int continuous_drop_off;
/**
* Empty constructor for deserialization
diff --git a/src/test/java/com/conveyal/gtfs/graphql/GTFSGraphQLTest.java b/src/test/java/com/conveyal/gtfs/graphql/GTFSGraphQLTest.java
index 8a9227003..240683306 100644
--- a/src/test/java/com/conveyal/gtfs/graphql/GTFSGraphQLTest.java
+++ b/src/test/java/com/conveyal/gtfs/graphql/GTFSGraphQLTest.java
@@ -148,6 +148,14 @@ public void canFetchAgencies() {
});
}
+ /** Tests that the attributions of a feed can be fetched. */
+ @Test
+ public void canFetchATtributions() {
+ assertTimeout(Duration.ofMillis(TEST_TIMEOUT), () -> {
+ MatcherAssert.assertThat(queryGraphQL("feedAttributions.txt"), matchesSnapshot());
+ });
+ }
+
/** Tests that the calendars of a feed can be fetched. */
@Test
public void canFetchCalendars() {
@@ -196,6 +204,14 @@ public void canFetchTrips() {
});
}
+ /** Tests that the translations of a feed can be fetched. */
+ @Test
+ public void canFetchTranslations() {
+ assertTimeout(Duration.ofMillis(TEST_TIMEOUT), () -> {
+ MatcherAssert.assertThat(queryGraphQL("feedTranslations.txt"), matchesSnapshot());
+ });
+ }
+
// TODO: make tests for schedule_exceptions / calendar_dates
/** Tests that the stop times of a feed can be fetched. */
diff --git a/src/test/java/com/conveyal/gtfs/loader/ConditionallyRequiredTest.java b/src/test/java/com/conveyal/gtfs/loader/ConditionallyRequiredTest.java
new file mode 100644
index 000000000..530f4499e
--- /dev/null
+++ b/src/test/java/com/conveyal/gtfs/loader/ConditionallyRequiredTest.java
@@ -0,0 +1,160 @@
+package com.conveyal.gtfs.loader;
+
+import com.conveyal.gtfs.TestUtils;
+import com.conveyal.gtfs.error.NewGTFSErrorType;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import javax.sql.DataSource;
+import java.io.IOException;
+import java.util.stream.Stream;
+
+import static com.conveyal.gtfs.GTFS.load;
+import static com.conveyal.gtfs.GTFS.validate;
+import static com.conveyal.gtfs.TestUtils.assertThatSqlCountQueryYieldsExpectedCount;
+import static com.conveyal.gtfs.error.NewGTFSErrorType.CONDITIONALLY_REQUIRED;
+import static com.conveyal.gtfs.error.NewGTFSErrorType.REFERENTIAL_INTEGRITY;
+import static com.conveyal.gtfs.error.NewGTFSErrorType.AGENCY_ID_REQUIRED_FOR_MULTI_AGENCY_FEEDS;
+
+public class ConditionallyRequiredTest {
+ private static String testDBName;
+ private static DataSource testDataSource;
+ private static String testNamespace;
+
+ @BeforeAll
+ public static void setUpClass() throws IOException {
+ // Create a new database
+ testDBName = TestUtils.generateNewDB();
+ String dbConnectionUrl = String.format("jdbc:postgresql://localhost/%s", testDBName);
+ testDataSource = TestUtils.createTestDataSource(dbConnectionUrl);
+ // load feed into db
+ String zipFileName = TestUtils.zipFolderFiles(
+ "real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks",
+ true);
+ FeedLoadResult feedLoadResult = load(zipFileName, testDataSource);
+ testNamespace = feedLoadResult.uniqueIdentifier;
+ validate(testNamespace, testDataSource);
+ }
+
+ @AfterAll
+ public static void tearDownClass() {
+ TestUtils.dropDB(testDBName);
+ }
+
+ @Test
+ public void stopTimeTableMissingConditionallyRequiredArrivalDepartureTimes() {
+ checkFeedHasOneError(
+ CONDITIONALLY_REQUIRED,
+ "StopTime",
+ "10",
+ "1",
+ "First and last stop times are required to have both an arrival and departure time."
+ );
+ }
+
+ @ParameterizedTest
+ @MethodSource("createStopTableChecks")
+ public void stopTableConditionallyRequiredTests(
+ NewGTFSErrorType errorType,
+ String entityType,
+ String lineNumber,
+ String entityId,
+ String badValue
+ ) {
+ checkFeedHasOneError(errorType, entityType, lineNumber, entityId, badValue);
+ }
+
+ private static Stream createStopTableChecks() {
+ return Stream.of(
+ Arguments.of(CONDITIONALLY_REQUIRED, "Stop", "2", "4957", "stop_name is required when location_type value is between 0 and 2."),
+ Arguments.of(CONDITIONALLY_REQUIRED, "Stop", "5", "1266", "parent_station is required when location_type value is between 2 and 4."),
+ Arguments.of(CONDITIONALLY_REQUIRED, "Stop", "3", "691", "stop_lat is required when location_type value is between 0 and 2."),
+ Arguments.of(CONDITIONALLY_REQUIRED, "Stop", "4", "692", "stop_lon is required when location_type value is between 0 and 2."),
+ Arguments.of(REFERENTIAL_INTEGRITY, "FareRule", "3", "1", "contains_id:zone_id:4"),
+ Arguments.of(REFERENTIAL_INTEGRITY, "FareRule", "3", "1", "destination_id:zone_id:3"),
+ Arguments.of(REFERENTIAL_INTEGRITY, "FareRule", "3", "1", "origin_id:zone_id:2")
+ );
+ }
+
+ @ParameterizedTest
+ @MethodSource("createTranslationTableChecks")
+ public void translationTableConditionallyRequiredTests(
+ String entityType,
+ String lineNumber,
+ String entityId,
+ String badValue
+ ) {
+ checkFeedHasOneError(CONDITIONALLY_REQUIRED, entityType, lineNumber, entityId, badValue);
+ }
+
+ private static Stream createTranslationTableChecks() {
+ return Stream.of(
+ Arguments.of("Translation", "2", "stops", "record_id is required when field_value is empty."),
+ Arguments.of("Translation", "3", "stops", "field_value is required when record_id is empty."),
+ Arguments.of("Translation", "4", "stops", "record_sub_id is required and must match stop_times when record_id is provided.")
+ );
+ }
+
+ @Test
+ public void agencyTableMissingConditionallyRequiredAgencyId() {
+ checkFeedHasOneError(
+ AGENCY_ID_REQUIRED_FOR_MULTI_AGENCY_FEEDS,
+ "Agency",
+ "2",
+ null,
+ "agency_id");
+ }
+
+ @Test
+ public void tripTableMissingConditionallyRequiredShapeId() {
+ checkFeedHasOneError(
+ CONDITIONALLY_REQUIRED,
+ "Trip",
+ "2",
+ "1",
+ "shape_id is required when a trip has continuous behavior defined."
+ );
+ }
+
+ @Test
+ public void routeTableMissingConditionallyRequiredAgencyId() {
+ checkFeedHasOneError(
+ AGENCY_ID_REQUIRED_FOR_MULTI_AGENCY_FEEDS,
+ "Route",
+ "2",
+ "21",
+ null
+ );
+ }
+
+ @Test
+ public void fareAttributeTableMissingConditionallyRequiredAgencyId() {
+ checkFeedHasOneError(
+ AGENCY_ID_REQUIRED_FOR_MULTI_AGENCY_FEEDS,
+ "FareAttribute",
+ "2",
+ "1",
+ null
+ );
+ }
+
+ /**
+ * Check that the test feed has exactly one error for the provided values.
+ */
+ private void checkFeedHasOneError(NewGTFSErrorType errorType, String entityType, String lineNumber, String entityId, String badValue) {
+ String sql = String.format("select count(*) from %s.errors where error_type = '%s' and entity_type = '%s' and line_number = '%s'",
+ testNamespace,
+ errorType,
+ entityType,
+ lineNumber);
+
+ if (entityId != null) sql += String.format(" and entity_id = '%s'", entityId);
+ if (badValue != null) sql += String.format(" and bad_value = '%s'", badValue);
+
+ assertThatSqlCountQueryYieldsExpectedCount(testDataSource, sql,1);
+ }
+}
diff --git a/src/test/java/com/conveyal/gtfs/loader/JDBCTableWriterTest.java b/src/test/java/com/conveyal/gtfs/loader/JDBCTableWriterTest.java
index 660c00798..c9f396301 100644
--- a/src/test/java/com/conveyal/gtfs/loader/JDBCTableWriterTest.java
+++ b/src/test/java/com/conveyal/gtfs/loader/JDBCTableWriterTest.java
@@ -183,8 +183,14 @@ public void canPreventSQLInjection() throws IOException, SQLException, InvalidNa
feedInfoInput.feed_publisher_name = publisherName;
feedInfoInput.feed_publisher_url = "example.com";
feedInfoInput.feed_lang = "en";
+ feedInfoInput.feed_start_date = "07052021";
+ feedInfoInput.feed_end_date = "09052021";
+ feedInfoInput.feed_lang = "en";
feedInfoInput.default_route_color = "1c8edb";
feedInfoInput.default_route_type = "3";
+ feedInfoInput.default_lang = "en";
+ feedInfoInput.feed_contact_email = "a@b.com";
+ feedInfoInput.feed_contact_url = "example.com";
// convert object to json and save it
JdbcTableWriter createTableWriter = createTestTableWriter(Table.FEED_INFO);
@@ -416,11 +422,19 @@ public void canCreateUpdateAndDeleteScheduleExceptions() throws IOException, SQL
}
/**
- * This test verifies that stop_times#shape_dist_traveled (and other "linked fields") are updated when a pattern
+ * This test verifies that stop_times#shape_dist_traveled and other linked fields are updated when a pattern
* is updated.
*/
@Test
- public void shouldUpdateStopTimeShapeDistTraveledOnPatternStopUpdate() throws IOException, SQLException, InvalidNamespaceException {
+ public void shouldUpdateStopTimeOnPatternStopUpdate() throws IOException, SQLException, InvalidNamespaceException {
+ final String[] STOP_TIMES_LINKED_FIELDS = new String[] {
+ "shape_dist_traveled",
+ "timepoint",
+ "drop_off_type",
+ "pickup_type",
+ "continuous_pickup",
+ "continuous_drop_off"
+ };
String routeId = newUUID();
String patternId = newUUID();
int startTime = 6 * 60 * 60; // 6 AM
@@ -453,10 +467,12 @@ public void shouldUpdateStopTimeShapeDistTraveledOnPatternStopUpdate() throws IO
assertNotNull(uuid);
// Check that trip exists.
assertThatSqlQueryYieldsRowCount(getColumnsForId(createdTrip.id, Table.TRIPS), 1);
- // Check the stop_time's initial shape_dist_traveled value. TODO test that other linked fields are updated?
+
+ // Check the stop_time's initial shape_dist_traveled value and other linked fields.
PreparedStatement statement = testDataSource.getConnection().prepareStatement(
String.format(
- "select shape_dist_traveled from %s.stop_times where stop_sequence=1 and trip_id='%s'",
+ "select %s from %s.stop_times where stop_sequence=1 and trip_id='%s'",
+ String.join(", ", STOP_TIMES_LINKED_FIELDS),
testNamespace,
createdTrip.trip_id
)
@@ -465,11 +481,22 @@ public void shouldUpdateStopTimeShapeDistTraveledOnPatternStopUpdate() throws IO
ResultSet resultSet = statement.executeQuery();
while (resultSet.next()) {
// First stop_time shape_dist_traveled should be zero.
- assertThat(resultSet.getInt(1), equalTo(0));
+ // Other linked fields should be interpreted as zero too.
+ for (int i = 1; i <= STOP_TIMES_LINKED_FIELDS.length; i++) {
+ assertThat(resultSet.getInt(i), equalTo(0));
+ }
}
+
// Update pattern_stop#shape_dist_traveled and check that the stop_time's shape_dist value is updated.
final double updatedShapeDistTraveled = 45.5;
- pattern.pattern_stops[1].shape_dist_traveled = updatedShapeDistTraveled;
+ PatternStopDTO pattern_stop = pattern.pattern_stops[1];
+ pattern_stop.shape_dist_traveled = updatedShapeDistTraveled;
+ // Assign an arbitrary value (the order of appearance in STOP_TIMES_LINKED_FIELDS) for the other linked fields.
+ pattern_stop.timepoint = 2;
+ pattern_stop.drop_off_type = 3;
+ pattern_stop.pickup_type = 4;
+ pattern_stop.continuous_pickup = 5;
+ pattern_stop.continuous_drop_off = 6;
JdbcTableWriter patternUpdater = createTestTableWriter(Table.PATTERNS);
String updatedPatternOutput = patternUpdater.update(pattern.id, mapper.writeValueAsString(pattern), true);
LOG.info("Updated pattern: {}", updatedPatternOutput);
@@ -477,6 +504,11 @@ public void shouldUpdateStopTimeShapeDistTraveledOnPatternStopUpdate() throws IO
while (resultSet2.next()) {
// First stop_time shape_dist_traveled should be updated.
assertThat(resultSet2.getDouble(1), equalTo(updatedShapeDistTraveled));
+
+ // Other linked fields should be as set above.
+ for (int i = 2; i <= STOP_TIMES_LINKED_FIELDS.length; i++) {
+ assertThat(resultSet2.getInt(i), equalTo(i));
+ }
}
}
diff --git a/src/test/resources/fake-agency/attributions.txt b/src/test/resources/fake-agency/attributions.txt
new file mode 100644
index 000000000..1d77168ac
--- /dev/null
+++ b/src/test/resources/fake-agency/attributions.txt
@@ -0,0 +1,2 @@
+attribution_id,agency_id,route_id,trip_id,organization_name,is_producer,is_operator,is_authority,attribution_url,attribution_email,attribution_phone
+1,1,,,Fake Transit,1,,,https://www.faketransit.org,customer.service@faketransit.org,
\ No newline at end of file
diff --git a/src/test/resources/fake-agency/translations.txt b/src/test/resources/fake-agency/translations.txt
new file mode 100644
index 000000000..da8fe2407
--- /dev/null
+++ b/src/test/resources/fake-agency/translations.txt
@@ -0,0 +1,2 @@
+table_name,field_name,language,translation,record_id,record_sub_id,field_value
+stops,stop_desc,FR,en direction du nord,4u6g,,
\ No newline at end of file
diff --git a/src/test/resources/graphql/feedAttributions.txt b/src/test/resources/graphql/feedAttributions.txt
new file mode 100644
index 000000000..6972e2757
--- /dev/null
+++ b/src/test/resources/graphql/feedAttributions.txt
@@ -0,0 +1,18 @@
+query ($namespace: String) {
+ feed(namespace: $namespace) {
+ feed_version
+ attributions {
+ attribution_id
+ agency_id
+ route_id
+ trip_id
+ organization_name
+ is_producer
+ is_operator
+ is_authority
+ attribution_url
+ attribution_email
+ attribution_phone
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/test/resources/graphql/feedTranslations.txt b/src/test/resources/graphql/feedTranslations.txt
new file mode 100644
index 000000000..4933e2e12
--- /dev/null
+++ b/src/test/resources/graphql/feedTranslations.txt
@@ -0,0 +1,14 @@
+query ($namespace: String) {
+ feed(namespace: $namespace) {
+ feed_version
+ translations {
+ table_name
+ field_name
+ language
+ translation
+ record_id
+ record_sub_id
+ field_value
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/agency.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/agency.txt
new file mode 100644
index 000000000..71811c1f9
--- /dev/null
+++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/agency.txt
@@ -0,0 +1,5 @@
+agency_id,agency_name,agency_url,agency_timezone,agency_lang,agency_phone,agency_fare_url,agency_email
+,VTA,https://www.vta.org,America/Los_Angeles,EN,408-321-2300,https://www.vta.org/go/fares,customer.service@vta.org
+VTA2,VTA2,https://www.vta.org,America/Los_Angeles,EN,408-321-2300,https://www.vta.org/go/fares,customer.service@vta.org
+VTA3,VTA3,https://www.vta.org,America/Los_Angeles,EN,408-321-2300,https://www.vta.org/go/fares,customer.service@vta.org
+
diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/attributions.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/attributions.txt
new file mode 100644
index 000000000..42a18d2c5
--- /dev/null
+++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/attributions.txt
@@ -0,0 +1,2 @@
+attribution_id,agency_id,route_id,trip_id,organization_name,is_producer,is_operator,is_authority,attribution_url,attribution_email,attribution_phone
+1,1,,,VTA,1,,,https://www.vta.org,customer.service@vta.org,
\ No newline at end of file
diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/calendar.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/calendar.txt
new file mode 100644
index 000000000..6077757f1
--- /dev/null
+++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/calendar.txt
@@ -0,0 +1,2 @@
+service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date
+1,1,1,1,1,1,0,0,20210208,20210611
\ No newline at end of file
diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/calendar_attributes.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/calendar_attributes.txt
new file mode 100644
index 000000000..a59162bca
--- /dev/null
+++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/calendar_attributes.txt
@@ -0,0 +1,2 @@
+service_id,service_description
+1,Weekday
\ No newline at end of file
diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/calendar_dates.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/calendar_dates.txt
new file mode 100644
index 000000000..bf2f0150e
--- /dev/null
+++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/calendar_dates.txt
@@ -0,0 +1,2 @@
+service_id,date,exception_type
+1,20210531,2
diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/directions.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/directions.txt
new file mode 100644
index 000000000..666d39239
--- /dev/null
+++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/directions.txt
@@ -0,0 +1,5 @@
+route_id,direction_id,direction
+21,0,East
+21,1,West
+22,0,East
+22,1,West
\ No newline at end of file
diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/fare_attributes.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/fare_attributes.txt
new file mode 100644
index 000000000..5ccbf0e80
--- /dev/null
+++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/fare_attributes.txt
@@ -0,0 +1,2 @@
+fare_id,price,currency_type,payment_method,transfers,agency_id,transfer_duration
+1,2.50000000,USD,0,,,
\ No newline at end of file
diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/fare_rules.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/fare_rules.txt
new file mode 100644
index 000000000..5591a375d
--- /dev/null
+++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/fare_rules.txt
@@ -0,0 +1,3 @@
+fare_id,route_id,origin_id,destination_id,contains_id
+1,21,1,,
+1,22,2,3,4
\ No newline at end of file
diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/feed_info.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/feed_info.txt
new file mode 100644
index 000000000..c59ff6d3e
--- /dev/null
+++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/feed_info.txt
@@ -0,0 +1,2 @@
+feed_publisher_name,feed_publisher_url,feed_lang,feed_start_date,feed_end_date,feed_version,feed_contact_email,feed_contact_url
+Santa Clara Valley Transportation Authority,https://www.vta.org,EN,20210208,20210613,2021-02-16_15:11,customer.service@vta.org,https://www.vta.org/about/contact
diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/realtime_routes.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/realtime_routes.txt
new file mode 100644
index 000000000..27b0949c5
--- /dev/null
+++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/realtime_routes.txt
@@ -0,0 +1,3 @@
+route_id,realtime_enabled,realtime_routename,realtime_routecode
+21,1,,
+22,1,,
\ No newline at end of file
diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/routes.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/routes.txt
new file mode 100644
index 000000000..250682bcd
--- /dev/null
+++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/routes.txt
@@ -0,0 +1,2 @@
+route_id,agency_id,route_short_name,route_long_name,route_desc,route_type,route_url,route_color,route_text_color,route_sort_order,ext_route_type,continuous_pickup,continuous_drop_off
+21,,21,Stanford Shopping Center - Santa Clara Transit Center,,3,https://www.vta.org/go/routes/21,29588c,FFFFFF,21,704,,
\ No newline at end of file
diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/stop_times.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/stop_times.txt
new file mode 100644
index 000000000..84e00fde9
--- /dev/null
+++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/stop_times.txt
@@ -0,0 +1,26 @@
+trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,continuous_pickup,continuous_drop_off,shape_dist_traveled,timepoint
+1,05:43:00,05:43:00,4957,1,,0,0,0,,,1
+1,05:44:00,05:44:00,691,2,,0,0,0,,0.58999997,0
+1,05:45:00,05:45:00,692,3,,0,0,0,,0.96569997,0
+1,05:46:00,05:46:00,1266,4,,0,0,0,,1.14470005,0
+1,05:47:00,05:47:00,1267,5,,0,0,0,,1.92729998,0
+1,05:48:00,05:48:00,1268,6,,0,0,0,,2.28160000,0
+1,05:49:00,05:49:00,1542,7,,0,0,0,,2.72530007,0
+1,05:50:00,05:50:00,1543,8,,0,0,0,,3.23429990,0
+1,,,1544,9,,0,0,0,,3.59170008,1
+2,05:52:00,05:52:00,1545,1,,0,0,,,3.88010001,0
+2,,,1546,2,,0,0,,,4.32210016,0
+2,,,1547,3,,0,0,,,4.82560015,0
+2,05:55:00,05:55:00,1548,4,,0,0,,,5.09070015,0
+3,05:55:00,05:55:00,1550,1,,0,0,,,5.59749985,0
+3,05:56:00,05:56:00,1562,2,,0,0,,,7.09219980,1
+4,05:55:00,05:55:00,1550,1,,0,0,,,5.59749985,0
+4,,,1558,2,,0,0,,,8.34879971,0
+4,,,1559,3,,0,0,,,8.68850040,0
+4,05:56:00,05:56:00,1562,4,,0,0,,,5.59749985,0
+5,00:00:00,00:00:00,4957,1,,0,0,,,,1
+5,01:00:00,01:00:00,1558,2,,0,0,,,8.34879971,0
+5,23:59:00,23:59:00,1562,3,,0,0,,,9.39290047,0
+6,00:00:00,00:00:00,4957,1,,0,0,,,,1
+6,,,1558,2,,0,0,,,8.34879971,0
+6,23:59:00,23:59:00,1562,3,,0,0,,,9.39290047,0
\ No newline at end of file
diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/stops.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/stops.txt
new file mode 100644
index 000000000..ecbf5ee72
--- /dev/null
+++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/stops.txt
@@ -0,0 +1,28 @@
+stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon,zone_id,stop_url,location_type,parent_station,wheelchair_boarding,platform_code,sign_dest
+4957,64957,,Southbound,37.40048600,-122.10892700,1,,0,1,1,,
+691,60691,San Antonio & El Camino,Northbound,,-122.11319800,,,0,,1,,
+692,60692,San Antonio & Miller,Northbound,37.40462900,,,,0,,1,,
+1266,61266,San Antonio & California,Northbound,37.40607000,-122.11050500,,,3,,1,,
+1267,61267,San Antonio & Nita,Northbound,37.41186000,-122.10592500,,,0,,1,,
+1268,61268,San Antonio & Nita,Northbound,37.41461600,-122.10392500,,,0,,1,,
+1542,61542,Middlefield & Montrose,Northbound,37.41752400,-122.10575900,,,0,,1,,
+1543,61543,Middlefield & Charleston,Westbound,37.42026600,-122.11023800,,,0,,1,,
+1544,61544,Middlefield & Mayview,Westbound,37.42291600,-122.11254400,,,0,,1,,
+1545,61545,Middlefield & Meadow,Westbound,37.42452600,-122.11510500,,,0,,1,,
+1546,61546,Middlefield & Ames,Westbound,37.42700600,-122.11903200,,,0,,1,,
+1547,61547,Middlefield & Layne,Westbound,37.42981800,-122.12349600,,,0,,1,,
+1548,61548,Middlefield & Matadero,Westbound,37.43129900,-122.12585600,,,0,,1,,
+1550,61550,Middlefield & Moreno,Westbound,37.43412800,-122.13037200,,,0,,1,,
+1551,61551,Middlefield & California,Westbound,37.43745600,-122.13566600,,,0,,1,,
+1552,61552,Middlefield & Seale,Westbound,37.43992600,-122.13953900,,,0,,2,,
+1553,61553,Middlefield & Embarcadero,Westbound,37.44248900,-122.14365200,,,0,,1,,
+1554,61554,Middlefield & Melville,Westbound,37.44458500,-122.14696700,,,0,,1,,
+1555,61555,Middlefield & Kingsley,Westbound,37.44538300,-122.14822500,,,0,,1,,
+1556,61556,Middlefield & Addison,Westbound,37.44694800,-122.15071600,,,0,,1,,
+1557,61557,Middlefield & Channing,Westbound,37.44775900,-122.15199200,,,0,,1,,
+1558,61558,Homer & Webster,Southbound,37.44699700,-122.15448100,,,0,,1,,
+1559,61559,Waverly & Homer,Westbound,37.44492200,-122.15685900,,,0,,1,,
+1560,61560,Hamilton & Waverly,Southbound,37.44612400,-122.15941900,,,0,,1,,
+1561,61561,Hamilton & Ramona,Southbound,37.44471500,-122.16085800,,,0,,1,,
+1562,61562,Hamilton & High,Southbound,37.44318800,-122.16232000,,,0,,1,,
+1563,61563,Palo Alto Transit Center (Bay 4),,37.44414700,-122.16663100,,,0,PS_PATC,1,4,
\ No newline at end of file
diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/translations.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/translations.txt
new file mode 100644
index 000000000..560db0832
--- /dev/null
+++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/translations.txt
@@ -0,0 +1,4 @@
+table_name,field_name,language,translation,record_id,record_sub_id,field_value
+stops,stop_desc,FR,vers le sud,,,
+stops,stop_desc,FR,en direction du nord,,,
+stops,stop_desc,FR,en direction du nord,stop_times,,
\ No newline at end of file
diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/trips.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/trips.txt
new file mode 100644
index 000000000..eac861720
--- /dev/null
+++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/trips.txt
@@ -0,0 +1,7 @@
+route_id,service_id,trip_id,trip_headsign,trip_short_name,direction_id,block_id,shape_id,wheelchair_accessible,bikes_allowed
+21,1,1,PALO ALTO TRANSIT CTR 1,,1,2145,,0,0
+21,1,2,PALO ALTO TRANSIT CTR 2,,1,2145,101395,0,0
+22,1,3,PALO ALTO TRANSIT CTR 3,,1,2145,101395,0,0
+23,1,4,PALO ALTO TRANSIT CTR 4,,1,2145,101395,0,0
+23,1,5,PALO ALTO TRANSIT CTR 5,,1,2145,101395,0,0
+23,1,6,PALO ALTO TRANSIT CTR 6,,1,2145,101395,0,0
diff --git a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchAttributions-0.json b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchAttributions-0.json
new file mode 100644
index 000000000..3092058c5
--- /dev/null
+++ b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchAttributions-0.json
@@ -0,0 +1,20 @@
+{
+ "data" : {
+ "feed" : {
+ "attributions" : [ {
+ "attribution_id" : 1,
+ "agency_id" : "1",
+ "route_id" : null,
+ "trip_id" : null,
+ "organization_name" : "Fake Transit",
+ "is_producer" : 1,
+ "is_operator" : null,
+ "is_authority" : null,
+ "attribution_url" : "https://www.faketransit.org",
+ "attribution_email" : "customer.service@faketransit.org",
+ "attribution_phone" : null
+ } ],
+ "feed_version" : "1.0"
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchTranslations-0.json b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchTranslations-0.json
new file mode 100644
index 000000000..1bae5c5e1
--- /dev/null
+++ b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchTranslations-0.json
@@ -0,0 +1,16 @@
+{
+ "data" : {
+ "feed" : {
+ "translations" : [ {
+ "table_name" : "stops",
+ "field_name" : "stop_desc",
+ "language" : "FR",
+ "translation" : "en direction du nord",
+ "record_id" : "4u6g",
+ "record_sub_id" : null,
+ "field_value" : null
+ } ],
+ "feed_version" : "1.0"
+ }
+ }
+}
\ No newline at end of file