diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 695b467c4..d30bb6689 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -48,10 +48,6 @@ jobs: key: maven-local-repo - name: Build with Maven run: mvn --no-transfer-progress package - - name: Codecov - # this first codecov run will upload a report associated with the commit set through CI environment variables - uses: codecov/codecov-action@v1.2.0 - continue-on-error: true - name: Clear contents of the target directory # Avoids issues where maven-semantic-release attempts to upload # multiple versions/builds (and fails due to the pre-existence of the version on maven central). @@ -71,7 +67,3 @@ jobs: GH_TOKEN: ${{ secrets.GH_TOKEN }} run: | semantic-release --prepare @conveyal/maven-semantic-release --publish @semantic-release/github,@conveyal/maven-semantic-release --verify-conditions @semantic-release/github,@conveyal/maven-semantic-release --verify-release @conveyal/maven-semantic-release --use-conveyal-workflow --dev-branch=dev --skip-maven-deploy - if [[ "$GITHUB_REF_SLUG" = "master" ]]; then - bash <(curl -s https://codecov.io/bash) -C "$(git rev-parse HEAD)" - bash <(curl -s https://codecov.io/bash) -C "$(git rev-parse HEAD^)" - fi diff --git a/.gitignore b/.gitignore index 928ad1b18..dc693fe79 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ *.iml .idea/ target/ + +lambda$*.json \ No newline at end of file diff --git a/README.md b/README.md index 1daff3de6..8dc117d4b 100644 --- a/README.md +++ b/README.md @@ -19,15 +19,24 @@ A gtfs-lib GTFSFeed object should faithfully represent the contents of a single gtfs-lib can be used as a Java library or run via the command line. If using this library with PostgreSQL for persistence, you must use at least version 9.6 of PostgreSQL. ### Library (maven) - +Include gtfs-lib as a library in your project with the following dependency in your `pom.xml`. ```xml - com.conveyal + + com.github.conveyal gtfs-lib ${choose-a-version} ``` +#### Jitpack + +gtfs-lib builds are hosted on [jitpack](https://jitpack.io/#conveyal/gtfs-lib). + +[Release versions](https://github.com/conveyal/gtfs-lib/releases) are available by default. + +Branch- (e.g. `dev-SNAPSHOT`) or commit-specific (using a 10 character commit ID like `a04294e420`) snapshot builds can be triggered by clicking `Get` for the build of your choice on jitpack's website or visiting https://jitpack.io/#conveyal/gtfs-lib/YOUR_VERSION. + ### Command line ```bash diff --git a/maven-artifact-signing-key.asc.enc b/maven-artifact-signing-key.asc.enc deleted file mode 100644 index c611046ba..000000000 Binary files a/maven-artifact-signing-key.asc.enc and /dev/null differ diff --git a/maven-settings.xml b/maven-settings.xml deleted file mode 100644 index eac060f03..000000000 --- a/maven-settings.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - - - ossrh - ${env.OSSRH_JIRA_USERNAME} - ${env.OSSRH_JIRA_PASSWORD} - - - - - ossrh - - true - - - gpg - ${env.GPG_KEY_NAME} - ${env.GPG_PASSPHRASE} - - - - - diff --git a/pom.xml b/pom.xml index d51c95e50..2bb3afa99 100644 --- a/pom.xml +++ b/pom.xml @@ -123,22 +123,6 @@ - - org.apache.maven.plugins - maven-gpg-plugin - 1.6 - - - sign-artifacts - - verify - - sign - - - - - org.apache.maven.plugins @@ -234,6 +218,13 @@ 5.7.0 test + + + org.junit.jupiter + junit-jupiter-params + 5.6.2 + test + com.beust @@ -292,7 +283,7 @@ commons-io commons-io - 2.4 + 2.7 diff --git a/src/main/java/com/conveyal/gtfs/GTFSFeed.java b/src/main/java/com/conveyal/gtfs/GTFSFeed.java index c07c11a5f..8e6de45e2 100644 --- a/src/main/java/com/conveyal/gtfs/GTFSFeed.java +++ b/src/main/java/com/conveyal/gtfs/GTFSFeed.java @@ -14,7 +14,6 @@ import org.locationtech.jts.algorithm.ConvexHull; import org.locationtech.jts.geom.*; import org.locationtech.jts.index.strtree.STRtree; -import org.locationtech.jts.simplify.DouglasPeuckerSimplifier; import org.mapdb.BTreeMap; import org.mapdb.DB; import org.mapdb.DBMaker; @@ -66,6 +65,8 @@ public class GTFSFeed implements Cloneable, Closeable { public final Map stops; public final Map transfers; public final BTreeMap trips; + public final Map translations; + public final Map attributions; public final Set transitIds = new HashSet<>(); /** CRC32 of the GTFS file this was loaded from */ @@ -636,6 +637,8 @@ private GTFSFeed (DB db) { fares = db.getTreeMap("fares"); services = db.getTreeMap("services"); shape_points = db.getTreeMap("shape_points"); + translations = db.getTreeMap("translations"); + attributions = db.getTreeMap("attributions"); feedId = db.getAtomicString("feed_id").get(); checksum = db.getAtomicLong("checksum").get(); diff --git a/src/main/java/com/conveyal/gtfs/TripPatternKey.java b/src/main/java/com/conveyal/gtfs/TripPatternKey.java index e06a5f6ab..9331483e0 100644 --- a/src/main/java/com/conveyal/gtfs/TripPatternKey.java +++ b/src/main/java/com/conveyal/gtfs/TripPatternKey.java @@ -24,6 +24,8 @@ public class TripPatternKey { public TIntList arrivalTimes = new TIntArrayList(); public TIntList departureTimes = new TIntArrayList(); public TIntList timepoints = new TIntArrayList(); + public TIntList continuous_pickup = new TIntArrayList(); + public TIntList continuous_drop_off = new TIntArrayList(); public TDoubleList shapeDistances = new TDoubleArrayList(); public TripPatternKey (String routeId) { @@ -39,6 +41,8 @@ public void addStopTime (StopTime st) { departureTimes.add(st.departure_time); timepoints.add(st.timepoint); shapeDistances.add(st.shape_dist_traveled); + continuous_pickup.add(st.continuous_pickup); + continuous_drop_off.add(st.continuous_drop_off); } @Override diff --git a/src/main/java/com/conveyal/gtfs/error/NewGTFSError.java b/src/main/java/com/conveyal/gtfs/error/NewGTFSError.java index e350fe671..98e61895f 100644 --- a/src/main/java/com/conveyal/gtfs/error/NewGTFSError.java +++ b/src/main/java/com/conveyal/gtfs/error/NewGTFSError.java @@ -1,5 +1,6 @@ package com.conveyal.gtfs.error; +import com.conveyal.gtfs.loader.LineContext; import com.conveyal.gtfs.loader.Table; import com.conveyal.gtfs.model.Entity; import org.slf4j.Logger; @@ -82,6 +83,14 @@ public static NewGTFSError forLine (Table table, int lineNumber, NewGTFSErrorTyp return error; } + // Factory Builder for cases where an entity has not yet been constructed, but we know the line number. + public static NewGTFSError forLine(LineContext lineContext, NewGTFSErrorType errorType, String badValue) { + NewGTFSError error = new NewGTFSError(lineContext.table.getEntityClass(), errorType); + error.lineNumber = lineContext.lineNumber; + error.badValue = badValue; + return error; + } + // Factory Builder for cases where the entity has already been decoded and an error is discovered during validation public static NewGTFSError forEntity(Entity entity, NewGTFSErrorType errorType) { NewGTFSError error = new NewGTFSError(entity.getClass(), errorType); diff --git a/src/main/java/com/conveyal/gtfs/error/NewGTFSErrorType.java b/src/main/java/com/conveyal/gtfs/error/NewGTFSErrorType.java index dd8f4be99..49f44fa3f 100644 --- a/src/main/java/com/conveyal/gtfs/error/NewGTFSErrorType.java +++ b/src/main/java/com/conveyal/gtfs/error/NewGTFSErrorType.java @@ -8,9 +8,11 @@ */ public enum NewGTFSErrorType { // Standard errors. + AGENCY_ID_REQUIRED_FOR_MULTI_AGENCY_FEEDS(Priority.HIGH, "For GTFS feeds with more than one agency, agency_id is required."), BOOLEAN_FORMAT(Priority.MEDIUM, "A GTFS boolean field must contain the value 1 or 0."), COLOR_FORMAT(Priority.MEDIUM, "A color should be specified with six-characters (three two-digit hexadecimal numbers)."), COLUMN_NAME_UNSAFE(Priority.HIGH, "Column header contains characters not safe in SQL, it was renamed."), + CONDITIONALLY_REQUIRED(Priority.HIGH, "A conditionally required field was missing in a particular row."), CURRENCY_UNKNOWN(Priority.MEDIUM, "The currency code was not recognized."), DATE_FORMAT(Priority.MEDIUM, "Date format should be YYYYMMDD."), DATE_NO_SERVICE(Priority.MEDIUM, "No service_ids were active on a date within the range of dates with defined service."), diff --git a/src/main/java/com/conveyal/gtfs/graphql/GraphQLGtfsSchema.java b/src/main/java/com/conveyal/gtfs/graphql/GraphQLGtfsSchema.java index e752660cb..acd077d3d 100644 --- a/src/main/java/com/conveyal/gtfs/graphql/GraphQLGtfsSchema.java +++ b/src/main/java/com/conveyal/gtfs/graphql/GraphQLGtfsSchema.java @@ -144,9 +144,12 @@ public class GraphQLGtfsSchema { .description("A GTFS feed_info object") .field(MapFetcher.field("id", GraphQLInt)) .field(MapFetcher.field("feed_id")) + .field(MapFetcher.field("feed_contact_email")) + .field(MapFetcher.field("feed_contact_url")) .field(MapFetcher.field("feed_publisher_name")) .field(MapFetcher.field("feed_publisher_url")) .field(MapFetcher.field("feed_lang")) + .field(MapFetcher.field("default_lang")) .field(MapFetcher.field("feed_start_date")) .field(MapFetcher.field("feed_end_date")) .field(MapFetcher.field("feed_version")) @@ -257,10 +260,37 @@ public class GraphQLGtfsSchema { .field(MapFetcher.field("timepoint", GraphQLInt)) .field(MapFetcher.field("drop_off_type", GraphQLInt)) .field(MapFetcher.field("pickup_type", GraphQLInt)) - // Editor-specific fields + .field(MapFetcher.field("continuous_drop_off", GraphQLInt)) + .field(MapFetcher.field("continuous_pickup", GraphQLInt)) .field(MapFetcher.field("shape_dist_traveled", GraphQLFloat)) .build(); + // Represents rows from attributions.txt + public static final GraphQLObjectType attributionsType = newObject().name("attributions") + .field(MapFetcher.field("attribution_id")) + .field(MapFetcher.field("agency_id")) + .field(MapFetcher.field("route_id")) + .field(MapFetcher.field("trip_id")) + .field(MapFetcher.field("organization_name")) + .field(MapFetcher.field("is_producer", GraphQLInt)) + .field(MapFetcher.field("is_operator", GraphQLInt)) + .field(MapFetcher.field("is_authority", GraphQLInt)) + .field(MapFetcher.field("attribution_url")) + .field(MapFetcher.field("attribution_email")) + .field(MapFetcher.field("attribution_phone")) + .build(); + + // Represents rows from translations.txt + public static final GraphQLObjectType translationsType = newObject().name("translations") + .field(MapFetcher.field("table_name")) + .field(MapFetcher.field("field_name")) + .field(MapFetcher.field("language")) + .field(MapFetcher.field("translation")) + .field(MapFetcher.field("record_id")) + .field(MapFetcher.field("record_sub_id")) + .field(MapFetcher.field("field_value")) + .build(); + // Represents rows from routes.txt public static final GraphQLObjectType routeType = newObject().name("route") .description("A line from a GTFS routes.txt table") @@ -272,7 +302,8 @@ public class GraphQLGtfsSchema { .field(MapFetcher.field("route_desc")) .field(MapFetcher.field("route_url")) .field(MapFetcher.field("route_branding_url")) - // TODO route_type as enum or int + .field(MapFetcher.field("continuous_drop_off", GraphQLInt)) + .field(MapFetcher.field("continuous_pickup", GraphQLInt)) .field(MapFetcher.field("route_type", GraphQLInt)) .field(MapFetcher.field("route_color")) .field(MapFetcher.field("route_text_color")) @@ -341,6 +372,7 @@ public class GraphQLGtfsSchema { .field(MapFetcher.field("stop_url")) .field(MapFetcher.field("stop_timezone")) .field(MapFetcher.field("parent_station")) + .field(MapFetcher.field("platform_code")) .field(MapFetcher.field("location_type", GraphQLInt)) .field(MapFetcher.field("wheelchair_boarding", GraphQLInt)) // Returns all stops that reference parent stop's stop_id @@ -403,6 +435,8 @@ public class GraphQLGtfsSchema { .field(MapFetcher.field("shape_dist_traveled", GraphQLFloat)) .field(MapFetcher.field("drop_off_type", GraphQLInt)) .field(MapFetcher.field("pickup_type", GraphQLInt)) + .field(MapFetcher.field("continuous_drop_off", GraphQLInt)) + .field(MapFetcher.field("continuous_pickup", GraphQLInt)) .field(MapFetcher.field("stop_sequence", GraphQLInt)) .field(MapFetcher.field("timepoint", GraphQLInt)) // FIXME: This will only returns a list with one stop entity (unless there is a referential integrity issue) @@ -763,6 +797,26 @@ public class GraphQLGtfsSchema { .dataFetcher(new JDBCFetcher("services")) .build() ) + .field(newFieldDefinition() + .name("attributions") + .type(new GraphQLList(GraphQLGtfsSchema.attributionsType)) + .argument(stringArg("namespace")) // FIXME maybe these nested namespace arguments are not doing anything. + .argument(intArg(ID_ARG)) + .argument(intArg(LIMIT_ARG)) + .argument(intArg(OFFSET_ARG)) + .dataFetcher(new JDBCFetcher("attributions")) + .build() + ) + .field(newFieldDefinition() + .name("translations") + .type(new GraphQLList(GraphQLGtfsSchema.translationsType)) + .argument(stringArg("namespace")) // FIXME maybe these nested namespace arguments are not doing anything. + .argument(intArg(ID_ARG)) + .argument(intArg(LIMIT_ARG)) + .argument(intArg(OFFSET_ARG)) + .dataFetcher(new JDBCFetcher("translations")) + .build() + ) .build(); /** diff --git a/src/main/java/com/conveyal/gtfs/loader/EntityPopulator.java b/src/main/java/com/conveyal/gtfs/loader/EntityPopulator.java index 762176262..7105c6c71 100644 --- a/src/main/java/com/conveyal/gtfs/loader/EntityPopulator.java +++ b/src/main/java/com/conveyal/gtfs/loader/EntityPopulator.java @@ -64,6 +64,8 @@ public interface EntityPopulator { patternStop.stop_sequence = getIntIfPresent(result, "stop_sequence", columnForName); patternStop.timepoint = getIntIfPresent(result, "timepoint", columnForName); patternStop.shape_dist_traveled = getDoubleIfPresent(result, "shape_dist_traveled", columnForName); + patternStop.continuous_pickup = getIntIfPresent (result, "continuous_pickup", columnForName); + patternStop.continuous_drop_off = getIntIfPresent (result, "continuous_drop_off", columnForName); return patternStop; }; @@ -139,17 +141,19 @@ public interface EntityPopulator { }; EntityPopulator ROUTE = (result, columnForName) -> { - Route route = new Route(); - route.route_id = getStringIfPresent(result, "route_id", columnForName); - route.agency_id = getStringIfPresent(result, "agency_id", columnForName); - route.route_short_name = getStringIfPresent(result, "route_short_name", columnForName); - route.route_long_name = getStringIfPresent(result, "route_long_name", columnForName); - route.route_desc = getStringIfPresent(result, "route_desc", columnForName); - route.route_type = getIntIfPresent (result, "route_type", columnForName); - route.route_color = getStringIfPresent(result, "route_color", columnForName); - route.route_text_color = getStringIfPresent(result, "route_text_color", columnForName); - route.route_url = getUrlIfPresent (result, "route_url", columnForName); - route.route_branding_url = getUrlIfPresent (result, "route_branding_url", columnForName); + Route route = new Route(); + route.route_id = getStringIfPresent(result, "route_id", columnForName); + route.agency_id = getStringIfPresent(result, "agency_id", columnForName); + route.route_short_name = getStringIfPresent(result, "route_short_name", columnForName); + route.route_long_name = getStringIfPresent(result, "route_long_name", columnForName); + route.route_desc = getStringIfPresent(result, "route_desc", columnForName); + route.route_type = getIntIfPresent (result, "route_type", columnForName); + route.route_color = getStringIfPresent(result, "route_color", columnForName); + route.route_text_color = getStringIfPresent(result, "route_text_color", columnForName); + route.route_url = getUrlIfPresent (result, "route_url", columnForName); + route.route_branding_url = getUrlIfPresent (result, "route_branding_url", columnForName); + route.continuous_pickup = getIntIfPresent (result, "continuous_pickup", columnForName); + route.continuous_drop_off = getIntIfPresent (result, "continuous_drop_off", columnForName); return route; }; @@ -166,7 +170,8 @@ public interface EntityPopulator { stop.stop_timezone = getStringIfPresent(result, "stop_timezone", columnForName); stop.stop_url = getUrlIfPresent (result, "stop_url", columnForName); stop.location_type = getIntIfPresent (result, "location_type", columnForName); - stop.wheelchair_boarding = Integer.toString(getIntIfPresent(result, "wheelchair_boarding", columnForName)); + stop.wheelchair_boarding = getIntIfPresent(result, "wheelchair_boarding", columnForName); + stop.platform_code = getStringIfPresent(result, "platform_code", columnForName); return stop; }; @@ -205,6 +210,8 @@ public interface EntityPopulator { stopTime.stop_headsign = getStringIfPresent(result, "stop_headsign", columnForName); stopTime.pickup_type = getIntIfPresent (result, "pickup_type", columnForName); stopTime.drop_off_type = getIntIfPresent (result, "drop_off_type", columnForName); + stopTime.continuous_pickup = getIntIfPresent (result, "continuous_pickup", columnForName); + stopTime.continuous_drop_off = getIntIfPresent (result, "continuous_drop_off", columnForName); stopTime.timepoint = getIntIfPresent (result, "timepoint", columnForName); stopTime.shape_dist_traveled = getDoubleIfPresent(result, "shape_dist_traveled", columnForName); return stopTime; diff --git a/src/main/java/com/conveyal/gtfs/loader/FeedLoadResult.java b/src/main/java/com/conveyal/gtfs/loader/FeedLoadResult.java index 69d62d9f7..3a61fdd50 100644 --- a/src/main/java/com/conveyal/gtfs/loader/FeedLoadResult.java +++ b/src/main/java/com/conveyal/gtfs/loader/FeedLoadResult.java @@ -34,6 +34,8 @@ public class FeedLoadResult implements Serializable { public TableLoadResult stopTimes; public TableLoadResult transfers; public TableLoadResult trips; + public TableLoadResult translations; + public TableLoadResult attributions; public long loadTimeMillis; public long completionTime; @@ -59,5 +61,7 @@ public FeedLoadResult (boolean constructTableResults) { stopTimes = new TableLoadResult(); transfers = new TableLoadResult(); trips = new TableLoadResult(); + translations = new TableLoadResult(); + attributions = new TableLoadResult(); } } diff --git a/src/main/java/com/conveyal/gtfs/loader/Field.java b/src/main/java/com/conveyal/gtfs/loader/Field.java index 2440c4624..52cda789c 100644 --- a/src/main/java/com/conveyal/gtfs/loader/Field.java +++ b/src/main/java/com/conveyal/gtfs/loader/Field.java @@ -2,6 +2,7 @@ import com.conveyal.gtfs.error.NewGTFSError; import com.conveyal.gtfs.error.NewGTFSErrorType; +import com.conveyal.gtfs.loader.conditions.ConditionalRequirement; import com.google.common.collect.ImmutableSet; import java.sql.PreparedStatement; @@ -48,6 +49,9 @@ public abstract class Field { public Table referenceTable = null; private boolean shouldBeIndexed; private boolean emptyValuePermitted; + private boolean isConditionallyRequired; + private boolean isForeign; + public ConditionalRequirement[] conditions; public Field(String name, Requirement requirement) { this.name = name; @@ -137,6 +141,25 @@ public boolean isForeignReference () { return this.referenceTable != null; } + /** + * Fluent method to mark a field as having foreign references. If flagged, the field value is added to + * {@link ReferenceTracker#uniqueValuesForFields} to be used as a look-up for reference matches. Note: this is intended only for + * special cases (e.g., zone_id) where the field being referenced does not exist as the primary key of a table. + */ + Field hasForeignReferences() { + isForeign = true; + return this; + } + + /** + * Indicates that this field has foreign references. Note: this is intentionally distinct from + * {@link #isForeignReference()} and is intended only for special cases (e.g., zone_id) where the field being + * referenced does not exist as the primary key of a table. + */ + public boolean isForeign() { + return isForeign; + } + /** * Fluent method that indicates that a newly constructed field should be indexed after the table is loaded. * FIXME: should shouldBeIndexed be determined based on presence of referenceTable? @@ -152,7 +175,8 @@ public boolean shouldBeIndexed() { } /** - * Fluent method indicates that this field is a reference to an entry in the table provided as an argument. + * Fluent method that indicates that this field is a reference to an entry in the table provided as an argument + * (i.e. it is a foreign reference). * @param table * @return this same Field instance */ @@ -180,10 +204,26 @@ public boolean isEmptyValuePermitted() { /** * Get the expression used to select this column from the database based on the prefix. The csvOutput parameter is - * needed in overriden method implementations that have special ways of outputting certain fields. The prefix + * needed in overridden method implementations that have special ways of outputting certain fields. The prefix * parameter is assumed to be either null or a string in the format: `schema.` */ public String getColumnExpression(String prefix, boolean csvOutput) { return prefix != null ? String.format("%s%s", prefix, name) : name; } + + /** + * Mark this field as conditionally required. If needed an optional list of conditions can be provided. + */ + public Field requireConditions(ConditionalRequirement...conditions) { + this.isConditionallyRequired = true; + this.conditions = conditions; + return this; + } + + /** + * Indicates that this field is conditionally required. + */ + public boolean isConditionallyRequired() { + return isConditionallyRequired; + } } diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsLoader.java b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsLoader.java index 37036657d..b17a812c0 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsLoader.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsLoader.java @@ -67,7 +67,7 @@ public class JdbcGtfsLoader { public static final long INSERT_BATCH_SIZE = 500; // Represents null in Postgres text format - private static final String POSTGRES_NULL_TEXT = "\\N"; + public static final String POSTGRES_NULL_TEXT = "\\N"; private static final Logger LOG = LoggerFactory.getLogger(JdbcGtfsLoader.class); private String gtfsFilePath; @@ -92,7 +92,9 @@ public JdbcGtfsLoader(String gtfsFilePath, DataSource dataSource) { this.dataSource = dataSource; } - /** Get SQL string for creating the feed registry table (AKA, the "feeds" table). */ + /** + * Get SQL string for creating the feed registry table (AKA, the "feeds" table). + */ public static String getCreateFeedRegistrySQL() { return "create table if not exists feeds (namespace varchar primary key, md5 varchar, " + "sha1 varchar, feed_id varchar, feed_version varchar, filename varchar, loaded_date timestamp, " + @@ -110,7 +112,7 @@ public static String getCreateFeedRegistrySQL() { // SHA1 took 1072 msec, 9fb356af4be2750f20955203787ec6f95d32ef22 // There appears to be no advantage to loading tables in parallel, as the whole loading process is I/O bound. - public FeedLoadResult loadTables () { + public FeedLoadResult loadTables() { // This result object will be returned to the caller to summarize the feed and report any critical errors. FeedLoadResult result = new FeedLoadResult(); @@ -135,7 +137,7 @@ public FeedLoadResult loadTables () { this.tablePrefix = randomIdString(); result.filename = gtfsFilePath; result.uniqueIdentifier = tablePrefix; - + // The order of the following four lines should not be changed because the schema needs to be in place // before the error storage can be constructed, which in turn needs to exist in case any errors are // encountered during the loading process. @@ -155,14 +157,16 @@ public FeedLoadResult loadTables () { result.calendarDates = load(Table.CALENDAR_DATES); result.routes = load(Table.ROUTES); result.fareAttributes = load(Table.FARE_ATTRIBUTES); - result.fareRules = load(Table.FARE_RULES); result.feedInfo = load(Table.FEED_INFO); result.shapes = load(Table.SHAPES); result.stops = load(Table.STOPS); + result.fareRules = load(Table.FARE_RULES); result.transfers = load(Table.TRANSFERS); result.trips = load(Table.TRIPS); // refs routes result.frequencies = load(Table.FREQUENCIES); // refs trips result.stopTimes = load(Table.STOP_TIMES); + result.translations = load(Table.TRANSLATIONS); + result.attributions = load(Table.ATTRIBUTIONS); result.errorCount = errorStorage.getErrorCount(); // This will commit and close the single connection that has been shared between all preceding load steps. errorStorage.commitAndClose(); @@ -180,15 +184,15 @@ public FeedLoadResult loadTables () { } return result; } - + /** * Creates a schema/namespace in the database WITHOUT committing the changes. * This does *not* setup any other tables or enter the schema name in a registry (@see #registerFeed). - * + * * @param connection Connection to the database to create the schema on. * @param schemaName Name of the schema (i.e. table prefix). Should not include the dot suffix. */ - static void createSchema (Connection connection, String schemaName) { + static void createSchema(Connection connection, String schemaName) { try { Statement statement = connection.createStatement(); // FIXME do the following only on databases that support schemas. @@ -205,13 +209,13 @@ static void createSchema (Connection connection, String schemaName) { * Add a line to the list of loaded feeds showing that this feed has been loaded. * We used to inspect feed_info here so we could make our table prefix based on feed ID and version. * Now we just load feed_info like any other table. - * // Create a row in the table of loaded feeds for this feed + * // Create a row in the table of loaded feeds for this feed * Really this is not just making the table prefix - it's loading the feed_info and should also calculate hashes. * * Originally we were flattening all feed_info files into one root-level table, but that forces us to drop any * custom fields in feed_info. */ - private void registerFeed (File gtfsFile) { + private void registerFeed(File gtfsFile) { // FIXME is this extra CSV reader used anymore? Check comment below. // First, inspect feed_info.txt to extract the ID and version. @@ -243,7 +247,7 @@ private void registerFeed (File gtfsFile) { // current_timestamp seems to be the only standard way to get the current time across all common databases. // Record total load processing time? PreparedStatement insertStatement = connection.prepareStatement( - "insert into feeds values (?, ?, ?, ?, ?, ?, current_timestamp, null, false)"); + "insert into feeds values (?, ?, ?, ?, ?, ?, current_timestamp, null, false)"); insertStatement.setString(1, tablePrefix); insertStatement.setString(2, md5Hex); insertStatement.setString(3, shaHex); @@ -271,7 +275,7 @@ static void createFeedRegistryIfNotExists(Connection connection) throws SQLExcep /** * This wraps the main internal table loader method to catch exceptions and figure out how many errors happened. */ - private TableLoadResult load (Table table) { + private TableLoadResult load(Table table) { // This object will be returned to the caller to summarize the contents of the table and any errors. TableLoadResult tableLoadResult = new TableLoadResult(); int initialErrorCount = errorStorage.getErrorCount(); @@ -311,9 +315,10 @@ private int getTableSize(Table table) { /** * This function will throw any exception that occurs. Those exceptions will be handled by the outer load method. + * * @return number of rows that were loaded. */ - private int loadInternal (Table table) throws Exception { + private int loadInternal(Table table) throws Exception { CsvReader csvReader = table.getCsvReader(zip, errorStorage); if (csvReader == null) { LOG.info(String.format("file %s.txt not found in gtfs zipfile", table.name)); @@ -363,7 +368,7 @@ private int loadInternal (Table table) throws Exception { // When outputting text, accumulate transformed strings to allow skipping rows when errors are encountered. // One extra position in the array for the CSV line number. String[] transformedStrings = new String[cleanFields.length + 1]; - + boolean tableHasConditionalRequirements = table.hasConditionalRequirements(); // Iterate over each record and prepare the record for storage in the table either through batch insert // statements or postgres text copy operation. while (csvReader.readRecord()) { @@ -373,6 +378,7 @@ private int loadInternal (Table table) throws Exception { errorStorage.storeError(NewGTFSError.forTable(table, TABLE_TOO_LONG)); break; } + // Line 1 is considered the header row, so the first actual row of data will be line 2. int lineNumber = ((int) csvReader.getCurrentRecord()) + 2; if (lineNumber % 500_000 == 0) LOG.info("Processed {}", human(lineNumber)); if (csvReader.getColumnCount() != fields.length) { @@ -403,10 +409,10 @@ private int loadInternal (Table table) throws Exception { // error. if ( table.name.equals("calendar_dates") && - "service_id".equals(field.name) && - "1".equals(csvReader.get(Field.getFieldIndex(fields, "exception_type"))) + "service_id".equals(field.name) && + "1".equals(csvReader.get(Field.getFieldIndex(fields, "exception_type"))) - ){ + ) { for (NewGTFSError error : errors) { if (NewGTFSErrorType.REFERENTIAL_INTEGRITY.equals(error.errorType)) { // Do not record bad service_id reference errors for calendar date entries that add service @@ -430,6 +436,12 @@ private int loadInternal (Table table) throws Exception { // Increment column index. columnIndex += 1; } + if (tableHasConditionalRequirements) { + LineContext lineContext = new LineContext(table, fields, transformedStrings, lineNumber); + errorStorage.storeErrors( + referenceTracker.checkConditionallyRequiredFields(lineContext) + ); + } if (postgresText) { // Print a new line in the standard postgres text format: // https://www.postgresql.org/docs/9.1/static/sql-copy.html#AEN64380 @@ -481,7 +493,7 @@ public static void copyFromFile(Connection connection, File file, String targetT InputStream stream = new BufferedInputStream(new FileInputStream(file.getAbsolutePath())); // Our connection pool wraps the Connection objects, so we need to unwrap the Postgres connection interface. CopyManager copyManager = new CopyManager(connection.unwrap(BaseConnection.class)); - copyManager.copyIn(copySql, stream, 1024*1024); + copyManager.copyIn(copySql, stream, 1024 * 1024); stream.close(); // It is also possible to load from local file if this code is running on the database server. // statement.execute(String.format("copy %s from '%s'", table.name, tempTextFile.getAbsolutePath())); @@ -515,9 +527,12 @@ public void setValueForField(Table table, int fieldIndex, int lineNumber, Field if (postgresText) { ValidateFieldResult result = field.validateAndConvert(string); // If the result is null, use the null-setting method. - if (result.clean == null) setFieldToNull(postgresText, transformedStrings, fieldIndex, field); - // Otherwise, set the cleaned field according to its index. - else transformedStrings[fieldIndex + 1] = result.clean; + if (result.clean == null) { + setFieldToNull(postgresText, transformedStrings, fieldIndex, field); + } else { + // Otherwise, set the cleaned field according to its index. + transformedStrings[fieldIndex + 1] = result.clean; + } errors = result.errors; } else { errors = field.setParameter(insertStatement, fieldIndex + 2, string); @@ -543,15 +558,18 @@ public void setValueForField(Table table, int fieldIndex, int lineNumber, Field * Sets field to null in statement or string array depending on whether postgres is being used. */ private void setFieldToNull(boolean postgresText, String[] transformedStrings, int fieldIndex, Field field) { - if (postgresText) transformedStrings[fieldIndex + 1] = POSTGRES_NULL_TEXT; - // Adjust parameter index by two: indexes are one-based and the first one is the CSV line number. - else try { - // LOG.info("setting {} index to null", fieldIndex + 2); - field.setNull(insertStatement, fieldIndex + 2); - } catch (SQLException e) { - e.printStackTrace(); - // FIXME: store error here? It appears that an exception should only be thrown if the type value is invalid, - // the connection is closed, or the index is out of bounds. So storing an error may be unnecessary. + if (postgresText) { + transformedStrings[fieldIndex + 1] = POSTGRES_NULL_TEXT; + } else { + // Adjust parameter index by two: indexes are one-based and the first one is the CSV line number. + try { + // LOG.info("setting {} index to null", fieldIndex + 2); + field.setNull(insertStatement, fieldIndex + 2); + } catch (SQLException e) { + e.printStackTrace(); + // FIXME: store error here? It appears that an exception should only be thrown if the type value is invalid, + // the connection is closed, or the index is out of bounds. So storing an error may be unnecessary. + } } } @@ -563,7 +581,7 @@ private void setFieldToNull(boolean postgresText, String[] transformedStrings, i * * TODO add a test including SQL injection text (quote and semicolon) */ - public static String sanitize (String string, SQLErrorStorage errorStorage) { + public static String sanitize(String string, SQLErrorStorage errorStorage) { String clean = string.replaceAll("[^\\p{Alnum}_]", ""); if (!clean.equals(string)) { LOG.warn("SQL identifier '{}' was sanitized to '{}'", string, clean); diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsSnapshotter.java b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsSnapshotter.java index 610a4f8ac..3b36a9b8e 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsSnapshotter.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsSnapshotter.java @@ -112,6 +112,8 @@ public SnapshotResult copyTables() { result.stopTimes = copy(Table.STOP_TIMES, true); result.transfers = copy(Table.TRANSFERS, true); result.trips = copy(Table.TRIPS, true); + result.attributions = copy(Table.ATTRIBUTIONS, true); + result.translations = copy(Table.TRANSLATIONS, true); result.completionTime = System.currentTimeMillis(); result.loadTimeMillis = result.completionTime - startTime; LOG.info("Copying tables took {} sec", (result.loadTimeMillis) / 1000); diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java index 14665a7c0..666ba77bf 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java @@ -634,6 +634,8 @@ private String updateChildTable( "timepoint", "drop_off_type", "pickup_type", + "continuous_pickup", + "continuous_drop_off", "shape_dist_traveled" ); } @@ -1146,6 +1148,8 @@ private void insertBlankStopTimes( stopTime.pickup_type = patternStop.pickup_type; stopTime.timepoint = patternStop.timepoint; stopTime.shape_dist_traveled = patternStop.shape_dist_traveled; + stopTime.continuous_drop_off = patternStop.continuous_drop_off; + stopTime.continuous_pickup = patternStop.continuous_pickup; stopTime.stop_sequence = i; // Update stop time with each trip ID and add to batch. for (String tripId : tripIds) { diff --git a/src/main/java/com/conveyal/gtfs/loader/LineContext.java b/src/main/java/com/conveyal/gtfs/loader/LineContext.java new file mode 100644 index 000000000..2ef0f1900 --- /dev/null +++ b/src/main/java/com/conveyal/gtfs/loader/LineContext.java @@ -0,0 +1,46 @@ +package com.conveyal.gtfs.loader; + +/** + * Wrapper class that provides access to row values and line context (e.g., line number) for a particular row of GTFS + * data. + */ +public class LineContext { + public final Table table; + private final Field[] fields; + /** + * The row data has one extra value at the beginning of the array that represents the line number. + */ + private final String[] rowDataWithLineNumber; + public final int lineNumber; + + public LineContext(Table table, Field[] fields, String[] rowDataWithLineNumber, int lineNumber) { + this.table = table; + this.fields = fields; + this.rowDataWithLineNumber = rowDataWithLineNumber; + this.lineNumber = lineNumber; + } + + /** + * Get value for a particular column index from a set of row data. Note: the row data here has one extra value at + * the beginning of the array that represents the line number (hence the +1). This is because the data is formatted + * for batch insertion into a postgres table. + */ + public String getValueForRow(int columnIndex) { + return rowDataWithLineNumber[columnIndex + 1]; + } + + /** + * Overloaded method to provide value for the current line for a particular field. + */ + public String getValueForRow(String fieldName) { + int fieldIndex = Field.getFieldIndex(fields, fieldName); + return rowDataWithLineNumber[fieldIndex + 1]; + } + + /** + * Overloaded method to provide value for the current line for the key field. + */ + public String getEntityId() { + return getValueForRow(table.getKeyFieldIndex(fields)); + } +} diff --git a/src/main/java/com/conveyal/gtfs/loader/ReferenceTracker.java b/src/main/java/com/conveyal/gtfs/loader/ReferenceTracker.java index 7a01e25dd..acd1178c7 100644 --- a/src/main/java/com/conveyal/gtfs/loader/ReferenceTracker.java +++ b/src/main/java/com/conveyal/gtfs/loader/ReferenceTracker.java @@ -1,9 +1,12 @@ package com.conveyal.gtfs.loader; import com.conveyal.gtfs.error.NewGTFSError; +import com.conveyal.gtfs.loader.conditions.ConditionalRequirement; +import com.google.common.collect.HashMultimap; import java.util.Collections; import java.util.HashSet; +import java.util.Map; import java.util.Set; import static com.conveyal.gtfs.error.NewGTFSErrorType.DUPLICATE_ID; @@ -21,6 +24,7 @@ */ public class ReferenceTracker { public final Set transitIds = new HashSet<>(); + public final HashMultimap uniqueValuesForFields = HashMultimap.create(); public final Set transitIdsWithSequence = new HashSet<>(); /** @@ -62,6 +66,13 @@ public Set checkReferencesAndUniqueness(String keyValue, int lineN : !table.hasUniqueKeyField ? null : keyField; String transitId = String.join(":", keyField, keyValue); + // Unique key values are needed for referential integrity checks as part of checks for fields that have + // conditional requirements. This also tracks "special" foreign keys like stop#zone_id that are not primary keys + // of the table they exist in. + if ((field.name.equals(keyField) && keyField.equals(uniqueKeyField)) || field.isForeign()) { + uniqueValuesForFields.put(field.name, value); + } + // If the field is optional and there is no value present, skip check. if (!field.isRequired() && "".equals(value)) return Collections.emptySet(); @@ -136,4 +147,28 @@ public Set checkReferencesAndUniqueness(String keyValue, int lineN } return errors; } + + + /** + * Work through each conditionally required check assigned to fields within a table. First check the reference field + * to confirm if it meets the conditions whereby the conditional field is required. If the conditional field is + * required confirm that a value has been provided, if not, log an error. + */ + public Set checkConditionallyRequiredFields(LineContext lineContext) { + Set errors = new HashSet<>(); + Map fieldsToCheck = lineContext.table.getConditionalRequirements(); + + // Work through each field that has been assigned a conditional requirement. + for (Map.Entry entry : fieldsToCheck.entrySet()) { + Field referenceField = entry.getKey(); + ConditionalRequirement[] conditionalRequirements = entry.getValue(); + // Work through each field's conditional requirements. + for (ConditionalRequirement conditionalRequirement : conditionalRequirements) { + errors.addAll( + conditionalRequirement.check(lineContext, referenceField, uniqueValuesForFields) + ); + } + } + return errors; + } } diff --git a/src/main/java/com/conveyal/gtfs/loader/Table.java b/src/main/java/com/conveyal/gtfs/loader/Table.java index 6ea3c2bcf..b397d9958 100644 --- a/src/main/java/com/conveyal/gtfs/loader/Table.java +++ b/src/main/java/com/conveyal/gtfs/loader/Table.java @@ -2,7 +2,15 @@ import com.conveyal.gtfs.error.NewGTFSError; import com.conveyal.gtfs.error.SQLErrorStorage; +import com.conveyal.gtfs.loader.conditions.AgencyHasMultipleRowsCheck; +import com.conveyal.gtfs.loader.conditions.ConditionalRequirement; +import com.conveyal.gtfs.loader.conditions.FieldInRangeCheck; +import com.conveyal.gtfs.loader.conditions.FieldIsEmptyCheck; +import com.conveyal.gtfs.loader.conditions.FieldNotEmptyAndMatchesValueCheck; +import com.conveyal.gtfs.loader.conditions.ForeignRefExistsCheck; +import com.conveyal.gtfs.loader.conditions.ReferenceFieldShouldBeProvidedCheck; import com.conveyal.gtfs.model.Agency; +import com.conveyal.gtfs.model.Attribution; import com.conveyal.gtfs.model.Calendar; import com.conveyal.gtfs.model.CalendarDate; import com.conveyal.gtfs.model.Entity; @@ -18,6 +26,7 @@ import com.conveyal.gtfs.model.Stop; import com.conveyal.gtfs.model.StopTime; import com.conveyal.gtfs.model.Transfer; +import com.conveyal.gtfs.model.Translation; import com.conveyal.gtfs.model.Trip; import com.conveyal.gtfs.storage.StorageException; import com.csvreader.CsvReader; @@ -38,16 +47,16 @@ import java.util.Arrays; import java.util.Collections; import java.util.Enumeration; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import static com.conveyal.gtfs.error.NewGTFSErrorType.DUPLICATE_HEADER; -import static com.conveyal.gtfs.error.NewGTFSErrorType.DUPLICATE_ID; -import static com.conveyal.gtfs.error.NewGTFSErrorType.REFERENTIAL_INTEGRITY; import static com.conveyal.gtfs.error.NewGTFSErrorType.TABLE_IN_SUBDIRECTORY; import static com.conveyal.gtfs.loader.JdbcGtfsLoader.sanitize; import static com.conveyal.gtfs.loader.Requirement.EDITOR; @@ -98,15 +107,19 @@ public Table (String name, Class entityClass, Requirement requ } public static final Table AGENCY = new Table("agency", Agency.class, REQUIRED, - new StringField("agency_id", OPTIONAL), // FIXME? only required if there are more than one - new StringField("agency_name", REQUIRED), - new URLField("agency_url", REQUIRED), - new StringField("agency_timezone", REQUIRED), // FIXME new field type for time zones? + new StringField("agency_id", OPTIONAL).requireConditions( + // If there is more than one agency, the agency_id must be provided + // https://developers.google.com/transit/gtfs/reference#agencytxt + new AgencyHasMultipleRowsCheck() + ).hasForeignReferences(), + new StringField("agency_name", REQUIRED), + new URLField("agency_url", REQUIRED), + new StringField("agency_timezone", REQUIRED), // FIXME new field type for time zones? new StringField("agency_lang", OPTIONAL), // FIXME new field type for languages? - new StringField("agency_phone", OPTIONAL), - new URLField("agency_branding_url", OPTIONAL), - new URLField("agency_fare_url", OPTIONAL), - new StringField("agency_email", OPTIONAL) // FIXME new field type for emails? + new StringField("agency_phone", OPTIONAL), + new URLField("agency_branding_url", OPTIONAL), + new URLField("agency_fare_url", OPTIONAL), + new StringField("agency_email", OPTIONAL) // FIXME new field type for emails? ).restrictDelete().addPrimaryKey(); // The GTFS spec says this table is required, but in practice it is not required if calendar_dates is present. @@ -147,7 +160,11 @@ public Table (String name, Class entityClass, Requirement requ new CurrencyField("currency_type", REQUIRED), new ShortField("payment_method", REQUIRED, 1), new ShortField("transfers", REQUIRED, 2).permitEmptyValue(), - new StringField("agency_id", OPTIONAL), // FIXME? only required if there are more than one + new StringField("agency_id", OPTIONAL).requireConditions( + // If there is more than one agency, this agency_id is required. + // https://developers.google.com/transit/gtfs/reference#fare_attributestxt + new ReferenceFieldShouldBeProvidedCheck("agency_id") + ), new IntegerField("transfer_duration", OPTIONAL) ).addPrimaryKey(); @@ -156,7 +173,7 @@ public Table (String name, Class entityClass, Requirement requ public static final Table FEED_INFO = new Table("feed_info", FeedInfo.class, OPTIONAL, new StringField("feed_publisher_name", REQUIRED), // feed_id is not the first field because that would label it as the key field, which we do not want because the - // key field cannot be optional. + // key field cannot be optional. feed_id is not part of the GTFS spec, but is required by OTP to associate static GTFS with GTFS-rt feeds. new StringField("feed_id", OPTIONAL), new URLField("feed_publisher_url", REQUIRED), new LanguageField("feed_lang", REQUIRED), @@ -166,41 +183,40 @@ public Table (String name, Class entityClass, Requirement requ // Editor-specific field that represents default route values for use in editing. new ColorField("default_route_color", EDITOR), // FIXME: Should the route type max value be equivalent to GTFS spec's max? - new IntegerField("default_route_type", EDITOR, 999) + new IntegerField("default_route_type", EDITOR, 999), + new LanguageField("default_lang", OPTIONAL), + new StringField("feed_contact_email", OPTIONAL), + new URLField("feed_contact_url", OPTIONAL) ).keyFieldIsNotUnique(); public static final Table ROUTES = new Table("routes", Route.class, REQUIRED, new StringField("route_id", REQUIRED), - new StringField("agency_id", OPTIONAL).isReferenceTo(AGENCY), - new StringField("route_short_name", OPTIONAL), // one of short or long must be provided - new StringField("route_long_name", OPTIONAL), - new StringField("route_desc", OPTIONAL), + new StringField("agency_id", OPTIONAL).isReferenceTo(AGENCY).requireConditions( + // If there is more than one agency, this agency_id is required. + // https://developers.google.com/transit/gtfs/reference#routestxt + new ReferenceFieldShouldBeProvidedCheck("agency_id") + ), + new StringField("route_short_name", OPTIONAL), // one of short or long must be provided + new StringField("route_long_name", OPTIONAL), + new StringField("route_desc", OPTIONAL), // Max route type according to the GTFS spec is 7; however, there is a GTFS proposal that could see this // max value grow to around 1800: https://groups.google.com/forum/#!msg/gtfs-changes/keT5rTPS7Y0/71uMz2l6ke0J new IntegerField("route_type", REQUIRED, 1800), - new URLField("route_url", OPTIONAL), - new URLField("route_branding_url", OPTIONAL), - new ColorField("route_color", OPTIONAL), // really this is an int in hex notation - new ColorField("route_text_color", OPTIONAL), + new URLField("route_url", OPTIONAL), + new URLField("route_branding_url", OPTIONAL), + new ColorField("route_color", OPTIONAL), // really this is an int in hex notation + new ColorField("route_text_color", OPTIONAL), // Editor fields below. new ShortField("publicly_visible", EDITOR, 1), // wheelchair_accessible is an exemplar field applied to all trips on a route. new ShortField("wheelchair_accessible", EDITOR, 2).permitEmptyValue(), new IntegerField("route_sort_order", OPTIONAL, 0, Integer.MAX_VALUE), // Status values are In progress (0), Pending approval (1), and Approved (2). - new ShortField("status", EDITOR, 2) + new ShortField("status", EDITOR, 2), + new ShortField("continuous_pickup", OPTIONAL,3), + new ShortField("continuous_drop_off", OPTIONAL,3) ).addPrimaryKey(); - public static final Table FARE_RULES = new Table("fare_rules", FareRule.class, OPTIONAL, - new StringField("fare_id", REQUIRED).isReferenceTo(FARE_ATTRIBUTES), - new StringField("route_id", OPTIONAL).isReferenceTo(ROUTES), - // FIXME: referential integrity check for zone_id for below three fields? - new StringField("origin_id", OPTIONAL), - new StringField("destination_id", OPTIONAL), - new StringField("contains_id", OPTIONAL)) - .withParentTable(FARE_ATTRIBUTES) - .addPrimaryKey().keyFieldIsNotUnique(); - public static final Table SHAPES = new Table("shapes", ShapePoint.class, OPTIONAL, new StringField("shape_id", REQUIRED), new IntegerField("shape_pt_sequence", REQUIRED), @@ -226,20 +242,53 @@ public Table (String name, Class entityClass, Requirement requ ).addPrimaryKey(); public static final Table STOPS = new Table("stops", Stop.class, REQUIRED, - new StringField("stop_id", REQUIRED), - new StringField("stop_code", OPTIONAL), - new StringField("stop_name", REQUIRED), - new StringField("stop_desc", OPTIONAL), - new DoubleField("stop_lat", REQUIRED, -80, 80, 6), - new DoubleField("stop_lon", REQUIRED, -180, 180, 6), - new StringField("zone_id", OPTIONAL), - new URLField("stop_url", OPTIONAL), - new ShortField("location_type", OPTIONAL, 2), - // FIXME: Need self-reference check during referential integrity check - new StringField("parent_station", OPTIONAL), //.isReferenceToSelf() - new StringField("stop_timezone", OPTIONAL), - new ShortField("wheelchair_boarding", OPTIONAL, 2) - ).restrictDelete().addPrimaryKey(); + new StringField("stop_id", REQUIRED), + new StringField("stop_code", OPTIONAL), + // The actual conditions that will be acted upon are within the location_type field. + new StringField("stop_name", OPTIONAL).requireConditions(), + new StringField("stop_desc", OPTIONAL), + // The actual conditions that will be acted upon are within the location_type field. + new DoubleField("stop_lat", OPTIONAL, -80, 80, 6).requireConditions(), + // The actual conditions that will be acted upon are within the location_type field. + new DoubleField("stop_lon", OPTIONAL, -180, 180, 6).requireConditions(), + new StringField("zone_id", OPTIONAL).hasForeignReferences(), + new URLField("stop_url", OPTIONAL), + new ShortField("location_type", OPTIONAL, 4).requireConditions( + // If the location type is defined and within range, the dependent fields are required. + // https://developers.google.com/transit/gtfs/reference#stopstxt + new FieldInRangeCheck(0, 2, "stop_name"), + new FieldInRangeCheck(0, 2, "stop_lat"), + new FieldInRangeCheck(0, 2, "stop_lon"), + new FieldInRangeCheck(2, 4, "parent_station") + ), + // The actual conditions that will be acted upon are within the location_type field. + new StringField("parent_station", OPTIONAL).requireConditions(), + new StringField("stop_timezone", OPTIONAL), + new ShortField("wheelchair_boarding", OPTIONAL, 2), + new StringField("platform_code", OPTIONAL) + ) + .restrictDelete() + .addPrimaryKey(); + + // GTFS reference: https://developers.google.com/transit/gtfs/reference#fare_rulestxt + public static final Table FARE_RULES = new Table("fare_rules", FareRule.class, OPTIONAL, + new StringField("fare_id", REQUIRED).isReferenceTo(FARE_ATTRIBUTES), + new StringField("route_id", OPTIONAL).isReferenceTo(ROUTES), + new StringField("origin_id", OPTIONAL).requireConditions( + // If the origin_id is defined, its value must exist as a zone_id in stops.txt. + new ForeignRefExistsCheck("zone_id", "fare_rules") + ), + new StringField("destination_id", OPTIONAL).requireConditions( + // If the destination_id is defined, its value must exist as a zone_id in stops.txt. + new ForeignRefExistsCheck("zone_id", "fare_rules") + ), + new StringField("contains_id", OPTIONAL).requireConditions( + // If the contains_id is defined, its value must exist as a zone_id in stops.txt. + new ForeignRefExistsCheck("zone_id", "fare_rules") + ) + ) + .withParentTable(FARE_ATTRIBUTES) + .addPrimaryKey().keyFieldIsNotUnique(); public static final Table PATTERN_STOP = new Table("pattern_stops", PatternStop.class, OPTIONAL, new StringField("pattern_id", REQUIRED).isReferenceTo(PATTERNS), @@ -252,7 +301,9 @@ public Table (String name, Class entityClass, Requirement requ new IntegerField("drop_off_type", EDITOR, 2), new IntegerField("pickup_type", EDITOR, 2), new DoubleField("shape_dist_traveled", EDITOR, 0, Double.POSITIVE_INFINITY, -1), - new ShortField("timepoint", EDITOR, 1) + new ShortField("timepoint", EDITOR, 1), + new ShortField("continuous_pickup", OPTIONAL,3), + new ShortField("continuous_drop_off", OPTIONAL,3) ).withParentTable(PATTERNS); public static final Table TRANSFERS = new Table("transfers", Transfer.class, OPTIONAL, @@ -266,16 +317,16 @@ public Table (String name, Class entityClass, Requirement requ .hasCompoundKey(); public static final Table TRIPS = new Table("trips", Trip.class, REQUIRED, - new StringField("trip_id", REQUIRED), - new StringField("route_id", REQUIRED).isReferenceTo(ROUTES).indexThisColumn(), + new StringField("trip_id", REQUIRED), + new StringField("route_id", REQUIRED).isReferenceTo(ROUTES).indexThisColumn(), // FIXME: Should this also optionally reference CALENDAR_DATES? // FIXME: Do we need an index on service_id - new StringField("service_id", REQUIRED).isReferenceTo(CALENDAR), - new StringField("trip_headsign", OPTIONAL), - new StringField("trip_short_name", OPTIONAL), + new StringField("service_id", REQUIRED).isReferenceTo(CALENDAR), + new StringField("trip_headsign", OPTIONAL), + new StringField("trip_short_name", OPTIONAL), new ShortField("direction_id", OPTIONAL, 1), - new StringField("block_id", OPTIONAL), - new StringField("shape_id", OPTIONAL).isReferenceTo(SHAPES), + new StringField("block_id", OPTIONAL), + new StringField("shape_id", OPTIONAL).isReferenceTo(SHAPES), new ShortField("wheelchair_accessible", OPTIONAL, 2), new ShortField("bikes_allowed", OPTIONAL, 2), // Editor-specific fields below. @@ -295,6 +346,8 @@ public Table (String name, Class entityClass, Requirement requ new StringField("stop_headsign", OPTIONAL), new ShortField("pickup_type", OPTIONAL, 3), new ShortField("drop_off_type", OPTIONAL, 3), + new ShortField("continuous_pickup", OPTIONAL, 3), + new ShortField("continuous_drop_off", OPTIONAL, 3), new DoubleField("shape_dist_traveled", OPTIONAL, 0, Double.POSITIVE_INFINITY, 2), new ShortField("timepoint", OPTIONAL, 1), new IntegerField("fare_units_traveled", EXTENSION) // OpenOV NL extension @@ -313,24 +366,59 @@ public Table (String name, Class entityClass, Requirement requ .withParentTable(TRIPS) .keyFieldIsNotUnique(); + // GTFS reference: https://developers.google.com/transit/gtfs/reference#attributionstxt + public static final Table TRANSLATIONS = new Table("translations", Translation.class, OPTIONAL, + new StringField("table_name", REQUIRED), + new StringField("field_name", REQUIRED), + new LanguageField("language", REQUIRED), + new StringField("translation", REQUIRED), + new StringField("record_id", OPTIONAL).requireConditions( + // If the field_value is empty the record_id is required. + new FieldIsEmptyCheck("field_value") + ), + new StringField("record_sub_id", OPTIONAL).requireConditions( + // If the record_id is not empty and the value is stop_times the record_sub_id is required. + new FieldNotEmptyAndMatchesValueCheck("record_id", "stop_times") + ), + new StringField("field_value", OPTIONAL).requireConditions( + // If the record_id is empty the field_value is required. + new FieldIsEmptyCheck("record_id") + )) + .keyFieldIsNotUnique(); + + public static final Table ATTRIBUTIONS = new Table("attributions", Attribution.class, OPTIONAL, + new StringField("attribution_id", OPTIONAL), + new StringField("agency_id", OPTIONAL).isReferenceTo(AGENCY), + new LanguageField("route_id", OPTIONAL).isReferenceTo(ROUTES), + new StringField("trip_id", OPTIONAL).isReferenceTo(TRIPS), + new StringField("organization_name", REQUIRED), + new ShortField("is_producer", OPTIONAL, 1), + new ShortField("is_operator", OPTIONAL, 1), + new ShortField("is_authority", OPTIONAL, 1), + new URLField("attribution_url", OPTIONAL), + new StringField("attribution_email", OPTIONAL), + new StringField("attribution_phone", OPTIONAL)); + /** List of tables in order needed for checking referential integrity during load stage. */ public static final Table[] tablesInOrder = { - AGENCY, - CALENDAR, - SCHEDULE_EXCEPTIONS, - CALENDAR_DATES, - FARE_ATTRIBUTES, - FEED_INFO, - ROUTES, - FARE_RULES, - PATTERNS, - SHAPES, - STOPS, - PATTERN_STOP, - TRANSFERS, - TRIPS, - STOP_TIMES, - FREQUENCIES + AGENCY, + CALENDAR, + SCHEDULE_EXCEPTIONS, + CALENDAR_DATES, + FARE_ATTRIBUTES, + FEED_INFO, + ROUTES, + PATTERNS, + SHAPES, + STOPS, + FARE_RULES, + PATTERN_STOP, + TRANSFERS, + TRIPS, + STOP_TIMES, + FREQUENCIES, + TRANSLATIONS, + ATTRIBUTIONS }; /** @@ -996,4 +1084,19 @@ public int getKeyFieldIndex(Field[] fields) { String keyField = getKeyFieldName(); return Field.getFieldIndex(fields, keyField); } + + public boolean hasConditionalRequirements() { + return !getConditionalRequirements().isEmpty(); + } + + public Map getConditionalRequirements() { + Map fieldsWithConditions = new HashMap<>(); + for (Field field : fields) { + if (field.isConditionallyRequired()) { + fieldsWithConditions.put(field, field.conditions); + } + } + return fieldsWithConditions; + } + } diff --git a/src/main/java/com/conveyal/gtfs/loader/conditions/AgencyHasMultipleRowsCheck.java b/src/main/java/com/conveyal/gtfs/loader/conditions/AgencyHasMultipleRowsCheck.java new file mode 100644 index 000000000..3b3b6ca9b --- /dev/null +++ b/src/main/java/com/conveyal/gtfs/loader/conditions/AgencyHasMultipleRowsCheck.java @@ -0,0 +1,62 @@ +package com.conveyal.gtfs.loader.conditions; + +import com.conveyal.gtfs.error.NewGTFSError; +import com.conveyal.gtfs.loader.Field; +import com.conveyal.gtfs.loader.LineContext; +import com.google.common.collect.HashMultimap; + +import java.util.HashSet; +import java.util.Set; + +import static com.conveyal.gtfs.error.NewGTFSErrorType.AGENCY_ID_REQUIRED_FOR_MULTI_AGENCY_FEEDS; +import static com.conveyal.gtfs.loader.JdbcGtfsLoader.POSTGRES_NULL_TEXT; + +/** + * Conditional requirement to check that an agency_id has been provided if more than one row exists in agency.txt. + */ +public class AgencyHasMultipleRowsCheck extends ConditionalRequirement { + + private final int FIRST_ROW = 2; + private final int SECOND_ROW = 3; + + public AgencyHasMultipleRowsCheck() { + this.dependentFieldName = "agency_id"; + } + + /** + * Flag an error if there are multiple rows in agency.txt and the agency_id is missing for any rows. + */ + public Set check( + LineContext lineContext, + Field referenceField, + HashMultimap uniqueValuesForFields + ) { + String dependentFieldValue = lineContext.getValueForRow(dependentFieldName); + Set errors = new HashSet<>(); + Set agencyIdValues = uniqueValuesForFields.get(dependentFieldName); + // Do some awkward checks to determine if the first or second row (or another) is missing the agency_id. + boolean firstOrSecondMissingId = lineContext.lineNumber == SECOND_ROW && agencyIdValues.contains(""); + boolean currentRowMissingId = POSTGRES_NULL_TEXT.equals(dependentFieldValue); + boolean secondRowMissingId = firstOrSecondMissingId && currentRowMissingId; + if (firstOrSecondMissingId || (lineContext.lineNumber > SECOND_ROW && currentRowMissingId)) { + // The check on the agency table is carried out whilst the agency table is being loaded so it + // is possible to compare the number of agencyIdValues added against the number of rows loaded to + // accurately determine missing agency_id values. + int lineNumber = secondRowMissingId + ? SECOND_ROW + : firstOrSecondMissingId + ? FIRST_ROW + : lineContext.lineNumber; + errors.add( + NewGTFSError.forLine( + lineContext.table, + lineNumber, + AGENCY_ID_REQUIRED_FOR_MULTI_AGENCY_FEEDS, + dependentFieldName + ) + ); + } + return errors; + } + +} diff --git a/src/main/java/com/conveyal/gtfs/loader/conditions/ConditionalRequirement.java b/src/main/java/com/conveyal/gtfs/loader/conditions/ConditionalRequirement.java new file mode 100644 index 000000000..25fe7087d --- /dev/null +++ b/src/main/java/com/conveyal/gtfs/loader/conditions/ConditionalRequirement.java @@ -0,0 +1,28 @@ +package com.conveyal.gtfs.loader.conditions; + +import com.conveyal.gtfs.error.NewGTFSError; +import com.conveyal.gtfs.loader.Field; +import com.conveyal.gtfs.loader.LineContext; +import com.google.common.collect.HashMultimap; + +import java.util.Set; + +/** + * An abstract class which primarily defines a method used by implementing classes to define specific conditional + * requirement checks. + */ +public abstract class ConditionalRequirement { + /** The name of the dependent field, which is a field that requires a specific value if the reference and + * (in some cases) dependent field checks meet certain conditions. + */ + protected String dependentFieldName; + + /** + * All sub classes must implement this method and provide related conditional checks. + */ + public abstract Set check( + LineContext lineContext, + Field referenceField, + HashMultimap uniqueValuesForFields + ); +} diff --git a/src/main/java/com/conveyal/gtfs/loader/conditions/FieldInRangeCheck.java b/src/main/java/com/conveyal/gtfs/loader/conditions/FieldInRangeCheck.java new file mode 100644 index 000000000..47f752c45 --- /dev/null +++ b/src/main/java/com/conveyal/gtfs/loader/conditions/FieldInRangeCheck.java @@ -0,0 +1,88 @@ +package com.conveyal.gtfs.loader.conditions; + +import com.conveyal.gtfs.error.NewGTFSError; +import com.conveyal.gtfs.loader.Field; +import com.conveyal.gtfs.loader.LineContext; +import com.google.common.collect.HashMultimap; + +import java.util.HashSet; +import java.util.Set; + +import static com.conveyal.gtfs.error.NewGTFSErrorType.CONDITIONALLY_REQUIRED; +import static com.conveyal.gtfs.loader.JdbcGtfsLoader.POSTGRES_NULL_TEXT; + +/** + * Conditional requirement to check that a reference field value is within a defined range and the conditional field + * value has not be defined. + */ +public class FieldInRangeCheck extends ConditionalRequirement { + /** The minimum reference field value if a range check is being performed. */ + protected int minReferenceValue; + /** The maximum reference field value if a range check is being performed. */ + protected int maxReferenceValue; + + public FieldInRangeCheck( + int minReferenceValue, + int maxReferenceValue, + String dependentFieldName + ) { + this.minReferenceValue = minReferenceValue; + this.maxReferenceValue = maxReferenceValue; + this.dependentFieldName = dependentFieldName; + } + + /** + * If the reference field value is within a defined range and the conditional field value has not been defined, flag + * an error. + */ + public Set check( + LineContext lineContext, + Field referenceField, + HashMultimap uniqueValuesForFields + ) { + Set errors = new HashSet<>(); + String referenceFieldValue = lineContext.getValueForRow(referenceField.name); + String conditionalFieldValue = lineContext.getValueForRow(dependentFieldName); + boolean referenceValueMeetsRangeCondition = + !POSTGRES_NULL_TEXT.equals(referenceFieldValue) && + // TODO use pre-existing method in ShortField? + isValueInRange(referenceFieldValue, minReferenceValue, maxReferenceValue); + + if (!referenceValueMeetsRangeCondition) { + // If ref value does not meet the range condition, there is no need to check this conditional + // value for (e.g.) an empty value. Continue to the next check. + return errors; + } + + if (POSTGRES_NULL_TEXT.equals(conditionalFieldValue)) { + // Reference value in range and conditionally required field is empty. + String message = String.format( + "%s is required when %s value is between %d and %d.", + dependentFieldName, + referenceField.name, + minReferenceValue, + maxReferenceValue + ); + errors.add( + NewGTFSError.forLine( + lineContext, + CONDITIONALLY_REQUIRED, + message).setEntityId(lineContext.getEntityId()) + ); + } + return errors; + } + + /** + * Check if the provided value is within the min and max values. If the field value can not be converted + * to a number it is assumed that the value is not a number and will therefore never be within the min/max range. + */ + private boolean isValueInRange(String referenceFieldValue, int min, int max) { + try { + int fieldValue = Integer.parseInt(referenceFieldValue); + return fieldValue >= min && fieldValue <= max; + } catch (NumberFormatException e) { + return false; + } + } +} diff --git a/src/main/java/com/conveyal/gtfs/loader/conditions/FieldIsEmptyCheck.java b/src/main/java/com/conveyal/gtfs/loader/conditions/FieldIsEmptyCheck.java new file mode 100644 index 000000000..3ba3c291d --- /dev/null +++ b/src/main/java/com/conveyal/gtfs/loader/conditions/FieldIsEmptyCheck.java @@ -0,0 +1,55 @@ +package com.conveyal.gtfs.loader.conditions; + +import com.conveyal.gtfs.error.NewGTFSError; +import com.conveyal.gtfs.loader.Field; +import com.conveyal.gtfs.loader.LineContext; +import com.google.common.collect.HashMultimap; + +import java.util.HashSet; +import java.util.Set; + +import static com.conveyal.gtfs.error.NewGTFSErrorType.CONDITIONALLY_REQUIRED; +import static com.conveyal.gtfs.loader.JdbcGtfsLoader.POSTGRES_NULL_TEXT; + +/** + * Conditional requirement to check that if a dependent field value is empty then the reference field value is provided. + */ +public class FieldIsEmptyCheck extends ConditionalRequirement { + + public FieldIsEmptyCheck(String dependentFieldName) { + this.dependentFieldName = dependentFieldName; + } + + /** + * Check the dependent field value. If it is empty, the reference field value must be provided. + */ + public Set check( + LineContext lineContext, + Field referenceField, + HashMultimap uniqueValuesForFields + ) { + Set errors = new HashSet<>(); + String dependentFieldValue = lineContext.getValueForRow(dependentFieldName); + String referenceFieldValue = lineContext.getValueForRow(referenceField.name); + if ( + POSTGRES_NULL_TEXT.equals(dependentFieldValue) && + POSTGRES_NULL_TEXT.equals(referenceFieldValue) + ) { + // The reference field is required when the dependent field is empty. + String message = String.format( + "%s is required when %s is empty.", + referenceField.name, + dependentFieldName + ); + errors.add( + NewGTFSError.forLine( + lineContext, + CONDITIONALLY_REQUIRED, + message).setEntityId(lineContext.getEntityId()) + ); + + } + return errors; + } + +} diff --git a/src/main/java/com/conveyal/gtfs/loader/conditions/FieldNotEmptyAndMatchesValueCheck.java b/src/main/java/com/conveyal/gtfs/loader/conditions/FieldNotEmptyAndMatchesValueCheck.java new file mode 100644 index 000000000..c693120fe --- /dev/null +++ b/src/main/java/com/conveyal/gtfs/loader/conditions/FieldNotEmptyAndMatchesValueCheck.java @@ -0,0 +1,62 @@ +package com.conveyal.gtfs.loader.conditions; + +import com.conveyal.gtfs.error.NewGTFSError; +import com.conveyal.gtfs.loader.Field; +import com.conveyal.gtfs.loader.LineContext; +import com.google.common.collect.HashMultimap; + +import java.util.HashSet; +import java.util.Set; + +import static com.conveyal.gtfs.error.NewGTFSErrorType.CONDITIONALLY_REQUIRED; +import static com.conveyal.gtfs.loader.JdbcGtfsLoader.POSTGRES_NULL_TEXT; + +/** + * Conditional requirement to check that a dependent field value is not empty and matches an expected value. + */ +public class FieldNotEmptyAndMatchesValueCheck extends ConditionalRequirement { + /** The expected dependent field value. */ + private String requiredDependentFieldValue; + + public FieldNotEmptyAndMatchesValueCheck( + String dependentFieldName, + String requiredDependentFieldValue + ) { + this.dependentFieldName = dependentFieldName; + this.requiredDependentFieldValue = requiredDependentFieldValue; + } + + /** + * Check the dependent field value is not empty and matches the expected value. + */ + public Set check( + LineContext lineContext, + Field referenceField, + HashMultimap uniqueValuesForFields + ) { + Set errors = new HashSet<>(); + String dependentFieldValue = lineContext.getValueForRow(dependentFieldName); + String referenceFieldValue = lineContext.getValueForRow(referenceField.name); + if ( + !POSTGRES_NULL_TEXT.equals(dependentFieldValue) && + dependentFieldValue.equals(requiredDependentFieldValue) && + POSTGRES_NULL_TEXT.equals(referenceFieldValue) + ) { + String message = String.format( + "%s is required and must match %s when %s is provided.", + referenceField.name, + requiredDependentFieldValue, + dependentFieldName + ); + errors.add( + NewGTFSError.forLine( + lineContext, + CONDITIONALLY_REQUIRED, + message).setEntityId(lineContext.getEntityId()) + ); + + } + return errors; + } + +} diff --git a/src/main/java/com/conveyal/gtfs/loader/conditions/ForeignRefExistsCheck.java b/src/main/java/com/conveyal/gtfs/loader/conditions/ForeignRefExistsCheck.java new file mode 100644 index 000000000..23c9d5e5d --- /dev/null +++ b/src/main/java/com/conveyal/gtfs/loader/conditions/ForeignRefExistsCheck.java @@ -0,0 +1,62 @@ +package com.conveyal.gtfs.loader.conditions; + +import com.conveyal.gtfs.error.NewGTFSError; +import com.conveyal.gtfs.loader.Field; +import com.conveyal.gtfs.loader.LineContext; +import com.conveyal.gtfs.loader.ReferenceTracker; +import com.google.common.collect.HashMultimap; + +import java.util.HashSet; +import java.util.Set; + +import static com.conveyal.gtfs.error.NewGTFSErrorType.REFERENTIAL_INTEGRITY; +import static com.conveyal.gtfs.loader.JdbcGtfsLoader.POSTGRES_NULL_TEXT; + +/** + * Conditional requirement to check that an expected foreign field value matches a conditional field value. + */ +public class ForeignRefExistsCheck extends ConditionalRequirement { + /** The reference table name. */ + private String referenceTableName; + + public ForeignRefExistsCheck(String dependentFieldName, String referenceTableName) { + this.dependentFieldName = dependentFieldName; + this.referenceTableName = referenceTableName; + } + + /** + * Check that an expected foreign field value matches a conditional field value. Selected foreign field values are + * added to {@link ReferenceTracker#uniqueValuesForFields} as part of the load process and are used here to check + * conditional fields which have a dependency on them. e.g. stop#zone_id does not exist in stops table, but is + * required by fare_rules records (e.g. origin_id). + */ + public Set check( + LineContext lineContext, + Field referenceField, + HashMultimap uniqueValuesForFields + ) { + Set errors = new HashSet<>(); + String referenceFieldValue = lineContext.getValueForRow(referenceField.name); + // Expected reference in foreign field id list. + String foreignFieldReference = + String.join( + ":", + dependentFieldName, + referenceFieldValue + ); + if (lineContext.table.name.equals(referenceTableName) && + !POSTGRES_NULL_TEXT.equals(referenceFieldValue) && + !uniqueValuesForFields.get(dependentFieldName).contains(foreignFieldReference) + ) { + errors.add( + NewGTFSError.forLine( + lineContext, + REFERENTIAL_INTEGRITY, + String.join(":", referenceField.name, foreignFieldReference) + ).setEntityId(lineContext.getEntityId()) + ); + } + return errors; + } + +} diff --git a/src/main/java/com/conveyal/gtfs/loader/conditions/ReferenceFieldShouldBeProvidedCheck.java b/src/main/java/com/conveyal/gtfs/loader/conditions/ReferenceFieldShouldBeProvidedCheck.java new file mode 100644 index 000000000..33fd55716 --- /dev/null +++ b/src/main/java/com/conveyal/gtfs/loader/conditions/ReferenceFieldShouldBeProvidedCheck.java @@ -0,0 +1,53 @@ +package com.conveyal.gtfs.loader.conditions; + +import com.conveyal.gtfs.error.NewGTFSError; +import com.conveyal.gtfs.loader.Field; +import com.conveyal.gtfs.loader.LineContext; +import com.google.common.collect.HashMultimap; + +import java.util.HashSet; +import java.util.Set; + +import static com.conveyal.gtfs.error.NewGTFSErrorType.AGENCY_ID_REQUIRED_FOR_MULTI_AGENCY_FEEDS; +import static com.conveyal.gtfs.loader.JdbcGtfsLoader.POSTGRES_NULL_TEXT; + +/** + * Conditional requirement to check that the reference field is not empty when the dependent field/table has multiple + * rows. + */ +public class ReferenceFieldShouldBeProvidedCheck extends ConditionalRequirement { + + public ReferenceFieldShouldBeProvidedCheck(String dependentFieldName) { + this.dependentFieldName = dependentFieldName; + } + + /** + * Checks that the reference field is not empty when the dependent field/table has multiple rows. This is + * principally designed for checking that routes#agency_id is filled when multiple agencies exist. + */ + public Set check( + LineContext lineContext, + Field referenceField, + HashMultimap uniqueValuesForFields + ) { + String referenceFieldValue = lineContext.getValueForRow(referenceField.name); + Set errors = new HashSet<>(); + int dependentFieldCount = uniqueValuesForFields.get(dependentFieldName).size(); + if (dependentFieldCount > 1) { + // If there are multiple entries for the dependent field (including empty strings to account for any + // potentially missing values), the reference field must not be empty. + boolean referenceFieldIsEmpty = POSTGRES_NULL_TEXT.equals(referenceFieldValue); + if (referenceFieldIsEmpty) { + errors.add( + NewGTFSError.forLine( + lineContext, + AGENCY_ID_REQUIRED_FOR_MULTI_AGENCY_FEEDS, + null + ).setEntityId(lineContext.getEntityId()) + ); + } + } + return errors; + } + +} diff --git a/src/main/java/com/conveyal/gtfs/model/Attribution.java b/src/main/java/com/conveyal/gtfs/model/Attribution.java new file mode 100644 index 000000000..deaa30be4 --- /dev/null +++ b/src/main/java/com/conveyal/gtfs/model/Attribution.java @@ -0,0 +1,119 @@ +package com.conveyal.gtfs.model; + +import com.conveyal.gtfs.GTFSFeed; + +import java.io.IOException; +import java.net.URL; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.Iterator; + +public class Attribution extends Entity { + + String attribution_id; + String agency_id; + String route_id; + String trip_id; + String organization_name; + int is_producer; + int is_operator; + int is_authority; + URL attribution_url; + String attribution_email; + String attribution_phone; + + @Override + public String getId () { + return attribution_id; + } + + /** + * Sets the parameters for a prepared statement following the parameter order defined in + * {@link com.conveyal.gtfs.loader.Table#ATTRIBUTIONS}. JDBC prepared statement parameters use a one-based index. + */ + @Override + public void setStatementParameters(PreparedStatement statement, boolean setDefaultId) throws SQLException { + int oneBasedIndex = 1; + if (!setDefaultId) statement.setInt(oneBasedIndex++, id); + statement.setString(oneBasedIndex++, attribution_id); + statement.setString(oneBasedIndex++, agency_id); + statement.setString(oneBasedIndex++, route_id); + statement.setString(oneBasedIndex++, trip_id); + statement.setString(oneBasedIndex++, organization_name); + setIntParameter(statement, oneBasedIndex++, is_producer); + setIntParameter(statement, oneBasedIndex++, is_operator); + setIntParameter(statement, oneBasedIndex++, is_authority); + statement.setString(oneBasedIndex++, attribution_url != null ? attribution_url .toString() : null); + statement.setString(oneBasedIndex++, attribution_email); + statement.setString(oneBasedIndex++, attribution_phone); + } + + public static class Loader extends Entity.Loader { + + public Loader(GTFSFeed feed) { + super(feed, "attributions"); + } + + @Override + protected boolean isRequired() { + return false; + } + + @Override + public void loadOneRow() throws IOException { + Attribution a = new Attribution(); + a.id = row + 1; // offset line number by 1 to account for 0-based row index + a.attribution_id = getStringField("attribution_id", false); + a.agency_id = getStringField("agency_id", false); + a.route_id = getStringField("route_id", true); + a.trip_id = getStringField("trip_id", false); + a.organization_name = getStringField("organization_name", true); + a.is_producer = getIntField("is_producer", false, 0, 1); + a.is_operator = getIntField("is_operator", false, 0, 1); + a.is_authority = getIntField("is_authority", false, 0, 1); + a.attribution_url = getUrlField("attribution_url", false); + a.attribution_email = getStringField("attribution_email", false); + a.attribution_phone = getStringField("attribution_phone", false); + + // TODO clooge due to not being able to have null keys in mapdb + if (a.attribution_id == null) a.attribution_id = "NONE"; + + feed.attributions.put(a.attribution_id, a); + } + } + + public static class Writer extends Entity.Writer { + public Writer (GTFSFeed feed) { + super(feed, "attribution"); + } + + @Override + protected void writeHeaders() throws IOException { + writer.writeRecord(new String[] {"attribution_id", "agency_id", "route_id", "trip_id", "organization_name", + "is_producer", "is_operator", "is_authority", "attribution_url", "attribution_email", "attribution_phone"}); + } + + @Override + protected void writeOneRow(Attribution a) throws IOException { + writeStringField(a.attribution_id); + writeStringField(a.agency_id); + writeStringField(a.route_id); + writeStringField(a.trip_id); + writeStringField(a.organization_name); + writeIntField(a.is_producer); + writeIntField(a.is_operator); + writeIntField(a.is_authority); + writeUrlField(a.attribution_url); + writeStringField(a.attribution_email); + writeStringField(a.attribution_phone); + endRecord(); + } + + @Override + protected Iterator iterator() { + return feed.attributions.values().iterator(); + } + } + + +} diff --git a/src/main/java/com/conveyal/gtfs/model/FeedInfo.java b/src/main/java/com/conveyal/gtfs/model/FeedInfo.java index 42fc61cbf..1cc94008d 100644 --- a/src/main/java/com/conveyal/gtfs/model/FeedInfo.java +++ b/src/main/java/com/conveyal/gtfs/model/FeedInfo.java @@ -23,6 +23,9 @@ public class FeedInfo extends Entity implements Cloneable { public LocalDate feed_start_date; public LocalDate feed_end_date; public String feed_version; + public String default_lang; + public String feed_contact_email; + public URL feed_contact_url; public FeedInfo clone () { try { @@ -50,6 +53,11 @@ public void setStatementParameters(PreparedStatement statement, boolean setDefau feedStartDateField.setParameter(statement, oneBasedIndex++, feed_start_date); feedEndDateField.setParameter(statement, oneBasedIndex++, feed_end_date); statement.setString(oneBasedIndex++, feed_version); + statement.setString(oneBasedIndex++, default_lang); + statement.setString(oneBasedIndex++, feed_contact_email); + String feedContactUrl = feed_contact_url != null ? feed_contact_url.toString() : null; + statement.setString(oneBasedIndex++, feedContactUrl); + } public static class Loader extends Entity.Loader { @@ -74,6 +82,9 @@ public void loadOneRow() throws IOException { fi.feed_start_date = getDateField("feed_start_date", false); fi.feed_end_date = getDateField("feed_end_date", false); fi.feed_version = getStringField("feed_version", false); + fi.default_lang = getStringField("default_lang", false); + fi.feed_contact_email = getStringField("feed_contact_email", false); + fi.feed_contact_url = getUrlField("feed_contact_url", false); fi.feed = feed; if (feed.feedInfo.isEmpty()) { feed.feedInfo.put("NONE", fi); @@ -93,7 +104,7 @@ public Writer(GTFSFeed feed) { @Override public void writeHeaders() throws IOException { writer.writeRecord(new String[] {"feed_id", "feed_publisher_name", "feed_publisher_url", "feed_lang", - "feed_start_date", "feed_end_date", "feed_version"}); + "feed_start_date", "feed_end_date", "feed_version", "default_lang", "feed_contact_email", "feed_contact_url"}); } @Override @@ -110,6 +121,9 @@ public void writeOneRow(FeedInfo i) throws IOException { else writeStringField(""); writeStringField(i.feed_version); + writeStringField(i.default_lang); + writeStringField(i.feed_contact_email); + writeUrlField(i.feed_contact_url); endRecord(); } diff --git a/src/main/java/com/conveyal/gtfs/model/PatternStop.java b/src/main/java/com/conveyal/gtfs/model/PatternStop.java index 547280c23..62f343c66 100644 --- a/src/main/java/com/conveyal/gtfs/model/PatternStop.java +++ b/src/main/java/com/conveyal/gtfs/model/PatternStop.java @@ -23,6 +23,8 @@ public class PatternStop extends Entity { public int pickup_type; public int drop_off_type; public int timepoint; + public int continuous_pickup = INT_MISSING; + public int continuous_drop_off = INT_MISSING; public PatternStop () {} diff --git a/src/main/java/com/conveyal/gtfs/model/Route.java b/src/main/java/com/conveyal/gtfs/model/Route.java index c5eb05fbb..3c669e8e8 100644 --- a/src/main/java/com/conveyal/gtfs/model/Route.java +++ b/src/main/java/com/conveyal/gtfs/model/Route.java @@ -34,6 +34,8 @@ public class Route extends Entity { // implements Entity.Factory public String route_text_color; public URL route_branding_url; public String feed_id; + public int continuous_pickup = INT_MISSING; + public int continuous_drop_off = INT_MISSING; @Override public String getId () { @@ -64,6 +66,8 @@ public void setStatementParameters(PreparedStatement statement, boolean setDefau // route_sort_order setIntParameter(statement, oneBasedIndex++, route_sort_order); setIntParameter(statement, oneBasedIndex++, 0); + setIntParameter(statement, oneBasedIndex++, continuous_pickup); + setIntParameter(statement, oneBasedIndex++, continuous_drop_off); } public static class Loader extends Entity.Loader { @@ -104,6 +108,8 @@ public void loadOneRow() throws IOException { r.route_color = getStringField("route_color", false); r.route_text_color = getStringField("route_text_color", false); r.route_branding_url = getUrlField("route_branding_url", false); + r.continuous_pickup = getIntField("continuous_pickup", true, 0, 3); + r.continuous_pickup = getIntField("continuous_drop_off", true, 0, 3); r.feed = feed; r.feed_id = feed.feedId; // Attempting to put a null key or value will cause an NPE in BTreeMap @@ -130,6 +136,8 @@ public void writeHeaders() throws IOException { writeStringField("route_text_color"); writeStringField("route_branding_url"); writeStringField("route_sort_order"); + writeStringField("continuous_pickup"); + writeStringField("continuous_drop_off"); endRecord(); } @@ -146,6 +154,8 @@ public void writeOneRow(Route r) throws IOException { writeStringField(r.route_text_color); writeUrlField(r.route_branding_url); writeIntField(r.route_sort_order); + writeIntField(r.continuous_pickup); + writeIntField(r.continuous_drop_off); endRecord(); } diff --git a/src/main/java/com/conveyal/gtfs/model/Stop.java b/src/main/java/com/conveyal/gtfs/model/Stop.java index 25a3ce82a..24923380e 100644 --- a/src/main/java/com/conveyal/gtfs/model/Stop.java +++ b/src/main/java/com/conveyal/gtfs/model/Stop.java @@ -22,9 +22,9 @@ public class Stop extends Entity { public int location_type; public String parent_station; public String stop_timezone; - // TODO should be int - public String wheelchair_boarding; + public int wheelchair_boarding; public String feed_id; + public String platform_code; @Override public String getId () { @@ -39,12 +39,6 @@ public String getId () { public void setStatementParameters(PreparedStatement statement, boolean setDefaultId) throws SQLException { int oneBasedIndex = 1; if (!setDefaultId) statement.setInt(oneBasedIndex++, id); - int wheelchairBoarding = 0; - try { - wheelchairBoarding = Integer.parseInt(wheelchair_boarding); - } catch (NumberFormatException e) { - // Do nothing, wheelchairBoarding will remain zero. - } statement.setString(oneBasedIndex++, stop_id); statement.setString(oneBasedIndex++, stop_code); statement.setString(oneBasedIndex++, stop_name); @@ -56,8 +50,8 @@ public void setStatementParameters(PreparedStatement statement, boolean setDefau setIntParameter(statement, oneBasedIndex++, location_type); statement.setString(oneBasedIndex++, parent_station); statement.setString(oneBasedIndex++, stop_timezone); - // FIXME: For some reason wheelchair boarding type is String - setIntParameter(statement, oneBasedIndex++, wheelchairBoarding); + setIntParameter(statement, oneBasedIndex++, wheelchair_boarding); + statement.setString(oneBasedIndex++, platform_code); } public static class Loader extends Entity.Loader { @@ -86,9 +80,10 @@ public void loadOneRow() throws IOException { s.location_type = getIntField("location_type", false, 0, 1); s.parent_station = getStringField("parent_station", false); s.stop_timezone = getStringField("stop_timezone", false); - s.wheelchair_boarding = getStringField("wheelchair_boarding", false); + s.wheelchair_boarding = getIntField("wheelchair_boarding", false, 0, 2); s.feed = feed; s.feed_id = feed.feedId; + s.platform_code = getStringField("platform_code", false); /* TODO check ref integrity later, this table self-references via parent_station */ // Attempting to put a null key or value will cause an NPE in BTreeMap if (s.stop_id != null) feed.stops.put(s.stop_id, s); @@ -104,7 +99,7 @@ public Writer (GTFSFeed feed) { @Override public void writeHeaders() throws IOException { writer.writeRecord(new String[] {"stop_id", "stop_code", "stop_name", "stop_desc", "stop_lat", "stop_lon", "zone_id", - "stop_url", "location_type", "parent_station", "stop_timezone", "wheelchair_boarding"}); + "stop_url", "location_type", "parent_station", "stop_timezone", "wheelchair_boarding", "platform_code"}); } @Override @@ -120,7 +115,8 @@ public void writeOneRow(Stop s) throws IOException { writeIntField(s.location_type); writeStringField(s.parent_station); writeStringField(s.stop_timezone); - writeStringField(s.wheelchair_boarding); + writeIntField(s.wheelchair_boarding); + writeStringField(s.platform_code); endRecord(); } diff --git a/src/main/java/com/conveyal/gtfs/model/StopTime.java b/src/main/java/com/conveyal/gtfs/model/StopTime.java index 02445e84f..1a227a393 100644 --- a/src/main/java/com/conveyal/gtfs/model/StopTime.java +++ b/src/main/java/com/conveyal/gtfs/model/StopTime.java @@ -27,6 +27,8 @@ public class StopTime extends Entity implements Cloneable, Serializable { public String stop_headsign; public int pickup_type; public int drop_off_type; + public int continuous_pickup = INT_MISSING; + public int continuous_drop_off = INT_MISSING; public double shape_dist_traveled = DOUBLE_MISSING; public int timepoint = INT_MISSING; @@ -56,6 +58,8 @@ public void setStatementParameters(PreparedStatement statement, boolean setDefau statement.setString(oneBasedIndex++, stop_headsign); setIntParameter(statement, oneBasedIndex++, pickup_type); setIntParameter(statement, oneBasedIndex++, drop_off_type); + setIntParameter(statement, oneBasedIndex++, continuous_pickup); + setIntParameter(statement, oneBasedIndex++, continuous_drop_off); statement.setDouble(oneBasedIndex++, shape_dist_traveled); setIntParameter(statement, oneBasedIndex++, timepoint); } @@ -85,6 +89,8 @@ public void loadOneRow() throws IOException { st.stop_headsign = getStringField("stop_headsign", false); st.pickup_type = getIntField("pickup_type", false, 0, 3); // TODO add ranges as parameters st.drop_off_type = getIntField("drop_off_type", false, 0, 3); + st.continuous_pickup = getIntField("continuous_pickup", true, 0, 3); + st.continuous_pickup = getIntField("continuous_drop_off", true, 0, 3); st.shape_dist_traveled = getDoubleField("shape_dist_traveled", false, 0D, Double.MAX_VALUE); // FIXME using both 0 and NaN for "missing", define DOUBLE_MISSING st.timepoint = getIntField("timepoint", false, 0, 1, INT_MISSING); st.feed = null; // this could circular-serialize the whole feed @@ -108,7 +114,7 @@ public Writer (GTFSFeed feed) { @Override protected void writeHeaders() throws IOException { writer.writeRecord(new String[] {"trip_id", "arrival_time", "departure_time", "stop_id", "stop_sequence", "stop_headsign", - "pickup_type", "drop_off_type", "shape_dist_traveled", "timepoint"}); + "pickup_type", "drop_off_type", "continuous_pickup", "continuous_drop_off", "shape_dist_traveled", "timepoint"}); } @Override @@ -121,6 +127,8 @@ protected void writeOneRow(StopTime st) throws IOException { writeStringField(st.stop_headsign); writeIntField(st.pickup_type); writeIntField(st.drop_off_type); + writeIntField(st.continuous_pickup); + writeIntField(st.continuous_drop_off); writeDoubleField(st.shape_dist_traveled); writeIntField(st.timepoint); endRecord(); diff --git a/src/main/java/com/conveyal/gtfs/model/Translation.java b/src/main/java/com/conveyal/gtfs/model/Translation.java new file mode 100644 index 000000000..13d9ad981 --- /dev/null +++ b/src/main/java/com/conveyal/gtfs/model/Translation.java @@ -0,0 +1,107 @@ +package com.conveyal.gtfs.model; + +import com.conveyal.gtfs.GTFSFeed; + +import java.io.IOException; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.Iterator; + +public class Translation extends Entity { + + public String table_name; + public String field_name; + public String language; + public String translation; + public String record_id; + public String record_sub_id; + public String field_value; + + @Override + public String getId() { + return createId(table_name, field_name, language); + } + + /** + * Sets the parameters for a prepared statement following the parameter order defined in + * {@link com.conveyal.gtfs.loader.Table#TRANSLATIONS}. JDBC prepared statement parameters use a one-based index. + */ + @Override + public void setStatementParameters(PreparedStatement statement, boolean setDefaultId) throws SQLException { + int oneBasedIndex = 1; + if (!setDefaultId) statement.setInt(oneBasedIndex++, id); + statement.setString(oneBasedIndex++, table_name); + statement.setString(oneBasedIndex++, field_name); + statement.setString(oneBasedIndex++, language); + statement.setString(oneBasedIndex++, translation); + statement.setString(oneBasedIndex++, record_id); + statement.setString(oneBasedIndex++, record_sub_id); + statement.setString(oneBasedIndex++, field_value); + } + + public static class Loader extends Entity.Loader { + + public Loader(GTFSFeed feed) { + super(feed, "translation"); + } + + @Override + protected boolean isRequired() { + return false; + } + + @Override + public void loadOneRow() throws IOException { + Translation t = new Translation(); + t.id = row + 1; // offset line number by 1 to account for 0-based row index + t.table_name = getStringField("table_name", true); + t.field_name = getStringField("field_name", true); + t.field_name = getStringField("language", true); + t.translation = getStringField("translation", true); + t.record_id = getStringField("record_id", false); + t.record_sub_id = getStringField("record_sub_id", false); + t.field_value = getStringField("field_value", false); + feed.translations.put( + createId(t.table_name, t.field_name, t.language), + t + ); + } + } + + public static class Writer extends Entity.Writer { + public Writer (GTFSFeed feed) { + super(feed, "translation"); + } + + @Override + protected void writeHeaders() throws IOException { + writer.writeRecord(new String[] {"table_name", "field_name", "language", "translation", "record_id", + "record_sub_id", "field_value"}); + } + + @Override + protected void writeOneRow(Translation t) throws IOException { + writeStringField(t.table_name); + writeStringField(t.field_name); + writeStringField(t.language); + writeStringField(t.translation); + writeStringField(t.record_id); + writeStringField(t.record_sub_id); + writeStringField(t.field_value); + endRecord(); + } + + @Override + protected Iterator iterator() { + return feed.translations.values().iterator(); + } + } + + /** + * Translation entries have no ID in GTFS so we define one based on the fields in the translation entry. + */ + private static String createId(String table_name, String field_name, String language) { + return String.format("%s_%s_%s", table_name, field_name, language); + } + +} diff --git a/src/main/java/com/conveyal/gtfs/validator/NewTripTimesValidator.java b/src/main/java/com/conveyal/gtfs/validator/NewTripTimesValidator.java index cf395e674..dffdc2c44 100644 --- a/src/main/java/com/conveyal/gtfs/validator/NewTripTimesValidator.java +++ b/src/main/java/com/conveyal/gtfs/validator/NewTripTimesValidator.java @@ -4,18 +4,21 @@ import com.conveyal.gtfs.loader.Feed; import com.conveyal.gtfs.model.Entity; import com.conveyal.gtfs.model.Route; -import com.conveyal.gtfs.model.ShapePoint; import com.conveyal.gtfs.model.Stop; import com.conveyal.gtfs.model.StopTime; import com.conveyal.gtfs.model.Trip; -import com.google.common.collect.ListMultimap; -import com.google.common.collect.MultimapBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; -import static com.conveyal.gtfs.error.NewGTFSErrorType.*; +import static com.conveyal.gtfs.error.NewGTFSErrorType.CONDITIONALLY_REQUIRED; +import static com.conveyal.gtfs.error.NewGTFSErrorType.MISSING_ARRIVAL_OR_DEPARTURE; +import static com.conveyal.gtfs.error.NewGTFSErrorType.TRIP_TOO_FEW_STOP_TIMES; /** * Check that the travel times between adjacent stops in trips are reasonable. @@ -114,6 +117,8 @@ private boolean fixInitialFinal (StopTime stopTime) { registerError(stopTime, MISSING_ARRIVAL_OR_DEPARTURE); fixMissingTimes(stopTime); if (missingEitherTime(stopTime)) { + //TODO: Is this even needed? Already covered by MISSING_ARRIVAL_OR_DEPARTURE. + registerError(stopTime, CONDITIONALLY_REQUIRED, "First and last stop times are required to have both an arrival and departure time."); return true; } } @@ -135,17 +140,22 @@ private void processTrip (List stopTimes) { // This error should already have been caught TODO verify. return; } + // Our code should only call this method with non-null stopTimes. if (stopTimes.size() < 2) { registerError(trip, TRIP_TOO_FEW_STOP_TIMES); return; } + boolean hasContinuousBehavior = false; // Make a parallel list of stops based on the stop_times for this trip. // We will remove any stop_times for stops that don't exist in the feed. // We could ask the SQL server to do the join between stop_times and stops, but we want to check references. List stops = new ArrayList<>(); for (Iterator it = stopTimes.iterator(); it.hasNext(); ) { StopTime stopTime = it.next(); + if (hasContinuousBehavior(stopTime.continuous_drop_off, stopTime.continuous_pickup)) { + hasContinuousBehavior = true; + } Stop stop = stopById.get(stopTime.stop_id); if (stop == null) { // All bad references should have been recorded at import, we can just remove them from the trips. @@ -166,6 +176,18 @@ private void processTrip (List stopTimes) { // All bad references should have been recorded at import and null trip check is handled above, we can just // ignore nulls. Route route = routeById.get(trip.route_id); + if (route != null && + hasContinuousBehavior(route.continuous_drop_off, route.continuous_pickup)) { + hasContinuousBehavior = true; + } + + if (trip.shape_id == null && hasContinuousBehavior) { + registerError( + trip, + CONDITIONALLY_REQUIRED, + "shape_id is required when a trip has continuous behavior defined." + ); + } // Pass these same cleaned lists of stop_times and stops into each trip validator in turn. for (TripValidator tripValidator : tripValidators) tripValidator.validateTrip(trip, route, stopTimes, stops); } @@ -181,4 +203,17 @@ public void complete (ValidationResult validationResult) { } } + /** + * Determine if a trip has continuous behaviour by checking the values that have been defined for continuous drop + * off and pickup. + */ + private boolean hasContinuousBehavior(int continuousDropOff, int continuousPickup) { + return + continuousDropOff == 0 || + continuousDropOff == 2 || + continuousDropOff == 3 || + continuousPickup == 0 || + continuousPickup == 2 || + continuousPickup == 3; + } } diff --git a/src/main/java/com/conveyal/gtfs/validator/PatternFinderValidator.java b/src/main/java/com/conveyal/gtfs/validator/PatternFinderValidator.java index 9584b9e61..f24ac1c44 100644 --- a/src/main/java/com/conveyal/gtfs/validator/PatternFinderValidator.java +++ b/src/main/java/com/conveyal/gtfs/validator/PatternFinderValidator.java @@ -158,6 +158,8 @@ public void complete(ValidationResult validationResult) { setIntParameter(insertPatternStopStatement,7, key.pickupTypes.get(i)); setDoubleParameter(insertPatternStopStatement, 8, key.shapeDistances.get(i)); setIntParameter(insertPatternStopStatement,9, key.timepoints.get(i)); + setIntParameter(insertPatternStopStatement,10, key.continuous_pickup.get(i)); + setIntParameter(insertPatternStopStatement,11, key.continuous_drop_off.get(i)); patternStopTracker.addBatch(); } // Finally, update all trips on this pattern to reference this pattern's ID. diff --git a/src/test/java/com/conveyal/gtfs/GTFSTest.java b/src/test/java/com/conveyal/gtfs/GTFSTest.java index 87a2a60d1..dca6a72bc 100644 --- a/src/test/java/com/conveyal/gtfs/GTFSTest.java +++ b/src/test/java/com/conveyal/gtfs/GTFSTest.java @@ -224,6 +224,8 @@ public void canLoadAndExportSimpleAgencyInSubDirectory() { new ErrorExpectation(NewGTFSErrorType.TABLE_IN_SUBDIRECTORY), new ErrorExpectation(NewGTFSErrorType.TABLE_IN_SUBDIRECTORY), new ErrorExpectation(NewGTFSErrorType.TABLE_IN_SUBDIRECTORY), + new ErrorExpectation(NewGTFSErrorType.TABLE_IN_SUBDIRECTORY), + new ErrorExpectation(NewGTFSErrorType.TABLE_IN_SUBDIRECTORY), new ErrorExpectation(NewGTFSErrorType.ROUTE_LONG_NAME_CONTAINS_SHORT_NAME), new ErrorExpectation(NewGTFSErrorType.FEED_TRAVEL_TIMES_ROUNDED), new ErrorExpectation(NewGTFSErrorType.STOP_UNUSED), @@ -560,8 +562,8 @@ private boolean runIntegrationTestOnZipFile( ErrorExpectation[] errorExpectations, FeedValidatorCreator... customValidators ) { - String newDBName = TestUtils.generateNewDB(); - String dbConnectionUrl = String.join("/", JDBC_URL, newDBName); + String testDBName = TestUtils.generateNewDB(); + String dbConnectionUrl = String.join("/", JDBC_URL, testDBName); DataSource dataSource = TestUtils.createTestDataSource(dbConnectionUrl); String namespace; diff --git a/src/test/java/com/conveyal/gtfs/dto/FeedInfoDTO.java b/src/test/java/com/conveyal/gtfs/dto/FeedInfoDTO.java index fb3feac2b..52ede95c3 100644 --- a/src/test/java/com/conveyal/gtfs/dto/FeedInfoDTO.java +++ b/src/test/java/com/conveyal/gtfs/dto/FeedInfoDTO.java @@ -14,4 +14,7 @@ public class FeedInfoDTO { public String feed_version; public String default_route_color; public String default_route_type; + public String default_lang; + public String feed_contact_email; + public String feed_contact_url; } diff --git a/src/test/java/com/conveyal/gtfs/dto/PatternStopDTO.java b/src/test/java/com/conveyal/gtfs/dto/PatternStopDTO.java index 12888f4d6..f61b68cb5 100644 --- a/src/test/java/com/conveyal/gtfs/dto/PatternStopDTO.java +++ b/src/test/java/com/conveyal/gtfs/dto/PatternStopDTO.java @@ -11,6 +11,8 @@ public class PatternStopDTO { public Integer pickup_type; public Integer stop_sequence; public Integer timepoint; + public Integer continuous_pickup; + public Integer continuous_drop_off; /** Empty constructor for deserialization */ public PatternStopDTO() {} diff --git a/src/test/java/com/conveyal/gtfs/dto/RouteDTO.java b/src/test/java/com/conveyal/gtfs/dto/RouteDTO.java index 0ea408213..ec15aca45 100644 --- a/src/test/java/com/conveyal/gtfs/dto/RouteDTO.java +++ b/src/test/java/com/conveyal/gtfs/dto/RouteDTO.java @@ -24,4 +24,6 @@ public class RouteDTO { /** This field is incorrectly set to String in order to test how empty string literals are persisted to the database. */ public String route_sort_order; public Integer status; + public int continuous_pickup; + public int continuous_drop_off; } diff --git a/src/test/java/com/conveyal/gtfs/dto/StopDTO.java b/src/test/java/com/conveyal/gtfs/dto/StopDTO.java index 21ec9e4fa..66a333863 100644 --- a/src/test/java/com/conveyal/gtfs/dto/StopDTO.java +++ b/src/test/java/com/conveyal/gtfs/dto/StopDTO.java @@ -14,4 +14,5 @@ public class StopDTO { public String parent_station; public Integer location_type; public Integer wheelchair_boarding; + public String platform_code; } diff --git a/src/test/java/com/conveyal/gtfs/dto/StopTimeDTO.java b/src/test/java/com/conveyal/gtfs/dto/StopTimeDTO.java index 9e385e3ca..7941c68ee 100644 --- a/src/test/java/com/conveyal/gtfs/dto/StopTimeDTO.java +++ b/src/test/java/com/conveyal/gtfs/dto/StopTimeDTO.java @@ -11,6 +11,8 @@ public class StopTimeDTO { public Integer drop_off_type; public Integer pickup_type; public Double shape_dist_traveled; + public int continuous_pickup; + public int continuous_drop_off; /** * Empty constructor for deserialization diff --git a/src/test/java/com/conveyal/gtfs/graphql/GTFSGraphQLTest.java b/src/test/java/com/conveyal/gtfs/graphql/GTFSGraphQLTest.java index 8a9227003..240683306 100644 --- a/src/test/java/com/conveyal/gtfs/graphql/GTFSGraphQLTest.java +++ b/src/test/java/com/conveyal/gtfs/graphql/GTFSGraphQLTest.java @@ -148,6 +148,14 @@ public void canFetchAgencies() { }); } + /** Tests that the attributions of a feed can be fetched. */ + @Test + public void canFetchATtributions() { + assertTimeout(Duration.ofMillis(TEST_TIMEOUT), () -> { + MatcherAssert.assertThat(queryGraphQL("feedAttributions.txt"), matchesSnapshot()); + }); + } + /** Tests that the calendars of a feed can be fetched. */ @Test public void canFetchCalendars() { @@ -196,6 +204,14 @@ public void canFetchTrips() { }); } + /** Tests that the translations of a feed can be fetched. */ + @Test + public void canFetchTranslations() { + assertTimeout(Duration.ofMillis(TEST_TIMEOUT), () -> { + MatcherAssert.assertThat(queryGraphQL("feedTranslations.txt"), matchesSnapshot()); + }); + } + // TODO: make tests for schedule_exceptions / calendar_dates /** Tests that the stop times of a feed can be fetched. */ diff --git a/src/test/java/com/conveyal/gtfs/loader/ConditionallyRequiredTest.java b/src/test/java/com/conveyal/gtfs/loader/ConditionallyRequiredTest.java new file mode 100644 index 000000000..530f4499e --- /dev/null +++ b/src/test/java/com/conveyal/gtfs/loader/ConditionallyRequiredTest.java @@ -0,0 +1,160 @@ +package com.conveyal.gtfs.loader; + +import com.conveyal.gtfs.TestUtils; +import com.conveyal.gtfs.error.NewGTFSErrorType; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import javax.sql.DataSource; +import java.io.IOException; +import java.util.stream.Stream; + +import static com.conveyal.gtfs.GTFS.load; +import static com.conveyal.gtfs.GTFS.validate; +import static com.conveyal.gtfs.TestUtils.assertThatSqlCountQueryYieldsExpectedCount; +import static com.conveyal.gtfs.error.NewGTFSErrorType.CONDITIONALLY_REQUIRED; +import static com.conveyal.gtfs.error.NewGTFSErrorType.REFERENTIAL_INTEGRITY; +import static com.conveyal.gtfs.error.NewGTFSErrorType.AGENCY_ID_REQUIRED_FOR_MULTI_AGENCY_FEEDS; + +public class ConditionallyRequiredTest { + private static String testDBName; + private static DataSource testDataSource; + private static String testNamespace; + + @BeforeAll + public static void setUpClass() throws IOException { + // Create a new database + testDBName = TestUtils.generateNewDB(); + String dbConnectionUrl = String.format("jdbc:postgresql://localhost/%s", testDBName); + testDataSource = TestUtils.createTestDataSource(dbConnectionUrl); + // load feed into db + String zipFileName = TestUtils.zipFolderFiles( + "real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks", + true); + FeedLoadResult feedLoadResult = load(zipFileName, testDataSource); + testNamespace = feedLoadResult.uniqueIdentifier; + validate(testNamespace, testDataSource); + } + + @AfterAll + public static void tearDownClass() { + TestUtils.dropDB(testDBName); + } + + @Test + public void stopTimeTableMissingConditionallyRequiredArrivalDepartureTimes() { + checkFeedHasOneError( + CONDITIONALLY_REQUIRED, + "StopTime", + "10", + "1", + "First and last stop times are required to have both an arrival and departure time." + ); + } + + @ParameterizedTest + @MethodSource("createStopTableChecks") + public void stopTableConditionallyRequiredTests( + NewGTFSErrorType errorType, + String entityType, + String lineNumber, + String entityId, + String badValue + ) { + checkFeedHasOneError(errorType, entityType, lineNumber, entityId, badValue); + } + + private static Stream createStopTableChecks() { + return Stream.of( + Arguments.of(CONDITIONALLY_REQUIRED, "Stop", "2", "4957", "stop_name is required when location_type value is between 0 and 2."), + Arguments.of(CONDITIONALLY_REQUIRED, "Stop", "5", "1266", "parent_station is required when location_type value is between 2 and 4."), + Arguments.of(CONDITIONALLY_REQUIRED, "Stop", "3", "691", "stop_lat is required when location_type value is between 0 and 2."), + Arguments.of(CONDITIONALLY_REQUIRED, "Stop", "4", "692", "stop_lon is required when location_type value is between 0 and 2."), + Arguments.of(REFERENTIAL_INTEGRITY, "FareRule", "3", "1", "contains_id:zone_id:4"), + Arguments.of(REFERENTIAL_INTEGRITY, "FareRule", "3", "1", "destination_id:zone_id:3"), + Arguments.of(REFERENTIAL_INTEGRITY, "FareRule", "3", "1", "origin_id:zone_id:2") + ); + } + + @ParameterizedTest + @MethodSource("createTranslationTableChecks") + public void translationTableConditionallyRequiredTests( + String entityType, + String lineNumber, + String entityId, + String badValue + ) { + checkFeedHasOneError(CONDITIONALLY_REQUIRED, entityType, lineNumber, entityId, badValue); + } + + private static Stream createTranslationTableChecks() { + return Stream.of( + Arguments.of("Translation", "2", "stops", "record_id is required when field_value is empty."), + Arguments.of("Translation", "3", "stops", "field_value is required when record_id is empty."), + Arguments.of("Translation", "4", "stops", "record_sub_id is required and must match stop_times when record_id is provided.") + ); + } + + @Test + public void agencyTableMissingConditionallyRequiredAgencyId() { + checkFeedHasOneError( + AGENCY_ID_REQUIRED_FOR_MULTI_AGENCY_FEEDS, + "Agency", + "2", + null, + "agency_id"); + } + + @Test + public void tripTableMissingConditionallyRequiredShapeId() { + checkFeedHasOneError( + CONDITIONALLY_REQUIRED, + "Trip", + "2", + "1", + "shape_id is required when a trip has continuous behavior defined." + ); + } + + @Test + public void routeTableMissingConditionallyRequiredAgencyId() { + checkFeedHasOneError( + AGENCY_ID_REQUIRED_FOR_MULTI_AGENCY_FEEDS, + "Route", + "2", + "21", + null + ); + } + + @Test + public void fareAttributeTableMissingConditionallyRequiredAgencyId() { + checkFeedHasOneError( + AGENCY_ID_REQUIRED_FOR_MULTI_AGENCY_FEEDS, + "FareAttribute", + "2", + "1", + null + ); + } + + /** + * Check that the test feed has exactly one error for the provided values. + */ + private void checkFeedHasOneError(NewGTFSErrorType errorType, String entityType, String lineNumber, String entityId, String badValue) { + String sql = String.format("select count(*) from %s.errors where error_type = '%s' and entity_type = '%s' and line_number = '%s'", + testNamespace, + errorType, + entityType, + lineNumber); + + if (entityId != null) sql += String.format(" and entity_id = '%s'", entityId); + if (badValue != null) sql += String.format(" and bad_value = '%s'", badValue); + + assertThatSqlCountQueryYieldsExpectedCount(testDataSource, sql,1); + } +} diff --git a/src/test/java/com/conveyal/gtfs/loader/JDBCTableWriterTest.java b/src/test/java/com/conveyal/gtfs/loader/JDBCTableWriterTest.java index 660c00798..c9f396301 100644 --- a/src/test/java/com/conveyal/gtfs/loader/JDBCTableWriterTest.java +++ b/src/test/java/com/conveyal/gtfs/loader/JDBCTableWriterTest.java @@ -183,8 +183,14 @@ public void canPreventSQLInjection() throws IOException, SQLException, InvalidNa feedInfoInput.feed_publisher_name = publisherName; feedInfoInput.feed_publisher_url = "example.com"; feedInfoInput.feed_lang = "en"; + feedInfoInput.feed_start_date = "07052021"; + feedInfoInput.feed_end_date = "09052021"; + feedInfoInput.feed_lang = "en"; feedInfoInput.default_route_color = "1c8edb"; feedInfoInput.default_route_type = "3"; + feedInfoInput.default_lang = "en"; + feedInfoInput.feed_contact_email = "a@b.com"; + feedInfoInput.feed_contact_url = "example.com"; // convert object to json and save it JdbcTableWriter createTableWriter = createTestTableWriter(Table.FEED_INFO); @@ -416,11 +422,19 @@ public void canCreateUpdateAndDeleteScheduleExceptions() throws IOException, SQL } /** - * This test verifies that stop_times#shape_dist_traveled (and other "linked fields") are updated when a pattern + * This test verifies that stop_times#shape_dist_traveled and other linked fields are updated when a pattern * is updated. */ @Test - public void shouldUpdateStopTimeShapeDistTraveledOnPatternStopUpdate() throws IOException, SQLException, InvalidNamespaceException { + public void shouldUpdateStopTimeOnPatternStopUpdate() throws IOException, SQLException, InvalidNamespaceException { + final String[] STOP_TIMES_LINKED_FIELDS = new String[] { + "shape_dist_traveled", + "timepoint", + "drop_off_type", + "pickup_type", + "continuous_pickup", + "continuous_drop_off" + }; String routeId = newUUID(); String patternId = newUUID(); int startTime = 6 * 60 * 60; // 6 AM @@ -453,10 +467,12 @@ public void shouldUpdateStopTimeShapeDistTraveledOnPatternStopUpdate() throws IO assertNotNull(uuid); // Check that trip exists. assertThatSqlQueryYieldsRowCount(getColumnsForId(createdTrip.id, Table.TRIPS), 1); - // Check the stop_time's initial shape_dist_traveled value. TODO test that other linked fields are updated? + + // Check the stop_time's initial shape_dist_traveled value and other linked fields. PreparedStatement statement = testDataSource.getConnection().prepareStatement( String.format( - "select shape_dist_traveled from %s.stop_times where stop_sequence=1 and trip_id='%s'", + "select %s from %s.stop_times where stop_sequence=1 and trip_id='%s'", + String.join(", ", STOP_TIMES_LINKED_FIELDS), testNamespace, createdTrip.trip_id ) @@ -465,11 +481,22 @@ public void shouldUpdateStopTimeShapeDistTraveledOnPatternStopUpdate() throws IO ResultSet resultSet = statement.executeQuery(); while (resultSet.next()) { // First stop_time shape_dist_traveled should be zero. - assertThat(resultSet.getInt(1), equalTo(0)); + // Other linked fields should be interpreted as zero too. + for (int i = 1; i <= STOP_TIMES_LINKED_FIELDS.length; i++) { + assertThat(resultSet.getInt(i), equalTo(0)); + } } + // Update pattern_stop#shape_dist_traveled and check that the stop_time's shape_dist value is updated. final double updatedShapeDistTraveled = 45.5; - pattern.pattern_stops[1].shape_dist_traveled = updatedShapeDistTraveled; + PatternStopDTO pattern_stop = pattern.pattern_stops[1]; + pattern_stop.shape_dist_traveled = updatedShapeDistTraveled; + // Assign an arbitrary value (the order of appearance in STOP_TIMES_LINKED_FIELDS) for the other linked fields. + pattern_stop.timepoint = 2; + pattern_stop.drop_off_type = 3; + pattern_stop.pickup_type = 4; + pattern_stop.continuous_pickup = 5; + pattern_stop.continuous_drop_off = 6; JdbcTableWriter patternUpdater = createTestTableWriter(Table.PATTERNS); String updatedPatternOutput = patternUpdater.update(pattern.id, mapper.writeValueAsString(pattern), true); LOG.info("Updated pattern: {}", updatedPatternOutput); @@ -477,6 +504,11 @@ public void shouldUpdateStopTimeShapeDistTraveledOnPatternStopUpdate() throws IO while (resultSet2.next()) { // First stop_time shape_dist_traveled should be updated. assertThat(resultSet2.getDouble(1), equalTo(updatedShapeDistTraveled)); + + // Other linked fields should be as set above. + for (int i = 2; i <= STOP_TIMES_LINKED_FIELDS.length; i++) { + assertThat(resultSet2.getInt(i), equalTo(i)); + } } } diff --git a/src/test/resources/fake-agency/attributions.txt b/src/test/resources/fake-agency/attributions.txt new file mode 100644 index 000000000..1d77168ac --- /dev/null +++ b/src/test/resources/fake-agency/attributions.txt @@ -0,0 +1,2 @@ +attribution_id,agency_id,route_id,trip_id,organization_name,is_producer,is_operator,is_authority,attribution_url,attribution_email,attribution_phone +1,1,,,Fake Transit,1,,,https://www.faketransit.org,customer.service@faketransit.org, \ No newline at end of file diff --git a/src/test/resources/fake-agency/translations.txt b/src/test/resources/fake-agency/translations.txt new file mode 100644 index 000000000..da8fe2407 --- /dev/null +++ b/src/test/resources/fake-agency/translations.txt @@ -0,0 +1,2 @@ +table_name,field_name,language,translation,record_id,record_sub_id,field_value +stops,stop_desc,FR,en direction du nord,4u6g,, \ No newline at end of file diff --git a/src/test/resources/graphql/feedAttributions.txt b/src/test/resources/graphql/feedAttributions.txt new file mode 100644 index 000000000..6972e2757 --- /dev/null +++ b/src/test/resources/graphql/feedAttributions.txt @@ -0,0 +1,18 @@ +query ($namespace: String) { + feed(namespace: $namespace) { + feed_version + attributions { + attribution_id + agency_id + route_id + trip_id + organization_name + is_producer + is_operator + is_authority + attribution_url + attribution_email + attribution_phone + } + } +} \ No newline at end of file diff --git a/src/test/resources/graphql/feedTranslations.txt b/src/test/resources/graphql/feedTranslations.txt new file mode 100644 index 000000000..4933e2e12 --- /dev/null +++ b/src/test/resources/graphql/feedTranslations.txt @@ -0,0 +1,14 @@ +query ($namespace: String) { + feed(namespace: $namespace) { + feed_version + translations { + table_name + field_name + language + translation + record_id + record_sub_id + field_value + } + } +} \ No newline at end of file diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/agency.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/agency.txt new file mode 100644 index 000000000..71811c1f9 --- /dev/null +++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/agency.txt @@ -0,0 +1,5 @@ +agency_id,agency_name,agency_url,agency_timezone,agency_lang,agency_phone,agency_fare_url,agency_email +,VTA,https://www.vta.org,America/Los_Angeles,EN,408-321-2300,https://www.vta.org/go/fares,customer.service@vta.org +VTA2,VTA2,https://www.vta.org,America/Los_Angeles,EN,408-321-2300,https://www.vta.org/go/fares,customer.service@vta.org +VTA3,VTA3,https://www.vta.org,America/Los_Angeles,EN,408-321-2300,https://www.vta.org/go/fares,customer.service@vta.org + diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/attributions.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/attributions.txt new file mode 100644 index 000000000..42a18d2c5 --- /dev/null +++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/attributions.txt @@ -0,0 +1,2 @@ +attribution_id,agency_id,route_id,trip_id,organization_name,is_producer,is_operator,is_authority,attribution_url,attribution_email,attribution_phone +1,1,,,VTA,1,,,https://www.vta.org,customer.service@vta.org, \ No newline at end of file diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/calendar.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/calendar.txt new file mode 100644 index 000000000..6077757f1 --- /dev/null +++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/calendar.txt @@ -0,0 +1,2 @@ +service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date +1,1,1,1,1,1,0,0,20210208,20210611 \ No newline at end of file diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/calendar_attributes.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/calendar_attributes.txt new file mode 100644 index 000000000..a59162bca --- /dev/null +++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/calendar_attributes.txt @@ -0,0 +1,2 @@ +service_id,service_description +1,Weekday \ No newline at end of file diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/calendar_dates.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/calendar_dates.txt new file mode 100644 index 000000000..bf2f0150e --- /dev/null +++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/calendar_dates.txt @@ -0,0 +1,2 @@ +service_id,date,exception_type +1,20210531,2 diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/directions.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/directions.txt new file mode 100644 index 000000000..666d39239 --- /dev/null +++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/directions.txt @@ -0,0 +1,5 @@ +route_id,direction_id,direction +21,0,East +21,1,West +22,0,East +22,1,West \ No newline at end of file diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/fare_attributes.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/fare_attributes.txt new file mode 100644 index 000000000..5ccbf0e80 --- /dev/null +++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/fare_attributes.txt @@ -0,0 +1,2 @@ +fare_id,price,currency_type,payment_method,transfers,agency_id,transfer_duration +1,2.50000000,USD,0,,, \ No newline at end of file diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/fare_rules.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/fare_rules.txt new file mode 100644 index 000000000..5591a375d --- /dev/null +++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/fare_rules.txt @@ -0,0 +1,3 @@ +fare_id,route_id,origin_id,destination_id,contains_id +1,21,1,, +1,22,2,3,4 \ No newline at end of file diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/feed_info.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/feed_info.txt new file mode 100644 index 000000000..c59ff6d3e --- /dev/null +++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/feed_info.txt @@ -0,0 +1,2 @@ +feed_publisher_name,feed_publisher_url,feed_lang,feed_start_date,feed_end_date,feed_version,feed_contact_email,feed_contact_url +Santa Clara Valley Transportation Authority,https://www.vta.org,EN,20210208,20210613,2021-02-16_15:11,customer.service@vta.org,https://www.vta.org/about/contact diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/realtime_routes.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/realtime_routes.txt new file mode 100644 index 000000000..27b0949c5 --- /dev/null +++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/realtime_routes.txt @@ -0,0 +1,3 @@ +route_id,realtime_enabled,realtime_routename,realtime_routecode +21,1,, +22,1,, \ No newline at end of file diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/routes.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/routes.txt new file mode 100644 index 000000000..250682bcd --- /dev/null +++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/routes.txt @@ -0,0 +1,2 @@ +route_id,agency_id,route_short_name,route_long_name,route_desc,route_type,route_url,route_color,route_text_color,route_sort_order,ext_route_type,continuous_pickup,continuous_drop_off +21,,21,Stanford Shopping Center - Santa Clara Transit Center,,3,https://www.vta.org/go/routes/21,29588c,FFFFFF,21,704,, \ No newline at end of file diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/stop_times.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/stop_times.txt new file mode 100644 index 000000000..84e00fde9 --- /dev/null +++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/stop_times.txt @@ -0,0 +1,26 @@ +trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,continuous_pickup,continuous_drop_off,shape_dist_traveled,timepoint +1,05:43:00,05:43:00,4957,1,,0,0,0,,,1 +1,05:44:00,05:44:00,691,2,,0,0,0,,0.58999997,0 +1,05:45:00,05:45:00,692,3,,0,0,0,,0.96569997,0 +1,05:46:00,05:46:00,1266,4,,0,0,0,,1.14470005,0 +1,05:47:00,05:47:00,1267,5,,0,0,0,,1.92729998,0 +1,05:48:00,05:48:00,1268,6,,0,0,0,,2.28160000,0 +1,05:49:00,05:49:00,1542,7,,0,0,0,,2.72530007,0 +1,05:50:00,05:50:00,1543,8,,0,0,0,,3.23429990,0 +1,,,1544,9,,0,0,0,,3.59170008,1 +2,05:52:00,05:52:00,1545,1,,0,0,,,3.88010001,0 +2,,,1546,2,,0,0,,,4.32210016,0 +2,,,1547,3,,0,0,,,4.82560015,0 +2,05:55:00,05:55:00,1548,4,,0,0,,,5.09070015,0 +3,05:55:00,05:55:00,1550,1,,0,0,,,5.59749985,0 +3,05:56:00,05:56:00,1562,2,,0,0,,,7.09219980,1 +4,05:55:00,05:55:00,1550,1,,0,0,,,5.59749985,0 +4,,,1558,2,,0,0,,,8.34879971,0 +4,,,1559,3,,0,0,,,8.68850040,0 +4,05:56:00,05:56:00,1562,4,,0,0,,,5.59749985,0 +5,00:00:00,00:00:00,4957,1,,0,0,,,,1 +5,01:00:00,01:00:00,1558,2,,0,0,,,8.34879971,0 +5,23:59:00,23:59:00,1562,3,,0,0,,,9.39290047,0 +6,00:00:00,00:00:00,4957,1,,0,0,,,,1 +6,,,1558,2,,0,0,,,8.34879971,0 +6,23:59:00,23:59:00,1562,3,,0,0,,,9.39290047,0 \ No newline at end of file diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/stops.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/stops.txt new file mode 100644 index 000000000..ecbf5ee72 --- /dev/null +++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/stops.txt @@ -0,0 +1,28 @@ +stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon,zone_id,stop_url,location_type,parent_station,wheelchair_boarding,platform_code,sign_dest +4957,64957,,Southbound,37.40048600,-122.10892700,1,,0,1,1,, +691,60691,San Antonio & El Camino,Northbound,,-122.11319800,,,0,,1,, +692,60692,San Antonio & Miller,Northbound,37.40462900,,,,0,,1,, +1266,61266,San Antonio & California,Northbound,37.40607000,-122.11050500,,,3,,1,, +1267,61267,San Antonio & Nita,Northbound,37.41186000,-122.10592500,,,0,,1,, +1268,61268,San Antonio & Nita,Northbound,37.41461600,-122.10392500,,,0,,1,, +1542,61542,Middlefield & Montrose,Northbound,37.41752400,-122.10575900,,,0,,1,, +1543,61543,Middlefield & Charleston,Westbound,37.42026600,-122.11023800,,,0,,1,, +1544,61544,Middlefield & Mayview,Westbound,37.42291600,-122.11254400,,,0,,1,, +1545,61545,Middlefield & Meadow,Westbound,37.42452600,-122.11510500,,,0,,1,, +1546,61546,Middlefield & Ames,Westbound,37.42700600,-122.11903200,,,0,,1,, +1547,61547,Middlefield & Layne,Westbound,37.42981800,-122.12349600,,,0,,1,, +1548,61548,Middlefield & Matadero,Westbound,37.43129900,-122.12585600,,,0,,1,, +1550,61550,Middlefield & Moreno,Westbound,37.43412800,-122.13037200,,,0,,1,, +1551,61551,Middlefield & California,Westbound,37.43745600,-122.13566600,,,0,,1,, +1552,61552,Middlefield & Seale,Westbound,37.43992600,-122.13953900,,,0,,2,, +1553,61553,Middlefield & Embarcadero,Westbound,37.44248900,-122.14365200,,,0,,1,, +1554,61554,Middlefield & Melville,Westbound,37.44458500,-122.14696700,,,0,,1,, +1555,61555,Middlefield & Kingsley,Westbound,37.44538300,-122.14822500,,,0,,1,, +1556,61556,Middlefield & Addison,Westbound,37.44694800,-122.15071600,,,0,,1,, +1557,61557,Middlefield & Channing,Westbound,37.44775900,-122.15199200,,,0,,1,, +1558,61558,Homer & Webster,Southbound,37.44699700,-122.15448100,,,0,,1,, +1559,61559,Waverly & Homer,Westbound,37.44492200,-122.15685900,,,0,,1,, +1560,61560,Hamilton & Waverly,Southbound,37.44612400,-122.15941900,,,0,,1,, +1561,61561,Hamilton & Ramona,Southbound,37.44471500,-122.16085800,,,0,,1,, +1562,61562,Hamilton & High,Southbound,37.44318800,-122.16232000,,,0,,1,, +1563,61563,Palo Alto Transit Center (Bay 4),,37.44414700,-122.16663100,,,0,PS_PATC,1,4, \ No newline at end of file diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/translations.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/translations.txt new file mode 100644 index 000000000..560db0832 --- /dev/null +++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/translations.txt @@ -0,0 +1,4 @@ +table_name,field_name,language,translation,record_id,record_sub_id,field_value +stops,stop_desc,FR,vers le sud,,, +stops,stop_desc,FR,en direction du nord,,, +stops,stop_desc,FR,en direction du nord,stop_times,, \ No newline at end of file diff --git a/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/trips.txt b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/trips.txt new file mode 100644 index 000000000..eac861720 --- /dev/null +++ b/src/test/resources/real-world-gtfs-feeds/VTA-gtfs-conditionally-required-checks/trips.txt @@ -0,0 +1,7 @@ +route_id,service_id,trip_id,trip_headsign,trip_short_name,direction_id,block_id,shape_id,wheelchair_accessible,bikes_allowed +21,1,1,PALO ALTO TRANSIT CTR 1,,1,2145,,0,0 +21,1,2,PALO ALTO TRANSIT CTR 2,,1,2145,101395,0,0 +22,1,3,PALO ALTO TRANSIT CTR 3,,1,2145,101395,0,0 +23,1,4,PALO ALTO TRANSIT CTR 4,,1,2145,101395,0,0 +23,1,5,PALO ALTO TRANSIT CTR 5,,1,2145,101395,0,0 +23,1,6,PALO ALTO TRANSIT CTR 6,,1,2145,101395,0,0 diff --git a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchAttributions-0.json b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchAttributions-0.json new file mode 100644 index 000000000..3092058c5 --- /dev/null +++ b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchAttributions-0.json @@ -0,0 +1,20 @@ +{ + "data" : { + "feed" : { + "attributions" : [ { + "attribution_id" : 1, + "agency_id" : "1", + "route_id" : null, + "trip_id" : null, + "organization_name" : "Fake Transit", + "is_producer" : 1, + "is_operator" : null, + "is_authority" : null, + "attribution_url" : "https://www.faketransit.org", + "attribution_email" : "customer.service@faketransit.org", + "attribution_phone" : null + } ], + "feed_version" : "1.0" + } + } +} \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchTranslations-0.json b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchTranslations-0.json new file mode 100644 index 000000000..1bae5c5e1 --- /dev/null +++ b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchTranslations-0.json @@ -0,0 +1,16 @@ +{ + "data" : { + "feed" : { + "translations" : [ { + "table_name" : "stops", + "field_name" : "stop_desc", + "language" : "FR", + "translation" : "en direction du nord", + "record_id" : "4u6g", + "record_sub_id" : null, + "field_value" : null + } ], + "feed_version" : "1.0" + } + } +} \ No newline at end of file