Merge pull request #133 from ibi-group/dev

Merge IBI dev branch into catalogueglobal dev

.github/CODEOWNERS

@@ -0,0 +1,4 @@
+# See https://help.github.com/articles/about-codeowners/
+
+# A Conveyal employee is required to approve PR merges
+* @conveyal/employees

.github/issue_template.md

@@ -0,0 +1,21 @@
+_**NOTE:** This issue system is intended for reporting bugs and tracking progress in software development. Although this software is licensed with an open-source license, any issue opened here may not be responded to in a timely manner. [Conveyal](https://www.conveyal.com) is unable to provide technical support for custom deployments of this software unless your company has a support contract with us. Please remove this note when creating the issue._
+
+## Observed behavior
+
+Please explain what is being observed within the application here.
+
+## Expected behavior
+
+Please explain what should happen instead.
+
+## Steps to reproduce the problem
+
+Please be as specific as possible.
+
+## Any special notes on configuration used
+
+Please describe any applicable config files that were used
+
+## Version of datatools-server and datatools-ui if applicable (exact commit hash or branch name)
+
+If using this in conjunction with [datatools-ui](https://github.com/conveyal/datatools-ui), this info can be found by clicking on the gear icon on the sidebar.

.github/pull_request_template.md

@@ -0,0 +1,14 @@
+### Checklist
+
+- [ ] Appropriate branch selected _(all PRs must first be merged to `dev` before they can be merged to `master`)_
+- [ ] Any modified or new methods or classes have helpful JavaDoc and code is thoroughly commented
+- [ ] The description lists all applicable issues this PR seeks to resolve
+- [ ] The description lists any configuration setting(s) that differ from the default settings
+- [ ] All tests and CI builds passing
+- [ ] The description lists all relevant PRs included in this release _(remove this if not merging to master)_
+- [ ] e2e tests are all passing _(remove this if not merging to master)_
+- [ ] Code coverage improves or is at 100% _(remove this if not merging to master)_
+
+### Description
+
+Please explain the changes you made here and, if not immediately obvious from the code, how they resolve any referenced issues. Be sure to include all issues being resolved and any special configuration settings that are need for the software to run properly with these changes. If merging to master, please also list the PRs that are to be included.

.travis.yml

@@ -1,39 +1,69 @@
+dist: trusty # jdk 8 not available on xenial
 language: java
-jdk:
-- oraclejdk8
+java:
+  - oraclejdk8
 install: true
 sudo: false
 # Install mongoDB to perform persistence tests
-services: mongodb
+services:
+  - mongodb
+  - postgresql
+addons:
+  postgresql: 9.6
 cache:
   directories:
-  - "$HOME/.m2"
+    - $HOME/.m2
+    - $HOME/.cache/yarn
 # Install semantic-release
 before_script:
-  - yarn global add @conveyal/maven-semantic-release semantic-release
+  - yarn global add @conveyal/maven-semantic-release semantic-release@15
+  # Create dir for GTFS+ files (used during testing)
+  - mkdir /tmp/gtfsplus
 before_install:
 #- sed -i.bak -e 's|https://nexus.codehaus.org/snapshots/|https://oss.sonatype.org/content/repositories/codehaus-snapshots/|g' ~/.m2/settings.xml
 # set region in AWS config for S3 setup
 - mkdir ~/.aws && printf '%s\n' '[default]' 'aws_access_key_id=foo' 'aws_secret_access_key=bar' 'region=us-east-1' > ~/.aws/config
 - cp configurations/default/server.yml.tmp configurations/default/server.yml
+# create database for tests
+- psql -U postgres -c 'CREATE DATABASE catalogue;'
 script:
 # package jar
 - mvn package
 after_success:
-  - semantic-release --prepare @conveyal/maven-semantic-release --publish @semantic-release/github,@conveyal/maven-semantic-release --verify-conditions @semantic-release/github,@conveyal/maven-semantic-release --verify-release @conveyal/maven-semantic-release --use-conveyal-workflow --dev-branch=dev --skip-maven-deploy
-  # Run codecov after semantic-release because maven-semantic-release creates extra commits that
-  # codecov will need a report on to reference in future PRs to the release branch
+  # this first codecov run will upload a report associated with the commit set through Travis CI environment variables
   - bash <(curl -s https://codecov.io/bash)
-# notify slack channel of build status
+  # run maven-semantic-release to potentially create a new release of datatools-server. The flag --skip-maven-deploy is
+  # used to avoid deploying to maven central. So essentially, this just creates a release with a changelog on github.
+  #
+  # If maven-semantic-release finishes successfully and the current branch is master, upload coverage reports for the
+  # commits that maven-semantic-release generated. Since the above codecov run is associated with the commit that
+  # initiated the Travis build, the report will not be associated with the commits that maven-semantic-release performed
+  # (if it ended up creating a release and the two commits that were a part of that workflow). Therefore, if on master
+  # codecov needs to be ran two more times to create codecov reports for the commits made by maven-semantic-release.
+  # See https://github.com/conveyal/gtfs-lib/issues/193.
+  #
+  # The git commands get the commit hash of the HEAD commit and the commit just before HEAD.
+  - |
+    semantic-release --prepare @conveyal/maven-semantic-release --publish @semantic-release/github,@conveyal/maven-semantic-release --verify-conditions @semantic-release/github,@conveyal/maven-semantic-release --verify-release @conveyal/maven-semantic-release --use-conveyal-workflow --dev-branch=dev --skip-maven-deploy
+    if [[ "$TRAVIS_BRANCH" = "master" ]]; then
+      bash <(curl -s https://codecov.io/bash) -C "$(git rev-parse HEAD)"
+      bash <(curl -s https://codecov.io/bash) -C "$(git rev-parse HEAD^)"
+    fi
 notifications:
+  # notify slack channel of build status
   slack: conveyal:WQxmWiu8PdmujwLw4ziW72Gc
 before_deploy:
 # get branch name of current branch for use in jar name: https://graysonkoonce.com/getting-the-current-branch-name-during-a-pull-request-in-travis-ci/
 - export BRANCH=$(if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then echo $TRAVIS_BRANCH; else echo $TRAVIS_PULL_REQUEST_BRANCH; fi)
-# copy packaged jars over to deploy dir
+# Create directory that will contain artifacts to deploy to s3.
 - mkdir deploy
+# Display contents of target directory (for logging purposes only).
+- ls target/*.jar
+# Copy packaged jars over to deploy dir.
 - cp target/dt-*.jar deploy/
-- cp "target/dt-$(git describe --always).jar" "deploy/dt-latest-${BRANCH}.jar"
+# FIXME: Do not create a branch-specific jar for now. Having a jar that changes contents but keeps the same name
+#   may cause confusion down the road and may be undesirable.
+# - cp "target/dt-$(git describe --always).jar" "deploy/dt-latest-${BRANCH}.jar"
 deploy:
   provider: s3
   skip_cleanup: true
@@ -45,5 +75,5 @@ deploy:
   local-dir: deploy
   acl: public_read
   on:
-    repo: conveyal/datatools-server
+    repo: ibi-group/datatools-server
     all_branches: true

configurations/default/env.yml.tmp

@@ -1,10 +1,13 @@
+# This client ID refers to the UI client in Auth0.
 AUTH0_CLIENT_ID: your-auth0-client-id
 AUTH0_DOMAIN: your-auth0-domain
 # Note: One of AUTH0_SECRET or AUTH0_PUBLIC_KEY should be used depending on the signing algorithm set on the client.
 # It seems that newer Auth0 accounts (2017 and later) might default to RS256 (public key).
 AUTH0_SECRET: your-auth0-secret # uses HS256 signing algorithm
 # AUTH0_PUBLIC_KEY: /path/to/auth0.pem # uses RS256 signing algorithm
-AUTH0_TOKEN: your-auth0-token
+# This client/secret pair refer to a machine-to-machine Auth0 application used to access the Management API.
+AUTH0_API_CLIENT: your-api-client-id
+AUTH0_API_SECRET: your-api-secret-id
 DISABLE_AUTH: false
 OSM_VEX: http://localhost:1000
 SPARKPOST_KEY: your-sparkpost-key

configurations/default/server.yml.tmp

@@ -15,6 +15,9 @@ modules:
     enabled: false
   user_admin:
     enabled: true
+  # Enable GTFS+ module for testing purposes
+  gtfsplus:
+    enabled: true
   gtfsapi:
     enabled: true
     load_on_fetch: false
@@ -29,3 +32,6 @@ extensions:
     enabled: true
     api: http://api.transitfeeds.com/v1/getFeeds
     key: your-api-key
+  # Enable MTC for testing purposes
+  mtc:
+    enabled: true

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.conveyal</groupId>
     <artifactId>datatools-server</artifactId>
-    <version>3.2.1-SNAPSHOT</version>
+    <version>3.5.1-SNAPSHOT</version>
 
     <licenses>
         <license>
@@ -16,42 +16,33 @@
     </licenses>
 
     <!-- Developer entries are provided for primary developers.
-   For other contributors, see https://github.com/conveyal/datatools-server/graphs/contributors -->
+   For other contributors, see https://github.com/ibi-group/datatools-server/graphs/contributors -->
     <developers>
         <developer>
             <name>Landon Reed</name>
-            <email>[email protected]</email>
-            <organization>Conveyal</organization>
-            <organizationUrl>http://conveyal.com/</organizationUrl>
-        </developer>
-        <developer>
-            <name>Andrew Byrd</name>
-            <email>[email protected]</email>
-            <organization>Conveyal</organization>
-            <organizationUrl>http://conveyal.com/</organizationUrl>
-        </developer>
-        <developer>
-            <name>David Emory</name>
-            <email>[email protected]</email>
-            <organization>Conveyal</organization>
-            <organizationUrl>http://conveyal.com/</organizationUrl>
+            <email>[email protected]</email>
+            <organization>IBI Group</organization>
+            <organizationUrl>https://ibigroup.com/</organizationUrl>
         </developer>
         <developer>
             <name>Evan Siroky</name>
-            <email>esiroky@conveyal.com</email>
-            <organization>Conveyal</organization>
-            <organizationUrl>http://conveyal.com/</organizationUrl>
+            <email>evan.siroky@ibigroup.com</email>
+            <organization>IBI Group</organization>
+            <organizationUrl>https://ibigroup.com/</organizationUrl>
         </developer>
     </developers>
 
     <!-- Define where the source code for project lives -->
     <scm>
-        <connection>scm:git:https://github.com/conveyal/datatools-server.git</connection>
-        <developerConnection>scm:git:ssh://[email protected]/conveyal/datatools-server.git</developerConnection>
-        <url>https://github.com/conveyal/datatools-server.git</url>
+        <connection>scm:git:https://github.com/ibi-group/datatools-server.git</connection>
+        <developerConnection>scm:git:ssh://[email protected]/ibi-group/datatools-server.git</developerConnection>
+        <url>https://github.com/ibi-group/datatools-server.git</url>
     </scm>
     <properties>
-        <jackson.version>2.9.8</jackson.version>
+        <jackson.version>2.9.9</jackson.version>
+        <!-- Using the latest version of geotools (e.g, 20) seems to cause issues with the shapefile
+        plugin where the_geom for each feature is null. -->
+        <geotools.version>17.5</geotools.version>
     </properties>
     <build>
         <resources>
@@ -130,6 +121,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-jar-plugin</artifactId>
+                <version>3.1.2</version>
                 <configuration>
                     <archive>
                         <manifest>
@@ -158,23 +150,6 @@
                     </execution>
                 </executions>
             </plugin>
-            <!-- This plugin makes sure junit 5 tests run -->
-            <plugin>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <version>2.22.0</version>
-                <dependencies>
-                    <dependency>
-                        <groupId>org.junit.platform</groupId>
-                        <artifactId>junit-platform-surefire-provider</artifactId>
-                        <version>1.3.1</version>
-                    </dependency>
-                    <dependency>
-                        <groupId>org.junit.jupiter</groupId>
-                        <artifactId>junit-jupiter-engine</artifactId>
-                        <version>5.3.1</version>
-                    </dependency>
-                </dependencies>
-            </plugin>
         </plugins>
     </build>
     <repositories>
@@ -218,6 +193,11 @@
                 <updatePolicy>always</updatePolicy>
             </snapshots>
         </repository>
+        <!--  used for importing java projects from github -->
+        <repository>
+            <id>jitpack.io</id>
+            <url>https://jitpack.io</url>
+        </repository>
     </repositories>
 
     <dependencies>
@@ -238,7 +218,7 @@
         <dependency>
             <groupId>ch.qos.logback</groupId>
             <artifactId>logback-classic</artifactId>
-            <version>1.1.3</version>
+            <version>1.2.3</version>
         </dependency>
 
         <!-- Used to connect to and import legacy editor MapDBs -->
@@ -255,11 +235,31 @@
             <version>2.1.0</version>
         </dependency>
 
-        <!-- Used for loading/fetching/writing GTFS entities (also provides access to commons-io and AWS S3 SDK). -->
+        <!-- Used for testing (note: this should match the version in gtfs-lib). -->
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.12</version>
+            <scope>test</scope>
+        </dependency>
+
+        <!-- Used for loading/fetching/writing GTFS entities. gtfs-lib also provides access to:
+         - commons-io - generic utilities
+         - AWS S3 SDK - putting/getting objects into/out of S3.
+         -->
         <dependency>
             <groupId>com.conveyal</groupId>
             <artifactId>gtfs-lib</artifactId>
-            <version>4.2.2</version>
+            <version>5.0.2</version>
+            <!-- Exclusions added in order to silence SLF4J warnings about multiple bindings:
+                http://www.slf4j.org/codes.html#multiple_bindings
+            -->
+            <exclusions>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-simple</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <!-- Used for data-tools application database -->
@@ -290,11 +290,11 @@
             <artifactId>jackson-dataformat-yaml</artifactId>
             <version>${jackson.version}</version>
         </dependency>
-
+        <!-- This version does not use ${jackson.version} because v2.9.9.1 does not exist for jackson-annotations. -->
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
-            <version>${jackson.version}</version>
+            <version>2.9.9.1</version>
         </dependency>
 
         <dependency>
@@ -322,15 +322,18 @@
         <dependency>
             <groupId>org.geotools</groupId>
             <artifactId>gt-shapefile</artifactId>
-            <version>19.2</version>
+            <version>${geotools.version}</version>
         </dependency>
-
-        <!-- Unit testing -->
+        <!-- gt-metadata and gt-api contains some dependencies required by gt-shapefile -->
         <dependency>
-            <groupId>org.junit.jupiter</groupId>
-            <artifactId>junit-jupiter-api</artifactId>
-            <version>5.3.1</version>
-            <scope>test</scope>
+            <groupId>org.geotools</groupId>
+            <artifactId>gt-metadata</artifactId>
+            <version>${geotools.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.geotools</groupId>
+            <artifactId>gt-api</artifactId>
+            <version>${geotools.version}</version>
         </dependency>
 
         <!-- Error reporting -->
@@ -359,8 +362,32 @@
             <groupId>org.hamcrest</groupId>
             <artifactId>java-hamcrest</artifactId>
             <version>2.0.0.0</version>
+            <scope>test</scope>
+        </dependency>
+        <!-- Snapshotting library for testing -->
+        <dependency>
+            <groupId>com.github.conveyal</groupId>
+            <artifactId>java-snapshot-matcher</artifactId>
+            <version>3495b32f7b4d3f82590e0a2284029214070b6984</version>
+            <scope>test</scope>
+        </dependency>
+        <!-- wiremock is used to mock http requests -->
+        <dependency>
+            <groupId>com.github.tomakehurst</groupId>
+            <artifactId>wiremock-standalone</artifactId>
+            <version>2.14.0</version>
+            <scope>test</scope>
+        </dependency>
+        <!-- Used for writing csv for merged feeds. Note: this appears to be one of the only
+        CSV libraries that will only quote values when necessary (e.g., there is a comma character
+        contained within the value) and that will work with an output stream writer when writing
+        directly to a zip output stream.
+         -->
+        <dependency>
+            <groupId>net.sf.supercsv</groupId>
+            <artifactId>super-csv</artifactId>
+            <version>2.4.0</version>
         </dependency>
-
     </dependencies>
 
 </project>