Merge branch 'ibi-dev-2.x' into gmap-mobility-profile

.github/workflows/cibuild.yml

@@ -1,21 +1,15 @@
 name: OTP CI Build
-# On [push, pull_request] causes double-builds when creating PRs.
-# But triggering on push only will miss pull requests from outside authors.
-# The push event's ref is the name of the pushed branch;
-# The pull_request event's branch name is the merge target branch.
+
 on:
   push:
-    branches:
-      - master
-      - dev-1.x
-      - dev-2.x
   pull_request:
     branches:
-      - master
-      - dev-1.x
-      - dev-2.x
+      - ibi-dev
+      - ibi-dev-2.x
 jobs:
   build-linux:
+    env:
+      MAVEN_OPTS: -Xmx2g
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
@@ -31,7 +25,7 @@ jobs:
         with:
           node-version: 18
 
-      # Java setup step completes very fast, no need to run in a preconfigured docker container
+      # Java setup step completes very fast, no need to run in a preconfigured docker container.
       - name: Set up JDK 21
         uses: actions/setup-java@v4
         with:
@@ -46,7 +40,7 @@ jobs:
         # https://github.com/actions/runner-images/issues/1499
         # we set nodePath and npmPath to skip downloading the node binary, which frequently times out
         run: |
-          mvn --batch-mode jacoco:prepare-agent test jacoco:report -P prettierCheck -Dprettier.nodePath=node -Dprettier.npmPath=npm
+          mvn --batch-mode jacoco:prepare-agent test jacoco:report -P prettierCheck,unit-tests -Dprettier.nodePath=node -Dprettier.npmPath=npm
           mvn --batch-mode package -Dmaven.test.skip -P prettierSkip
 
       - name: Send coverage data to codecov.io
@@ -61,6 +55,55 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: mvn --batch-mode deploy --settings maven-settings.xml -DskipTests -DGITHUB_REPOSITORY=$GITHUB_REPOSITORY -P prettierSkip -P deployGitHub
 
+      ## IBI start
+      # So that the branch name can be reference via $GITHUB_REF_SLUG.
+      - name: Inject slug/short variables
+        uses: rlespinasse/[email protected]
+      # Define AWS profile credentials for use by subsequent steps.
+      - name: Add profile credentials to ~/.aws/credentials
+        if: github.event_name == 'push'
+        run: ./scripts/add-aws-credentials.sh
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: ${{ secrets.AWS_REGION }}
+      # Only prepare deploy files on push events to avoid duplicate pushes to s3 when a commit is
+      # pushed to a branch with an open pull request.
+      - name: Prepare build files for deployment to s3
+        if: github.event_name == 'push'
+        run: ./scripts/prep-build-for-deploy-from-github-actions.sh
+      # Copy build files to AWS.
+      - name: Copy build files to the otp-repo bucket
+        if: github.event_name == 'push'
+        run: |
+          aws s3 cp ./deploy s3://otp-repo --recursive --acl public-read
+      # Install mkdocs using pip for use by subsequent steps.
+      - name: Install mkdocs
+        if: github.event_name == 'push' && env.GITHUB_REF_SLUG == 'ibi-dev-2.x'
+        run: |
+          pip install mkdocs
+      # Using mkdocs produce docs for deployment.
+# FIXME: there is an error while trying to run enunciate to generate the docs
+#      - name: Prepare doc files for deployment to s3
+#        if: github.event_name == 'push' && env.GITHUB_REF_SLUG == 'ibi-dev-2.x'
+#        run: ./scripts/prep-docs-for-deploy-from-github-actions.sh
+#      # If on ibi-dev branch, then upload docs to IBI s3 buckets & cloudfront.
+# FIXME: figure out whether the same bucket should be used, or a different one should be used for 2.x docs
+#      - name: Upload docs to IBI s3 buckets & cloudfront
+#        if: github.event_name == 'push' && env.GITHUB_REF_SLUG == 'ibi-dev-2.x'
+#        run: |
+#          aws s3 cp ./target/mkdocs s3://otp-docs.ibi-transit.com --recursive --acl public-read
+      # Push notification to Teams.
+      - name: Notify us of the build status on MS Teams
+        # Release https://github.com/toko-bifrost/ms-teams-deploy-card/releases/tag/3.1.2 this matches
+        # the full length commit SHA below. This must not be changed without first reviewing the code of the newer
+        # version for potential security risks.
+        uses: toko-bifrost/ms-teams-deploy-card@dcc94e4ce4088b1e6b6de5c9a3cda4ddcbe97d2e
+        if: always()
+        with:
+          github-token: ${{ github.token }}
+          webhook-uri: ${{ secrets.MSTEAMS_WEBHOOK }}
+
   build-windows:
     timeout-minutes: 20
     runs-on: windows-latest
@@ -196,7 +239,6 @@ jobs:
     if: github.repository_owner == 'opentripplanner' && github.event_name == 'push' && (github.ref == 'refs/heads/dev-2.x' || github.ref == 'refs/heads/master')
     runs-on: ubuntu-latest
     needs:
-      - build-windows
       - build-linux
     steps:
       - uses: actions/checkout@v4

.github/workflows/smoke-tests.yml

@@ -0,0 +1,74 @@
+name: Smoke Tests
+
+on:
+  push:
+    branches:
+      - ibi-dev-2.x
+  pull_request:
+    branches:
+      - ibi-dev
+      - ibi-dev-2.x
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    env:
+      MAVEN_OPTS: -Xmx3g
+
+    strategy:
+      matrix:
+        locations:
+          - name: seattle
+            sleep: 15
+          - name: atlanta
+            sleep: 15
+          - name: houston
+            sleep: 15
+          - name: denver
+            sleep: 15
+          - name: septa
+            sleep: 15
+          - name: portland
+            # have to sleep longer since computing geofencing zones takes a while
+            sleep: 125
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Java
+        uses: actions/setup-java@v4
+        with:
+          java-version: 21
+          distribution: temurin
+          cache: maven
+
+      - name: Build jar file, compile tests
+        run: |
+          mvn --no-transfer-progress clean package -Dmaven.test.skip -P prettierSkip
+          mvn org.apache.maven.plugins:maven-compiler-plugin:testCompile
+
+      - name: Cache OSM files
+        uses: actions/cache@v3
+        with:
+          path: smoke-tests/**/*.pbf
+          key: osm-${{ matrix.locations.name }}-2023-10-31
+
+      - name: Build graph
+        working-directory: smoke-tests
+        run: |
+          make build-${{ matrix.locations.name }}
+
+      - name: Run smoke tests
+        working-directory: smoke-tests
+        run: |
+          make run-${{ matrix.locations.name }} &
+
+          # OTP needs a little while to start up so we sleep
+          sleep ${{ matrix.locations.sleep }}
+
+          cd ..
+
+          # run the actual smoke tests
+          # we run surefire:test in order to not recompile the tests for each city
+          mvn --no-transfer-progress surefire:test -Djunit.tags.included="${{ matrix.locations.name }}" -Djunit.tags.excluded="" -P prettierSkip
+
+          # shutting down the OTP instance running in the background (via make)
+          killall make

.gitignore

@@ -46,3 +46,8 @@ streetGraph.obj
 graph.obj
 # IntelliJ creates these pid files when you attach the debugger to tests
 .attach_pid*
+
+smoke-tests/*.jar
+smoke-tests/**/*.obj
+smoke-tests/**/*.pbf
+smoke-tests/**/*.zip

pom.xml

@@ -81,6 +81,8 @@
         <plugin.prettier.version>0.22</plugin.prettier.version>
         <plugin.prettier.goal>write</plugin.prettier.goal>
 
+        <junit.tags.included></junit.tags.included>
+        <junit.tags.excluded>smoke-test</junit.tags.excluded>
     </properties>
 
     <distributionManagement>
@@ -153,6 +155,9 @@
                             <version>${google.dagger.version}</version>
                         </path>
                     </annotationProcessorPaths>
+                    <compilerArgs>
+                        <arg>-parameters</arg>
+                    </compilerArgs>
                 </configuration>
             </plugin>
             <plugin>
@@ -306,6 +311,8 @@
                         <printStderrOnFailure>true</printStderrOnFailure>
                         <printStderrOnSuccess>false</printStderrOnSuccess>
                     </statelessTestsetInfoReporter>
+                    <excludedGroups>${junit.tags.excluded}</excludedGroups>
+                    <groups>${junit.tags.included}</groups>
                 </configuration>
             </plugin>
             <!-- code coverage report -->
@@ -804,6 +811,13 @@
             <version>${jackson.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>com.fasterxml.jackson.module</groupId>
+            <artifactId>jackson-module-parameter-names</artifactId>
+            <version>${jackson.version}</version>
+            <scope>test</scope>
+        </dependency>
+
         <!--
           Google library imports
           The version is imported using google libraries-bom, see dep.management
@@ -933,6 +947,12 @@
             <version>1.26.1</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.opentripplanner</groupId>
+            <artifactId>otp-client</artifactId>
+            <version>0.0.27</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <profiles>

scripts/add-aws-credentials.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+# This script will create the AWS credentials file if it does not exist.
+# It is only meant to be run on CI (to create the proper
+# environment for E2E tests).
+mkdir -p ~/.aws
+
+# If credentials do not exist, create file setting values to
+# environment variables (which must be defined in CI).
+# This should avoid any accidental overwrite on your local dev machine :)
+if [ ! -f ~/.aws/credentials ]; then
+cat > ~/.aws/credentials << EOL
+[default]
+aws_access_key_id = ${AWS_ACCESS_KEY_ID}
+aws_secret_access_key = ${AWS_SECRET_ACCESS_KEY}
+region = ${AWS_REGION}
+EOL
+fi

scripts/prep-build-for-deploy-from-github-actions.sh

@@ -0,0 +1,20 @@
+# get branch name of current branch for use in jar name
+export BRANCH=$(
+  if [ -n "$GITHUB_HEAD_REF" ]; then
+    echo ${GITHUB_HEAD_REF#refs/heads/};
+  else
+    echo ${GITHUB_REF#refs/heads/};
+  fi
+)
+echo $BRANCH
+# `git describe --tags --always HEAD` returns a string of the form v0.0.0-52-ge10d02d.
+# It assumes you have pushed a tag on a commit on github (e.g. a commit on the dev branch).
+# If for some reason git can't find a tag, fallback with --always to a commit sha.
+export JAR_VERSION=$(git describe --tags --always HEAD)
+echo $JAR_VERSION
+# Create a deployment folder, and a folder for the branch.
+mkdir deploy
+# Add the JAR file.
+cp target/*-shaded.jar deploy/otp-$JAR_VERSION.jar
+cp target/*-shaded.jar deploy/otp-latest-$BRANCH.jar
+

scripts/prep-docs-for-deploy-from-github-actions.sh

@@ -0,0 +1,10 @@
+# build markdown docs using mkdocs
+mkdocs build
+# build enunciate and JavaDoc
+mvn -DskipTests site
+mkdir -p target/mkdocs/api
+mkdir -p target/mkdocs/JavaDoc
+# copy enunciate API docs into mkdocs folder
+cp -R target/site/enunciate/apidocs target/mkdocs/api
+# copy JavaDoc into mkdocs folder
+cp -R target/apidocs target/mkdocs/JavaDoc

smoke-tests/Makefile

@@ -0,0 +1,30 @@
+CURL:=curl -L -\# --fail --create-dirs
+
+otp.jar:
+	cp ../target/otp-*-SNAPSHOT-shaded.jar otp.jar
+
+atlanta/osm.pbf:
+	${CURL} http://osm.ibi-transit.com/vex/33.654066160697056,-84.759521484375,33.994611584814606,-84.30976867675781.pbf -o $@
+
+houston/osm.pbf:
+	${CURL} http://osm.ibi-transit.com/vex/29.65285233056443,-95.48492431640624,29.857914952075095,-95.22193908691406.pbf -o $@
+
+denver/osm.pbf:
+	${CURL} http://osm.ibi-transit.com/vex/39.65857056750545,-105.0567626953125,39.8578370694061,-104.86862182617188.pbf -o $@
+
+septa/osm.pbf:
+	${CURL} http://osm.ibi-transit.com/vex/39.85441086976069,-75.29136657714844,40.0433862595006,-75.06202697753906.pbf -o $@
+
+portland/osm.pbf:
+	${CURL} http://osm.ibi-transit.com/vex/45.424388,-122.885443,45.684746,-122.44084.pbf -o $@
+
+seattle/osm.pbf:
+	${CURL} http://osm.ibi-transit.com/vex/47.50925,-122.47458,47.786105,-122.224433.pbf -o $@
+
+build-%: otp.jar %/osm.pbf
+	java -Xmx6G -jar otp.jar --build --save $*
+
+run-%: otp.jar
+	java -Xmx6G -jar otp.jar --load --serve $*
+
+

smoke-tests/Readme.md

@@ -0,0 +1,65 @@
+## IBI Group OTP Smoke Tests
+
+As of 2023 upstream OTP has over 3000 unit tests but is intentionally light on integration tests
+as these tend to be brittle and hard to maintain over time. They also don't supply the desired
+granularity to figure out a problem during development.
+
+For this reason this folder contains scripts and configuration files for running an OTP instance for 
+each OTP2 deployment that can then be used to run smoke tests against. 
+These tests are intended as a last sanity check when we merge upstream changes because these tend to 
+be very large. 
+It reduces the burden on the reviewer and increases the confidence that the upstream merge won't 
+break anything.
+
+### Details
+
+Each smoke test follows these steps:
+
+1. Takes the latest upstream code
+2. Downloads up-to-date GTFS and OSM files
+3. Builds a complete graph
+4. Starts an OTP instance
+5. Possibly configures realtime updates for transit and vehicle rental
+6. Sends API requests against OTP and asserts that the result is within expected parameters
+
+Since the input data is variable, there is some maintenance involved because expected results can 
+change over time. However, the upside is that catastrophic data errors are also caught early on.
+
+### Example results
+
+Here we see a screenshot of a typical smoke test run with the current locations (as of 2023) visible
+in the left panel.
+
+![Smoke tests screnshot](smoke-tests.png)
+
+### Commands
+
+If you want to prepare an OTP instance for the smoke tests, run the following commands to build the
+jar, download all OSM and GTFS files and then build the graph:
+
+```
+# start in root OTP dir
+cd OpenTripPlanner
+mvn package -DskipTests
+cd smoke-tests
+make build-atlanta
+```
+
+After the graph is built you can then run OTP with:
+
+```
+make run-atlanta
+```
+
+And finally run the smoke tests with
+
+```
+# go back to the root directory
+cd ..
+mvn test -Djunit.tags.included="atlanta" -Djunit.tags.excluded=""
+```
+
+### CI
+
+The CI workflow that does all of this automatically can be found
+at [smoke-tests.yml](../.github/workflows/smoke-tests.yml).