Merge remote-tracking branch 'origin/dev' into Update/kafka-version

.circleci/config.yml

@@ -17,7 +17,7 @@ version: 2.1
 jobs:
   build:
     docker:
-      - image: 'circleci/openjdk:14.0.2-jdk-buster-node'
+      - image: 'cimg/openjdk:21.0.2-node'
     steps:
       - checkout
       - run:

.devcontainer/post-create.sh

@@ -6,17 +6,55 @@ bin/kafka-server-start.sh -daemon config/server.properties
 # wait 2 seconds for the server to start and be able to add partitions
 sleep 2s
 # add topics
+
+# BSM
 bin/kafka-topics.sh --create --topic "topic.OdeBsmPojo" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
 bin/kafka-topics.sh --create --topic "topic.OdeBsmJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
 bin/kafka-topics.sh --create --topic "topic.FilteredOdeBsmJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.OdeRawEncodedBSMJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+
+# TIM
 bin/kafka-topics.sh --create --topic "topic.OdeTimJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
 bin/kafka-topics.sh --create --topic "topic.OdeTimBroadcastJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
 bin/kafka-topics.sh --create --topic "topic.J2735TimBroadcastJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
-bin/kafka-topics.sh --create --topic "topic.OdeDriverAlertJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.OdeTIMCertExpirationTimeJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.OdeRawEncodedTIMJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+
+# PSM
+bin/kafka-topics.sh --create --topic "topic.OdePsmPojo" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.OdePsmJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.OdeRawEncodedPSMJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+
+# SSM
+bin/kafka-topics.sh --create --topic "topic.OdeSsmPojo" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.OdeSsmJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.OdeRawEncodedSSMJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+
+# SRM
+bin/kafka-topics.sh --create --topic "topic.OdeSrmPojo" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.OdeSrmJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.OdeRawEncodedSRMJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+
+# MAP
+bin/kafka-topics.sh --create --topic "topic.OdeRawEncodedMAPJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.OdeMapTxPojo" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.OdeMapJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+
+# SPaT
+bin/kafka-topics.sh --create --topic "topic.OdeSpatTxPojo" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.OdeSpatPojo" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.OdeSpatJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.FilteredOdeSpatJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.OdeSpatRxJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.OdeSpatRxPojo" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+
+# ASN1
 bin/kafka-topics.sh --create --topic "topic.Asn1DecoderInput" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
 bin/kafka-topics.sh --create --topic "topic.Asn1DecoderOutput" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
 bin/kafka-topics.sh --create --topic "topic.Asn1EncoderInput" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
 bin/kafka-topics.sh --create --topic "topic.Asn1EncoderOutput" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+
+# ETC
 bin/kafka-topics.sh --create --topic "topic.SDWDepositorInput" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
-bin/kafka-topics.sh --create --topic "topic.OdeTIMCertExpirationTimeJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
-bin/kafka-topics.sh --create --topic "topic.OdeRawEncodedMessageJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.OdeRawEncodedMessageJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.OdeDriverAlertJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1

.github/workflows/ci.yml

@@ -7,18 +7,18 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Build
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
 
   sonar:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: recursive
       - name: Set up JDK
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: "21"
           distribution: "temurin"

docker-compose-confluent-cloud.yml

@@ -1,5 +1,3 @@
-# DOCKER_SHARED_VOLUME_WINDOWS should be defined for Windows host machine as C: and not defined for Linux hosts
-
 version: '3'
 services:
   ode:

docker-compose-ppm-nsv.yml

@@ -1,8 +1,6 @@
 # This docker-compose file configures PPM modules with self contained configuration and map files
 # So the module can run without reliance on a shared volume. This was done to avoid corporate network
 # and machine dress-code restrictions that would not permit the container sharing volume with host.
-#
-# DOCKER_SHARED_VOLUME_WINDOWS should be defined for Windows host machine as C: and not defined for Linux hosts
 
 version: '3'
 services:

docker-compose.yml

@@ -1,5 +1,3 @@
-# DOCKER_SHARED_VOLUME_WINDOWS should be defined for Windows host machine as C: and not defined for Linux hosts
-
 version: '3'
 services:
   kafka:

docs/Release_notes.md

@@ -1,6 +1,32 @@
 JPO-ODE Release Notes
 ----------------------------
 
+Version 1.6.0, released February 2024
+----------------------------------------
+### **Summary**
+The updates for the jpo-ode 1.6.0 release includes an update for java, switching over to targeting J2735 2020 and some data flow diagram updates.
+
+Enhancements in this release:
+- CDOT PR 50: Updated Java to version 21.
+- CDOT PR 51: Updated the ODE to target J2735 2020.
+- CDOT PR 52: Updated the data flow diagrams to reflect offloaded file path
+- CDOT PR 53: Removed DOCKER_SHARED_VOLUME_WINDOWS environment variable
+
+Known Issues:
+- No known issues at this time.
+
+
+Version 1.5.1, released November 2023
+----------------------------------------
+
+### **Summary**
+The updates for the jpo-ode 1.5.1 release includes a hotfix for nodes getting duplicated during JSON serialization/deserialization.
+- Moved JsonProperty declaration to getters to avoid duplicate fetch in serialization/deserialization scenarios
+
+Known Issues:
+- No known issues at this time.
+
+
 Version 1.5.0, released November 2023
 ----------------------------------------
 

jpo-ode-common/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>usdot.jpo.ode</groupId>
     <artifactId>jpo-ode</artifactId>
-    <version>1.3.0</version>
+    <version>1.6.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>jpo-ode-common</artifactId>

jpo-ode-core/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>usdot.jpo.ode</groupId>
     <artifactId>jpo-ode</artifactId>
-    <version>1.3.0</version>
+    <version>1.6.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>jpo-ode-core</artifactId>
@@ -23,12 +23,12 @@
     <dependency>
       <groupId>usdot.jpo.ode</groupId>
       <artifactId>jpo-ode-common</artifactId>
-      <version>1.3.0</version>
+      <version>1.6.0-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>usdot.jpo.ode</groupId>
       <artifactId>jpo-ode-plugins</artifactId>
-      <version>1.3.0</version>
+      <version>1.6.0-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>org.apache.httpcomponents</groupId>

jpo-ode-core/src/main/resources/schemas/schema-bsm.json

@@ -2171,7 +2171,6 @@
                                 ]
                             },
                             "maxItems": 8,
-                            "minItems": 1,
                             "type": [
                                 "array",
                                 "null"

jpo-ode-core/src/main/resources/schemas/schema-psm.json

@@ -9,6 +9,12 @@
                     "type": "string"
                 },
                 "psmSource": {
+                    "enum": [
+                        "RSU",
+                        "V2X",
+                        "MMITSS",
+                        "unknown"
+                    ],
                     "type": "string"
                 },
                 "logFileName": {
@@ -27,6 +33,14 @@
                             "type": "null"
                         },
                         "rxSource": {
+                            "enum": [
+                                "RSU",
+                                "SAT",
+                                "RV",
+                                "SNMP",
+                                "NA",
+                                "UNKNOWN"
+                            ],
                             "type": "string"
                         }
                     },

jpo-ode-plugins/pom.xml

@@ -11,7 +11,7 @@
   <parent>
     <groupId>usdot.jpo.ode</groupId>
     <artifactId>jpo-ode</artifactId>
-    <version>1.3.0</version>
+    <version>1.6.0-SNAPSHOT</version>
   </parent>
   <properties>
     <!-- SonarQube Properties -->
@@ -27,7 +27,7 @@
     <dependency>
       <groupId>usdot.jpo.ode</groupId>
       <artifactId>jpo-ode-common</artifactId>
-      <version>1.3.0</version>
+      <version>1.6.0-SNAPSHOT</version>
     </dependency>
     <!-- TODO open-ode
     <dependency>

jpo-ode-plugins/src/main/java/us/dot/its/jpo/ode/plugin/SituationDataWarehouse.java

@@ -38,6 +38,7 @@ public enum TimeToLive {
       private String deliverystart;
       private String deliverystop;
       private String recordId = null;
+      private String estimatedRemovalDate;
 
       public OdeGeoRegion getServiceRegion() {
          return serviceRegion;
@@ -88,6 +89,14 @@ public void setRecordId(String recordId) {
          this.recordId = recordId;
       }
 
+      public String getEstimatedRemovalDate(){
+         return estimatedRemovalDate;
+      }
+
+      public void setEstimatedRemovalDate(String estimatedRemovalDate){
+         this.estimatedRemovalDate = estimatedRemovalDate;
+      }
+
       @Override
       public int hashCode() {
          final int prime = 31;
@@ -98,6 +107,7 @@ public int hashCode() {
          result = prime * result + ((recordId == null) ? 0 : recordId.hashCode());
          result = prime * result + ((serviceRegion == null) ? 0 : serviceRegion.hashCode());
          result = prime * result + ((ttl == null) ? 0 : ttl.hashCode());
+         result = prime * result + ((estimatedRemovalDate == null) ? 0 : estimatedRemovalDate.hashCode());
          return result;
       }
 
@@ -137,6 +147,11 @@ public boolean equals(Object obj) {
             return false;
          if (ttl != other.ttl)
             return false;
+             if (estimatedRemovalDate == null) {
+            if (other.estimatedRemovalDate != null)
+               return false;
+         } else if (!estimatedRemovalDate.equals(other.estimatedRemovalDate))
+            return false;
          return true;
       }
 

jpo-ode-svcs/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>usdot.jpo.ode</groupId>
     <artifactId>jpo-ode</artifactId>
-    <version>1.3.0</version>
+    <version>1.6.0-SNAPSHOT</version>
   </parent>
   <artifactId>jpo-ode-svcs</artifactId>
   <packaging>jar</packaging>
@@ -102,12 +102,12 @@
     <dependency>
       <groupId>usdot.jpo.ode</groupId>
       <artifactId>jpo-ode-core</artifactId>
-      <version>1.3.0</version>
+      <version>1.6.0-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>usdot.jpo.ode</groupId>
       <artifactId>jpo-ode-plugins</artifactId>
-      <version>1.3.0</version>
+      <version>1.6.0-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>org.springframework</groupId>

jpo-ode-svcs/run.bat

@@ -1 +1 @@
-java -jar target\jpo-ode-svcs-1.3.0.jar 
+java -jar target\jpo-ode-svcs-1.6.0-SNAPSHOT.jar 

jpo-ode-svcs/run.sh

@@ -1,2 +1,2 @@
 #!/bin/bash
-java -jar target/jpo-ode-svcs-1.3.0.jar 
+java -jar target/jpo-ode-svcs-1.6.0-SNAPSHOT.jar 

jpo-ode-svcs/src/main/java/us/dot/its/jpo/ode/coder/stream/LogFileToAsn1CodecPublisher.java

@@ -53,6 +53,9 @@
 import us.dot.its.jpo.ode.util.JsonUtils.JsonUtilsException;
 
 public class LogFileToAsn1CodecPublisher implements Asn1CodecPublisher {
+	private static final String BSM_FLAG = "0014";
+    private static final String TIM_FLAG = "001f";
+    private static final String MAP_FLAG = "0012";
 
 	public static class LogFileToAsn1CodecPublisherException extends Exception {
 
@@ -74,9 +77,9 @@ public LogFileToAsn1CodecPublisherException(String string, Exception e) {
 	public LogFileToAsn1CodecPublisher(StringPublisher dataPub) {
 		this.publisher = dataPub;
 		this.serialId = new SerialId();
-		msgStartFlags.put("BSM", "0014");
-		msgStartFlags.put("TIM", "001f");
-		msgStartFlags.put("MAP", "0012");
+		msgStartFlags.put("BSM", BSM_FLAG);
+		msgStartFlags.put("TIM", TIM_FLAG);
+		msgStartFlags.put("MAP", MAP_FLAG);
 	}
 
 	public List<OdeData> publish(BufferedInputStream bis, String fileName, ImporterFileType fileType)
@@ -191,6 +194,15 @@ private void publishList(XmlUtils xmlUtils, List<OdeData> dataList) throws JsonP
 		}
 	}
 
+
+	/**
+		* Checks the header of the OdeMsgPayload and determines the encoding rule to be used in the Asn1DecoderInput XML.
+		* The payload is checked for various message start flags. It will add the encoding rule to the Asn1DecoderInput XML
+		* to tell the ASN1 codec to extract data from the header into the output message.
+		* 
+		* @param payload The OdeMsgPayload to check the header of.
+		* @return The encoding rule to be used in the Asn1DecoderInput XML.
+		*/
 	public String checkHeader(OdeMsgPayload payload) {
 		JSONObject payloadJson;
 		String header = null;

jpo-ode-svcs/src/main/java/us/dot/its/jpo/ode/importer/parser/PayloadParser.java

@@ -33,7 +33,12 @@ public class PayloadParser extends LogFileParser {
    private static Logger logger = LoggerFactory.getLogger(PayloadParser.class);
    private static HashMap<String, String> msgStartFlags = new HashMap<String, String>();
 
-   private static final int HEADER_SIZE = 20; 
+   // Maximum header size for 1609 payload headers
+   private static final int HEADER_SIZE_1609 = 20; 
+
+   // Maximum header size for WSMP payload headers
+   private static final int HEADER_SIZE_WSMP = 35; 
+
    public static final int PAYLOAD_LENGTH = 2;
 
    protected short payloadLength;
@@ -113,24 +118,24 @@ public byte[] removeHeader(byte[] packet) {
       for (String key : msgStartFlags.keySet()) {
          String startFlag = msgStartFlags.get(key);
          int startIndex = hexPacket.toLowerCase().indexOf(startFlag);
-         if (hexPacketParsed == ""){
+         if (hexPacketParsed.equals("")) {
             logger.debug("Start index for: " + key + " is: " + startIndex);
             if (startIndex == -1) {
                logger.debug("Message does not have header for: " + key);
                break;
-            } else if (startIndex <= HEADER_SIZE) {
+            } else if (startIndex <= HEADER_SIZE_1609) {
                logger.debug("Message has supported header. startIndex: " + startIndex + " msgFlag: " + startFlag);
                hexPacketParsed = hexPacket;
-            // Using a value of 35 as the largest index from preliminary testing data.
-            } else if (startIndex > HEADER_SIZE && startIndex < 35) {
-               int trueStartIndex = HEADER_SIZE
-                     + hexPacket.substring(HEADER_SIZE, hexPacket.length()).indexOf(startFlag);
+            // If the header type is WSMP, the header will be stripped from the payload.
+            } else if (startIndex > HEADER_SIZE_1609 && startIndex < HEADER_SIZE_WSMP) {
+               int trueStartIndex = HEADER_SIZE_1609
+                     + hexPacket.substring(HEADER_SIZE_1609, hexPacket.length()).indexOf(startFlag);
                logger.debug("Found payload start at: " + trueStartIndex);
                hexPacketParsed = hexPacket.substring(trueStartIndex, hexPacket.length());
             }
          }
       }
-      if (hexPacketParsed == ""){
+      if (hexPacketParsed.equals("")) {
          hexPacketParsed = hexPacket;
          logger.debug("Could not identify a Header in the following packet: " + hexPacketParsed);
       } else {