Merge pull request usdot-jpo-ode#57 from usdot-jpo-ode/candidate_r1

Merge candidate_r1 into master

.github/workflows/ci.yml

@@ -0,0 +1,24 @@
+name: CI
+on: 
+  pull_request:
+    types: [opened, reopened, synchronize]
+  push:
+    branches: [develop, master]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Set up JDK
+        uses: actions/setup-java@v4
+        with:
+          java-version: "21"
+          distribution: "temurin"
+      - name: Run Sonar
+        env:
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+        run: |
+          mvn -e -X clean org.jacoco:jacoco-maven-plugin:prepare-agent package sonar:sonar -Dsonar.projectKey=usdot-jpo-ode_jpo-s3-deposit -Dsonar.projectName=jpo-s3-deposit -Dsonar.organization=usdot-jpo-ode -Dsonar.host.url=https://sonarcloud.io -Dsonar.branch.name=$GITHUB_REF_NAME

.gitignore

@@ -1,12 +1,14 @@
-target/
-
-.DS_Store
-
-idea/
-/.idea
-consumer-example.iml
-/.classpath
-/.project
-/.settings/*
-/bin
-/*.log
+target/
+
+.DS_Store
+
+idea/
+/.idea
+consumer-example.iml
+/.classpath
+/.project
+/.settings/*
+/bin
+/*.log
+
+.env

Dockerfile

@@ -1,28 +1,17 @@
-FROM maven:3.5.4-jdk-8-alpine as builder
-MAINTAINER [email protected]
+FROM maven:3.8-eclipse-temurin-21-alpine as builder
 
 WORKDIR /home
 COPY ./pom.xml .
 COPY ./src ./src
 
 RUN mvn clean package assembly:single
 
-FROM eclipse-temurin:11-jre-alpine
+FROM eclipse-temurin:21-jre-alpine
 
 COPY --from=builder /home/src/main/resources/logback.xml /home
+COPY --from=builder /home/src/main/resources/log4j.properties /home
 COPY --from=builder /home/target/jpo-aws-depositor-jar-with-dependencies.jar /home
 
+
 CMD java -Dlogback.configurationFile=/home/logback.xml \
-	-jar /home/jpo-aws-depositor-jar-with-dependencies.jar \
-	--bootstrap-server $DOCKER_HOST_IP:9092 \
-	-g $DEPOSIT_GROUP \
-	-t $DEPOSIT_TOPIC \
-	-b $DEPOSIT_BUCKET_NAME \
-	-k $DEPOSIT_KEY_NAME \
-	-i $K_AWS_ACCESS_KEY_ID \
-	-a $K_AWS_SECRET_ACCESS_SECRET \
-	-n $K_AWS_SESSION_TOKEN \
-	-e $K_AWS_EXPIRATION \
-	-u $API_ENDPOINT \
-	-h $HEADER_ACCEPT \
-	-x $HEADER_X_API_KEY
+	-jar /home/jpo-aws-depositor-jar-with-dependencies.jar \

docker-compose-confluent-cloud.yml

@@ -5,11 +5,13 @@ services:
       context: .
       dockerfile: Dockerfile
     environment:
-      DOCKER_HOST_IP: ${DOCKER_HOST_IP}
+      BOOTSTRAP_SERVER: ${BOOTSTRAP_SERVER}
       API_ENDPOINT: ${API_ENDPOINT}
       DEPOSIT_TOPIC: ${DEPOSIT_TOPIC}
       DEPOSIT_GROUP: ${DEPOSIT_GROUP}
+      DESTINATION: ${DESTINATION}
       DEPOSIT_BUCKET_NAME: ${DEPOSIT_BUCKET_NAME}
+      REGION: ${REGION}
       DEPOSIT_KEY_NAME: ${DEPOSIT_KEY_NAME}
       AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}
       AWS_EXPIRATION: ${AWS_EXPIRATION}

docker-compose-mongo.yml

@@ -0,0 +1,79 @@
+# NOTE: This docker-compose spins up kafka, MongoDB, and a kafka-connect pod. This does NOT start up the S3-depositor module.
+version: '3'
+services:
+  zookeeper:
+    image: wurstmeister/zookeeper
+    ports:
+      - "2181:2181"
+
+  kafka:
+    image: wurstmeister/kafka
+    ports:
+      - "9092:9092"
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: ${DOCKER_HOST_IP}
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_CREATE_TOPICS: "topic.OdeSpatJson:1:1,topic.OdeBsmJson:1:1,topic.OdeTimJson:1:1,topic.OdePsmJson:1:1,topic.OdeRawEncodedPsmJson,topic.OdeRawEncodedBSMJson:1:1,topic.OdeRawEncodedSPATJson:1:1,topic.OdeRawEncodedTIMJson:1:1,topic.OdeRawEncodedMAPJson:1:1,topic.OdeMapJson:1:1,topic.OdeRawEncodedPSMJson:1:1,topic.OdePsmJson:1:1"
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" # Allows kakfa connect to create its own topics
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+
+  mongodb_container:
+    image: mongo:6
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: ${MONGO_DB_USER}
+      MONGO_INITDB_ROOT_PASSWORD: ${MONGO_DB_PASS}
+      MONGO_IP: ${MONGO_IP}
+      MONGO_DB_NAME: ${MONGO_DB_NAME}
+      MONGO_COLLECTION_TTL: ${MONGO_COLLECTION_TTL}
+    ports:
+      - "27017:27017"
+    volumes:
+      - mongodb_data_container:/MongoDB_data/db
+      - ./mongo-connector/create_indexes.js:/docker-entrypoint-initdb.d/create_indexes.js
+    healthcheck:
+      test: | 
+        echo 'db.runCommand("ping").ok' | mongosh localhost:27017/test --quiet
+      interval: 10s
+      start_period: 30s
+
+  kafka-connect:
+    image: cp-kafka-connect:6.1.9
+    build:
+      context: ./mongo-connector
+      dockerfile: Dockerfile
+    ports:
+      - "8083:8083"
+    depends_on:
+      mongodb_container:
+        condition: service_healthy
+    environment:
+      MONGO_URI: ${MONGO_URI}
+      MONGO_DB_NAME: ${MONGO_DB_NAME}
+      CONNECT_BOOTSTRAP_SERVERS: ${DOCKER_HOST_IP}:9092
+      CONNECT_REST_ADVERTISED_HOST_NAME: connect
+      CONNECT_REST_PORT: 8083
+      CONNECT_GROUP_ID: compose-connect-group
+      CONNECT_CONFIG_STORAGE_TOPIC: topic.kafka-connect-configs
+      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
+      CONNECT_CONFIG_STORAGE_CLEANUP_POLICY: compact
+      CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
+      CONNECT_OFFSET_STORAGE_TOPIC: topic.kafka-connect-offsets
+      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
+      CONNECT_OFFSET_STORAGE_CLEANUP_POLICY: compact
+      CONNECT_STATUS_STORAGE_TOPIC: topic.kafka-connect-status
+      CONNECT_STATUS_STORAGE_CLEANUP_POLICY: compact
+      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
+      CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
+      CONNECT_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
+      CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
+      CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
+      CONNECT_LOG4J_ROOT_LOGLEVEL: "ERROR"
+      CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=ERROR,org.reflections=ERROR,com.mongodb.kafka=ERROR"
+      CONNECT_PLUGIN_PATH: /usr/share/confluent-hub-components
+      CONNECT_ZOOKEEPER_CONNECT: "zookeeper:2181"
+    volumes:
+      - ./mongo-connector/connect_start.sh:/scripts/connect_start.sh
+
+volumes:
+  mongodb_data_container:

docker-compose.yml

@@ -21,10 +21,12 @@ services:
       context: .
       dockerfile: Dockerfile
     environment:
-      DOCKER_HOST_IP: ${DOCKER_HOST_IP}
+      BOOTSTRAP_SERVER: ${BOOTSTRAP_SERVER}
       API_ENDPOINT: ${API_ENDPOINT}
       DEPOSIT_TOPIC: ${DEPOSIT_TOPIC}
       DEPOSIT_GROUP: ${DEPOSIT_GROUP}
+      DESTINATION: ${DESTINATION}
+      REGION: ${REGION}
       DEPOSIT_BUCKET_NAME: ${DEPOSIT_BUCKET_NAME}
       DEPOSIT_KEY_NAME: ${DEPOSIT_KEY_NAME}
       AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}

docs/Release_notes.md

@@ -1,6 +1,22 @@
 Jpo-s3-deposit Release Notes
 ----------------------------
 
+Version 1.4.0, released February 2024
+----------------------------------------
+
+### **Summary**
+The changes for the jpo-s3-deposit 1.4.0 release include a log4j initialization fix, an update for Java, dockerhub image documentation & a MongoDB connector implementation.
+
+Enhancements in this release:
+- CDOT PR 12: Fixed log4j initialization issues
+- CDOT PR 11: Updated Java to v21
+- CDOT PR 10: Added dockerhub image documentation
+- CDOT PR 9: Implemented MongoDB Connector
+
+Known Issues:
+- No known issues at this time.
+
+
 Version 1.3.0, released November 2023
 ----------------------------------------
 

docs/dockerhub.md

@@ -0,0 +1,78 @@
+# jpo-s3-deposit
+
+## GitHub Repository Link
+https://github.com/usdot-jpo-ode/jpo-s3-deposit
+
+## Purpose
+The purpose of the jpo-s3-deposit program is to deposit messages to permanent storage, such as an Amazon S3 bucket, MongoDB instance or other similar system.
+
+## How to pull the latest image
+The latest image can be pulled using the following command:
+> docker pull usdotjpoode/jpo-s3-deposit:latest
+
+## Required environment variables
+- DOCKER_HOST_IP
+- DEPOSIT_GROUP
+- DEPOSIT_TOPIC
+- DEPOSIT_BUCKET_NAME
+- DEPOSIT_KEY_NAME
+- API_ENDPOINT
+- HEADER_ACCEPT
+- HEADER_X_API_KEY
+
+## Direct Dependencies
+The S3D will fail to start up if the following containers are not already present:
+- Kafka
+- Zookeeper (relied on by Kafka)
+
+## Indirect Dependencies
+The S3D will not receive messages to process if the ODE is not running.
+
+## Example docker-compose.yml with direct dependencies:
+```
+version: '2'
+services:
+  zookeeper:
+    image: wurstmeister/zookeeper
+    ports:
+      - "2181:2181"
+
+  kafka:
+    image: wurstmeister/kafka
+    ports:
+      - "9092:9092"
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: ${DOCKER_HOST_IP}
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_CREATE_TOPICS: "test:1:1"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+
+  s3d:
+    image: usdotjpoode/jpo-s3-deposit:release_q3
+    environment:
+      # required
+      DOCKER_HOST_IP: ${DOCKER_HOST_IP}
+      DEPOSIT_GROUP: ${DEPOSIT_GROUP}
+      DEPOSIT_TOPIC: ${DEPOSIT_TOPIC}
+      DEPOSIT_BUCKET_NAME: ${DEPOSIT_BUCKET_NAME}
+      DEPOSIT_KEY_NAME: ${DEPOSIT_KEY_NAME}
+      API_ENDPOINT: ${API_ENDPOINT}
+      HEADER_ACCEPT: ${HEADER_ACCEPT}
+      HEADER_X_API_KEY: ${HEADER_X_API_KEY}
+      # optional
+      AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}
+      AWS_EXPIRATION: ${AWS_EXPIRATION}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+    depends_on:
+      - kafka
+```
+
+## Expected startup output.
+The latest logs should look like this:
+```
+log4j:WARN No appenders could be found for logger (us.dot.its.jpo.ode.aws.depositor.AwsDepositor).
+log4j:WARN Please initialize the log4j system properly.
+log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
+```

mongo-connector/Dockerfile

@@ -0,0 +1,11 @@
+# Note that this image requires a version of Mongodb of 3.6 or later
+FROM confluentinc/cp-kafka-connect:6.1.9
+
+COPY connect_wait.sh /scripts/connect_wait.sh
+
+# Docs: https://www.mongodb.com/docs/kafka-connector/current/
+RUN confluent-hub install --no-prompt mongodb/kafka-connect-mongodb:1.11.1
+# Docs: https://docs.confluent.io/platform/current/connect/transforms/overview.html
+RUN confluent-hub install --no-prompt confluentinc/connect-transforms:1.4.3
+
+CMD ["bash", "-c", "/scripts/connect_wait.sh"]