fix: validate cw log events size before uploading (#201)

src/main/java/com/aws/greengrass/logmanager/CloudWatchAttemptLogsProcessor.java

@@ -67,6 +67,7 @@ public class CloudWatchAttemptLogsProcessor {
     private static final int TIMESTAMP_BYTES = 8;
     private static final int MAX_BATCH_SIZE = 1024 * 1024;
     private static final int MAX_EVENT_LENGTH = 1024 * 256 - TIMESTAMP_BYTES - EVENT_STORAGE_OVERHEAD;
+    private static final int MAX_NUM_OF_LOG_EVENTS = 10_000;
     private static final ObjectMapper DESERIALIZER = new ObjectMapper()
             .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES, true);
     private static final ThreadLocal<SimpleDateFormat> DATE_FORMATTER =
@@ -379,6 +380,10 @@ private Pair<Boolean, AtomicInteger> addNewLogEvent(AtomicInteger totalBytesRead
             return new Pair<>(true, new AtomicInteger());
         }
 
+        if (attemptLogInformation.getLogEvents().size() >= MAX_NUM_OF_LOG_EVENTS) {
+            return new Pair<>(true, new AtomicInteger());
+        }
+
         // If log line is over maximum allowed log event size, break it into chunks of allowed size before constructing
         // log events.
         if (dataSize > MAX_EVENT_LENGTH) {

src/test/java/com/aws/greengrass/logmanager/CloudWatchAttemptLogsProcessorTest.java

@@ -75,6 +75,7 @@ class CloudWatchAttemptLogsProcessorTest extends GGServiceTestUtil {
     private static final int EVENT_STORAGE_OVERHEAD = 26;
     private static final int TIMESTAMP_BYTES = 8;
     private static final int MAX_EVENT_LENGTH = 1024 * 256 - TIMESTAMP_BYTES - EVENT_STORAGE_OVERHEAD;
+    private static final int MAX_NUM_OF_LOG_EVENTS = 10000;
     static {
         DATE_FORMATTER.setTimeZone(TimeZone.getTimeZone("UTC"));
     }
@@ -223,6 +224,75 @@ void GIVEN_one_component_WHEN_one_file_less_than_max_THEN_reads_entire_file(
         }
     }
 
+
+    @Test
+    void GIVEN_one_component_WHEN_log_events_more_than_max_THEN_read_only_max_num_of_log_events(ExtensionContext context1)
+            throws IOException {
+        ignoreExceptionOfType(context1, DateTimeParseException.class);
+        File file = new File(directoryPath.resolve("greengrass_test.log").toUri());
+        assertTrue(file.createNewFile());
+        assertTrue(file.setReadable(true));
+        assertTrue(file.setWritable(true));
+        /* LOG_EVENT_LENGTH = MESSAGE_LENGTH + TIMESTAMP_BYTES + EVENT_STORAGE_OVERHEAD => MESSAGE_LENGTH + 34
+        * LOG_EVENT_LENGTH * (MAX_NUM_OF_LOG_EVENTS) <= MAX_BATCH_SIZE => ((MESSAGE_LENGTH +34)*10000)<=1024*1024
+        * MESSAGE_LENGTH <= ~72
+         */
+        int logEventMessageLength = 70; // "\r\n" takes 2 bytes
+        try (OutputStream fileOutputStream = Files.newOutputStream(file.toPath())) {
+            for (int i = 0; i <= 10000; i++) { // 10001 log events
+                boolean useLetters = true;
+                boolean useNumbers = false;
+                StringBuilder generatedString = new StringBuilder(RandomStringUtils.random(logEventMessageLength, useLetters, useNumbers));
+                generatedString.append("\r\n");
+                fileOutputStream.write(generatedString.toString().getBytes(StandardCharsets.UTF_8));
+            }
+        }
+        LogFile logFile = LogFile.of(file);
+        String fileHash = logFile.hashString();
+        try {
+            List<LogFileInformation> logFileInformationSet = new ArrayList<>();
+            logFileInformationSet.add(LogFileInformation.builder().startPosition(0).logFile(logFile).fileHash(fileHash).build());
+            ComponentLogFileInformation componentLogFileInformation = ComponentLogFileInformation.builder()
+                    .name("TestComponent")
+                    .desiredLogLevel(Level.INFO)
+                    .componentType(ComponentType.GreengrassSystemComponent)
+                    .logFileInformationList(logFileInformationSet)
+                    .build();
+
+            logsProcessor = new CloudWatchAttemptLogsProcessor(mockDeviceConfiguration, defaultClock);
+            CloudWatchAttempt attempt = logsProcessor.processLogFiles(componentLogFileInformation);
+            assertNotNull(attempt);
+
+            assertNotNull(attempt.getLogStreamsToLogEventsMap());
+            assertThat(attempt.getLogStreamsToLogEventsMap().entrySet(), IsNot.not(IsEmptyCollection.empty()));
+            String logGroup = calculateLogGroupName(ComponentType.GreengrassSystemComponent, "testRegion", "TestComponent");
+            assertEquals(attempt.getLogGroupName(), logGroup);
+            String logStream = calculateLogStreamName("testThing");
+            assertTrue(attempt.getLogStreamsToLogEventsMap().containsKey(logStream));
+            CloudWatchAttemptLogInformation logEventsForStream1 = attempt.getLogStreamsToLogEventsMap().get(logStream);
+            assertNotNull(logEventsForStream1.getLogEvents());
+            assertEquals(MAX_NUM_OF_LOG_EVENTS, logEventsForStream1.getLogEvents().size());
+            assertTrue(logEventsForStream1.getAttemptLogFileInformationMap().containsKey(fileHash));
+            assertEquals(0, logEventsForStream1.getAttemptLogFileInformationMap().get(fileHash).getStartPosition());
+            assertEquals((logEventMessageLength+2)*10000, logEventsForStream1.getAttemptLogFileInformationMap().get(fileHash).getBytesRead());
+            assertEquals("TestComponent", logEventsForStream1.getComponentName());
+            LocalDateTime localDateTimeNow = LocalDateTime.now(ZoneOffset.UTC);
+            for (InputLogEvent logEvent: logEventsForStream1.getLogEvents()) {
+                Instant logTimestamp = Instant.ofEpochMilli(logEvent.timestamp());
+                assertTrue(logTimestamp.isBefore(Instant.now()));
+                LocalDateTime localDate = LocalDateTime.ofInstant(logTimestamp, ZoneOffset.UTC);
+                assertEquals(localDateTimeNow.getYear(), localDate.getYear());
+                assertEquals(localDateTimeNow.getMonth().getValue(), localDate.getMonth().getValue());
+                assertEquals(localDateTimeNow.getDayOfMonth(), localDate.getDayOfMonth());
+            }
+        } finally {
+            assertTrue(file.delete());
+        }
+
+
+    }
+
+
     @Test
     void GIVEN_one_component_one_file_more_than_max_WHEN_merge_THEN_reads_partial_file(ExtensionContext context1)
             throws IOException {

uat/testing-features/pom.xml

@@ -30,7 +30,7 @@
         <dependency>
             <groupId>com.aws.greengrass</groupId>
             <artifactId>aws-greengrass-testing-standalone</artifactId>
-            <version>1.1.0-SNAPSHOT</version>
+            <version>1.2.0-SNAPSHOT</version>
             <scope>compile</scope>
         </dependency>
         <dependency>

uat/testing-features/src/main/resources/greengrass/features/log-manager-1.feature

@@ -281,4 +281,4 @@ Feature: Greengrass V2 LogManager
         And the local Greengrass deployment is SUCCEEDED on the device after 180 seconds
         Then I verify that it created a log group of type GreengrassSystemComponent for component System, with streams within 300 seconds in CloudWatch
         And I verify that it created a log group of type UserComponent for component UserComponentW, with streams within 300 seconds in CloudWatch
-        And I verify 5 logs for UserComponentW of type UserComponent have been uploaded to Cloudwatch within 80 seconds
+        And I verify 10000 logs for UserComponentW of type UserComponent have been uploaded to Cloudwatch within 120 seconds