From 5c32a08b24af8dab5775b37dab609977fecc4690 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Tue, 22 Oct 2024 23:54:16 +0900 Subject: [PATCH] update persistent volume db tests --- ...istent_volume_logs_database_client_test.go | 492 +++++++++++++++--- 1 file changed, 434 insertions(+), 58 deletions(-) diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go index 6661060ca0..87f8d03010 100644 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go @@ -53,6 +53,7 @@ const ( defaultYear = 2023 defaultDay = 0 // sunday + defaultHour = 5 startingWeek = 4 defaultShouldReturnAllLogs = true @@ -131,9 +132,10 @@ func TestStreamUserServiceLogsPerWeek_WithFilters(t *testing.T) { testUserService3Uuid: true, } - underlyingFs := createFilledPerWeekFilesystem(startingWeek) mockTime := logs_clock.NewMockLogsClockPerDay(defaultYear, startingWeek, defaultDay) - perWeekStreamStrategy := stream_logs_strategy.NewStreamLogsStrategyImpl(mockTime, persistent_volume_helpers.ConvertWeeksToDuration(retentionPeriodInWeeksForTesting), file_layout.NewPerWeekFileLayout(mockTime, volume_consts.LogsStorageDirpath)) + perWeekFileLayout := file_layout.NewPerWeekFileLayout(mockTime, volume_consts.LogsStorageDirpath) + underlyingFs := createFilledFilesystem(perWeekFileLayout, mockTime.Now()) + perWeekStreamStrategy := stream_logs_strategy.NewStreamLogsStrategyImpl(mockTime, persistent_volume_helpers.ConvertWeeksToDuration(retentionPeriodInWeeksForTesting), perWeekFileLayout) receivedUserServiceLogsByUuid, testEvaluationErr := executeStreamCallAndGetReceivedServiceLogLines( t, @@ -155,6 +157,56 @@ func TestStreamUserServiceLogsPerWeek_WithFilters(t *testing.T) { require.NoError(t, testEvaluationErr) } +func TestStreamUserServiceLogsPerHour_WithFilters(t *testing.T) { + expectedServiceAmountLogLinesByServiceUuid := map[service.ServiceUUID]int{ + testUserService1Uuid: 2, + testUserService2Uuid: 2, + testUserService3Uuid: 2, + } + + firstTextFilter := logline.NewDoesContainTextLogLineFilter(firstFilterText) + secondTextFilter := logline.NewDoesNotContainTextLogLineFilter(secondFilterText) + regexFilter := logline.NewDoesContainMatchRegexLogLineFilter(firstMatchRegexFilterStr) + + logLinesFilters := []logline.LogLineFilter{ + *firstTextFilter, + *secondTextFilter, + *regexFilter, + } + + expectedFirstLogLine := "Starting feature 'runs idempotently'" + + userServiceUuids := map[service.ServiceUUID]bool{ + testUserService1Uuid: true, + testUserService2Uuid: true, + testUserService3Uuid: true, + } + + mockTime := logs_clock.NewMockLogsClockPerHour(defaultYear, startingWeek, defaultDay, defaultHour) + perHourFileLayout := file_layout.NewPerHourFileLayout(mockTime, volume_consts.LogsStorageDirpath) + underlyingFs := createFilledFilesystem(perHourFileLayout, mockTime.Now()) + perHourStreamStrategy := stream_logs_strategy.NewStreamLogsStrategyImpl(mockTime, persistent_volume_helpers.ConvertWeeksToDuration(retentionPeriodInWeeksForTesting), perHourFileLayout) + + receivedUserServiceLogsByUuid, testEvaluationErr := executeStreamCallAndGetReceivedServiceLogLines( + t, + logLinesFilters, + userServiceUuids, + expectedServiceAmountLogLinesByServiceUuid, + doNotFollowLogs, + underlyingFs, + perHourStreamStrategy, + ) + + for serviceUuid, serviceLogLines := range receivedUserServiceLogsByUuid { + expectedAmountLogLines, found := expectedServiceAmountLogLinesByServiceUuid[serviceUuid] + require.True(t, found) + require.Equal(t, expectedAmountLogLines, len(serviceLogLines)) + require.Equal(t, expectedFirstLogLine, serviceLogLines[0].GetContent()) + } + + require.NoError(t, testEvaluationErr) +} + func TestStreamUserServiceLogs_NoLogsFromPersistentVolume(t *testing.T) { expectedServiceAmountLogLinesByServiceUuid := map[service.ServiceUUID]int{ testUserService1Uuid: 0, @@ -214,9 +266,10 @@ func TestStreamUserServiceLogsPerWeek_NoLogsFromPersistentVolume(t *testing.T) { testUserService3Uuid: true, } - underlyingFs := createEmptyPerWeekFilesystem(startingWeek) mockTime := logs_clock.NewMockLogsClockPerDay(defaultYear, startingWeek, defaultDay) - perWeekStreamStrategy := stream_logs_strategy.NewStreamLogsStrategyImpl(mockTime, persistent_volume_helpers.ConvertWeeksToDuration(retentionPeriodInWeeksForTesting), file_layout.NewPerWeekFileLayout(mockTime, volume_consts.LogsStorageDirpath)) + perWeekFileLayout := file_layout.NewPerWeekFileLayout(mockTime, volume_consts.LogsStorageDirpath) + underlyingFs := createEmptyFilesystem(perWeekFileLayout, mockTime.Now()) + perWeekStreamStrategy := stream_logs_strategy.NewStreamLogsStrategyImpl(mockTime, persistent_volume_helpers.ConvertWeeksToDuration(retentionPeriodInWeeksForTesting), perWeekFileLayout) receivedUserServiceLogsByUuid, testEvaluationErr := executeStreamCallAndGetReceivedServiceLogLines( t, @@ -236,6 +289,48 @@ func TestStreamUserServiceLogsPerWeek_NoLogsFromPersistentVolume(t *testing.T) { } } +func TestStreamUserServiceLogsPerHour_NoLogsFromPersistentVolume(t *testing.T) { + expectedServiceAmountLogLinesByServiceUuid := map[service.ServiceUUID]int{ + testUserService1Uuid: 0, + testUserService2Uuid: 0, + testUserService3Uuid: 0, + } + + firstTextFilter := logline.NewDoesContainTextLogLineFilter(notFoundedFilterText) + + logLinesFilters := []logline.LogLineFilter{ + *firstTextFilter, + } + + userServiceUuids := map[service.ServiceUUID]bool{ + testUserService1Uuid: true, + testUserService2Uuid: true, + testUserService3Uuid: true, + } + + mockTime := logs_clock.NewMockLogsClockPerHour(defaultYear, startingWeek, defaultDay, defaultHour) + perHourFileLayout := file_layout.NewPerWeekFileLayout(mockTime, volume_consts.LogsStorageDirpath) + underlyingFs := createEmptyFilesystem(perHourFileLayout, mockTime.Now()) + perHourStreamStrategy := stream_logs_strategy.NewStreamLogsStrategyImpl(mockTime, persistent_volume_helpers.ConvertWeeksToDuration(retentionPeriodInWeeksForTesting), perHourFileLayout) + + receivedUserServiceLogsByUuid, testEvaluationErr := executeStreamCallAndGetReceivedServiceLogLines( + t, + logLinesFilters, + userServiceUuids, + expectedServiceAmountLogLinesByServiceUuid, + doNotFollowLogs, + underlyingFs, + perHourStreamStrategy, + ) + require.NoError(t, testEvaluationErr) + + for serviceUuid, serviceLogLines := range receivedUserServiceLogsByUuid { + expectedAmountLogLines, found := expectedServiceAmountLogLinesByServiceUuid[serviceUuid] + require.True(t, found) + require.Equal(t, expectedAmountLogLines, len(serviceLogLines)) + } +} + func TestStreamUserServiceLogs_ThousandsOfLogLinesSuccessfulExecution(t *testing.T) { expectedAmountLogLines := 10_000 @@ -312,16 +407,15 @@ func TestStreamUserServiceLogsPerWeek_ThousandsOfLogLinesSuccessfulExecution(t * } underlyingFs := volume_filesystem.NewMockedVolumeFilesystem() - // %02d to format week num with leading zeros so 1-9 are converted to 01-09 for %V format - formattedWeekNum := fmt.Sprintf("%02d", startingWeek) - file1PathStr := fmt.Sprintf(file_layout.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), formattedWeekNum, string(enclaveUuid), testUserService1Uuid, volume_consts.Filetype) + mockTime := logs_clock.NewMockLogsClockPerDay(defaultYear, startingWeek, defaultDay) + perWeekFileLayout := file_layout.NewPerWeekFileLayout(mockTime, volume_consts.LogsStorageDirpath) + file1PathStr := perWeekFileLayout.GetLogFilePath(mockTime.Now(), testEnclaveUuid, testUserService1Uuid) file1, err := underlyingFs.Create(file1PathStr) require.NoError(t, err) _, err = file1.WriteString(logLinesStr) require.NoError(t, err) - mockTime := logs_clock.NewMockLogsClockPerDay(defaultYear, startingWeek, defaultDay) - perWeekStreamStrategy := stream_logs_strategy.NewStreamLogsStrategyImpl(mockTime, persistent_volume_helpers.ConvertWeeksToDuration(retentionPeriodInWeeksForTesting), file_layout.NewPerWeekFileLayout(mockTime, volume_consts.LogsStorageDirpath)) + perWeekStreamStrategy := stream_logs_strategy.NewStreamLogsStrategyImpl(mockTime, persistent_volume_helpers.ConvertWeeksToDuration(retentionPeriodInWeeksForTesting), perWeekFileLayout) receivedUserServiceLogsByUuid, testEvaluationErr := executeStreamCallAndGetReceivedServiceLogLines( t, @@ -342,6 +436,59 @@ func TestStreamUserServiceLogsPerWeek_ThousandsOfLogLinesSuccessfulExecution(t * } } +func TestStreamUserServiceLogsPerHour_ThousandsOfLogLinesSuccessfulExecution(t *testing.T) { + expectedAmountLogLines := 10_000 + + expectedServiceAmountLogLinesByServiceUuid := map[service.ServiceUUID]int{ + testUserService1Uuid: expectedAmountLogLines, + } + + var emptyFilters []logline.LogLineFilter + + expectedFirstLogLine := "Starting feature 'centralized logs'" + + var logLines []string + + for i := 0; i <= expectedAmountLogLines; i++ { + logLines = append(logLines, logLine1) + } + + logLinesStr := strings.Join(logLines, "\n") + + userServiceUuids := map[service.ServiceUUID]bool{ + testUserService1Uuid: true, + } + + underlyingFs := volume_filesystem.NewMockedVolumeFilesystem() + mockTime := logs_clock.NewMockLogsClockPerHour(defaultYear, startingWeek, defaultDay, defaultHour) + perHourFileLayout := file_layout.NewPerHourFileLayout(mockTime, volume_consts.LogsStorageDirpath) + file1PathStr := perHourFileLayout.GetLogFilePath(mockTime.Now(), testEnclaveUuid, testUserService1Uuid) + file1, err := underlyingFs.Create(file1PathStr) + require.NoError(t, err) + _, err = file1.WriteString(logLinesStr) + require.NoError(t, err) + + perHourStreamStrategy := stream_logs_strategy.NewStreamLogsStrategyImpl(mockTime, persistent_volume_helpers.ConvertWeeksToDuration(retentionPeriodInWeeksForTesting), perHourFileLayout) + + receivedUserServiceLogsByUuid, testEvaluationErr := executeStreamCallAndGetReceivedServiceLogLines( + t, + emptyFilters, + userServiceUuids, + expectedServiceAmountLogLinesByServiceUuid, + doNotFollowLogs, + underlyingFs, + perHourStreamStrategy, + ) + require.NoError(t, testEvaluationErr) + + for serviceUuid, serviceLogLines := range receivedUserServiceLogsByUuid { + expectedAmountLogLines, found := expectedServiceAmountLogLinesByServiceUuid[serviceUuid] + require.True(t, found) + require.Equal(t, expectedAmountLogLines, len(serviceLogLines)) + require.Equal(t, expectedFirstLogLine, serviceLogLines[0].GetContent()) + } +} + func TestStreamUserServiceLogs_EmptyLogLines(t *testing.T) { expectedAmountLogLines := 0 @@ -400,15 +547,15 @@ func TestStreamUserServiceLogsPerWeek_EmptyLogLines(t *testing.T) { logLinesStr := "" underlyingFs := volume_filesystem.NewMockedVolumeFilesystem() - formattedWeekNum := fmt.Sprintf("%02d", startingWeek) - file1PathStr := fmt.Sprintf(file_layout.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), formattedWeekNum, string(enclaveUuid), testUserService1Uuid, volume_consts.Filetype) + mockTime := logs_clock.NewMockLogsClockPerDay(defaultYear, startingWeek, defaultDay) + perWeekFileLayout := file_layout.NewPerWeekFileLayout(mockTime, volume_consts.LogsStorageDirpath) + file1PathStr := perWeekFileLayout.GetLogFilePath(mockTime.Now(), testEnclaveUuid, testUserService1Uuid) file1, err := underlyingFs.Create(file1PathStr) require.NoError(t, err) _, err = file1.WriteString(logLinesStr) require.NoError(t, err) - mockTime := logs_clock.NewMockLogsClockPerDay(defaultYear, startingWeek, defaultDay) - perWeekStreamStrategy := stream_logs_strategy.NewStreamLogsStrategyImpl(mockTime, persistent_volume_helpers.ConvertWeeksToDuration(retentionPeriodInWeeksForTesting), file_layout.NewPerWeekFileLayout(mockTime, volume_consts.LogsStorageDirpath)) + perWeekStreamStrategy := stream_logs_strategy.NewStreamLogsStrategyImpl(mockTime, persistent_volume_helpers.ConvertWeeksToDuration(retentionPeriodInWeeksForTesting), perWeekFileLayout) receivedUserServiceLogsByUuid, testEvaluationErr := executeStreamCallAndGetReceivedServiceLogLines( t, @@ -428,6 +575,50 @@ func TestStreamUserServiceLogsPerWeek_EmptyLogLines(t *testing.T) { } } +func TestStreamUserServiceLogsPerHour_EmptyLogLines(t *testing.T) { + expectedAmountLogLines := 0 + + expectedServiceAmountLogLinesByServiceUuid := map[service.ServiceUUID]int{ + testUserService1Uuid: expectedAmountLogLines, + } + + var emptyFilters []logline.LogLineFilter + + userServiceUuids := map[service.ServiceUUID]bool{ + testUserService1Uuid: true, + } + + logLinesStr := "" + + underlyingFs := volume_filesystem.NewMockedVolumeFilesystem() + mockTime := logs_clock.NewMockLogsClockPerHour(defaultYear, startingWeek, defaultDay, defaultHour) + perHourFileLayout := file_layout.NewPerHourFileLayout(mockTime, volume_consts.LogsStorageDirpath) + file1PathStr := perHourFileLayout.GetLogFilePath(mockTime.Now(), testEnclaveUuid, testUserService1Uuid) + file1, err := underlyingFs.Create(file1PathStr) + require.NoError(t, err) + _, err = file1.WriteString(logLinesStr) + require.NoError(t, err) + + perHourStreamStrategy := stream_logs_strategy.NewStreamLogsStrategyImpl(mockTime, persistent_volume_helpers.ConvertWeeksToDuration(retentionPeriodInWeeksForTesting), perHourFileLayout) + + receivedUserServiceLogsByUuid, testEvaluationErr := executeStreamCallAndGetReceivedServiceLogLines( + t, + emptyFilters, + userServiceUuids, + expectedServiceAmountLogLinesByServiceUuid, + doNotFollowLogs, + underlyingFs, + perHourStreamStrategy, + ) + require.NoError(t, testEvaluationErr) + + for serviceUuid, serviceLogLines := range receivedUserServiceLogsByUuid { + expectedAmountLogLines, found := expectedServiceAmountLogLinesByServiceUuid[serviceUuid] + require.True(t, found) + require.Equal(t, expectedAmountLogLines, len(serviceLogLines)) + } +} + func TestStreamUserServiceLogsPerWeek_WithLogsAcrossWeeks(t *testing.T) { expectedAmountLogLines := 8 @@ -456,26 +647,26 @@ func TestStreamUserServiceLogsPerWeek_WithLogsAcrossWeeks(t *testing.T) { logLine4} underlyingFs := volume_filesystem.NewMockedVolumeFilesystem() + mockTime := logs_clock.NewMockLogsClockPerDay(defaultYear, 4, defaultDay) + perWeekFileLayout := file_layout.NewPerWeekFileLayout(mockTime, volume_consts.LogsStorageDirpath) week3logLinesStr := strings.Join(week3logLines, "\n") + "\n" week4logLinesStr := strings.Join(week4logLines, "\n") - formattedWeekFour := fmt.Sprintf("%02d", 4) - week4filepath := fmt.Sprintf(file_layout.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), formattedWeekFour, testEnclaveUuid, testUserService1Uuid, volume_consts.Filetype) + week4filepath := perWeekFileLayout.GetLogFilePath(mockTime.Now(), testEnclaveUuid, testUserService1Uuid) week4, err := underlyingFs.Create(week4filepath) require.NoError(t, err) _, err = week4.WriteString(week4logLinesStr) require.NoError(t, err) - formattedWeekThree := fmt.Sprintf("%02d", 3) - week3filepath := fmt.Sprintf(file_layout.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), formattedWeekThree, testEnclaveUuid, testUserService1Uuid, volume_consts.Filetype) + week3Time := logs_clock.NewMockLogsClockPerDay(defaultYear, 3, defaultDay) + week3filepath := perWeekFileLayout.GetLogFilePath(week3Time.Now(), testEnclaveUuid, testUserService1Uuid) week3, err := underlyingFs.Create(week3filepath) require.NoError(t, err) _, err = week3.WriteString(week3logLinesStr) require.NoError(t, err) - mockTime := logs_clock.NewMockLogsClockPerDay(defaultYear, 4, defaultDay) - perWeekStreamStrategy := stream_logs_strategy.NewStreamLogsStrategyImpl(mockTime, persistent_volume_helpers.ConvertWeeksToDuration(retentionPeriodInWeeksForTesting), file_layout.NewPerWeekFileLayout(mockTime, volume_consts.LogsStorageDirpath)) + perWeekStreamStrategy := stream_logs_strategy.NewStreamLogsStrategyImpl(mockTime, persistent_volume_helpers.ConvertWeeksToDuration(retentionPeriodInWeeksForTesting), perWeekFileLayout) receivedUserServiceLogsByUuid, testEvaluationErr := executeStreamCallAndGetReceivedServiceLogLines( t, @@ -494,7 +685,74 @@ func TestStreamUserServiceLogsPerWeek_WithLogsAcrossWeeks(t *testing.T) { require.Equal(t, expectedAmountLogLines, len(serviceLogLines)) require.Equal(t, expectedFirstLogLine, serviceLogLines[0].GetContent()) } +} + +func TestStreamUserServiceLogsPerHour_WithLogsAcrossHours(t *testing.T) { + expectedAmountLogLines := 8 + + expectedServiceAmountLogLinesByServiceUuid := map[service.ServiceUUID]int{ + testUserService1Uuid: expectedAmountLogLines, + } + var logLinesFilters []logline.LogLineFilter + + userServiceUuids := map[service.ServiceUUID]bool{ + testUserService1Uuid: true, + } + + expectedFirstLogLine := "Starting feature 'centralized logs'" + + hour4logLines := []string{ + logLine5, + logLine6, + logLine7, + logLine8} + hour3logLines := []string{ + logLine1, + logLine2, + logLine3a, + logLine3b, + logLine4} + + underlyingFs := volume_filesystem.NewMockedVolumeFilesystem() + hour4Time := logs_clock.NewMockLogsClockPerHour(defaultYear, startingWeek, defaultDay, 4) + perHourFileLayout := file_layout.NewPerHourFileLayout(hour4Time, volume_consts.LogsStorageDirpath) + + hour3logLinesStr := strings.Join(hour3logLines, "\n") + "\n" + hour4logLinesStr := strings.Join(hour4logLines, "\n") + + hour4filepath := perHourFileLayout.GetLogFilePath(hour4Time.Now(), testEnclaveUuid, testUserService1Uuid) + hour4, err := underlyingFs.Create(hour4filepath) + require.NoError(t, err) + _, err = hour4.WriteString(hour4logLinesStr) + require.NoError(t, err) + + hour3Time := logs_clock.NewMockLogsClockPerHour(defaultYear, startingWeek, defaultDay, 3) + hour3filepath := perHourFileLayout.GetLogFilePath(hour3Time.Now(), testEnclaveUuid, testUserService1Uuid) + hour3, err := underlyingFs.Create(hour3filepath) + require.NoError(t, err) + _, err = hour3.WriteString(hour3logLinesStr) + require.NoError(t, err) + + perHourStreamStrategy := stream_logs_strategy.NewStreamLogsStrategyImpl(hour4Time, persistent_volume_helpers.ConvertWeeksToDuration(retentionPeriodInWeeksForTesting), perHourFileLayout) + + receivedUserServiceLogsByUuid, testEvaluationErr := executeStreamCallAndGetReceivedServiceLogLines( + t, + logLinesFilters, + userServiceUuids, + expectedServiceAmountLogLinesByServiceUuid, + doNotFollowLogs, + underlyingFs, + perHourStreamStrategy, + ) + require.NoError(t, testEvaluationErr) + + for serviceUuid, serviceLogLines := range receivedUserServiceLogsByUuid { + expectedAmountLogLines, found := expectedServiceAmountLogLinesByServiceUuid[serviceUuid] + require.True(t, found) + require.Equal(t, expectedAmountLogLines, len(serviceLogLines)) + require.Equal(t, expectedFirstLogLine, serviceLogLines[0].GetContent()) + } } func TestStreamUserServiceLogsPerWeek_WithLogLineAcrossWeeks(t *testing.T) { @@ -565,6 +823,74 @@ func TestStreamUserServiceLogsPerWeek_WithLogLineAcrossWeeks(t *testing.T) { } } +func TestStreamUserServiceLogsPerHour_WithLogLineAcrossHours(t *testing.T) { + expectedAmountLogLines := 8 + + expectedServiceAmountLogLinesByServiceUuid := map[service.ServiceUUID]int{ + testUserService1Uuid: expectedAmountLogLines, + } + + var logLinesFilters []logline.LogLineFilter + + userServiceUuids := map[service.ServiceUUID]bool{ + testUserService1Uuid: true, + } + + expectedFirstLogLine := "Starting feature 'centralized logs'" + + hour4logLines := []string{ + logLine3b, + logLine4, + logLine5, + logLine6, + logLine7, + logLine8} + hour3logLines := []string{ + logLine1, + logLine2, + logLine3a} + + underlyingFs := volume_filesystem.NewMockedVolumeFilesystem() + hour4Time := logs_clock.NewMockLogsClockPerHour(defaultYear, startingWeek, defaultDay, 4) + perHourFileLayout := file_layout.NewPerHourFileLayout(hour4Time, volume_consts.LogsStorageDirpath) + + hour3logLinesStr := strings.Join(hour3logLines, "\n") + "\n" + hour4logLinesStr := strings.Join(hour4logLines, "\n") + + hour4filepath := perHourFileLayout.GetLogFilePath(hour4Time.Now(), testEnclaveUuid, testUserService1Uuid) + hour4, err := underlyingFs.Create(hour4filepath) + require.NoError(t, err) + _, err = hour4.WriteString(hour4logLinesStr) + require.NoError(t, err) + + hour3Time := logs_clock.NewMockLogsClockPerHour(defaultYear, startingWeek, defaultDay, 3) + hour3filepath := perHourFileLayout.GetLogFilePath(hour3Time.Now(), testEnclaveUuid, testUserService1Uuid) + hour3, err := underlyingFs.Create(hour3filepath) + require.NoError(t, err) + _, err = hour3.WriteString(hour3logLinesStr) + require.NoError(t, err) + + perHourStreamStrategy := stream_logs_strategy.NewStreamLogsStrategyImpl(hour4Time, persistent_volume_helpers.ConvertWeeksToDuration(retentionPeriodInWeeksForTesting), perHourFileLayout) + + receivedUserServiceLogsByUuid, testEvaluationErr := executeStreamCallAndGetReceivedServiceLogLines( + t, + logLinesFilters, + userServiceUuids, + expectedServiceAmountLogLinesByServiceUuid, + doNotFollowLogs, + underlyingFs, + perHourStreamStrategy, + ) + require.NoError(t, testEvaluationErr) + + for serviceUuid, serviceLogLines := range receivedUserServiceLogsByUuid { + expectedAmountLogLines, found := expectedServiceAmountLogLinesByServiceUuid[serviceUuid] + require.True(t, found) + require.Equal(t, expectedAmountLogLines, len(serviceLogLines)) + require.Equal(t, expectedFirstLogLine, serviceLogLines[0].GetContent()) + } +} + func TestStreamUserServiceLogsPerWeekReturnsTimestampedLogLines(t *testing.T) { expectedAmountLogLines := 3 @@ -586,16 +912,16 @@ func TestStreamUserServiceLogsPerWeekReturnsTimestampedLogLines(t *testing.T) { timestampedLogLinesStr := strings.Join(timestampedLogLines, "\n") + "\n" underlyingFs := volume_filesystem.NewMockedVolumeFilesystem() + mockTime := logs_clock.NewMockLogsClockPerDay(defaultYear, startingWeek, defaultDay) + perWeekFileLayout := file_layout.NewPerWeekFileLayout(mockTime, volume_consts.LogsStorageDirpath) - formattedWeekNum := fmt.Sprintf("%02d", startingWeek) - filepath := fmt.Sprintf(file_layout.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), formattedWeekNum, testEnclaveUuid, testUserService1Uuid, volume_consts.Filetype) + filepath := perWeekFileLayout.GetLogFilePath(mockTime.Now(), testEnclaveUuid, testUserService1Uuid) file, err := underlyingFs.Create(filepath) require.NoError(t, err) _, err = file.WriteString(timestampedLogLinesStr) require.NoError(t, err) - mockTime := logs_clock.NewMockLogsClockPerDay(defaultYear, startingWeek, defaultDay) - perWeekStreamStrategy := stream_logs_strategy.NewStreamLogsStrategyImpl(mockTime, persistent_volume_helpers.ConvertWeeksToDuration(retentionPeriodInWeeksForTesting), file_layout.NewPerWeekFileLayout(mockTime, volume_consts.LogsStorageDirpath)) + perWeekStreamStrategy := stream_logs_strategy.NewStreamLogsStrategyImpl(mockTime, persistent_volume_helpers.ConvertWeeksToDuration(retentionPeriodInWeeksForTesting), perWeekFileLayout) expectedTime, err := time.Parse(utcFormat, defaultUTCTimestampStr) require.NoError(t, err) @@ -621,6 +947,62 @@ func TestStreamUserServiceLogsPerWeekReturnsTimestampedLogLines(t *testing.T) { } } +func TestStreamUserServiceLogsPerHourReturnsTimestampedLogLines(t *testing.T) { + expectedAmountLogLines := 3 + + expectedServiceAmountLogLinesByServiceUuid := map[service.ServiceUUID]int{ + testUserService1Uuid: expectedAmountLogLines, + } + + var logLinesFilters []logline.LogLineFilter + + userServiceUuids := map[service.ServiceUUID]bool{ + testUserService1Uuid: true, + } + + timedLogLine1 := fmt.Sprintf("{\"log\":\"Starting feature 'centralized logs'\", \"timestamp\":\"%v\"}", defaultUTCTimestampStr) + timedLogLine2 := fmt.Sprintf("{\"log\":\"Starting feature 'runs idempotently'\", \"timestamp\":\"%v\"}", defaultUTCTimestampStr) + timedLogLine3 := fmt.Sprintf("{\"log\":\"The enclave was created\", \"timestamp\":\"%v\"}", defaultUTCTimestampStr) + + timestampedLogLines := []string{timedLogLine1, timedLogLine2, timedLogLine3} + timestampedLogLinesStr := strings.Join(timestampedLogLines, "\n") + "\n" + + underlyingFs := volume_filesystem.NewMockedVolumeFilesystem() + mockTime := logs_clock.NewMockLogsClockPerHour(defaultYear, startingWeek, defaultDay, defaultHour) + perHourFileLayout := file_layout.NewPerWeekFileLayout(mockTime, volume_consts.LogsStorageDirpath) + + filepath := perHourFileLayout.GetLogFilePath(mockTime.Now(), testEnclaveUuid, testUserService1Uuid) + file, err := underlyingFs.Create(filepath) + require.NoError(t, err) + _, err = file.WriteString(timestampedLogLinesStr) + require.NoError(t, err) + + perHourStreamStrategy := stream_logs_strategy.NewStreamLogsStrategyImpl(mockTime, persistent_volume_helpers.ConvertWeeksToDuration(retentionPeriodInWeeksForTesting), perHourFileLayout) + + expectedTime, err := time.Parse(utcFormat, defaultUTCTimestampStr) + require.NoError(t, err) + + receivedUserServiceLogsByUuid, testEvaluationErr := executeStreamCallAndGetReceivedServiceLogLines( + t, + logLinesFilters, + userServiceUuids, + expectedServiceAmountLogLinesByServiceUuid, + doNotFollowLogs, + underlyingFs, + perHourStreamStrategy, + ) + require.NoError(t, testEvaluationErr) + + for serviceUuid, serviceLogLines := range receivedUserServiceLogsByUuid { + expectedAmountLogLines, found := expectedServiceAmountLogLinesByServiceUuid[serviceUuid] + require.True(t, found) + require.Equal(t, expectedAmountLogLines, len(serviceLogLines)) + for _, logLine := range serviceLogLines { + require.Equal(t, expectedTime, logLine.GetTimestamp()) + } + } +} + func TestStreamUserServiceLogsPerFileReturnsTimestampedLogLines(t *testing.T) { expectedAmountLogLines := 3 @@ -697,10 +1079,8 @@ func executeStreamCallAndGetReceivedServiceLogLines( kurtosisBackend := backend_interface.NewMockKurtosisBackend(t) - // no log file management is done in these tests so values for logFileManager aren't important - mockTime := logs_clock.NewMockLogsClockPerDay(0, 0, 0) - fileLayout := file_layout.NewPerWeekFileLayout(mockTime, volume_consts.LogsStorageDirpath) - logFileManager := log_file_manager.NewLogFileManager(kurtosisBackend, underlyingFs, fileLayout, mockTime, time.Duration(0), volume_consts.LogsStorageDirpath) + // no log file management is done in these tests so values provided to logFileManager aren't important + logFileManager := log_file_manager.NewLogFileManager(nil, nil, nil, nil, time.Duration(0), "") logsDatabaseClient := NewPersistentVolumeLogsDatabaseClient(kurtosisBackend, underlyingFs, logFileManager, streamStrategy) userServiceLogsByUuidChan, errChan, receivedCancelCtxFunc, err := logsDatabaseClient.StreamUserServiceLogs(ctx, enclaveUuid, userServiceUuids, logLinesFilters, shouldFollowLogs, defaultShouldReturnAllLogs, defaultNumLogLines) @@ -756,14 +1136,13 @@ func executeStreamCallAndGetReceivedServiceLogLines( return receivedServiceLogsByUuid, nil } -func createFilledPerFileFilesystem() volume_filesystem.VolumeFilesystem { +func createFilledFilesystem(fileLayout file_layout.LogFileLayout, time time.Time) volume_filesystem.VolumeFilesystem { logLines := []string{logLine1, logLine2, logLine3a, logLine3b, logLine4, logLine5, logLine6, logLine7, logLine8} - logLinesStr := strings.Join(logLines, "\n") - file1PathStr := fmt.Sprintf(volume_consts.PerFileFmtStr, volume_consts.LogsStorageDirpath, testEnclaveUuid, testUserService1Uuid, volume_consts.Filetype) - file2PathStr := fmt.Sprintf(volume_consts.PerFileFmtStr, volume_consts.LogsStorageDirpath, testEnclaveUuid, testUserService2Uuid, volume_consts.Filetype) - file3PathStr := fmt.Sprintf(volume_consts.PerFileFmtStr, volume_consts.LogsStorageDirpath, testEnclaveUuid, testUserService3Uuid, volume_consts.Filetype) + file1PathStr := fileLayout.GetLogFilePath(time, testEnclaveUuid, testUserService1Uuid) + file2PathStr := fileLayout.GetLogFilePath(time, testEnclaveUuid, testUserService2Uuid) + file3PathStr := fileLayout.GetLogFilePath(time, testEnclaveUuid, testUserService3Uuid) mapFs := volume_filesystem.NewMockedVolumeFilesystem() @@ -779,26 +1158,16 @@ func createFilledPerFileFilesystem() volume_filesystem.VolumeFilesystem { return mapFs } -func createFilledPerWeekFilesystem(week int) volume_filesystem.VolumeFilesystem { - logLines := []string{logLine1, logLine2, logLine3a, logLine3b, logLine4, logLine5, logLine6, logLine7, logLine8} - - logLinesStr := strings.Join(logLines, "\n") - // %02d to format week num with leading zeros so 1-9 are converted to 01-09 for %V format - formattedWeekNum := fmt.Sprintf("%02d", week) - file1PathStr := fmt.Sprintf(file_layout.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), formattedWeekNum, testEnclaveUuid, testUserService1Uuid, volume_consts.Filetype) - file2PathStr := fmt.Sprintf(file_layout.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), formattedWeekNum, testEnclaveUuid, testUserService2Uuid, volume_consts.Filetype) - file3PathStr := fmt.Sprintf(file_layout.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), formattedWeekNum, testEnclaveUuid, testUserService3Uuid, volume_consts.Filetype) +func createEmptyFilesystem(fileLayout file_layout.LogFileLayout, time time.Time) volume_filesystem.VolumeFilesystem { + file1PathStr := fileLayout.GetLogFilePath(time, testEnclaveUuid, testUserService1Uuid) + file2PathStr := fileLayout.GetLogFilePath(time, testEnclaveUuid, testUserService2Uuid) + file3PathStr := fileLayout.GetLogFilePath(time, testEnclaveUuid, testUserService3Uuid) mapFs := volume_filesystem.NewMockedVolumeFilesystem() - file1, _ := mapFs.Create(file1PathStr) - _, _ = file1.WriteString(logLinesStr) - - file2, _ := mapFs.Create(file2PathStr) - _, _ = file2.WriteString(logLinesStr) - - file3, _ := mapFs.Create(file3PathStr) - _, _ = file3.WriteString(logLinesStr) + _, _ = mapFs.Create(file1PathStr) + _, _ = mapFs.Create(file2PathStr) + _, _ = mapFs.Create(file3PathStr) return mapFs } @@ -817,18 +1186,25 @@ func createEmptyPerFileFilesystem() volume_filesystem.VolumeFilesystem { return mapFs } -func createEmptyPerWeekFilesystem(week int) volume_filesystem.VolumeFilesystem { - // %02d to format week num with leading zeros so 1-9 are converted to 01-09 for %V format - formattedWeekNum := fmt.Sprintf("%02d", week) - file1PathStr := fmt.Sprintf(file_layout.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), formattedWeekNum, testEnclaveUuid, testUserService1Uuid, volume_consts.Filetype) - file2PathStr := fmt.Sprintf(file_layout.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), formattedWeekNum, testEnclaveUuid, testUserService2Uuid, volume_consts.Filetype) - file3PathStr := fmt.Sprintf(file_layout.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), formattedWeekNum, testEnclaveUuid, testUserService3Uuid, volume_consts.Filetype) +func createFilledPerFileFilesystem() volume_filesystem.VolumeFilesystem { + logLines := []string{logLine1, logLine2, logLine3a, logLine3b, logLine4, logLine5, logLine6, logLine7, logLine8} + + logLinesStr := strings.Join(logLines, "\n") + + file1PathStr := fmt.Sprintf(volume_consts.PerFileFmtStr, volume_consts.LogsStorageDirpath, testEnclaveUuid, testUserService1Uuid, volume_consts.Filetype) + file2PathStr := fmt.Sprintf(volume_consts.PerFileFmtStr, volume_consts.LogsStorageDirpath, testEnclaveUuid, testUserService2Uuid, volume_consts.Filetype) + file3PathStr := fmt.Sprintf(volume_consts.PerFileFmtStr, volume_consts.LogsStorageDirpath, testEnclaveUuid, testUserService3Uuid, volume_consts.Filetype) mapFs := volume_filesystem.NewMockedVolumeFilesystem() - _, _ = mapFs.Create(file1PathStr) - _, _ = mapFs.Create(file2PathStr) - _, _ = mapFs.Create(file3PathStr) + file1, _ := mapFs.Create(file1PathStr) + _, _ = file1.WriteString(logLinesStr) + + file2, _ := mapFs.Create(file2PathStr) + _, _ = file2.WriteString(logLinesStr) + + file3, _ := mapFs.Create(file3PathStr) + _, _ = file3.WriteString(logLinesStr) return mapFs }