feat: Add Beekeeper history table (#184)

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.6.0] - 2024-11-29
+## Added
+- Added a db migration file and implementation of `beekeeper-history` table to track beekeeper activity.
+
 ## [3.5.8] - 2024-11-28
 ### Added
 - Added `IcebergTableListenerEventFilter` filter for Iceberg tables in `beekeeper-scheduler-apiary` to prevent scheduling paths and metadata for deletion. 

beekeeper-core/pom.xml

@@ -44,6 +44,27 @@
       <version>27.1-jre</version>
     </dependency>
 
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+      <version>${jackson.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-annotations</artifactId>
+      <version>${jackson.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-core</artifactId>
+      <version>${jackson.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.datatype</groupId>
+      <artifactId>jackson-datatype-jsr310</artifactId>
+      <version>${jackson.version}</version>
+    </dependency>
+
     <!-- micrometer -->
     <dependency>
       <groupId>io.micrometer</groupId>

beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingStatus.java

@@ -18,6 +18,8 @@
 public enum HousekeepingStatus {
   SCHEDULED,
   FAILED,
+  FAILED_TO_DELETE,
+  FAILED_TO_SCHEDULE,
   DELETED,
   DISABLED,
   SKIPPED

beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/BeekeeperHistory.java

@@ -0,0 +1,73 @@
+package com.expediagroup.beekeeper.core.model.history;
+
+import java.time.LocalDateTime;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.Table;
+
+import lombok.Builder;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+
+import com.expediagroup.beekeeper.core.monitoring.MetricTag;
+import com.expediagroup.beekeeper.core.monitoring.Taggable;
+
+@Data
+@NoArgsConstructor
+@Entity
+@Table(name = "beekeeper_history")
+public class BeekeeperHistory implements Taggable {
+
+  @Id
+  @GeneratedValue(strategy = GenerationType.IDENTITY)
+  private Long id;
+
+  @EqualsAndHashCode.Exclude
+  @Column(name = "event_timestamp", nullable = false, updatable = false)
+  private LocalDateTime eventTimestamp;
+
+  @Column(name = "database_name", nullable = false)
+  private String databaseName;
+
+  @Column(name = "table_name", nullable = false)
+  private String tableName;
+
+  @Column(name = "lifecycle_type", nullable = false)
+  private String lifecycleType;
+
+  @Column(name = "housekeeping_status", nullable = false)
+  private String housekeepingStatus;
+
+  @Column(name = "event_details", columnDefinition = "TEXT")
+  private String eventDetails;
+
+  @Builder
+  public BeekeeperHistory(
+      Long id,
+      LocalDateTime eventTimestamp,
+      String databaseName,
+      String tableName,
+      String lifecycleType,
+      String housekeepingStatus,
+      String eventDetails
+  ) {
+    this.id = id;
+    this.eventTimestamp = eventTimestamp;
+    this.databaseName = databaseName;
+    this.tableName = tableName;
+    this.lifecycleType = lifecycleType;
+    this.housekeepingStatus = housekeepingStatus;
+    this.eventDetails = eventDetails;
+  }
+
+  @Override
+  public MetricTag getMetricTag() {
+    return new MetricTag("table", String.join(".", databaseName, tableName));
+  }
+
+}

beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepository.java

@@ -0,0 +1,19 @@
+package com.expediagroup.beekeeper.core.repository;
+
+import org.springframework.data.domain.Pageable;
+import org.springframework.data.domain.Slice;
+import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
+import org.springframework.data.jpa.repository.Query;
+import org.springframework.data.repository.PagingAndSortingRepository;
+import org.springframework.data.repository.query.Param;
+
+import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory;
+
+public interface BeekeeperHistoryRepository extends PagingAndSortingRepository<BeekeeperHistory, Long>,
+    JpaSpecificationExecutor<BeekeeperHistory> {
+
+  @Query(value = "from BeekeeperHistory t where t.lifecycleType = :lifecycle")
+  Slice<BeekeeperHistory> findRecordsByLifecycleType(
+      @Param("lifecycle") String lifecycle,
+      Pageable pageable);
+}

beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java

@@ -0,0 +1,37 @@
+package com.expediagroup.beekeeper.core.service;
+
+import java.time.LocalDateTime;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.expediagroup.beekeeper.core.model.HousekeepingEntity;
+import com.expediagroup.beekeeper.core.model.HousekeepingStatus;
+import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory;
+import com.expediagroup.beekeeper.core.repository.BeekeeperHistoryRepository;
+
+public class BeekeeperHistoryService {
+
+  private static final Logger log = LoggerFactory.getLogger(BeekeeperHistoryService.class);
+
+  private final BeekeeperHistoryRepository beekeeperHistoryRepository;
+
+  public BeekeeperHistoryService(BeekeeperHistoryRepository beekeeperHistoryRepository) {
+    this.beekeeperHistoryRepository = beekeeperHistoryRepository;
+  }
+
+  public void saveHistory(HousekeepingEntity housekeepingEntity, HousekeepingStatus status) {
+    BeekeeperHistory event = BeekeeperHistory.builder()
+        .id(housekeepingEntity.getId())
+        .eventTimestamp(LocalDateTime.now())
+        .databaseName(housekeepingEntity.getDatabaseName())
+        .tableName(housekeepingEntity.getTableName())
+        .lifecycleType(housekeepingEntity.getLifecycleType())
+        .housekeepingStatus(status.name())
+        .eventDetails(housekeepingEntity.toString())
+        .build();
+
+    log.info("Saving activity in Beekeeper History table; {}", event);
+    beekeeperHistoryRepository.save(event);
+  }
+}

beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepositoryTest.java

@@ -0,0 +1,141 @@
+package com.expediagroup.beekeeper.core.repository;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED;
+import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.FAILED;
+import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED;
+
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.util.List;
+
+import org.assertj.core.util.Lists;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.domain.PageRequest;
+import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.TestPropertySource;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+import org.springframework.test.context.support.AnnotationConfigContextLoader;
+
+import com.expediagroup.beekeeper.core.TestApplication;
+import com.expediagroup.beekeeper.core.model.HousekeepingMetadata;
+import com.expediagroup.beekeeper.core.model.HousekeepingPath;
+import com.expediagroup.beekeeper.core.model.HousekeepingStatus;
+import com.expediagroup.beekeeper.core.model.PeriodDuration;
+import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory;
+
+@ExtendWith(SpringExtension.class)
+@TestPropertySource(properties = {
+    "hibernate.data-source.driver-class-name=org.h2.Driver",
+    "hibernate.dialect=org.hibernate.dialect.H2Dialect",
+    "hibernate.hbm2ddl.auto=create",
+    "spring.jpa.show-sql=true",
+    "spring.datasource.url=jdbc:h2:mem:beekeeper;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE;MODE=MySQL" })
+@ContextConfiguration(classes = { TestApplication.class }, loader = AnnotationConfigContextLoader.class)
+@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS)
+public class BeekeeperHistoryRepositoryTest {
+
+  protected static final String DATABASE_NAME = "database";
+  protected static final String TABLE_NAME = "table";
+  protected static final PeriodDuration CLEANUP_DELAY = PeriodDuration.parse("P3D");
+  protected static final LocalDateTime COLUMN_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC"));
+  protected static final LocalDateTime EVENT_TIMESTAMP = COLUMN_TIMESTAMP.plus(CLEANUP_DELAY);
+
+  private static final int PAGE = 0;
+  private static final int PAGE_SIZE = 500;
+
+  @Autowired
+  private BeekeeperHistoryRepository repository;
+
+  @BeforeEach
+  public void setupDb() {
+    repository.deleteAll();
+  }
+
+  @Test
+  public void typicalSave() {
+    BeekeeperHistory expiredEntry = createExpiredEvent(SCHEDULED);
+    BeekeeperHistory unreferencedEntry = createUnreferencedEvent(SCHEDULED);
+
+    repository.save(expiredEntry);
+    repository.save(unreferencedEntry);
+
+    List<BeekeeperHistory> historyList = Lists.newArrayList(
+        repository.findRecordsByLifecycleType("EXPIRED", PageRequest.of(PAGE, PAGE_SIZE)));
+    assertThat(historyList.size()).isEqualTo(1);
+
+    historyList = Lists.newArrayList(
+        repository.findRecordsByLifecycleType("UNREFERENCED", PageRequest.of(PAGE, PAGE_SIZE)));
+    assertThat(historyList.size()).isEqualTo(1);
+  }
+
+  @Test
+  public void expired_multipleStatuses() {
+    BeekeeperHistory scheduledEntry = createExpiredEvent(SCHEDULED);
+    BeekeeperHistory deletedEntry = createExpiredEvent(DELETED);
+    BeekeeperHistory failedEntry = createExpiredEvent(FAILED);
+
+    repository.save(scheduledEntry);
+    repository.save(deletedEntry);
+    repository.save(failedEntry);
+
+    List<BeekeeperHistory> historyList = Lists.newArrayList(
+        repository.findRecordsByLifecycleType("EXPIRED", PageRequest.of(PAGE, PAGE_SIZE)));
+    assertThat(historyList.size()).isEqualTo(3);
+  }
+
+  @Test
+  public void unreferenced_multipleStatuses() {
+    BeekeeperHistory scheduledEntry = createUnreferencedEvent(SCHEDULED);
+    BeekeeperHistory deletedEntry = createUnreferencedEvent(DELETED);
+    BeekeeperHistory failedEntry = createUnreferencedEvent(FAILED);
+
+    repository.save(scheduledEntry);
+    repository.save(deletedEntry);
+    repository.save(failedEntry);
+
+    List<BeekeeperHistory> historyList = Lists.newArrayList(
+        repository.findRecordsByLifecycleType("UNREFERENCED", PageRequest.of(PAGE, PAGE_SIZE)));
+    assertThat(historyList.size()).isEqualTo(3);
+  }
+
+  protected BeekeeperHistory createExpiredEvent(HousekeepingStatus status) {
+    HousekeepingMetadata entity = HousekeepingMetadata.builder()
+        .cleanupAttempts(3)
+        .cleanupDelay(PeriodDuration.parse("P1D"))
+        .partitionName("event_date")
+        .creationTimestamp(COLUMN_TIMESTAMP)
+        .modifiedTimestamp(COLUMN_TIMESTAMP)
+        .build();
+
+    return createHistoryEntry("EXPIRED", status, entity.toString());
+  }
+
+  protected BeekeeperHistory createUnreferencedEvent(HousekeepingStatus status) {
+    HousekeepingPath entity = HousekeepingPath.builder()
+        .cleanupAttempts(3)
+        .cleanupDelay(PeriodDuration.parse("P1D"))
+        .creationTimestamp(COLUMN_TIMESTAMP)
+        .modifiedTimestamp(COLUMN_TIMESTAMP)
+        .build();
+
+    return createHistoryEntry("UNREFERENCED", status, entity.toString());
+  }
+
+  protected BeekeeperHistory createHistoryEntry(String lifecycleType, HousekeepingStatus status,
+      String eventDetails) {
+    return BeekeeperHistory.builder()
+        .eventTimestamp(EVENT_TIMESTAMP)
+        .databaseName(DATABASE_NAME)
+        .tableName(TABLE_NAME)
+        .lifecycleType(lifecycleType)
+        .housekeepingStatus(status.name())
+        .eventDetails(eventDetails)
+        .build();
+  }
+}