decompose notifcation services, separate queue impl (#420)

finos · Nov 7, 2023 · d7c26a9 · d7c26a9
1 parent d5b91c4
commit d7c26a9
Show file tree

Hide file tree

Showing 80 changed files with 1,697 additions and 815 deletions.
diff --git a/legend-depot-artifacts-api/pom.xml b/legend-depot-artifacts-api/pom.xml
@@ -46,6 +46,10 @@
             <groupId>org.finos.legend.depot</groupId>
             <artifactId>legend-depot-generations-api</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.finos.legend.depot</groupId>
+            <artifactId>legend-depot-notifications-api</artifactId>
+        </dependency>
         <dependency>
             <groupId>${junit.groupId}</groupId>
             <artifactId>${junit.artifactId}</artifactId>

diff --git a/.../java/org/finos/legend/depot/services/api/artifacts/handlers/ProjectArtifactsHandler.java b/.../java/org/finos/legend/depot/services/api/artifacts/handlers/ProjectArtifactsHandler.java
@@ -15,15 +15,15 @@
 
 package org.finos.legend.depot.services.api.artifacts.handlers;
 
-import org.finos.legend.depot.domain.api.MetadataEventResponse;
+import org.finos.legend.depot.domain.notifications.MetadataNotificationResponse;
 
 import java.io.File;
 import java.util.List;
 
 public interface ProjectArtifactsHandler
 {
 
-    MetadataEventResponse refreshProjectVersionArtifacts(String groupId,String artifactId, String versionId, List<File> files);
+    MetadataNotificationResponse refreshProjectVersionArtifacts(String groupId, String artifactId, String versionId, List<File> files);
 
     void delete(String groupId, String artifactId, String versionId);
 }
diff --git a/.../main/java/org/finos/legend/depot/services/api/artifacts/purge/ArtifactsPurgeService.java b/.../main/java/org/finos/legend/depot/services/api/artifacts/purge/ArtifactsPurgeService.java
@@ -15,22 +15,22 @@
 
 package org.finos.legend.depot.services.api.artifacts.purge;
 
-import org.finos.legend.depot.domain.api.MetadataEventResponse;
+import org.finos.legend.depot.domain.notifications.MetadataNotificationResponse;
 
 public interface ArtifactsPurgeService
 {
-    MetadataEventResponse evictLeastRecentlyUsed(int ttlForVersionsInDays, int ttlForSnapshotsInDays);
+    MetadataNotificationResponse evictLeastRecentlyUsed(int ttlForVersionsInDays, int ttlForSnapshotsInDays);
 
-    MetadataEventResponse evictVersionsNotUsed();
+    MetadataNotificationResponse evictVersionsNotUsed();
 
-    MetadataEventResponse evictOldestProjectVersions(String groupId, String artifactId, int versionsToKeep);
+    MetadataNotificationResponse evictOldestProjectVersions(String groupId, String artifactId, int versionsToKeep);
 
     void evict(String groupId, String artifactId, String versionId);
 
     void delete(String groupId, String artifactId, String versionId);
 
-    MetadataEventResponse deprecate(String groupId, String artifactId, String version);
+    MetadataNotificationResponse deprecate(String groupId, String artifactId, String version);
 
-    MetadataEventResponse deprecateVersionsNotInRepository();
+    MetadataNotificationResponse deprecateVersionsNotInRepository();
 
   }
diff --git a/...n/java/org/finos/legend/depot/services/api/artifacts/refresh/ArtifactsRefreshService.java b/...n/java/org/finos/legend/depot/services/api/artifacts/refresh/ArtifactsRefreshService.java
@@ -15,22 +15,22 @@
 
 package org.finos.legend.depot.services.api.artifacts.refresh;
 
-import org.finos.legend.depot.domain.api.MetadataEventResponse;
+import org.finos.legend.depot.domain.notifications.MetadataNotificationResponse;
 
 
 public interface ArtifactsRefreshService
 {
-    default MetadataEventResponse refreshVersionForProject(String groupId, String artifactId, String versionId,boolean transitive, String parentEventId)
+    default MetadataNotificationResponse refreshVersionForProject(String groupId, String artifactId, String versionId, boolean transitive, String parentEventId)
     {
         return refreshVersionForProject(groupId, artifactId, versionId,false, transitive, parentEventId);
     }
 
-    MetadataEventResponse refreshVersionForProject(String groupId, String artifactId, String versionId,boolean fullUpdate, boolean transitive, String parentEventId);
+    MetadataNotificationResponse refreshVersionForProject(String groupId, String artifactId, String versionId, boolean fullUpdate, boolean transitive, String parentEventId);
 
-    MetadataEventResponse refreshAllVersionsForProject(String groupId, String artifactId, boolean fullUpdate,boolean allVersions,boolean transitive, String parentEventId);
+    MetadataNotificationResponse refreshAllVersionsForProject(String groupId, String artifactId, boolean fullUpdate, boolean allVersions, boolean transitive, String parentEventId);
 
-    MetadataEventResponse refreshAllVersionsForAllProjects(boolean fullUpdate,boolean allVersions,boolean transitive, String parentEventId);
+    MetadataNotificationResponse refreshAllVersionsForAllProjects(boolean fullUpdate, boolean allVersions, boolean transitive, String parentEventId);
 
-    MetadataEventResponse refreshDefaultSnapshotsForAllProjects(boolean fullUpdate, boolean transitive, String parentEventId);
+    MetadataNotificationResponse refreshDefaultSnapshotsForAllProjects(boolean fullUpdate, boolean transitive, String parentEventId);
 
 }
diff --git a/legend-depot-artifacts-services/pom.xml b/legend-depot-artifacts-services/pom.xml
@@ -55,12 +55,20 @@
         </dependency>
         <dependency>
             <groupId>org.finos.legend.depot</groupId>
-            <artifactId>legend-depot-store-notifications</artifactId>
+            <artifactId>legend-depot-notifications-api</artifactId>
         </dependency>
         <dependency>
             <groupId>org.finos.legend.depot</groupId>
             <artifactId>legend-depot-core-tracing</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.finos.legend.depot</groupId>
+            <artifactId>legend-depot-core-authorisation</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.finos.legend.depot</groupId>
+            <artifactId>legend-depot-core-schedules-api</artifactId>
+        </dependency>
         <!-- SDLC -->
         <dependency>
             <groupId>org.finos.legend.sdlc</groupId>
@@ -163,6 +171,11 @@
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.finos.legend.depot</groupId>
+            <artifactId>legend-depot-notifications-store-mongo</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.finos.legend.depot</groupId>
             <artifactId>legend-depot-metrics-query-services</artifactId>
@@ -173,6 +186,11 @@
             <artifactId>legend-depot-metrics-query-store-mongo</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.finos.legend.depot</groupId>
+            <artifactId>legend-depot-notifications-services</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>de.bwaldvogel</groupId>
             <artifactId>mongo-java-server</artifactId>

diff --git a/...s/legend/depot/services/artifacts/handlers/entities/AbstractEntityRefreshHandlerImpl.java b/...s/legend/depot/services/artifacts/handlers/entities/AbstractEntityRefreshHandlerImpl.java
@@ -15,7 +15,7 @@
 
 package org.finos.legend.depot.services.artifacts.handlers.entities;
 
-import org.finos.legend.depot.domain.api.MetadataEventResponse;
+import org.finos.legend.depot.domain.notifications.MetadataNotificationResponse;
 import org.finos.legend.depot.domain.version.VersionValidator;
 import org.finos.legend.depot.services.api.entities.ManageEntitiesService;
 import org.finos.legend.depot.services.api.artifacts.handlers.entties.EntityArtifactsProvider;
@@ -64,10 +64,10 @@ private String getGAVCoordinates(String groupId,String artifactId, String versio
     }
 
 
-    public MetadataEventResponse refreshVersionArtifacts(String groupId, String artifactId,String versionId, List<File> files)
+    public MetadataNotificationResponse refreshVersionArtifacts(String groupId, String artifactId, String versionId, List<File> files)
     {
 
-        MetadataEventResponse response = new MetadataEventResponse();
+        MetadataNotificationResponse response = new MetadataNotificationResponse();
         try
         {
             String gavCoordinates = getGAVCoordinates(groupId, artifactId, versionId);

diff --git a/...java/org/finos/legend/depot/services/artifacts/handlers/entities/EntitiesHandlerImpl.java b/...java/org/finos/legend/depot/services/artifacts/handlers/entities/EntitiesHandlerImpl.java
@@ -15,7 +15,7 @@
 
 package org.finos.legend.depot.services.artifacts.handlers.entities;
 
-import org.finos.legend.depot.domain.api.MetadataEventResponse;
+import org.finos.legend.depot.domain.notifications.MetadataNotificationResponse;
 import org.finos.legend.depot.services.api.entities.ManageEntitiesService;
 import org.finos.legend.depot.services.api.artifacts.handlers.entties.EntitiesArtifactsHandler;
 import org.finos.legend.depot.services.api.artifacts.handlers.entties.EntityArtifactsProvider;
@@ -33,7 +33,7 @@ public EntitiesHandlerImpl(ManageEntitiesService entitiesService, EntityArtifact
     }
 
     @Override
-    public MetadataEventResponse refreshProjectVersionArtifacts(String groupId, String artifactId, String versionId, List<File> files)
+    public MetadataNotificationResponse refreshProjectVersionArtifacts(String groupId, String artifactId, String versionId, List<File> files)
     {
         return super.refreshVersionArtifacts(groupId, artifactId, versionId, files);
     }

diff --git a/...finos/legend/depot/services/artifacts/handlers/generations/FileGenerationHandlerImpl.java b/...finos/legend/depot/services/artifacts/handlers/generations/FileGenerationHandlerImpl.java
@@ -18,7 +18,7 @@
 import org.apache.commons.io.FilenameUtils;
 import org.finos.legend.depot.services.api.artifacts.repository.ArtifactRepository;
 import org.finos.legend.depot.domain.artifacts.repository.ArtifactType;
-import org.finos.legend.depot.domain.api.MetadataEventResponse;
+import org.finos.legend.depot.domain.notifications.MetadataNotificationResponse;
 import org.finos.legend.depot.domain.generation.DepotGeneration;
 import org.finos.legend.depot.store.model.generations.StoredFileGeneration;
 import org.finos.legend.depot.domain.version.VersionValidator;
@@ -71,9 +71,9 @@ public FileGenerationHandlerImpl(ArtifactRepository repository, FileGenerationsA
 
 
 
-    public MetadataEventResponse refreshProjectVersionArtifacts(String groupId,String artifactId, String versionId, List<File> files)
+    public MetadataNotificationResponse refreshProjectVersionArtifacts(String groupId, String artifactId, String versionId, List<File> files)
     {
-        MetadataEventResponse response = new MetadataEventResponse();
+        MetadataNotificationResponse response = new MetadataNotificationResponse();
         try
         {
             List<StoredFileGeneration> newGenerations = new ArrayList<>();

diff --git a/.../main/java/org/finos/legend/depot/services/artifacts/purge/ArtifactsPurgeServiceImpl.java b/.../main/java/org/finos/legend/depot/services/artifacts/purge/ArtifactsPurgeServiceImpl.java
@@ -16,7 +16,7 @@
 package org.finos.legend.depot.services.artifacts.purge;
 
 import org.finos.legend.depot.domain.VersionedData;
-import org.finos.legend.depot.domain.api.MetadataEventResponse;
+import org.finos.legend.depot.domain.notifications.MetadataNotificationResponse;
 import org.finos.legend.depot.domain.artifacts.repository.ArtifactType;
 import org.finos.legend.depot.domain.project.ProjectVersion;
 import org.finos.legend.depot.services.api.artifacts.handlers.ProjectArtifactHandlerFactory;
@@ -138,11 +138,11 @@ public void evict(String groupId, String artifactId, String versionId)
     }
 
     @Override
-    public MetadataEventResponse deprecate(String groupId, String artifactId, String versionId)
+    public MetadataNotificationResponse deprecate(String groupId, String artifactId, String versionId)
     {
         return TracerFactory.get().executeWithTrace(DEPRECATE_VERSION, () ->
         {
-            MetadataEventResponse response = new MetadataEventResponse();
+            MetadataNotificationResponse response = new MetadataNotificationResponse();
             StoreProjectVersionData projectData = getProjectVersion(groupId, artifactId, versionId);
             projectData.getVersionData().setDeprecated(true);
             response.addMessage(String.format("%s-%s-%s deprecated", groupId, artifactId, versionId));
@@ -152,9 +152,9 @@ public MetadataEventResponse deprecate(String groupId, String artifactId, String
     }
 
     @Override
-    public MetadataEventResponse deprecateVersionsNotInRepository()
+    public MetadataNotificationResponse deprecateVersionsNotInRepository()
     {
-        MetadataEventResponse response = new MetadataEventResponse();
+        MetadataNotificationResponse response = new MetadataNotificationResponse();
         versionsMismatchService.findVersionsMismatches().parallelStream().forEach(versionMismatch ->
         {
             if (!versionMismatch.versionsNotInRepository.isEmpty())
@@ -171,12 +171,12 @@ public MetadataEventResponse deprecateVersionsNotInRepository()
     }
 
     @Override
-    public MetadataEventResponse evictOldestProjectVersions(String groupId, String artifactId, int versionsToKeep)
+    public MetadataNotificationResponse evictOldestProjectVersions(String groupId, String artifactId, int versionsToKeep)
     {
         projects.checkExists(groupId, artifactId);
         return TracerFactory.get().executeWithTrace(EVICT_OLDEST, () ->
         {
-            MetadataEventResponse response = new MetadataEventResponse();
+            MetadataNotificationResponse response = new MetadataNotificationResponse();
             List<String> versionIds = projects.getVersions(groupId, artifactId);
             int numberOfVersions = versionIds.size();
             try
@@ -202,11 +202,11 @@ public MetadataEventResponse evictOldestProjectVersions(String groupId, String a
     }
 
     @Override
-    public MetadataEventResponse evictLeastRecentlyUsed(int ttlForVersionsInDays, int ttlForSnapshotsInDays)
+    public MetadataNotificationResponse evictLeastRecentlyUsed(int ttlForVersionsInDays, int ttlForSnapshotsInDays)
     {
         Set<ProjectVersion> evictProjectVersions = new HashSet<>();
         LocalDateTime currentDateTime = LocalDateTime.now();
-        MetadataEventResponse response = new MetadataEventResponse();
+        MetadataNotificationResponse response = new MetadataNotificationResponse();
         try
         {
             LOGGER.info("Started finding eviction candidates for snapshot versions");
@@ -230,10 +230,10 @@ public MetadataEventResponse evictLeastRecentlyUsed(int ttlForVersionsInDays, in
     }
 
     @Override
-    public MetadataEventResponse evictVersionsNotUsed()
+    public MetadataNotificationResponse evictVersionsNotUsed()
     {
         Set<ProjectVersion> evictProjectVersions = new HashSet<>();
-        MetadataEventResponse response = new MetadataEventResponse();
+        MetadataNotificationResponse response = new MetadataNotificationResponse();
         try
         {
             LOGGER.info("Started finding versions not being used for eviction");