Storing latest version to project data (#409)

* Storing latest version to project data
Removing the pagination of projects data

* cleaning up MR for paginating project results

* add readme message

legend-depot-artifacts-services/src/main/java/org/finos/legend/depot/services/artifacts/refresh/ProjectVersionRefreshHandler.java

@@ -43,6 +43,7 @@
 import org.finos.legend.depot.services.api.artifacts.handlers.ProjectArtifactsHandler;
 import org.finos.legend.depot.core.services.tracing.TracerFactory;
 import org.finos.legend.depot.core.services.metrics.PrometheusMetricsFactory;
+import org.finos.legend.sdlc.domain.model.version.VersionId;
 import org.slf4j.Logger;
 
 import javax.inject.Inject;
@@ -242,6 +243,7 @@ MetadataEventResponse doRefresh(MetadataNotification event)
                     {
 
                         updateProjectVersionData(project, event.getVersionId(), newDependencies);
+                        updateProjectData(project, event.getVersionId());
                         //we let the version load but will check dependencies exists and report missing dependencies as errors
                         if (!event.isTransitive())
                         {
@@ -304,6 +306,32 @@ private void updateProjectVersionData(StoreProjectData project, String versionId
         LOGGER.info("Finished updating project data [{}-{}-{}]", project.getGroupId(), project.getArtifactId(), versionId);
     }
 
+    private void updateProjectData(StoreProjectData projectData, String versionId)
+    {
+        if (!VersionValidator.isSnapshotVersion(versionId))
+        {
+            String latestVersion = calculateLatestVersion(projectData.getLatestVersion(), versionId);
+            if (!latestVersion.equals(projectData.getLatestVersion()))
+            {
+                projectData.setLatestVersion(latestVersion);
+                projects.createOrUpdate(projectData);
+            }
+        }
+    }
+
+    private String calculateLatestVersion(String projectLatestVersion, String versionId)
+    {
+        if (projectLatestVersion == null)
+        {
+            return versionId;
+        }
+        else if (VersionId.parseVersionId(versionId).compareTo(VersionId.parseVersionId(projectLatestVersion)) > 1)
+        {
+            return versionId;
+        }
+        return projectLatestVersion;
+    }
+
     private String queueWorkToRefreshProjectVersion(StoreProjectData projectData, String versionId, boolean fullUpdate, boolean transitive, String parentEvent)
     {
         return String.format("queued: [%s-%s-%s], parentEventId :[%s], full/transitive :[%s/%s],event id :[%s] ",

legend-depot-artifacts-services/src/test/java/org/finos/legend/depot/services/artifacts/purge/TestArtifactsPurgeService.java

@@ -115,7 +115,7 @@ public void canEvictOldVersions()
         List<String> projectVersions = projectsService.getVersions(TEST_GROUP_ID, TEST_ARTIFACT_ID);
         Assert.assertEquals(3, projectVersions.size());
         Assert.assertEquals("2.0.0", projectVersions.get(0));
-        Assert.assertEquals("2.3.0", projectsService.getLatestVersion(TEST_GROUP_ID, TEST_ARTIFACT_ID).get().toVersionIdString());
+        Assert.assertEquals("2.3.0", project.getLatestVersion());
 
         Assert.assertEquals(2, entitiesStore.getAllEntities(TEST_GROUP_ID, TEST_ARTIFACT_ID, "2.0.0").size());
         purgeService.evictOldestProjectVersions(TEST_GROUP_ID,TEST_ARTIFACT_ID,1);

legend-depot-artifacts-services/src/test/java/org/finos/legend/depot/services/artifacts/refresh/TestArtifactsRefreshService.java

@@ -116,7 +116,6 @@ public void setUpData()
 
         projectsVersionsStore.createOrUpdate(new StoreProjectVersionData(TEST_GROUP_ID, TEST_DEPENDENCIES_ARTIFACT_ID, "2.0.0"));
         projectsVersionsStore.createOrUpdate(new StoreProjectVersionData(TEST_GROUP_ID, TEST_DEPENDENCIES_ARTIFACT_ID, BRANCH_SNAPSHOT("master")));
-        projectsVersionsStore.createOrUpdate(new StoreProjectVersionData(TEST_GROUP_ID, TEST_ARTIFACT_ID,"2.3.3"));
         projectsVersionsStore.createOrUpdate(new StoreProjectVersionData(TEST_GROUP_ID, TEST_ARTIFACT_ID, "1.0.0"));
         projectsVersionsStore.createOrUpdate(new StoreProjectVersionData(TEST_GROUP_ID, TEST_ARTIFACT_ID, BRANCH_SNAPSHOT("master")));
 
@@ -202,8 +201,6 @@ public void canUpdateCreateMasterRevision()
     @Test
     public void canRefreshProjectVersion()
     {
-        Assert.assertEquals(0, entitiesStore.getAllEntities(TEST_GROUP_ID, TEST_ARTIFACT_ID, "2.3.3").size());
-
         Assert.assertEquals(0, entitiesStore.getAllEntities(TEST_GROUP_ID, TEST_DEPENDENCIES_ARTIFACT_ID, "1.0.0").size());
 
         MetadataEventResponse response = artifactsRefreshService.refreshVersionForProject(TEST_GROUP_ID, TEST_ARTIFACT_ID, "2.0.0",true,PARENT_EVENT_ID);
@@ -344,15 +341,15 @@ public void partialRefreshAllVersionForProjectOnlyRefreshLatest()
         notificationsQueueManager.handleAll();
 
         List<String> versions = projectsService.getVersions(TEST_GROUP_ID, TEST_ARTIFACT_ID);
-        Assert.assertEquals(3, versions.size());
-        Assert.assertEquals("2.3.3",projectsService.getLatestVersion(TEST_GROUP_ID, TEST_ARTIFACT_ID).get().toVersionIdString());
+        Assert.assertEquals(2, versions.size());
+        Assert.assertEquals("2.0.0",projectsService.findCoordinates(TEST_GROUP_ID, TEST_ARTIFACT_ID).get().getLatestVersion());
 
 
         projectsService.delete(TEST_GROUP_ID, TEST_ARTIFACT_ID, "1.0.0");
-        Assert.assertEquals("2.3.3",projectsService.getLatestVersion(TEST_GROUP_ID,TEST_ARTIFACT_ID).get().toVersionIdString());
+        Assert.assertEquals("2.0.0",projectsService.findCoordinates(TEST_GROUP_ID,TEST_ARTIFACT_ID).get().getLatestVersion());
 
         artifactsRefreshService.refreshAllVersionsForProject(TEST_GROUP_ID, TEST_ARTIFACT_ID, false, false, false,PARENT_EVENT_ID);
-        Assert.assertEquals("2.3.3",projectsService.getLatestVersion(TEST_GROUP_ID,TEST_ARTIFACT_ID).get().toVersionIdString());
+        Assert.assertEquals("2.0.0",projectsService.findCoordinates(TEST_GROUP_ID,TEST_ARTIFACT_ID).get().getLatestVersion());
         Assert.assertEquals(2,notificationsQueueManager.getAllInQueue().size());
         Assert.assertEquals("1.0.0",notificationsQueueManager.getAllInQueue().get(1).getVersionId());
         notificationsQueueManager.handleAll();
@@ -502,7 +499,7 @@ public void cantRefreshSameVersionTwice()
         notificationsQueueManager.handleAll();
         Assert.assertEquals(docsBefore,entitiesStore.getAllStoredEntities().size());
         Assert.assertTrue(projectsVersionsStore.find(TEST_GROUP_ID,TEST_ARTIFACT_ID,versionId).isPresent());
-        Assert.assertEquals(3,projectsVersionsStore.find(TEST_GROUP_ID,TEST_ARTIFACT_ID).stream().filter(x -> !x.getVersionId().equals(BRANCH_SNAPSHOT("master"))).collect(Collectors.toList()).size());
+        Assert.assertEquals(2,projectsVersionsStore.find(TEST_GROUP_ID,TEST_ARTIFACT_ID).stream().filter(x -> !x.getVersionId().equals(BRANCH_SNAPSHOT("master"))).collect(Collectors.toList()).size());
         Assert.assertEquals(0,notificationsQueueManager.getAllInQueue().size());
     }
 

legend-depot-artifacts-services/src/test/resources/data/projects.json

@@ -2,12 +2,14 @@
   {
     "projectId": "PROD-A",
     "groupId": "examples.metadata",
-    "artifactId": "test"
+    "artifactId": "test",
+    "latestVersion": "2.3.0"
   },
   {
     "projectId": "PROD-B",
     "groupId": "examples.metadata",
-    "artifactId": "test-dependencies"
+    "artifactId": "test-dependencies",
+    "latestVersion": "1.0.0"
   },
   {
     "projectId": "PROD-C",

legend-depot-core-data-api/src/main/java/org/finos/legend/depot/services/api/projects/ProjectsService.java

@@ -33,20 +33,13 @@ public interface ProjectsService
 
     List<StoreProjectVersionData> findByUpdatedDate(long updatedFrom, long updatedTo);
 
-    /**
-     * NOTE: page starting from 1
-     */
-    List<StoreProjectData> getProjects(int page, int pageSize);
-
     default List<String> getVersions(String groupId, String artifactId)
     {
         return getVersions(groupId, artifactId,false);
     }
 
     List<String> getVersions(String groupId, String artifactId,boolean includeSnapshots);
 
-    Optional<VersionId> getLatestVersion(String groupId, String artifactId);
-
     @Deprecated
     List<StoreProjectData> findByProjectId(String projectId);
 

legend-depot-core-data-api/src/main/java/org/finos/legend/depot/store/api/projects/Projects.java

@@ -24,11 +24,6 @@ public interface Projects
 {
     List<StoreProjectData> getAll();
 
-    /**
-     * NOTE: page starting from 1
-     */
-    List<StoreProjectData> getProjects(int page, int pageSize);
-
     Optional<StoreProjectData> find(String groupId, String artifactId);
 
     @Deprecated

legend-depot-core-data-api/src/main/java/org/finos/legend/depot/store/model/projects/StoreProjectData.java

@@ -32,15 +32,19 @@ public class StoreProjectData extends CoordinateData implements HasIdentifier
     @JsonProperty
     private String projectId;
 
+    @JsonProperty
+    private String latestVersion;
+
     public StoreProjectData()
     {
         super();
     }
 
-    public StoreProjectData(String projectId, String groupId, String artifactId, String defaultBranch)
+    public StoreProjectData(String projectId, String groupId, String artifactId, String defaultBranch, String latestVersion)
     {
         this(projectId, groupId, artifactId);
         this.defaultBranch = defaultBranch;
+        this.latestVersion = latestVersion;
     }
 
     public StoreProjectData(String projectId,String groupId,String artifactId)
@@ -64,6 +68,16 @@ public String getProjectId()
         return projectId;
     }
 
+    public String getLatestVersion()
+    {
+        return latestVersion;
+    }
+
+    public void setLatestVersion(String latestVersion)
+    {
+        this.latestVersion = latestVersion;
+    }
+
     @Override
     @JsonIgnore
     public String getId()

legend-depot-core-data-services/src/main/java/org/finos/legend/depot/services/projects/ProjectsServiceImpl.java

@@ -39,7 +39,6 @@
 import org.finos.legend.depot.store.api.projects.UpdateProjects;
 import org.finos.legend.depot.services.api.metrics.query.QueryMetricsRegistry;
 import org.finos.legend.depot.store.notifications.queue.api.Queue;
-import org.finos.legend.sdlc.domain.model.version.VersionId;
 
 import javax.inject.Inject;
 import javax.inject.Named;
@@ -101,12 +100,6 @@ public List<StoreProjectData> getAllProjectCoordinates()
         return projects.getAll();
     }
 
-    @Override
-    public List<StoreProjectData> getProjects(int page, int pageSize)
-    {
-        return projects.getProjects(page, pageSize);
-    }
-
     @Override
     public List<String> getVersions(String groupId, String artifactId,boolean includeSnapshots)
     {
@@ -160,7 +153,12 @@ public Optional<StoreProjectVersionData> find(String groupId, String artifactId,
     {
         if (VersionAlias.LATEST.getName().equals(versionId))
         {
-            return projectsVersions.find(groupId, artifactId).stream().filter(v -> !VersionValidator.isSnapshotVersion(v.getVersionId()) && !v.getVersionData().isExcluded()).max(Comparator.comparing(o -> VersionId.parseVersionId(o.getVersionId())));
+            Optional<StoreProjectData> projectData = this.findCoordinates(groupId, artifactId);
+            if (projectData.isPresent())
+            {
+                return projectsVersions.find(groupId, artifactId, projectData.get().getLatestVersion());
+            }
+            return Optional.empty();
         }
         else if (VersionAlias.HEAD.getName().equals(versionId))
         {
@@ -219,12 +217,6 @@ public void checkExists(String groupId, String artifactId) throws IllegalArgumen
         }
     }
 
-    @Override
-    public Optional<VersionId> getLatestVersion(String groupId, String artifactId)
-    {
-        return this.getVersions(groupId, artifactId,false).stream().map(v -> VersionId.parseVersionId(v)).max(VersionId::compareTo);
-    }
-
     @Override
     public Set<ProjectVersion> getDependencies(List<ProjectVersion> projectVersions, boolean transitive)
     {

legend-depot-core-data-services/src/main/java/org/finos/legend/depot/store/resources/projects/ManageProjectsResource.java

@@ -62,15 +62,15 @@ protected String getResourceName()
     @Path("/projects/{projectId}/{groupId}/{artifactId}")
     @ApiOperation(ResourceLoggingAndTracing.CREATE_UPDATE_PROJECT)
     @Produces(MediaType.APPLICATION_JSON)
-    public StoreProjectData updateProject(@PathParam("projectId") String projectId, @PathParam("groupId") String groupId, @PathParam("artifactId") String artifactId, @QueryParam("defaultBranch") @ApiParam String defaultBranch)
+    public StoreProjectData updateProject(@PathParam("projectId") String projectId, @PathParam("groupId") String groupId, @PathParam("artifactId") String artifactId, @QueryParam("defaultBranch") @ApiParam String defaultBranch, @QueryParam("latestVersion") @ApiParam String latestVersion)
     {
         return handle(
                 ResourceLoggingAndTracing.CREATE_UPDATE_PROJECT,
                 ResourceLoggingAndTracing.CREATE_UPDATE_PROJECT + projectId,
                 () ->
                 {
                     validateUser();
-                    return projectApi.createOrUpdate(new StoreProjectData(projectId, groupId, artifactId, defaultBranch));
+                    return projectApi.createOrUpdate(new StoreProjectData(projectId, groupId, artifactId, defaultBranch, latestVersion));
                 });
     }
 

legend-depot-core-data-services/src/test/java/org/finos/legend/depot/services/projects/TestProjectsService.java

@@ -281,7 +281,7 @@ public void canGetProjectCoordinatesByGA()
     {
         Optional<StoreProjectData> storeProjectData = projectsService.findCoordinates("examples.metadata", "test");
         Assert.assertTrue(storeProjectData.isPresent());
-        Assert.assertEquals(storeProjectData.get(), new StoreProjectData("PROD-A", "examples.metadata", "test"));
+        Assert.assertEquals(storeProjectData.get(), new StoreProjectData("PROD-A", "examples.metadata", "test", null, "2.3.1"));
 
         Optional<StoreProjectData> storeProjectData1 = projectsService.findCoordinates("dummy.dep", "test");
         Assert.assertFalse(storeProjectData1.isPresent());
@@ -352,11 +352,11 @@ public void canGetLatestVersionForProject()
         Assert.assertNotNull(fullVersions);
         Assert.assertEquals(2, fullVersions.size());
 
-        Assert.assertTrue(projectsService.getLatestVersion("examples.metadata", "test").isPresent());
-        Assert.assertEquals("2.3.1", projectsService.getLatestVersion("examples.metadata", "test").get().toVersionIdString());
+        Assert.assertTrue(projectsService.findCoordinates("examples.metadata", "test").isPresent());
+        Assert.assertEquals("2.3.1", projectsService.findCoordinates("examples.metadata", "test").get().getLatestVersion());
 
         Assert.assertTrue(projectsService.find("examples.metadata", "test","latest").isPresent());
-        Assert.assertEquals("2.3.1", projectsService.getLatestVersion("examples.metadata", "test").get().toVersionIdString());
+        Assert.assertEquals("2.3.1", projectsService.findCoordinates("examples.metadata", "test").get().getLatestVersion());
 
         Assert.assertFalse(projectsService.find("dont","exist","latest").isPresent());
 

legend-depot-core-data-store-mongo/src/main/java/org/finos/legend/depot/store/mongo/admin/CoreDataMigrations.java

@@ -55,5 +55,11 @@ public void addTransitiveDependenciesToVersionData()
     {
         new DependenciesMigration(mongoDatabase).addTransitiveDependenciesToVersionData();
     }
+
+    @Deprecated
+    public void addLatestVersionToProjectData()
+    {
+        new ProjectToProjectVersionMigration(mongoDatabase).addLatestVersionToProjectData();
+    }
 }
 

legend-depot-core-data-store-mongo/src/main/java/org/finos/legend/depot/store/mongo/admin/migrations/ProjectToProjectVersionMigration.java

@@ -15,6 +15,7 @@
 
 package org.finos.legend.depot.store.mongo.admin.migrations;
 
+import com.mongodb.client.FindIterable;
 import com.mongodb.client.MongoCollection;
 import com.mongodb.client.MongoDatabase;
 import com.mongodb.client.model.Filters;
@@ -27,12 +28,17 @@
 import org.finos.legend.depot.store.mongo.core.BaseMongo;
 import org.finos.legend.depot.store.mongo.projects.ProjectsMongo;
 import org.finos.legend.depot.store.mongo.projects.ProjectsVersionsMongo;
+import org.finos.legend.sdlc.domain.model.version.VersionId;
 import org.slf4j.Logger;
 
 import java.util.Collections;
 import java.util.List;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.Optional;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Consumer;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 
 import static org.finos.legend.depot.domain.version.VersionValidator.BRANCH_SNAPSHOT;
@@ -42,6 +48,7 @@ public final class ProjectToProjectVersionMigration
 {
     private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(ProjectToProjectVersionMigration.class);
     private final MongoDatabase mongoDatabase;
+    private static final String EXCLUDED = "versionData.excluded";
 
     public ProjectToProjectVersionMigration(MongoDatabase mongoDatabase)
     {
@@ -128,4 +135,43 @@ public void cleanUpProjectData()
         });
         LOGGER.info(String.format("versions updated [%s]", i.get()));
     }
+
+    @Deprecated
+    public void addLatestVersionToProjectData()
+    {
+        MongoCollection<Document> projectCollection = mongoDatabase.getCollection(ProjectsMongo.COLLECTION);
+        MongoCollection<Document> versionCollection = mongoDatabase.getCollection(ProjectsVersionsMongo.COLLECTION);
+        projectCollection.find().forEach((Consumer<Document>) document ->
+        {
+            AtomicInteger i = new AtomicInteger();
+            String groupId = document.getString(BaseMongo.GROUP_ID);
+            String artifactId = document.getString(BaseMongo.ARTIFACT_ID);
+            try
+            {
+                FindIterable<Document> versionDocument = versionCollection.find(Filters.and(
+                        Filters.eq(BaseMongo.GROUP_ID, groupId), Filters.eq(BaseMongo.ARTIFACT_ID, artifactId),
+                        Filters.not(Filters.regex(BaseMongo.VERSION_ID, BRANCH_SNAPSHOT(""))), Filters.eq(EXCLUDED, false)));
+                List<VersionId> parsedVersions  = new ArrayList<>();
+                versionDocument.forEach((Consumer<Document>) doc ->
+                {
+                    parsedVersions.add(VersionId.parseVersionId(doc.getString(BaseMongo.VERSION_ID)));
+                });
+                Optional<VersionId> latestVersion = parsedVersions.stream().max(Comparator.comparing(Function.identity()));
+                if (latestVersion.isPresent())
+                {
+                    LOGGER.info(String.format("%s-%s updated with latest version", groupId, artifactId));
+                    projectCollection.updateOne(Filters.and(Filters.eq(BaseMongo.GROUP_ID, groupId),
+                            Filters.eq(BaseMongo.ARTIFACT_ID, artifactId)), Updates.set("latestVersion", latestVersion.get().toVersionIdString()));
+                    LOGGER.info(String.format("%s-%s update completed",groupId,artifactId));
+                }
+                LOGGER.info(String.format("projects updated [%s]",i.incrementAndGet()));
+            }
+            catch (Exception e)
+            {
+                LOGGER.info("Error while updating data:" + e.getMessage());
+                LOGGER.info(String.format("projects updated [%s] before error",i.get()));
+                LOGGER.info(String.format("%s-%s update could not be completed",groupId, artifactId));
+            }
+        });
+    }
 }

legend-depot-core-data-store-mongo/src/main/java/org/finos/legend/depot/store/mongo/projects/ProjectsMongo.java

@@ -35,7 +35,6 @@
 import java.util.List;
 import java.util.Optional;
 
-
 public class ProjectsMongo extends BaseMongo<StoreProjectData> implements Projects, UpdateProjects
 {
 
@@ -87,12 +86,6 @@ public List<StoreProjectData> getAll()
         return getAllStoredEntities();
     }
 
-    @Override
-    public List<StoreProjectData> getProjects(int page, int pageSize)
-    {
-        return getStoredEntitiesByPage(page, pageSize);
-    }
-
     @Override
     public List<StoreProjectData> findByProjectId(String projectId)
     {