diff --git a/legend-depot-artifacts-services/src/main/java/org/finos/legend/depot/services/artifacts/refresh/ProjectVersionRefreshHandler.java b/legend-depot-artifacts-services/src/main/java/org/finos/legend/depot/services/artifacts/refresh/ProjectVersionRefreshHandler.java index 7b0518a8a..9f8ae60a1 100644 --- a/legend-depot-artifacts-services/src/main/java/org/finos/legend/depot/services/artifacts/refresh/ProjectVersionRefreshHandler.java +++ b/legend-depot-artifacts-services/src/main/java/org/finos/legend/depot/services/artifacts/refresh/ProjectVersionRefreshHandler.java @@ -43,6 +43,7 @@ import org.finos.legend.depot.services.api.artifacts.handlers.ProjectArtifactsHandler; import org.finos.legend.depot.core.services.tracing.TracerFactory; import org.finos.legend.depot.core.services.metrics.PrometheusMetricsFactory; +import org.finos.legend.sdlc.domain.model.version.VersionId; import org.slf4j.Logger; import javax.inject.Inject; @@ -242,6 +243,7 @@ MetadataEventResponse doRefresh(MetadataNotification event) { updateProjectVersionData(project, event.getVersionId(), newDependencies); + updateProjectData(project, event.getVersionId()); //we let the version load but will check dependencies exists and report missing dependencies as errors if (!event.isTransitive()) { @@ -304,6 +306,27 @@ private void updateProjectVersionData(StoreProjectData project, String versionId LOGGER.info("Finished updating project data [{}-{}-{}]", project.getGroupId(), project.getArtifactId(), versionId); } + private void updateProjectData(StoreProjectData projectData, String versionId) + { + if (!VersionValidator.isSnapshotVersion(versionId)) + { + if (projectData.getLatestVersion() == null) + { + projectData.setLatestVersion(versionId); + projects.createOrUpdate(projectData); + } + else + { + //finding if the new version is the latest version + if (VersionId.parseVersionId(versionId).compareTo(VersionId.parseVersionId(projectData.getLatestVersion())) > 1) + { + projectData.setLatestVersion(versionId); + projects.createOrUpdate(projectData); + } + } + } + } + private String queueWorkToRefreshProjectVersion(StoreProjectData projectData, String versionId, boolean fullUpdate, boolean transitive, String parentEvent) { return String.format("queued: [%s-%s-%s], parentEventId :[%s], full/transitive :[%s/%s],event id :[%s] ", diff --git a/legend-depot-artifacts-services/src/test/java/org/finos/legend/depot/services/artifacts/purge/TestArtifactsPurgeService.java b/legend-depot-artifacts-services/src/test/java/org/finos/legend/depot/services/artifacts/purge/TestArtifactsPurgeService.java index a2a8631fc..a31cdf76d 100644 --- a/legend-depot-artifacts-services/src/test/java/org/finos/legend/depot/services/artifacts/purge/TestArtifactsPurgeService.java +++ b/legend-depot-artifacts-services/src/test/java/org/finos/legend/depot/services/artifacts/purge/TestArtifactsPurgeService.java @@ -115,7 +115,7 @@ public void canEvictOldVersions() List projectVersions = projectsService.getVersions(TEST_GROUP_ID, TEST_ARTIFACT_ID); Assert.assertEquals(3, projectVersions.size()); Assert.assertEquals("2.0.0", projectVersions.get(0)); - Assert.assertEquals("2.3.0", projectsService.getLatestVersion(TEST_GROUP_ID, TEST_ARTIFACT_ID).get().toVersionIdString()); + Assert.assertEquals("2.3.0", project.getLatestVersion()); Assert.assertEquals(2, entitiesStore.getAllEntities(TEST_GROUP_ID, TEST_ARTIFACT_ID, "2.0.0").size()); purgeService.evictOldestProjectVersions(TEST_GROUP_ID,TEST_ARTIFACT_ID,1); diff --git a/legend-depot-artifacts-services/src/test/java/org/finos/legend/depot/services/artifacts/refresh/TestArtifactsRefreshService.java b/legend-depot-artifacts-services/src/test/java/org/finos/legend/depot/services/artifacts/refresh/TestArtifactsRefreshService.java index 4c135f37a..66e5d368c 100644 --- a/legend-depot-artifacts-services/src/test/java/org/finos/legend/depot/services/artifacts/refresh/TestArtifactsRefreshService.java +++ b/legend-depot-artifacts-services/src/test/java/org/finos/legend/depot/services/artifacts/refresh/TestArtifactsRefreshService.java @@ -116,7 +116,6 @@ public void setUpData() projectsVersionsStore.createOrUpdate(new StoreProjectVersionData(TEST_GROUP_ID, TEST_DEPENDENCIES_ARTIFACT_ID, "2.0.0")); projectsVersionsStore.createOrUpdate(new StoreProjectVersionData(TEST_GROUP_ID, TEST_DEPENDENCIES_ARTIFACT_ID, BRANCH_SNAPSHOT("master"))); - projectsVersionsStore.createOrUpdate(new StoreProjectVersionData(TEST_GROUP_ID, TEST_ARTIFACT_ID,"2.3.3")); projectsVersionsStore.createOrUpdate(new StoreProjectVersionData(TEST_GROUP_ID, TEST_ARTIFACT_ID, "1.0.0")); projectsVersionsStore.createOrUpdate(new StoreProjectVersionData(TEST_GROUP_ID, TEST_ARTIFACT_ID, BRANCH_SNAPSHOT("master"))); @@ -202,8 +201,6 @@ public void canUpdateCreateMasterRevision() @Test public void canRefreshProjectVersion() { - Assert.assertEquals(0, entitiesStore.getAllEntities(TEST_GROUP_ID, TEST_ARTIFACT_ID, "2.3.3").size()); - Assert.assertEquals(0, entitiesStore.getAllEntities(TEST_GROUP_ID, TEST_DEPENDENCIES_ARTIFACT_ID, "1.0.0").size()); MetadataEventResponse response = artifactsRefreshService.refreshVersionForProject(TEST_GROUP_ID, TEST_ARTIFACT_ID, "2.0.0",true,PARENT_EVENT_ID); @@ -344,15 +341,15 @@ public void partialRefreshAllVersionForProjectOnlyRefreshLatest() notificationsQueueManager.handleAll(); List versions = projectsService.getVersions(TEST_GROUP_ID, TEST_ARTIFACT_ID); - Assert.assertEquals(3, versions.size()); - Assert.assertEquals("2.3.3",projectsService.getLatestVersion(TEST_GROUP_ID, TEST_ARTIFACT_ID).get().toVersionIdString()); + Assert.assertEquals(2, versions.size()); + Assert.assertEquals("2.0.0",projectsService.findCoordinates(TEST_GROUP_ID, TEST_ARTIFACT_ID).get().getLatestVersion()); projectsService.delete(TEST_GROUP_ID, TEST_ARTIFACT_ID, "1.0.0"); - Assert.assertEquals("2.3.3",projectsService.getLatestVersion(TEST_GROUP_ID,TEST_ARTIFACT_ID).get().toVersionIdString()); + Assert.assertEquals("2.0.0",projectsService.findCoordinates(TEST_GROUP_ID,TEST_ARTIFACT_ID).get().getLatestVersion()); artifactsRefreshService.refreshAllVersionsForProject(TEST_GROUP_ID, TEST_ARTIFACT_ID, false, false, false,PARENT_EVENT_ID); - Assert.assertEquals("2.3.3",projectsService.getLatestVersion(TEST_GROUP_ID,TEST_ARTIFACT_ID).get().toVersionIdString()); + Assert.assertEquals("2.0.0",projectsService.findCoordinates(TEST_GROUP_ID,TEST_ARTIFACT_ID).get().getLatestVersion()); Assert.assertEquals(2,notificationsQueueManager.getAllInQueue().size()); Assert.assertEquals("1.0.0",notificationsQueueManager.getAllInQueue().get(1).getVersionId()); notificationsQueueManager.handleAll(); @@ -502,7 +499,7 @@ public void cantRefreshSameVersionTwice() notificationsQueueManager.handleAll(); Assert.assertEquals(docsBefore,entitiesStore.getAllStoredEntities().size()); Assert.assertTrue(projectsVersionsStore.find(TEST_GROUP_ID,TEST_ARTIFACT_ID,versionId).isPresent()); - Assert.assertEquals(3,projectsVersionsStore.find(TEST_GROUP_ID,TEST_ARTIFACT_ID).stream().filter(x -> !x.getVersionId().equals(BRANCH_SNAPSHOT("master"))).collect(Collectors.toList()).size()); + Assert.assertEquals(2,projectsVersionsStore.find(TEST_GROUP_ID,TEST_ARTIFACT_ID).stream().filter(x -> !x.getVersionId().equals(BRANCH_SNAPSHOT("master"))).collect(Collectors.toList()).size()); Assert.assertEquals(0,notificationsQueueManager.getAllInQueue().size()); } diff --git a/legend-depot-artifacts-services/src/test/resources/data/projects.json b/legend-depot-artifacts-services/src/test/resources/data/projects.json index dafb6f0c8..4c576298c 100644 --- a/legend-depot-artifacts-services/src/test/resources/data/projects.json +++ b/legend-depot-artifacts-services/src/test/resources/data/projects.json @@ -2,12 +2,14 @@ { "projectId": "PROD-A", "groupId": "examples.metadata", - "artifactId": "test" + "artifactId": "test", + "latestVersion": "2.3.0" }, { "projectId": "PROD-B", "groupId": "examples.metadata", - "artifactId": "test-dependencies" + "artifactId": "test-dependencies", + "latestVersion": "1.0.0" }, { "projectId": "PROD-C", diff --git a/legend-depot-core-data-api/src/main/java/org/finos/legend/depot/services/api/projects/ProjectsService.java b/legend-depot-core-data-api/src/main/java/org/finos/legend/depot/services/api/projects/ProjectsService.java index 737cd8a3e..4b1055b83 100644 --- a/legend-depot-core-data-api/src/main/java/org/finos/legend/depot/services/api/projects/ProjectsService.java +++ b/legend-depot-core-data-api/src/main/java/org/finos/legend/depot/services/api/projects/ProjectsService.java @@ -31,11 +31,6 @@ public interface ProjectsService { List getAllProjectCoordinates(); - /** - * NOTE: page starting from 1 - */ - List getProjects(int page, int pageSize); - default List getVersions(String groupId, String artifactId) { return getVersions(groupId, artifactId,false); @@ -43,8 +38,6 @@ default List getVersions(String groupId, String artifactId) List getVersions(String groupId, String artifactId,boolean includeSnapshots); - Optional getLatestVersion(String groupId, String artifactId); - @Deprecated List findByProjectId(String projectId); diff --git a/legend-depot-core-data-api/src/main/java/org/finos/legend/depot/store/api/projects/Projects.java b/legend-depot-core-data-api/src/main/java/org/finos/legend/depot/store/api/projects/Projects.java index 91e42de7d..178cb41ae 100644 --- a/legend-depot-core-data-api/src/main/java/org/finos/legend/depot/store/api/projects/Projects.java +++ b/legend-depot-core-data-api/src/main/java/org/finos/legend/depot/store/api/projects/Projects.java @@ -24,11 +24,6 @@ public interface Projects { List getAll(); - /** - * NOTE: page starting from 1 - */ - List getProjects(int page, int pageSize); - Optional find(String groupId, String artifactId); @Deprecated diff --git a/legend-depot-core-data-api/src/main/java/org/finos/legend/depot/store/model/projects/StoreProjectData.java b/legend-depot-core-data-api/src/main/java/org/finos/legend/depot/store/model/projects/StoreProjectData.java index 0da4e1c3e..ea8f5b2cd 100644 --- a/legend-depot-core-data-api/src/main/java/org/finos/legend/depot/store/model/projects/StoreProjectData.java +++ b/legend-depot-core-data-api/src/main/java/org/finos/legend/depot/store/model/projects/StoreProjectData.java @@ -32,15 +32,19 @@ public class StoreProjectData extends CoordinateData implements HasIdentifier @JsonProperty private String projectId; + @JsonProperty + private String latestVersion; + public StoreProjectData() { super(); } - public StoreProjectData(String projectId, String groupId, String artifactId, String defaultBranch) + public StoreProjectData(String projectId, String groupId, String artifactId, String defaultBranch, String latestVersion) { this(projectId, groupId, artifactId); this.defaultBranch = defaultBranch; + this.latestVersion = latestVersion; } public StoreProjectData(String projectId,String groupId,String artifactId) @@ -64,6 +68,16 @@ public String getProjectId() return projectId; } + public String getLatestVersion() + { + return latestVersion; + } + + public void setLatestVersion(String latestVersion) + { + this.latestVersion = latestVersion; + } + @Override @JsonIgnore public String getId() diff --git a/legend-depot-core-data-services/src/main/java/org/finos/legend/depot/services/projects/ProjectsServiceImpl.java b/legend-depot-core-data-services/src/main/java/org/finos/legend/depot/services/projects/ProjectsServiceImpl.java index 2534ac680..2b3d06226 100644 --- a/legend-depot-core-data-services/src/main/java/org/finos/legend/depot/services/projects/ProjectsServiceImpl.java +++ b/legend-depot-core-data-services/src/main/java/org/finos/legend/depot/services/projects/ProjectsServiceImpl.java @@ -39,7 +39,6 @@ import org.finos.legend.depot.store.api.projects.UpdateProjects; import org.finos.legend.depot.services.api.metrics.query.QueryMetricsRegistry; import org.finos.legend.depot.store.notifications.queue.api.Queue; -import org.finos.legend.sdlc.domain.model.version.VersionId; import javax.inject.Inject; import javax.inject.Named; @@ -101,12 +100,6 @@ public List getAllProjectCoordinates() return projects.getAll(); } - @Override - public List getProjects(int page, int pageSize) - { - return projects.getProjects(page, pageSize); - } - @Override public List getVersions(String groupId, String artifactId,boolean includeSnapshots) { @@ -154,7 +147,12 @@ public Optional find(String groupId, String artifactId, { if (VersionAlias.LATEST.getName().equals(versionId)) { - return projectsVersions.find(groupId, artifactId).stream().filter(v -> !VersionValidator.isSnapshotVersion(v.getVersionId()) && !v.getVersionData().isExcluded()).max(Comparator.comparing(o -> VersionId.parseVersionId(o.getVersionId()))); + Optional projectData = this.findCoordinates(groupId, artifactId); + if (projectData.isPresent()) + { + return projectsVersions.find(groupId, artifactId, projectData.get().getLatestVersion()); + } + return Optional.empty(); } else if (VersionAlias.HEAD.getName().equals(versionId)) { @@ -213,12 +211,6 @@ public void checkExists(String groupId, String artifactId) throws IllegalArgumen } } - @Override - public Optional getLatestVersion(String groupId, String artifactId) - { - return this.getVersions(groupId, artifactId,false).stream().map(v -> VersionId.parseVersionId(v)).max(VersionId::compareTo); - } - @Override public Set getDependencies(List projectVersions, boolean transitive) { diff --git a/legend-depot-core-data-services/src/main/java/org/finos/legend/depot/store/resources/projects/ManageProjectsResource.java b/legend-depot-core-data-services/src/main/java/org/finos/legend/depot/store/resources/projects/ManageProjectsResource.java index 31e55840f..7e03b225d 100644 --- a/legend-depot-core-data-services/src/main/java/org/finos/legend/depot/store/resources/projects/ManageProjectsResource.java +++ b/legend-depot-core-data-services/src/main/java/org/finos/legend/depot/store/resources/projects/ManageProjectsResource.java @@ -62,7 +62,7 @@ protected String getResourceName() @Path("/projects/{projectId}/{groupId}/{artifactId}") @ApiOperation(ResourceLoggingAndTracing.CREATE_UPDATE_PROJECT) @Produces(MediaType.APPLICATION_JSON) - public StoreProjectData updateProject(@PathParam("projectId") String projectId, @PathParam("groupId") String groupId, @PathParam("artifactId") String artifactId, @QueryParam("defaultBranch") @ApiParam String defaultBranch) + public StoreProjectData updateProject(@PathParam("projectId") String projectId, @PathParam("groupId") String groupId, @PathParam("artifactId") String artifactId, @QueryParam("defaultBranch") @ApiParam String defaultBranch, @QueryParam("latestVersion") @ApiParam String latestVersion) { return handle( ResourceLoggingAndTracing.CREATE_UPDATE_PROJECT, @@ -70,7 +70,7 @@ public StoreProjectData updateProject(@PathParam("projectId") String projectId, () -> { validateUser(); - return projectApi.createOrUpdate(new StoreProjectData(projectId, groupId, artifactId, defaultBranch)); + return projectApi.createOrUpdate(new StoreProjectData(projectId, groupId, artifactId, defaultBranch, latestVersion)); }); } diff --git a/legend-depot-core-data-services/src/test/java/org/finos/legend/depot/services/projects/TestProjectsService.java b/legend-depot-core-data-services/src/test/java/org/finos/legend/depot/services/projects/TestProjectsService.java index ca439f888..7ed3df47a 100644 --- a/legend-depot-core-data-services/src/test/java/org/finos/legend/depot/services/projects/TestProjectsService.java +++ b/legend-depot-core-data-services/src/test/java/org/finos/legend/depot/services/projects/TestProjectsService.java @@ -281,7 +281,7 @@ public void canGetProjectCoordinatesByGA() { Optional storeProjectData = projectsService.findCoordinates("examples.metadata", "test"); Assert.assertTrue(storeProjectData.isPresent()); - Assert.assertEquals(storeProjectData.get(), new StoreProjectData("PROD-A", "examples.metadata", "test")); + Assert.assertEquals(storeProjectData.get(), new StoreProjectData("PROD-A", "examples.metadata", "test", null, "2.3.1")); Optional storeProjectData1 = projectsService.findCoordinates("dummy.dep", "test"); Assert.assertFalse(storeProjectData1.isPresent()); @@ -352,11 +352,11 @@ public void canGetLatestVersionForProject() Assert.assertNotNull(fullVersions); Assert.assertEquals(2, fullVersions.size()); - Assert.assertTrue(projectsService.getLatestVersion("examples.metadata", "test").isPresent()); - Assert.assertEquals("2.3.1", projectsService.getLatestVersion("examples.metadata", "test").get().toVersionIdString()); + Assert.assertTrue(projectsService.findCoordinates("examples.metadata", "test").isPresent()); + Assert.assertEquals("2.3.1", projectsService.findCoordinates("examples.metadata", "test").get().getLatestVersion()); Assert.assertTrue(projectsService.find("examples.metadata", "test","latest").isPresent()); - Assert.assertEquals("2.3.1", projectsService.getLatestVersion("examples.metadata", "test").get().toVersionIdString()); + Assert.assertEquals("2.3.1", projectsService.findCoordinates("examples.metadata", "test").get().getLatestVersion()); Assert.assertFalse(projectsService.find("dont","exist","latest").isPresent()); diff --git a/legend-depot-core-data-store-mongo/src/main/java/org/finos/legend/depot/store/mongo/admin/CoreDataMigrations.java b/legend-depot-core-data-store-mongo/src/main/java/org/finos/legend/depot/store/mongo/admin/CoreDataMigrations.java index 430af7e0b..c7bb94013 100644 --- a/legend-depot-core-data-store-mongo/src/main/java/org/finos/legend/depot/store/mongo/admin/CoreDataMigrations.java +++ b/legend-depot-core-data-store-mongo/src/main/java/org/finos/legend/depot/store/mongo/admin/CoreDataMigrations.java @@ -55,5 +55,11 @@ public void addTransitiveDependenciesToVersionData() { new DependenciesMigration(mongoDatabase).addTransitiveDependenciesToVersionData(); } + + @Deprecated + public void addLatestVersionToProjectData() + { + new ProjectToProjectVersionMigration(mongoDatabase).addLatestVersionToProjectData(); + } } diff --git a/legend-depot-core-data-store-mongo/src/main/java/org/finos/legend/depot/store/mongo/admin/migrations/ProjectToProjectVersionMigration.java b/legend-depot-core-data-store-mongo/src/main/java/org/finos/legend/depot/store/mongo/admin/migrations/ProjectToProjectVersionMigration.java index ddec2e699..2ee1c1720 100644 --- a/legend-depot-core-data-store-mongo/src/main/java/org/finos/legend/depot/store/mongo/admin/migrations/ProjectToProjectVersionMigration.java +++ b/legend-depot-core-data-store-mongo/src/main/java/org/finos/legend/depot/store/mongo/admin/migrations/ProjectToProjectVersionMigration.java @@ -15,6 +15,7 @@ package org.finos.legend.depot.store.mongo.admin.migrations; +import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import com.mongodb.client.model.Filters; @@ -27,12 +28,17 @@ import org.finos.legend.depot.store.mongo.core.BaseMongo; import org.finos.legend.depot.store.mongo.projects.ProjectsMongo; import org.finos.legend.depot.store.mongo.projects.ProjectsVersionsMongo; +import org.finos.legend.sdlc.domain.model.version.VersionId; import org.slf4j.Logger; import java.util.Collections; import java.util.List; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; +import java.util.function.Function; import java.util.stream.Collectors; import static org.finos.legend.depot.domain.version.VersionValidator.BRANCH_SNAPSHOT; @@ -42,6 +48,7 @@ public final class ProjectToProjectVersionMigration { private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(ProjectToProjectVersionMigration.class); private final MongoDatabase mongoDatabase; + private static final String EXCLUDED = "versionData.excluded"; public ProjectToProjectVersionMigration(MongoDatabase mongoDatabase) { @@ -128,4 +135,43 @@ public void cleanUpProjectData() }); LOGGER.info(String.format("versions updated [%s]", i.get())); } + + @Deprecated + public void addLatestVersionToProjectData() + { + MongoCollection projectCollection = mongoDatabase.getCollection(ProjectsMongo.COLLECTION); + MongoCollection versionCollection = mongoDatabase.getCollection(ProjectsVersionsMongo.COLLECTION); + projectCollection.find().forEach((Consumer) document -> + { + AtomicInteger i = new AtomicInteger(); + String groupId = document.getString(BaseMongo.GROUP_ID); + String artifactId = document.getString(BaseMongo.ARTIFACT_ID); + try + { + FindIterable versionDocument = versionCollection.find(Filters.and( + Filters.eq(BaseMongo.GROUP_ID, groupId), Filters.eq(BaseMongo.ARTIFACT_ID, artifactId), + Filters.not(Filters.regex(BaseMongo.VERSION_ID, BRANCH_SNAPSHOT(""))), Filters.eq(EXCLUDED, false))); + List parsedVersions = new ArrayList<>(); + versionDocument.forEach((Consumer) doc -> + { + parsedVersions.add(VersionId.parseVersionId(doc.getString(BaseMongo.VERSION_ID))); + }); + Optional latestVersion = parsedVersions.stream().max(Comparator.comparing(Function.identity())); + if (latestVersion.isPresent()) + { + LOGGER.info(String.format("%s-%s updated with latest version", groupId, artifactId)); + projectCollection.updateOne(Filters.and(Filters.eq(BaseMongo.GROUP_ID, groupId), + Filters.eq(BaseMongo.ARTIFACT_ID, artifactId)), Updates.set("latestVersion", latestVersion.get().toVersionIdString())); + LOGGER.info(String.format("%s-%s update completed",groupId,artifactId)); + } + LOGGER.info(String.format("projects updated [%s]",i.incrementAndGet())); + } + catch (Exception e) + { + LOGGER.info("Error while updating data:" + e.getMessage()); + LOGGER.info(String.format("projects updated [%s] before error",i.get())); + LOGGER.info(String.format("%s-%s update could not be completed",groupId, artifactId)); + } + }); + } } diff --git a/legend-depot-core-data-store-mongo/src/main/java/org/finos/legend/depot/store/mongo/projects/ProjectsMongo.java b/legend-depot-core-data-store-mongo/src/main/java/org/finos/legend/depot/store/mongo/projects/ProjectsMongo.java index 77bb47bb5..b9abc2258 100644 --- a/legend-depot-core-data-store-mongo/src/main/java/org/finos/legend/depot/store/mongo/projects/ProjectsMongo.java +++ b/legend-depot-core-data-store-mongo/src/main/java/org/finos/legend/depot/store/mongo/projects/ProjectsMongo.java @@ -35,7 +35,6 @@ import java.util.List; import java.util.Optional; - public class ProjectsMongo extends BaseMongo implements Projects, UpdateProjects { @@ -87,12 +86,6 @@ public List getAll() return getAllStoredEntities(); } - @Override - public List getProjects(int page, int pageSize) - { - return getStoredEntitiesByPage(page, pageSize); - } - @Override public List findByProjectId(String projectId) { diff --git a/legend-depot-core-data-store-mongo/src/main/java/org/finos/legend/depot/store/mongo/resources/CoreDataStoreMigrationsResource.java b/legend-depot-core-data-store-mongo/src/main/java/org/finos/legend/depot/store/mongo/resources/CoreDataStoreMigrationsResource.java index ae34a1606..bda061907 100644 --- a/legend-depot-core-data-store-mongo/src/main/java/org/finos/legend/depot/store/mongo/resources/CoreDataStoreMigrationsResource.java +++ b/legend-depot-core-data-store-mongo/src/main/java/org/finos/legend/depot/store/mongo/resources/CoreDataStoreMigrationsResource.java @@ -111,4 +111,18 @@ public Response addTransitiveDependenciesToVersionData() }); } + @PUT + @Path("/migrations/addLatestVersionToProjectData") + @ApiOperation("Update project configurations with latest version") + @Deprecated + public Response addLatestVersionToProjectData() + { + return handle("Update project configurations with latest version", () -> + { + validateUser(); + mongoMigrations.addLatestVersionToProjectData(); + return Response.status(Response.Status.NO_CONTENT).build(); + }); + } + } diff --git a/legend-depot-core-data-store-mongo/src/test/java/org/finos/legend/depot/store/mongo/admin/migrations/TestProjectVersionMigrations.java b/legend-depot-core-data-store-mongo/src/test/java/org/finos/legend/depot/store/mongo/admin/migrations/TestProjectVersionMigrations.java index 91ce26ea7..8a8b17918 100644 --- a/legend-depot-core-data-store-mongo/src/test/java/org/finos/legend/depot/store/mongo/admin/migrations/TestProjectVersionMigrations.java +++ b/legend-depot-core-data-store-mongo/src/test/java/org/finos/legend/depot/store/mongo/admin/migrations/TestProjectVersionMigrations.java @@ -23,15 +23,19 @@ import org.finos.legend.depot.store.model.projects.StoreProjectVersionData; import org.finos.legend.depot.store.mongo.CoreDataMongoStoreTests; import org.finos.legend.depot.store.mongo.admin.CoreDataMigrations; +import org.finos.legend.depot.store.mongo.core.BaseMongo; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import java.io.InputStream; import java.net.URL; +import java.util.ArrayList; import java.util.List; +import java.util.function.Consumer; import static org.finos.legend.depot.domain.version.VersionValidator.BRANCH_SNAPSHOT; +import static org.finos.legend.depot.store.mongo.core.BaseMongo.buildDocument; import static org.finos.legend.depot.store.mongo.core.BaseMongo.convert; public class TestProjectVersionMigrations extends CoreDataMongoStoreTests @@ -110,4 +114,34 @@ public void testProjectDataCleanUp() Assert.assertEquals(result.getGroupId(), "examples.metadata"); Assert.assertEquals(result.getArtifactId(), "test"); } + + @Test + public void testProjectUpdatesWithLatestVersion() + { + Assert.assertEquals(0,mongoProvider.getCollection("versions").countDocuments()); + mongoAdminStore.migrationToProjectVersions(); + Assert.assertEquals(7,mongoProvider.getCollection("versions").countDocuments()); + mongoAdminStore.cleanUpProjectData(); + Assert.assertEquals(3,mongoProvider.getCollection("project-configurations").countDocuments()); + mongoAdminStore.addLatestVersionToProjectData(); + Assert.assertEquals(3,mongoProvider.getCollection("project-configurations").countDocuments()); + List result = new ArrayList<>(); + mongoProvider.getCollection("project-configurations").find().forEach((Consumer) doc -> result.add(convert(new ObjectMapper(), doc, StoreProjectData.class))); + Assert.assertEquals(result.get(0).getLatestVersion(), "2.3.1"); + Assert.assertEquals(result.get(1).getLatestVersion(), "1.0.0"); + Assert.assertEquals(result.get(2).getLatestVersion(), "2.0.1"); + } + + @Test + public void testProjectUpdatesForLatestWithNoVersion() + { + mongoProvider.getCollection("project-configurations").drop(); + mongoProvider.getCollection("project-configurations").insertOne(buildDocument(new StoreProjectData("PROD-A", "examples.project", "metadata"))); + mongoProvider.getCollection("versions").insertOne(BaseMongo.buildDocument(new StoreProjectVersionData("examples.project", "metadata", "master-snapshot"))); + + mongoAdminStore.addLatestVersionToProjectData(); + StoreProjectData result = convert(new ObjectMapper(),mongoProvider.getCollection("project-configurations").find().first(), StoreProjectData.class); + + Assert.assertNull(result.getLatestVersion()); + } } diff --git a/legend-depot-core-data-store-mongo/src/test/resources/data/projects.json b/legend-depot-core-data-store-mongo/src/test/resources/data/projects.json index a75775192..db56d0b63 100644 --- a/legend-depot-core-data-store-mongo/src/test/resources/data/projects.json +++ b/legend-depot-core-data-store-mongo/src/test/resources/data/projects.json @@ -2,12 +2,14 @@ { "projectId": "PROD-A", "groupId": "examples.metadata", - "artifactId": "test" + "artifactId": "test", + "latestVersion": "2.3.1" }, { "projectId": "PROD-B", "groupId": "examples.metadata", - "artifactId": "test-dependencies" + "artifactId": "test-dependencies", + "latestVersion": "1.0.0" }, { "projectId": "PROD-C", diff --git a/legend-depot-entities-services/src/main/java/org/finos/legend/depot/services/entities/EntityClassifierServiceImpl.java b/legend-depot-entities-services/src/main/java/org/finos/legend/depot/services/entities/EntityClassifierServiceImpl.java index 0d1816663..e52c27697 100644 --- a/legend-depot-entities-services/src/main/java/org/finos/legend/depot/services/entities/EntityClassifierServiceImpl.java +++ b/legend-depot-entities-services/src/main/java/org/finos/legend/depot/services/entities/EntityClassifierServiceImpl.java @@ -15,20 +15,19 @@ package org.finos.legend.depot.services.entities; -import org.eclipse.collections.impl.utility.ListIterate; import org.finos.legend.depot.domain.entity.DepotEntity; import org.finos.legend.depot.domain.project.ProjectVersion; import org.finos.legend.depot.domain.version.Scope; import org.finos.legend.depot.services.api.entities.EntityClassifierService; import org.finos.legend.depot.services.api.projects.ProjectsService; import org.finos.legend.depot.store.api.entities.Entities; -import org.finos.legend.sdlc.domain.model.version.VersionId; +import org.finos.legend.depot.store.model.projects.StoreProjectData; import org.slf4j.Logger; import javax.inject.Inject; import java.util.ArrayList; +import java.util.Collections; import java.util.List; -import java.util.Optional; import java.util.stream.Collectors; public class EntityClassifierServiceImpl implements EntityClassifierService @@ -44,13 +43,22 @@ public EntityClassifierServiceImpl(ProjectsService projects, Entities versions) this.entities = versions; } - private List getProjectsInfo(int page, int pageSize) + private List getLatestProjectVersionByPage(int page, int pageSize, List allProjects) { - return ListIterate.collect(projects.getProjects(page, pageSize), projectData -> + int beginIndex = page * pageSize - pageSize; + int lastIndex = page * pageSize - 1; + if (beginIndex >= allProjects.size() || beginIndex < 0) { - Optional latestVersion = projects.getLatestVersion(projectData.getGroupId(), projectData.getArtifactId()); - return new ProjectVersion(projectData.getGroupId(), projectData.getArtifactId(), latestVersion.isPresent() ? latestVersion.get().toVersionIdString() : null); - }).select(info -> info.getVersionId() != null); + return Collections.emptyList(); + } + else if (lastIndex >= allProjects.size()) + { + lastIndex = allProjects.size() - 1; + } + return allProjects.subList(beginIndex, lastIndex).stream() + .filter(project -> project.getLatestVersion() != null) + .map(project -> new ProjectVersion(project.getGroupId(), project.getArtifactId(), project.getLatestVersion())) + .collect(Collectors.toList()); } @Override @@ -63,7 +71,8 @@ public List getEntitiesByClassifierPath(String classifierPath, Stri List result = new ArrayList<>(); int PAGE_SIZE = 100; int currentPage = 1; - List projectVersions = this.getProjectsInfo(currentPage, PAGE_SIZE); + List allProjects = projects.getAllProjectCoordinates(); + List projectVersions = this.getLatestProjectVersionByPage(currentPage, PAGE_SIZE, allProjects); while (!projectVersions.isEmpty()) { List entities = this.findReleasedEntitiesByClassifier(classifierPath, search, projectVersions, limit, summary); @@ -73,7 +82,7 @@ public List getEntitiesByClassifierPath(String classifierPath, Stri break; } currentPage++; - projectVersions = this.getProjectsInfo(currentPage, PAGE_SIZE); + projectVersions = this.getLatestProjectVersionByPage(currentPage, PAGE_SIZE, allProjects); } if (limit != null) { diff --git a/legend-depot-entities-services/src/test/java/org/finos/legend/depot/services/entities/TestEntitiesService.java b/legend-depot-entities-services/src/test/java/org/finos/legend/depot/services/entities/TestEntitiesService.java index fd5a2df52..18f75257f 100644 --- a/legend-depot-entities-services/src/test/java/org/finos/legend/depot/services/entities/TestEntitiesService.java +++ b/legend-depot-entities-services/src/test/java/org/finos/legend/depot/services/entities/TestEntitiesService.java @@ -17,17 +17,19 @@ import org.eclipse.collections.api.tuple.Pair; import org.eclipse.collections.impl.tuple.Tuples; +import org.finos.legend.depot.services.api.projects.ManageProjectsService; +import org.finos.legend.depot.services.projects.ManageProjectsServiceImpl; import org.finos.legend.depot.store.api.entities.UpdateEntities; import org.finos.legend.depot.store.model.entities.EntityDefinition; import org.finos.legend.depot.domain.entity.ProjectVersionEntities; import org.finos.legend.depot.store.model.entities.StoredEntityData; import org.finos.legend.depot.store.model.entities.StoredEntityStringData; +import org.finos.legend.depot.store.model.projects.StoreProjectData; import org.finos.legend.depot.store.model.projects.StoreProjectVersionData; import org.finos.legend.depot.domain.project.ProjectVersion; import org.finos.legend.depot.domain.project.dependencies.VersionDependencyReport; import org.finos.legend.depot.services.TestBaseServices; import org.finos.legend.depot.services.api.entities.ManageEntitiesService; -import org.finos.legend.depot.services.projects.ProjectsServiceImpl; import org.finos.legend.depot.services.api.metrics.query.QueryMetricsRegistry; import org.finos.legend.depot.services.api.projects.configuration.ProjectsConfiguration; import org.finos.legend.depot.store.mongo.entities.EntitiesMongo; @@ -43,18 +45,21 @@ import java.util.List; import java.util.Map; import java.util.HashMap; +import java.util.Optional; import java.util.function.Predicate; import static org.mockito.Mockito.mock; import static org.finos.legend.depot.domain.version.VersionValidator.BRANCH_SNAPSHOT; +import static org.mockito.Mockito.when; public class TestEntitiesService extends TestBaseServices { private final QueryMetricsRegistry metrics = mock(QueryMetricsRegistry.class); private final Queue queue = mock(Queue.class); private EntitiesMongoTestUtils entityUtils = new EntitiesMongoTestUtils(mongoProvider); + private ManageProjectsService projectsService = new ManageProjectsServiceImpl(projectsVersionsStore, projectsStore, metrics, queue, new ProjectsConfiguration("master")); protected UpdateEntities entitiesStore = new EntitiesMongo(mongoProvider); - protected ManageEntitiesService entitiesService = new ManageEntitiesServiceImpl(entitiesStore, new ProjectsServiceImpl(projectsVersionsStore, projectsStore, metrics, queue, new ProjectsConfiguration("master"))); + protected ManageEntitiesService entitiesService = new ManageEntitiesServiceImpl(entitiesStore, projectsService); @Before public void setUpData() @@ -164,13 +169,16 @@ public void canQueryEntitiesWithVersionInPackage() @Test public void canQueryEntitiesWithLatestVersionAlias() { + //validation on project id , can't bypass PROD-D + projectsStore.createOrUpdate(new StoreProjectData("PROD-1","examples.metadata","test1",null, "1.0.0")); projectsVersionsStore.createOrUpdate(new StoreProjectVersionData("examples.metadata","test1","1.0.0")); + // latest is derived from project data entityUtils.loadEntities("PROD-D", "1.0.0"); String pkgName = "examples::metadata::test::dependency::v1_2_3"; Assert.assertEquals(2, entitiesService.getEntities("examples.metadata","test1","latest").size()); - Assert.assertEquals(2, entitiesService.getEntitiesByPackage("examples.metadata","test1","latest",pkgName, Collections.EMPTY_SET,true).size()); + Assert.assertEquals(2, entitiesService.getEntitiesByPackage("examples.metadata","test1","latest", pkgName, Collections.EMPTY_SET,true).size()); } diff --git a/legend-depot-generations-services/src/test/java/org/finos/legend/depot/services/generations/TestFileGenerationsService.java b/legend-depot-generations-services/src/test/java/org/finos/legend/depot/services/generations/TestFileGenerationsService.java index 2faa46b3f..d9efc93b0 100644 --- a/legend-depot-generations-services/src/test/java/org/finos/legend/depot/services/generations/TestFileGenerationsService.java +++ b/legend-depot-generations-services/src/test/java/org/finos/legend/depot/services/generations/TestFileGenerationsService.java @@ -93,7 +93,7 @@ public void loadData() throws Exception Assert.assertEquals(54, generations.getAll().size()); } - when(projectsStore.find("group.test","test")).thenReturn(Optional.of(new StoreProjectData("prod-1","group.test","test"))); + when(projectsStore.find("group.test","test")).thenReturn(Optional.of(new StoreProjectData("prod-1","group.test","test",null,"1.0.0"))); when(projectsStore.find("group.test.otherproject", "test")).thenReturn(Optional.of(new StoreProjectData("prod-2","group.test.otherproject", "test"))); when(projectsVersionsStore.find("group.test","test",BRANCH_SNAPSHOT("master"))).thenReturn(Optional.of(new StoreProjectVersionData("group-test","test",BRANCH_SNAPSHOT("master")))); when(projectsVersionsStore.find("group.test","test","1.0.0")).thenReturn(Optional.of(new StoreProjectVersionData("group-test","test","1.0.0"))); diff --git a/legend-depot-pure-model-context/src/test/java/org/finos/legend/depot/services/pure/model/context/TestPureModelContextService.java b/legend-depot-pure-model-context/src/test/java/org/finos/legend/depot/services/pure/model/context/TestPureModelContextService.java index edd37c9da..b0aa873c7 100644 --- a/legend-depot-pure-model-context/src/test/java/org/finos/legend/depot/services/pure/model/context/TestPureModelContextService.java +++ b/legend-depot-pure-model-context/src/test/java/org/finos/legend/depot/services/pure/model/context/TestPureModelContextService.java @@ -100,7 +100,7 @@ private String toString(PureModelContextData contextData) @Before public void setupMetadata() { - projectsStore.createOrUpdate(new StoreProjectData("PROD-1","test.legend","blank-prod")); + projectsStore.createOrUpdate(new StoreProjectData("PROD-1","test.legend","blank-prod",null, "2.0.0")); setUpProjectsVersionsFromFile(projects); Assert.assertEquals(4, projectsService.getAllProjectCoordinates().size()); entityUtils.loadEntities(versionedEntities); diff --git a/legend-depot-store-mongo/README.md b/legend-depot-store-mongo/README.md index 8180d05f2..c1c741a6b 100644 --- a/legend-depot-store-mongo/README.md +++ b/legend-depot-store-mongo/README.md @@ -47,4 +47,11 @@ Currently, with Mongo implementation there is limit on how much nesting of an ob In order to store such entities and also make the storage of entities scalable to other databases, we are going to be storing the content as string. With this change we have also introduced three kinds of entity storage which will further help the cause of scalability. ### Steps to follow: -1. Use the API /migrations/migrateToStoredEntityData, this will update the entities collection to be identified as one form of entity data. \ No newline at end of file +1. Use the API /migrations/migrateToStoredEntityData, this will update the entities collection to be identified as one form of entity data. + +## 4. Storing entity's content as string (for version 2.11.0 and up) +### Why is the storage of latest versions required in project data? +Currently, we compute latest version whenever it is required. It is computed by fetching all versions for the project coordinates and then find the maximum of all the versions. We wanted to make the APIs using latest version have a better performance. +The result is storing latest version to the data itself. We save the computation of this version. We use the refresh handler to keep the latest versions in sync. +### Steps to follow: +1. Use the API /migrations/addLatestVersionToProjectData, this will update the project configurations with their respective latest version from versions collection. \ No newline at end of file