From 260c757c23d35f3900ef100dc70025431c327ad6 Mon Sep 17 00:00:00 2001 From: Jayaprakash8887 Date: Tue, 4 Jul 2023 10:38:05 +0530 Subject: [PATCH] Issue #KN-889 fix: QR ImageURL bulk sync --- .../sync/tool/mgr/CassandraESSyncManager.java | 240 +++++++++--------- .../sync/tool/shell/SyncShellCommands.java | 5 +- .../sunbird/sync/tool/util/DialcodeSync.java | 53 ++-- .../tool/mgr/CassandraESSyncManagerTest.java | 8 +- .../sync/tool/util/DialcodeSyncTest.java | 4 +- .../src/test/resources/application.conf | 5 +- 6 files changed, 169 insertions(+), 146 deletions(-) diff --git a/platform-tools/spikes/sync-tool/src/main/java/org/sunbird/sync/tool/mgr/CassandraESSyncManager.java b/platform-tools/spikes/sync-tool/src/main/java/org/sunbird/sync/tool/mgr/CassandraESSyncManager.java index c5309efedc..4cbc802b63 100644 --- a/platform-tools/spikes/sync-tool/src/main/java/org/sunbird/sync/tool/mgr/CassandraESSyncManager.java +++ b/platform-tools/spikes/sync-tool/src/main/java/org/sunbird/sync/tool/mgr/CassandraESSyncManager.java @@ -76,11 +76,11 @@ public class CassandraESSyncManager { private List validCollectionStatus = Arrays.asList("Live", "Unlisted"); public CassandraESSyncManager() {} - + public CassandraESSyncManager(DialcodeSync dialcodeSync) { this.dialcodeSync = dialcodeSync; } - + @PostConstruct private void init() throws Exception { definitionDTO = util.getDefinition(graphId, objectType); @@ -95,7 +95,7 @@ public void syncAllIds(String graphId, List resourceIds, List bo System.out.println("Bookmark Id's shouldn't be provided for Multiple textbooks"); resourceIds.forEach(textbook->{ Boolean flag = syncByBookmarkId(graphId, textbook, null, false); - System.out.println("Textbook id : " + textbook + " Sync status : " + flag); + System.out.println("Textbook id : " + textbook + " Sync status : " + flag); }); } else resourceIds.forEach(textbook->{ @@ -151,20 +151,20 @@ public Boolean syncByBookmarkId(String graphId, String resourceId, List } private void updateLeafNodeCount(Map hierarchy, String resourceId) throws Exception { - //Update Collection leafNodesCount in the hierarchy - int collectionLeafNodesCount = getLeafNodesCount(hierarchy); - hierarchy.put("leafNodesCount", collectionLeafNodesCount); - // Update RootNode in Neo4j - updateTextBookNode(resourceId, "leafNodesCount", collectionLeafNodesCount); + //Update Collection leafNodesCount in the hierarchy + int collectionLeafNodesCount = getLeafNodesCount(hierarchy); + hierarchy.put("leafNodesCount", collectionLeafNodesCount); + // Update RootNode in Neo4j + updateTextBookNode(resourceId, "leafNodesCount", collectionLeafNodesCount); - //Update leafNodesCount of children in the hierarchy - updateLeafNodesCountInHierarchyMetadata((List>) hierarchy.get("children")); + //Update leafNodesCount of children in the hierarchy + updateLeafNodesCountInHierarchyMetadata((List>) hierarchy.get("children")); - //Update cassandra with updatedHierarchy - hierarchyStore.saveOrUpdateHierarchy(resourceId, hierarchy); + //Update cassandra with updatedHierarchy + hierarchyStore.saveOrUpdateHierarchy(resourceId, hierarchy); - //Clear Redis Cache of hierarchy data - RedisStoreUtil.delete("hierarchy_" + resourceId); + //Clear Redis Cache of hierarchy data + RedisStoreUtil.delete("hierarchy_" + resourceId); } private Boolean updateElasticSearch(List> units, List bookmarkIds, String resourceId) throws Exception { @@ -208,9 +208,9 @@ private void getUnitsToBeSynced(Map unitsMetadata, List { if (child.containsKey("visibility") && StringUtils.equalsIgnoreCase((String) child.get("visibility"), "parent")) { - if (flag || bookmarkIds.contains(child.get("identifier"))){ - populateESDoc(unitsMetadata, child); - } + if (flag || bookmarkIds.contains(child.get("identifier"))){ + populateESDoc(unitsMetadata, child); + } if (child.containsKey("children")) { List> newChildren = mapper.convertValue(child.get("children"), new TypeReference>>(){}); getUnitsToBeSynced(unitsMetadata, newChildren , bookmarkIds, flag); @@ -252,44 +252,44 @@ private void updateRelationData(Map message, Map } private Map refactorUnit(Map child) { - Map childData = new HashMap<>(); + Map childData = new HashMap<>(); childData.putAll(child); for(String property : relationshipProperties) { - if(childData.containsKey(property)) { - List> nextLevelNodes = (List>) childData.get(property); - List finalPropertyList = new ArrayList<>(); - if (CollectionUtils.isNotEmpty(nextLevelNodes)) { - finalPropertyList = nextLevelNodes.stream().map(nextLevelNode -> { - String identifier = (String)nextLevelNode.get("identifier"); - return identifier; - }).collect(Collectors.toList()); - } - childData.remove(property); - childData.put(property, finalPropertyList); - } + if(childData.containsKey(property)) { + List> nextLevelNodes = (List>) childData.get(property); + List finalPropertyList = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(nextLevelNodes)) { + finalPropertyList = nextLevelNodes.stream().map(nextLevelNode -> { + String identifier = (String)nextLevelNode.get("identifier"); + return identifier; + }).collect(Collectors.toList()); + } + childData.remove(property); + childData.put(property, finalPropertyList); + } } return childData; - } + } private List getFailedUnitIds(List> units, List bookmarkIds) { List failedUnits = new ArrayList<>(); if(CollectionUtils.isNotEmpty(bookmarkIds)) { - if (units.size() == bookmarkIds.size()) + if (units.size() == bookmarkIds.size()) return failedUnits; - failedUnits.addAll(bookmarkIds); + failedUnits.addAll(bookmarkIds); units.forEach(unit -> { if (bookmarkIds.contains(unit.get("identifier"))) - failedUnits.remove(unit.get("identifier")); + failedUnits.remove(unit.get("identifier")); }); } return failedUnits; } private List getSyncedUnitIds(List> units){ - List syncedUnits = new ArrayList<>(); - units.forEach(unit -> { - syncedUnits.add((String)unit.get("identifier")); - }); - return syncedUnits; + List syncedUnits = new ArrayList<>(); + units.forEach(unit -> { + syncedUnits.add((String)unit.get("identifier")); + }); + return syncedUnits; } private Map getTBMetaData(String textBookId) throws Exception { @@ -328,7 +328,7 @@ private void putAdditionalFields(Map unit, String identifier) { } private Map getDefinition() { - this.graphId = RequestValidatorUtil.isEmptyOrNull(graphId) ? "domain" : graphId; + this.graphId = RequestValidatorUtil.isEmptyOrNull(graphId) ? "domain" : graphId; if (RequestValidatorUtil.isEmptyOrNull(definitionDTO)) { throw new ServerException("ERR_DEFINITION_NOT_FOUND", "No Definition found for " + objectType); } @@ -461,88 +461,88 @@ private void getLeafNodesIds(Map data, Set leafNodeIds) } } } - + public void syncECMLContent(List contentIds) { - if(CollectionUtils.isNotEmpty(contentIds)) { - System.out.println("Content came for handling external link: " + contentIds.toString()); - List contentWithNoBody = new ArrayList<>(); - List contentWithExternalLink = new ArrayList<>(); - List contentWithNoExternalLink = new ArrayList<>(); - contentIds.stream().forEach(x -> handleAssetWithExternalLink(contentWithNoExternalLink, contentWithExternalLink, contentWithNoBody, x)); - System.out.println("Content Body not exists for content: " + contentWithNoBody.toString()); - System.out.println("Content with External Link: " + contentWithExternalLink.toString()); - System.out.println("Content with no External Link: " + contentWithNoExternalLink.toString()); - } - } - + if(CollectionUtils.isNotEmpty(contentIds)) { + System.out.println("Content came for handling external link: " + contentIds.toString()); + List contentWithNoBody = new ArrayList<>(); + List contentWithExternalLink = new ArrayList<>(); + List contentWithNoExternalLink = new ArrayList<>(); + contentIds.stream().forEach(x -> handleAssetWithExternalLink(contentWithNoExternalLink, contentWithExternalLink, contentWithNoBody, x)); + System.out.println("Content Body not exists for content: " + contentWithNoBody.toString()); + System.out.println("Content with External Link: " + contentWithExternalLink.toString()); + System.out.println("Content with no External Link: " + contentWithNoExternalLink.toString()); + } + } + public void handleAssetWithExternalLink(List contentWithNoExternalLink, List contentWithExternalLink, List contentWithNoBody, String contentId) { - String contentBody = contentStore.getContentBody(contentId); - - if(StringUtils.isNoneBlank(contentBody)) { - BaseInitializer baseInitializer = new BaseInitializer(); - Plugin plugin = null; - - try { - plugin = baseInitializer.getPlugin(contentBody); - }catch(Exception e) { - System.out.println("Exception while rendring body for content : " + contentId + " ** Exception: " + e); - } - - if (null != plugin) { - try { - Manifest manifest = plugin.getManifest(); - if (null != manifest) { - List medias = manifest.getMedias(); - if(CollectionUtils.isNotEmpty(medias)) { - List> externalLink = new ArrayList>(); - for (Media media: medias) { - TelemetryManager.log("Validating Asset for External link: " + media.getId()); - if(validateAssetMediaForExternalLink(media)) { - Map assetMap = new HashMap(); - assetMap.put("id", media.getId()); - assetMap.put("src", media.getSrc()); - assetMap.put("type", media.getType()); - externalLink.add(assetMap); - } - } - if(CollectionUtils.isNotEmpty(externalLink)) { - contentWithExternalLink.add(contentId); - contentStore.updateExternalLink(contentId, externalLink); - }else { - contentWithNoExternalLink.add(contentId); - } - } - } - }catch(Exception e) { - TelemetryManager.error("Error while pushing externalLink details of content Id: " + contentId +" into cassandra.", e); - } - } - }else { - contentWithNoBody.add(contentId); - } - } - - protected boolean validateAssetMediaForExternalLink(Media media){ - boolean isExternal = false; - UrlValidator validator = new UrlValidator(); - String urlLink = media.getSrc(); - if(StringUtils.isNotBlank(urlLink) && - validator.isValid(media.getSrc()) && - !StringUtils.contains(urlLink, CloudStore.getContainerName())) - isExternal = true; - return isExternal; - } - - public void syncDialcodesByIds(List dialcodes) throws Exception { - if(CollectionUtils.isEmpty(dialcodes)) { - System.out.println("CassandraESSyncManager:syncDialcodesByIds:No dialcodes for syncing."); - return; - } - System.out.println("CassandraESSyncManager:syncDialcodesByIds:No dialcodes for syncing: " + dialcodes.size()); - int dialcodeSyncedCount = dialcodeSync.sync(dialcodes); - System.out.println("CassandraESSyncManager:syncDialcodesByIds::dialcodeSyncedCount: " + dialcodeSyncedCount); - - } + String contentBody = contentStore.getContentBody(contentId); + + if(StringUtils.isNoneBlank(contentBody)) { + BaseInitializer baseInitializer = new BaseInitializer(); + Plugin plugin = null; + + try { + plugin = baseInitializer.getPlugin(contentBody); + }catch(Exception e) { + System.out.println("Exception while rendring body for content : " + contentId + " ** Exception: " + e); + } + + if (null != plugin) { + try { + Manifest manifest = plugin.getManifest(); + if (null != manifest) { + List medias = manifest.getMedias(); + if(CollectionUtils.isNotEmpty(medias)) { + List> externalLink = new ArrayList>(); + for (Media media: medias) { + TelemetryManager.log("Validating Asset for External link: " + media.getId()); + if(validateAssetMediaForExternalLink(media)) { + Map assetMap = new HashMap(); + assetMap.put("id", media.getId()); + assetMap.put("src", media.getSrc()); + assetMap.put("type", media.getType()); + externalLink.add(assetMap); + } + } + if(CollectionUtils.isNotEmpty(externalLink)) { + contentWithExternalLink.add(contentId); + contentStore.updateExternalLink(contentId, externalLink); + }else { + contentWithNoExternalLink.add(contentId); + } + } + } + }catch(Exception e) { + TelemetryManager.error("Error while pushing externalLink details of content Id: " + contentId +" into cassandra.", e); + } + } + }else { + contentWithNoBody.add(contentId); + } + } + + protected boolean validateAssetMediaForExternalLink(Media media){ + boolean isExternal = false; + UrlValidator validator = new UrlValidator(); + String urlLink = media.getSrc(); + if(StringUtils.isNotBlank(urlLink) && + validator.isValid(media.getSrc()) && + !StringUtils.contains(urlLink, CloudStore.getContainerName())) + isExternal = true; + return isExternal; + } + + public void syncDialcodesByIds(List dialcodes, List filenames) throws Exception { + if(CollectionUtils.isEmpty(dialcodes) && CollectionUtils.isEmpty(filenames)) { + System.out.println("CassandraESSyncManager:syncDialcodesByIds:No dialcodes for syncing."); + return; + } + + int dialcodeSyncedCount = dialcodeSync.sync(dialcodes, filenames); + System.out.println("CassandraESSyncManager:syncDialcodesByIds::dialcodeSyncedCount: " + dialcodeSyncedCount); + + } public void syncCollectionIds(String graphId, List resourceIds) { List success = new ArrayList(); diff --git a/platform-tools/spikes/sync-tool/src/main/java/org/sunbird/sync/tool/shell/SyncShellCommands.java b/platform-tools/spikes/sync-tool/src/main/java/org/sunbird/sync/tool/shell/SyncShellCommands.java index 0045ba87ad..0ee8b6fdb8 100644 --- a/platform-tools/spikes/sync-tool/src/main/java/org/sunbird/sync/tool/shell/SyncShellCommands.java +++ b/platform-tools/spikes/sync-tool/src/main/java/org/sunbird/sync/tool/shell/SyncShellCommands.java @@ -143,7 +143,8 @@ public void syncLeafNodesByIds( @CliCommand(value = "syncdialcodes", help = "Refresh leafNodes by Id(s) for Collection MimeTypes") public void syncDialcodes( - @CliOption(key = {"id","ids"}, mandatory = false, help = "Unique Id of node object") final String[] ids) + @CliOption(key = {"id","ids"}, mandatory = false, help = "Unique Id of node object") final String[] ids, + @CliOption(key = {"filename", "filenames"}, mandatory = false, help = "dialcode filename") final String[] filenames) throws Exception { long startTime = System.currentTimeMillis(); DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss"); @@ -151,7 +152,7 @@ public void syncDialcodes( if(null != ids && ids.length > 0) { System.out.println("SyncShellCommands:syncDialcodes:Total dialcodes for syncing:: " + ids); - syncManager.syncDialcodesByIds(new ArrayList(Arrays.asList(ids))); + syncManager.syncDialcodesByIds(new ArrayList(Arrays.asList(ids)), new ArrayList(Arrays.asList(filenames))); } long endTime = System.currentTimeMillis(); diff --git a/platform-tools/spikes/sync-tool/src/main/java/org/sunbird/sync/tool/util/DialcodeSync.java b/platform-tools/spikes/sync-tool/src/main/java/org/sunbird/sync/tool/util/DialcodeSync.java index 4526c9faeb..fb79ec282d 100644 --- a/platform-tools/spikes/sync-tool/src/main/java/org/sunbird/sync/tool/util/DialcodeSync.java +++ b/platform-tools/spikes/sync-tool/src/main/java/org/sunbird/sync/tool/util/DialcodeSync.java @@ -1,10 +1,8 @@ package org.sunbird.sync.tool.util; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - +import com.datastax.driver.core.ResultSet; +import com.datastax.driver.core.Row; +import com.datastax.driver.core.Session; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.sunbird.cassandra.connector.util.CassandraConnector; @@ -14,9 +12,11 @@ import org.sunbird.searchindex.elasticsearch.ElasticSearchUtil; import org.sunbird.telemetry.logger.TelemetryManager; -import com.datastax.driver.core.ResultSet; -import com.datastax.driver.core.Row; -import com.datastax.driver.core.Session; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; public class DialcodeSync { @@ -48,10 +48,9 @@ public DialcodeSync() { ElasticSearchUtil.initialiseESClient(indexName, Platform.config.getString("search.es_conn_info")); } - public int sync(List dialcodes) throws Exception { - System.out.println("DialcodeSync:sync:message:: Total number of Dialcodes to be fetched from cassandra: " + dialcodes.size()); + public int sync(List dialcodes, List filenames) throws Exception { // Get dialcodes data from cassandra - Map messages = getDialcodesFromIds(dialcodes); + Map messages = getDialcodesFromIds(dialcodes, filenames); if(MapUtils.isEmpty(messages)) { System.out.println("DialcodeSync:sync:message:: No dialcodes data fetched from cassandra."); return 0; @@ -66,15 +65,27 @@ private void upsertDocument( Map messages) throws Exception { ElasticSearchUtil.bulkIndexWithIndexId(indexName, documentType, messages); } - public Map getDialcodesFromIds(List identifiers) { + public Map getDialcodesFromIds(List identifiers, List filenames) { try { + List updateddialcodes = null; + HashMap filenamesmap = null; + + if(identifiers != null || !identifiers.isEmpty()) + updateddialcodes = identifiers; + else { + filenamesmap = (HashMap)filenames.stream().collect(Collectors.toMap(s->s.split("_")[1],Function.identity())); + updateddialcodes = filenamesmap.keySet().stream().collect(Collectors.toList()); + } + + System.out.println("DialcodeSync:sync:message:: Total number of Dialcodes to be fetched from cassandra: " + updateddialcodes.size()); + Map messages = new HashMap(); - ResultSet rs = getDialcodesFromDB(identifiers); + ResultSet rs = getDialcodesFromDB(updateddialcodes); if (null != rs) { Map dialCodesFromDB = new HashMap(); while(rs.iterator().hasNext()) { Row row = rs.iterator().next(); - String dialcodeId = (String)row.getString("identifier"); + String dialcodeId = row.getString("identifier"); dialCodesFromDB.put(dialcodeId, row); Map syncRequest = new HashMap(){{ @@ -89,7 +100,11 @@ public Map getDialcodesFromIds(List identifiers) { put("objectType", "DialCode"); }}; - String imageUrl = getQRImageFromDB(dialcodeId); + String imageUrl = ""; + if(filenamesmap!=null && !filenamesmap.isEmpty()) + imageUrl = getQRImageFromDB(filenamesmap.get(dialcodeId), true); + else imageUrl = getQRImageFromDB(dialcodeId, false); + System.out.println("Returned imageUrl: " + imageUrl); if(isReplaceString) { imageUrl = StringUtils.replaceEach(imageUrl, new String[]{replaceSrcStringDIALStore}, new String[]{replaceDestStringDIALStore}); @@ -120,8 +135,12 @@ private ResultSet getDialcodesFromDB(List identifiers) { return session.execute(query); } - private String getQRImageFromDB(String dialcodeId) { - String query = "SELECT url FROM " + qrImageKeyspace + "." + qrImageTable + " WHERE dialcode ='" + dialcodeId + "' ALLOW FILTERING;"; + private String getQRImageFromDB(String dialcodeId, boolean isFileName) { + String query = ""; + if(isFileName) + query = "SELECT url FROM " + qrImageKeyspace + "." + qrImageTable + " WHERE filename ='" + dialcodeId + "';"; + else + query = "SELECT url FROM " + qrImageKeyspace + "." + qrImageTable + " WHERE dialcode ='" + dialcodeId + "' ALLOW FILTERING;"; System.out.println("getQRImageFromDB query: " + query); Session session = CassandraConnector.getSession(); ResultSet rs = session.execute(query); diff --git a/platform-tools/spikes/sync-tool/src/test/java/org/sunbird/sync/tool/mgr/CassandraESSyncManagerTest.java b/platform-tools/spikes/sync-tool/src/test/java/org/sunbird/sync/tool/mgr/CassandraESSyncManagerTest.java index 015118d915..edddbd1626 100644 --- a/platform-tools/spikes/sync-tool/src/test/java/org/sunbird/sync/tool/mgr/CassandraESSyncManagerTest.java +++ b/platform-tools/spikes/sync-tool/src/test/java/org/sunbird/sync/tool/mgr/CassandraESSyncManagerTest.java @@ -31,21 +31,21 @@ public class CassandraESSyncManagerTest { @Test public void testsyncDialcodesByIdsWithDialcodes() throws Exception { DialcodeSync dialcodeSync = PowerMockito.mock(DialcodeSync.class); - PowerMockito.when(dialcodeSync.sync(Mockito.anyList())).thenReturn(1); + PowerMockito.when(dialcodeSync.sync(Mockito.anyList(), Mockito.anyList())).thenReturn(1); List dialcodes = Arrays.asList("A1B2C3"); CassandraESSyncManager cassandraESSyncManager = new CassandraESSyncManager(dialcodeSync); - cassandraESSyncManager.syncDialcodesByIds(dialcodes); + cassandraESSyncManager.syncDialcodesByIds(dialcodes,null); } @Test public void testsyncDialcodesByIdsWithoutDialcodes() throws Exception { DialcodeSync dialcodeSync = PowerMockito.mock(DialcodeSync.class); - PowerMockito.when(dialcodeSync.sync(Mockito.anyList())).thenReturn(1); + PowerMockito.when(dialcodeSync.sync(Mockito.anyList(),Mockito.anyList())).thenReturn(1); List dialcodes = null; CassandraESSyncManager cassandraESSyncManager = new CassandraESSyncManager(dialcodeSync); - cassandraESSyncManager.syncDialcodesByIds(dialcodes); + cassandraESSyncManager.syncDialcodesByIds(dialcodes,null); } @Test diff --git a/platform-tools/spikes/sync-tool/src/test/java/org/sunbird/sync/tool/util/DialcodeSyncTest.java b/platform-tools/spikes/sync-tool/src/test/java/org/sunbird/sync/tool/util/DialcodeSyncTest.java index 636d01c2ea..f6f8f487b2 100644 --- a/platform-tools/spikes/sync-tool/src/test/java/org/sunbird/sync/tool/util/DialcodeSyncTest.java +++ b/platform-tools/spikes/sync-tool/src/test/java/org/sunbird/sync/tool/util/DialcodeSyncTest.java @@ -37,7 +37,7 @@ public void testSyncWrongDialcodes() throws Exception { PowerMockito.when(CassandraConnector.getSession()).thenReturn(session); DialcodeSync dialcodeSync = new DialcodeSync(); - Assert.isTrue(dialcodeSync.sync(Arrays.asList("A1B2C3")) == 0); + Assert.isTrue(dialcodeSync.sync(Arrays.asList("A1B2C3"), null) == 0); } @Test @@ -73,6 +73,6 @@ public void testSyncCorrectDialcodes() throws Exception { DialcodeSync dialcodeSync = new DialcodeSync(); - Assert.isTrue(dialcodeSync.sync(Arrays.asList("A1B2C3")) == 1); + Assert.isTrue(dialcodeSync.sync(Arrays.asList("A1B2C3"), null) == 1); } } diff --git a/platform-tools/spikes/sync-tool/src/test/resources/application.conf b/platform-tools/spikes/sync-tool/src/test/resources/application.conf index 4c519eea34..294322dfee 100644 --- a/platform-tools/spikes/sync-tool/src/test/resources/application.conf +++ b/platform-tools/spikes/sync-tool/src/test/resources/application.conf @@ -40,4 +40,7 @@ search.fields.query=["name^100","title^100","lemma^100","code^100","tags^100","d search.fields.date=["lastUpdatedOn","createdOn","versionDate","lastSubmittedOn","lastPublishedOn"] search.batch.size=500 -batch.size=100 \ No newline at end of file +batch.size=100 + +replace_src_string_DIAL_store= "{{ sync_tool_replace_src_string_DIAL_store | default('DIAL_STORAGE_BASE_PATH') }}" +replace_dest_string_DIAL_store="{{ sync_tool_replace_dest_string_DIAL_store | default('https://sunbirddevbbpublic.blob.core.windows.net/dial') }}" \ No newline at end of file