From 212b0b50782c83177f39dc8d7357b1e4c869683f Mon Sep 17 00:00:00 2001 From: Reham Muzzamil Date: Mon, 12 Jul 2021 18:10:42 +0500 Subject: [PATCH 1/5] 893 : Expose end-point /getAll on SettingsResource v2 for ETL stuff --- .../postgres/SettingRepositoryImpl.java | 118 +++++++++++++++++- .../custom/CustomSettingMetadataMapper.java | 4 + .../xml/CustomSettingsMetadataMapper.xml | 54 ++++++++ .../org/opensrp/search/SettingSearchBean.java | 38 ++++++ 4 files changed, 210 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/opensrp/repository/postgres/SettingRepositoryImpl.java b/src/main/java/org/opensrp/repository/postgres/SettingRepositoryImpl.java index 26d3deb6e..c4afa2539 100755 --- a/src/main/java/org/opensrp/repository/postgres/SettingRepositoryImpl.java +++ b/src/main/java/org/opensrp/repository/postgres/SettingRepositoryImpl.java @@ -182,13 +182,21 @@ public SettingConfiguration findSetting(SettingSearchBean settingQueryBean, public List findSettings(SettingSearchBean settingQueryBean, int limit, Map> treeNodeHashMap) { + List settingConfigurations; if (settingQueryBean.isResolveSettings()) { - return resolveSettings(findSettingsAndSettingsMetadata(settingQueryBean, treeNodeHashMap, limit), - settingQueryBean.isV1Settings(), settingQueryBean.isResolveSettings()); + settingConfigurations = settingQueryBean.isETL() ? + resolveSettingsMetadata(findSettingsMetadata(settingQueryBean, treeNodeHashMap, limit), + settingQueryBean.isV1Settings(), settingQueryBean.isResolveSettings()) : + resolveSettings(findSettingsAndSettingsMetadata(settingQueryBean, treeNodeHashMap, limit), + settingQueryBean.isV1Settings(), settingQueryBean.isResolveSettings()); } else { - return convertToSettingConfigurations(findSettingsAndSettingsMetadata(settingQueryBean, treeNodeHashMap, limit), - settingQueryBean.isV1Settings(), settingQueryBean.isResolveSettings()); + settingConfigurations = settingQueryBean.isETL() ? + convertSettingsMetadataToSettingConfigurations(findSettingsMetadata(settingQueryBean, treeNodeHashMap, limit), + settingQueryBean.isV1Settings(), settingQueryBean.isResolveSettings()) : + convertToSettingConfigurations(findSettingsAndSettingsMetadata(settingQueryBean, treeNodeHashMap, limit), + settingQueryBean.isV1Settings(), settingQueryBean.isResolveSettings()); } + return settingConfigurations; } public SettingsAndSettingsMetadataJoined findSettingsAndSettingsMetadata(SettingSearchBean settingQueryBean, @@ -218,6 +226,10 @@ public List findSettingsAndSettingsMetadata(S criteria.andMetadataVersionGreaterThan(settingQueryBean.getMetadataVersion()); } + if(settingQueryBean.getOrderByType() != null && settingQueryBean.getOrderByFieldName() != null) { + metadataExample.setOrderByClause(getOrderByClause(settingQueryBean.getOrderByFieldName().name(),settingQueryBean.getOrderByType().name())); + } + if (settingQueryBean.isResolveSettings()) { return fetchSettingsPerLocation(settingQueryBean, metadataExample, treeNodeHashMap, limit, criteria); } else { @@ -228,6 +240,39 @@ public List findSettingsAndSettingsMetadata(S } } + public List findSettingsMetadata(SettingSearchBean settingQueryBean, Map> treeNodeHashMap, int limit) { + SettingsMetadataExample metadataExample = new SettingsMetadataExample(); + SettingsMetadataExample.Criteria criteria = metadataExample.createCriteria(); + String locationId = settingQueryBean.getLocationId(); + Long primaryKey = settingQueryBean.getPrimaryKey(); + updateCriteria(criteria, settingQueryBean); + + if (primaryKey != null) { + metadataExample.or(metadataExample.createCriteria().andSettingsIdEqualTo(primaryKey)); + } + + if (StringUtils.isBlank(settingQueryBean.getId())) { + criteria.andServerVersionGreaterThanOrEqualTo(settingQueryBean.getServerVersion()); + } + + if (settingQueryBean.getMetadataVersion() != null) { + criteria.andMetadataVersionGreaterThan(settingQueryBean.getMetadataVersion()); + } + + if (settingQueryBean.getOrderByType() != null && settingQueryBean.getOrderByFieldName() != null) { + metadataExample.setOrderByClause(getOrderByClause(settingQueryBean.getOrderByFieldName().name(), settingQueryBean.getOrderByType().name())); + } + + if (settingQueryBean.isResolveSettings()) { + return fetchSettingsMetadataPerLocation(settingQueryBean, metadataExample, treeNodeHashMap, limit, criteria); + } else { + if (StringUtils.isNotEmpty(locationId)) { + criteria.andLocationIdEqualTo(locationId); + } + return settingMetadataMapper.selectManySettingsMetadata(metadataExample, 0, limit); + } + } + private SettingsMetadataExample.Criteria updateCriteria(SettingsMetadataExample.Criteria criteria, SettingSearchBean settingQueryBean) { String providerId = settingQueryBean.getProviderId(); @@ -289,6 +334,28 @@ private List fetchSettingsPerLocation(Setting return settingsAndSettingsMetadataJoinedList; } + private List fetchSettingsMetadataPerLocation(SettingSearchBean settingSearchBean, + SettingsMetadataExample metadataExample, Map> treeNodeHashMap, int limit, + SettingsMetadataExample.Criteria criteria) { + + List settingsMetadataList = new ArrayList<>(); + if (StringUtils.isNotBlank(settingSearchBean.getLocationId())) { + locationUuid = settingSearchBean.getLocationId(); + + reformattedLocationHierarchy.clear(); + if (treeNodeHashMap != null && treeNodeHashMap.size() > 0) { + reformattedLocationHierarchy(treeNodeHashMap); + } + + if (reformattedLocationHierarchy.size() > 0) { + criteria.andLocationIdIn(reformattedLocationHierarchy); + settingsMetadataList = settingMetadataMapper.selectManySettingsMetadata(metadataExample, 0, limit); + } + } + + return settingsMetadataList; + } + private void reformattedLocationHierarchy(Map> parentLocation) { Map.Entry> stringMapEntry = parentLocation.entrySet().iterator().next(); String locationKey = stringMapEntry.getKey(); @@ -327,6 +394,30 @@ private List resolveSettings( return reconcileConfigurations(stringConfigurationMap, stringSettingsMap); } + private List resolveSettingsMetadata( + List settingsMetadataList, boolean isV1Settings, + boolean resolveSettings) { + Map stringConfigurationMap = new HashMap<>(); + Map stringSettingsMap = new HashMap<>(); + + List configurations = convertSettingsMetadataToSettingConfigurations(settingsMetadataList, + isV1Settings, resolveSettings); + + for (SettingConfiguration configuration : configurations) { + String configKey = + configuration.getId() + "_" + configuration.getLocationId() + "_" + configuration.getIdentifier(); + stringConfigurationMap.put(configKey, configuration); + + List settingsList = configuration.getSettings(); + for (Setting setting : settingsList) { + String settingKey = setting.getKey() + "_" + configuration.getIdentifier(); + stringSettingsMap.put(settingKey, setting); + } + } + + return reconcileConfigurations(stringConfigurationMap, stringSettingsMap); + } + private List reconcileConfigurations(Map stringConfigurationMap, Map stringSettingsMap) { List settingConfigurations = new ArrayList<>(); @@ -444,6 +535,25 @@ private List convertToSettingConfigurations( return settingConfigurations; } + private List convertSettingsMetadataToSettingConfigurations( + List settingsMetadataList, boolean isV1Settings, + boolean resolveSettings) { + List settingConfigurations = new ArrayList<>(); + if (settingsMetadataList == null || settingsMetadataList.isEmpty()) { + return settingConfigurations; + } + + SettingConfiguration settingConfiguration = new SettingConfiguration(); + settingConfiguration.setSettings(new ArrayList<>()); + for (SettingsMetadata settingsMetadata : settingsMetadataList) { + settingConfiguration.getSettings().add(convertToSetting(settingsMetadata, isV1Settings, resolveSettings)); + settingConfiguration.setDocumentId(settingConfiguration.getSettings().get(0).getDocumentId()); + settingConfiguration.setIdentifier(settingConfiguration.getSettings().get(0).getSettingIdentifier()); + } + settingConfigurations.add(settingConfiguration); + return settingConfigurations; + } + private Setting convertToSetting(SettingsMetadata settingsMetadata, boolean isV1Settings, boolean resolveSettings) { Setting setting = new Setting(); getSettingValue(settingsMetadata, setting); diff --git a/src/main/java/org/opensrp/repository/postgres/mapper/custom/CustomSettingMetadataMapper.java b/src/main/java/org/opensrp/repository/postgres/mapper/custom/CustomSettingMetadataMapper.java index 6a1fa0360..8f6958515 100644 --- a/src/main/java/org/opensrp/repository/postgres/mapper/custom/CustomSettingMetadataMapper.java +++ b/src/main/java/org/opensrp/repository/postgres/mapper/custom/CustomSettingMetadataMapper.java @@ -19,5 +19,9 @@ List selectMany(@Param("example") SettingsMet int insertMany(@Param("settingsMetadata") List settingsMetadata); int updateMany(@Param("settingsMetadata") List settingsMetadata); + + List selectManySettingsMetadata(@Param("example") SettingsMetadataExample settingsExample, + @Param("offset") int offset, + @Param("limit") int limit); } diff --git a/src/main/java/org/opensrp/repository/postgres/mapper/custom/xml/CustomSettingsMetadataMapper.xml b/src/main/java/org/opensrp/repository/postgres/mapper/custom/xml/CustomSettingsMetadataMapper.xml index 9e370f062..7a33a7a52 100644 --- a/src/main/java/org/opensrp/repository/postgres/mapper/custom/xml/CustomSettingsMetadataMapper.xml +++ b/src/main/java/org/opensrp/repository/postgres/mapper/custom/xml/CustomSettingsMetadataMapper.xml @@ -26,6 +26,32 @@ + + + + + + + + + + + + + + + + + + + + + + s.json as settings_configuration_json, sm.id as settings_metadata_id, sm.json as settings_metadata_configuration_json, sm.* @@ -36,6 +62,17 @@ sm.settings_id = s.id + + + id, settings_id, document_id, identifier, server_version, team, team_id, provider_id, + location_id, uuid, json, setting_type, setting_value, setting_key, setting_description, + inherited_from, metadata_version + + + + diff --git a/src/main/java/org/opensrp/search/SettingSearchBean.java b/src/main/java/org/opensrp/search/SettingSearchBean.java index 1d8f171ae..f9b41b9e0 100644 --- a/src/main/java/org/opensrp/search/SettingSearchBean.java +++ b/src/main/java/org/opensrp/search/SettingSearchBean.java @@ -14,6 +14,14 @@ @NoArgsConstructor @AllArgsConstructor public class SettingSearchBean extends BaseDataObject { + + public enum OrderByType { + ASC, DESC + }; + + public enum FieldName { + id, metadata_version, server_version + }; @JsonProperty private String identifier; @@ -47,6 +55,12 @@ public class SettingSearchBean extends BaseDataObject { private Integer limit; + private FieldName orderByFieldName; + + private OrderByType orderByType; + + private boolean isETL; + public String getProviderId() { return providerId; } @@ -142,4 +156,28 @@ public Integer getLimit() { public void setLimit(Integer limit) { this.limit = limit; } + + public FieldName getOrderByFieldName() { + return orderByFieldName; + } + + public void setOrderByFieldName(FieldName orderByFieldName) { + this.orderByFieldName = orderByFieldName; + } + + public OrderByType getOrderByType() { + return orderByType; + } + + public void setOrderByType(OrderByType orderByType) { + this.orderByType = orderByType; + } + + public boolean isETL() { + return isETL; + } + + public void setETL(boolean ETL) { + isETL = ETL; + } } From 7e7730662461679387a1e79bc030309f90cc6db8 Mon Sep 17 00:00:00 2001 From: Reham Muzzamil Date: Mon, 12 Jul 2021 18:11:34 +0500 Subject: [PATCH 2/5] Bump up version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ad91f99e4..d0fcfbe68 100755 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ opensrp-server-core jar - 2.12.3-SNAPSHOT + 2.12.4-SNAPSHOT opensrp-server-core OpenSRP Server Core module https://github.com/OpenSRP/opensrp-server-core From ad32440d7a8adf68a425f9aa1c37e25ae1d2ca10 Mon Sep 17 00:00:00 2001 From: Reham Muzzamil Date: Tue, 13 Jul 2021 14:00:03 +0500 Subject: [PATCH 3/5] Update unit test --- .../postgres/SettingRepositoryTest.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/test/java/org/opensrp/repository/postgres/SettingRepositoryTest.java b/src/test/java/org/opensrp/repository/postgres/SettingRepositoryTest.java index 2f90540eb..5f8b26c7c 100755 --- a/src/test/java/org/opensrp/repository/postgres/SettingRepositoryTest.java +++ b/src/test/java/org/opensrp/repository/postgres/SettingRepositoryTest.java @@ -121,6 +121,22 @@ public void testFindByCriteria() { assertEquals(5, settings.size()); + settingQueryBean = new SettingSearchBean(); + settingQueryBean.setServerVersion(0L); + settingQueryBean.setMetadataVersion(0L); + settingQueryBean.setETL(true); + settings = settingRepository.findSettings(settingQueryBean, null); + + assertEquals(1, settings.size()); + + settingQueryBean = new SettingSearchBean(); + settingQueryBean.setServerVersion(0L); + settingQueryBean.setLocationId("44de66fb-e6c6-4bae-92bb-386dfe626eba"); + settingQueryBean.setETL(true); + settings = settingRepository.findSettings(settingQueryBean, null); + + assertEquals(1, settings.size()); + } @Test From a3b47cba06b93703bdf6c388bf241b7681aaa0f9 Mon Sep 17 00:00:00 2001 From: Reham Muzzamil Date: Tue, 13 Jul 2021 14:33:12 +0500 Subject: [PATCH 4/5] Update unit test --- .../org/opensrp/repository/postgres/SettingRepositoryTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/org/opensrp/repository/postgres/SettingRepositoryTest.java b/src/test/java/org/opensrp/repository/postgres/SettingRepositoryTest.java index 5f8b26c7c..9565cf419 100755 --- a/src/test/java/org/opensrp/repository/postgres/SettingRepositoryTest.java +++ b/src/test/java/org/opensrp/repository/postgres/SettingRepositoryTest.java @@ -133,6 +133,8 @@ public void testFindByCriteria() { settingQueryBean.setServerVersion(0L); settingQueryBean.setLocationId("44de66fb-e6c6-4bae-92bb-386dfe626eba"); settingQueryBean.setETL(true); + settingQueryBean.setOrderByFieldName(SettingSearchBean.FieldName.metadata_version); + settingQueryBean.setOrderByType(SettingSearchBean.OrderByType.ASC); settings = settingRepository.findSettings(settingQueryBean, null); assertEquals(1, settings.size()); From 1e19955cb2d86951775a2c206fad031f42495efd Mon Sep 17 00:00:00 2001 From: Reham Muzzamil Date: Tue, 13 Jul 2021 15:40:24 +0500 Subject: [PATCH 5/5] Fix broken unit test --- .../postgres/mapper/custom/xml/CustomSettingsMetadataMapper.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/opensrp/repository/postgres/mapper/custom/xml/CustomSettingsMetadataMapper.xml b/src/main/java/org/opensrp/repository/postgres/mapper/custom/xml/CustomSettingsMetadataMapper.xml index 7a33a7a52..ee50fe408 100644 --- a/src/main/java/org/opensrp/repository/postgres/mapper/custom/xml/CustomSettingsMetadataMapper.xml +++ b/src/main/java/org/opensrp/repository/postgres/mapper/custom/xml/CustomSettingsMetadataMapper.xml @@ -213,7 +213,7 @@ - order by ${orderByClause} + order by ${example.orderByClause} LIMIT #{limit} OFFSET #{offset}