From 19482c095f4b3c800b50d91ce76b36aedd174681 Mon Sep 17 00:00:00 2001 From: saiprakash-egov Date: Mon, 21 Oct 2024 19:30:20 +0530 Subject: [PATCH] Fetch households from matview and other minor changes --- .../ReferralManagementConfiguration.java | 3 + .../repository/HouseholdRepository.java | 39 ++++ .../rowmapper/HouseholdRowMapper.java | 76 +++++++ .../service/DownsyncService.java | 212 +++++++++++------- .../src/main/resources/application.properties | 1 + 5 files changed, 250 insertions(+), 81 deletions(-) create mode 100644 health-services/referralmanagement/src/main/java/org/egov/referralmanagement/repository/HouseholdRepository.java create mode 100644 health-services/referralmanagement/src/main/java/org/egov/referralmanagement/repository/rowmapper/HouseholdRowMapper.java diff --git a/health-services/referralmanagement/src/main/java/org/egov/referralmanagement/config/ReferralManagementConfiguration.java b/health-services/referralmanagement/src/main/java/org/egov/referralmanagement/config/ReferralManagementConfiguration.java index e69325f19c5..da2b35d5a49 100644 --- a/health-services/referralmanagement/src/main/java/org/egov/referralmanagement/config/ReferralManagementConfiguration.java +++ b/health-services/referralmanagement/src/main/java/org/egov/referralmanagement/config/ReferralManagementConfiguration.java @@ -115,4 +115,7 @@ public class ReferralManagementConfiguration { @Value("${egov.mdms.search.endpoint}") private String mdmsSearchUrl; + @Value("${egov.enable.matview.search}") + private String enableMatviewSearch; + } diff --git a/health-services/referralmanagement/src/main/java/org/egov/referralmanagement/repository/HouseholdRepository.java b/health-services/referralmanagement/src/main/java/org/egov/referralmanagement/repository/HouseholdRepository.java new file mode 100644 index 00000000000..a623502ba98 --- /dev/null +++ b/health-services/referralmanagement/src/main/java/org/egov/referralmanagement/repository/HouseholdRepository.java @@ -0,0 +1,39 @@ +package org.egov.referralmanagement.repository; + +import org.egov.common.ds.Tuple; +import org.egov.common.models.household.Household; +import org.egov.referralmanagement.repository.rowmapper.HouseholdRowMapper; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Repository; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Repository +public class HouseholdRepository { + + @Autowired + private NamedParameterJdbcTemplate namedParameterJdbcTemplate; + + @Autowired + private HouseholdRowMapper householdRowMapper; + + public Tuple> findByView (String localityCode, Integer limit, Integer offset, String tenantId) { + + + String query = "select * from household_address_mv where localitycode=:localitycode and rank between :start and :end "; + + Map paramsMap = new HashMap<>(); + paramsMap.put("start", offset); + paramsMap.put("end", offset+limit); + paramsMap.put("localitycode", localityCode); + + Map paramsMapCount = new HashMap<>(); + paramsMapCount.put("localitycode", localityCode); + Integer maxRank = namedParameterJdbcTemplate.queryForObject("select max(rank) from household_address_mv where localitycode=:localitycode", paramsMapCount, Integer.class); + Long totalCount = maxRank == null ? 0L : Long.valueOf(maxRank); + return new Tuple<>(totalCount, this.namedParameterJdbcTemplate.query(query, paramsMap, householdRowMapper)); + } +} diff --git a/health-services/referralmanagement/src/main/java/org/egov/referralmanagement/repository/rowmapper/HouseholdRowMapper.java b/health-services/referralmanagement/src/main/java/org/egov/referralmanagement/repository/rowmapper/HouseholdRowMapper.java new file mode 100644 index 00000000000..ed8d351c3a4 --- /dev/null +++ b/health-services/referralmanagement/src/main/java/org/egov/referralmanagement/repository/rowmapper/HouseholdRowMapper.java @@ -0,0 +1,76 @@ +package org.egov.referralmanagement.repository.rowmapper; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import digit.models.coremodels.AuditDetails; +import org.egov.common.models.core.AdditionalFields; +import org.egov.common.models.household.Address; +import org.egov.common.models.household.AddressType; +import org.egov.common.models.core.Boundary; +import org.egov.common.models.household.Household; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.stereotype.Component; + +import java.sql.ResultSet; +import java.sql.SQLException; + +@Component +public class HouseholdRowMapper implements RowMapper { + private final ObjectMapper objectMapper = new ObjectMapper(); + + @Override + public Household mapRow(ResultSet resultSet, int i) throws SQLException { + try { + AuditDetails auditDetails = AuditDetails.builder() + .createdBy(resultSet.getString("createdBy")) + .createdTime(resultSet.getLong("createdTime")) + .lastModifiedBy(resultSet.getString("lastModifiedBy")) + .lastModifiedTime(resultSet.getLong("lastModifiedTime")) + .build(); + AuditDetails clientAuditDetails = AuditDetails.builder() + .createdTime(resultSet.getLong("clientCreatedTime")) + .createdBy(resultSet.getString("clientCreatedBy")) + .lastModifiedTime(resultSet.getLong("clientLastModifiedTime")) + .lastModifiedBy(resultSet.getString("clientLastModifiedBy")) + .build(); + Household household = Household.builder() + .id(resultSet.getString("id")) + .rowVersion(resultSet.getInt("rowVersion")) + .isDeleted(resultSet.getBoolean("isDeleted")) + .tenantId(resultSet.getString("tenantId")) + .memberCount(resultSet.getInt("numberOfMembers")) + .clientReferenceId(resultSet.getString("clientReferenceId")) + .auditDetails(auditDetails) + .clientAuditDetails(clientAuditDetails) + .additionalFields(resultSet.getString("additionalDetails") == null ? null : objectMapper.readValue(resultSet + .getString("additionalDetails"), AdditionalFields.class)) + .address(Address.builder() + .id(resultSet.getString("aid")) + .clientReferenceId(resultSet.getString("aclientreferenceid")) + .tenantId(resultSet.getString("atenantid")) + .doorNo(resultSet.getString("doorNo")) + .latitude(resultSet.getDouble("latitude")) + .longitude(resultSet.getDouble("longitude")) + .locationAccuracy(resultSet.getDouble("locationAccuracy")) + .type(AddressType.fromValue(resultSet.getString("type"))) + .addressLine1(resultSet.getString("addressLine1")) + .addressLine2(resultSet.getString("addressLine2")) + .landmark(resultSet.getString("landmark")) + .city(resultSet.getString("city")) + .pincode(resultSet.getString("pinCode")) + .buildingName(resultSet.getString("buildingName")) + .street(resultSet.getString("street")) + .locality(resultSet.getString("localityCode") != null ? + Boundary.builder().code(resultSet.getString("localityCode")).build() : null) + .build()) + .build(); + if (household.getAddress().getId() == null) { + household.setAddress(null); + } + return household; + } catch (JsonProcessingException e) { + throw new SQLException(e); + } + + } +} diff --git a/health-services/referralmanagement/src/main/java/org/egov/referralmanagement/service/DownsyncService.java b/health-services/referralmanagement/src/main/java/org/egov/referralmanagement/service/DownsyncService.java index 7db34924dd4..b56c1a9e56d 100644 --- a/health-services/referralmanagement/src/main/java/org/egov/referralmanagement/service/DownsyncService.java +++ b/health-services/referralmanagement/src/main/java/org/egov/referralmanagement/service/DownsyncService.java @@ -11,6 +11,7 @@ import lombok.extern.slf4j.Slf4j; import org.egov.common.contract.request.RequestInfo; +import org.egov.common.ds.Tuple; import org.egov.common.http.client.ServiceRequestClient; import org.egov.common.models.household.Household; import org.egov.common.models.household.HouseholdBulkResponse; @@ -42,6 +43,7 @@ import org.egov.common.models.referralmanagement.sideeffect.SideEffectSearch; import org.egov.common.models.referralmanagement.sideeffect.SideEffectSearchRequest; import org.egov.referralmanagement.config.ReferralManagementConfiguration; +import org.egov.referralmanagement.repository.HouseholdRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.stereotype.Service; @@ -63,13 +65,18 @@ public class DownsyncService { private MasterDataService masterDataService; + private HouseholdRepository householdRepository; + + private static final Integer SEARCH_MAX_COUNT = 1000; + @Autowired public DownsyncService( ServiceRequestClient serviceRequestClient, ReferralManagementConfiguration referralManagementConfiguration, NamedParameterJdbcTemplate jdbcTemplate, SideEffectService sideEffectService, ReferralManagementService referralService, - MasterDataService masterDataService ) { + MasterDataService masterDataService, + HouseholdRepository householdRepository) { this.restClient = serviceRequestClient; this.configs = referralManagementConfiguration; @@ -77,7 +84,7 @@ public DownsyncService( ServiceRequestClient serviceRequestClient, this.sideEffectService=sideEffectService; this.referralService=referralService; this.masterDataService=masterDataService; - + this.householdRepository = householdRepository; } /** @@ -90,8 +97,8 @@ public Downsync prepareDownsyncData(DownsyncRequest downsyncRequest) { Downsync downsync = new Downsync(); DownsyncCriteria downsyncCriteria = downsyncRequest.getDownsyncCriteria(); - List householdIds = null; - Set individualIds = null; + List households = null; + List householdClientRefIds = null; List individualClientRefIds = null; List beneficiaryClientRefIds = null; List taskClientRefIds = null; @@ -104,16 +111,16 @@ public Downsync prepareDownsyncData(DownsyncRequest downsyncRequest) { LinkedHashMap projectType = masterDataService.getProjectType(downsyncRequest); /* search household */ - householdIds = searchHouseholds(downsyncRequest, downsync); + households = searchHouseholds(downsyncRequest, downsync); + householdClientRefIds = households.stream().map(Household::getClientReferenceId).collect(Collectors.toList()); - /* search household member using household ids */ - if (isSyncTimeAvailable || !CollectionUtils.isEmpty(householdIds)) { - individualIds = searchMembers(downsyncRequest, downsync, householdIds); - } + if (!CollectionUtils.isEmpty(householdClientRefIds)) + /* search household member using household client ref ids */ + individualClientRefIds = searchMembers(downsyncRequest, downsync, householdClientRefIds); /* search individuals using individual ids */ - if (isSyncTimeAvailable || !CollectionUtils.isEmpty(individualIds) ) { - individualClientRefIds = searchIndividuals(downsyncRequest, downsync, individualIds); + if (isSyncTimeAvailable || !CollectionUtils.isEmpty(individualClientRefIds) ) { + individualClientRefIds = searchIndividuals(downsyncRequest, downsync, individualClientRefIds); } /* search beneficiary using individual ids OR household ids */ @@ -154,65 +161,77 @@ public Downsync prepareDownsyncData(DownsyncRequest downsyncRequest) { * @param downsync * @return */ - private List searchHouseholds(DownsyncRequest downsyncRequest, Downsync downsync) { + private List searchHouseholds(DownsyncRequest downsyncRequest, Downsync downsync) { DownsyncCriteria criteria = downsyncRequest.getDownsyncCriteria(); - RequestInfo requestInfo = downsyncRequest.getRequestInfo(); + List households = null; - StringBuilder householdUrl = new StringBuilder(configs.getHouseholdHost()) - .append(configs.getHouseholdSearchUrl()); - householdUrl = appendUrlParams(householdUrl, criteria, null, null, true); + if (configs.getEnableMatviewSearch()) { + Tuple> res = householdRepository.findByView(criteria.getLocality(), criteria.getLimit(), criteria.getOffset(), null); + households = res.getY(); + } else { + RequestInfo requestInfo = downsyncRequest.getRequestInfo(); - HouseholdSearch householdSearch = HouseholdSearch.builder() - .localityCode(criteria.getLocality()) - .build(); + StringBuilder householdUrl = new StringBuilder(configs.getHouseholdHost()) + .append(configs.getHouseholdSearchUrl()); + householdUrl = appendUrlParams(householdUrl, criteria, null, null, true); - HouseholdSearchRequest searchRequest = HouseholdSearchRequest.builder() - .household(householdSearch) - .requestInfo(requestInfo) - .build(); + HouseholdSearch householdSearch = HouseholdSearch.builder() + .localityCode(criteria.getLocality()) + .build(); - HouseholdBulkResponse res = restClient.fetchResult(householdUrl, searchRequest, HouseholdBulkResponse.class); - List households = res.getHouseholds(); + HouseholdSearchRequest searchRequest = HouseholdSearchRequest.builder() + .household(householdSearch) + .requestInfo(requestInfo) + .build(); + + HouseholdBulkResponse res = restClient.fetchResult(householdUrl, searchRequest, HouseholdBulkResponse.class); + households = res.getHouseholds(); + } downsync.setHouseholds(households); downsync.getDownsyncCriteria().setTotalCount(res.getTotalCount()); if(CollectionUtils.isEmpty(households)) return Collections.emptyList(); - return households.stream().map(Household::getId).collect(Collectors.toList()); + return households; } /** * * @param downsyncRequest * @param downsync - * @param individualIds + * @param individualClientRefIds * @return individual ClientReferenceIds */ private List searchIndividuals(DownsyncRequest downsyncRequest, Downsync downsync, - Set individualIds) { + List individualClientRefIds) { DownsyncCriteria criteria = downsyncRequest.getDownsyncCriteria(); RequestInfo requestInfo = downsyncRequest.getRequestInfo(); - StringBuilder url = new StringBuilder(configs.getIndividualHost()) - .append(configs.getIndividualSearchUrl()); + List individuals = new ArrayList<>(); + List> subLists = splitList(individualClientRefIds, SEARCH_MAX_COUNT); - url = appendUrlParams(url, criteria, 0, individualIds.size(),true); + for (List list : subLists) { + StringBuilder url = new StringBuilder(configs.getIndividualHost()) + .append(configs.getIndividualSearchUrl()); - IndividualSearch individualSearch = IndividualSearch.builder() - .build(); + url = appendUrlParams(url, criteria, 0, list.size(), true); - if(!CollectionUtils.isEmpty(individualIds)) - individualSearch.setId(new ArrayList<>(individualIds)); + IndividualSearch individualSearch = IndividualSearch.builder() + .clientReferenceId(list) + .build(); - IndividualSearchRequest searchRequest = IndividualSearchRequest.builder() - .individual(individualSearch) - .requestInfo(requestInfo) - .build(); + IndividualSearchRequest searchRequest = IndividualSearchRequest.builder() + .individual(individualSearch) + .requestInfo(requestInfo) + .build(); + + List individualsSublist = restClient.fetchResult(url, searchRequest, IndividualBulkResponse.class).getIndividual(); + individuals.addAll(individualsSublist); + } - List individuals = restClient.fetchResult(url, searchRequest, IndividualBulkResponse.class).getIndividual(); downsync.setIndividuals(individuals); return individuals.stream().map(Individual::getClientReferenceId).collect(Collectors.toList()); @@ -221,37 +240,43 @@ private List searchIndividuals(DownsyncRequest downsyncRequest, Downsync /** * * @param downsyncRequest - * @param householdIds + * @param householdClientRefIds * @return */ - private Set searchMembers(DownsyncRequest downsyncRequest, Downsync downsync, - List householdIds) { + private List searchMembers(DownsyncRequest downsyncRequest, Downsync downsync, + List householdClientRefIds) { Long lastChangedSince = downsyncRequest.getDownsyncCriteria().getLastSyncedTime(); - List memberids = getPrimaryIds(householdIds, "householdId","HOUSEHOLD_MEMBER",lastChangedSince); + List memberids = getPrimaryIds(householdClientRefIds, "householdClientReferenceId","HOUSEHOLD_MEMBER",lastChangedSince); if (CollectionUtils.isEmpty(memberids)) - return Collections.emptySet(); + return Collections.emptyList(); - StringBuilder memberUrl = new StringBuilder(configs.getHouseholdHost()) - .append(configs.getHouseholdMemberSearchUrl()); + List> subLists = splitList(memberids, SEARCH_MAX_COUNT); + List members = new ArrayList<>(); + for (List list : subLists) { + StringBuilder memberUrl = new StringBuilder(configs.getHouseholdHost()) + .append(configs.getHouseholdMemberSearchUrl()); - appendUrlParams(memberUrl, downsyncRequest.getDownsyncCriteria(), 0, householdIds.size(), false); + appendUrlParams(memberUrl, downsyncRequest.getDownsyncCriteria(), 0, list.size(), false); - HouseholdMemberSearch memberSearch = HouseholdMemberSearch.builder() - .id(memberids) - .build(); + HouseholdMemberSearch memberSearch = HouseholdMemberSearch.builder() + .id(list) + .build(); - HouseholdMemberSearchRequest searchRequest = HouseholdMemberSearchRequest.builder() - .householdMemberSearch(memberSearch) - .requestInfo(downsyncRequest.getRequestInfo()) - .build(); + HouseholdMemberSearchRequest searchRequest = HouseholdMemberSearchRequest.builder() + .householdMemberSearch(memberSearch) + .requestInfo(downsyncRequest.getRequestInfo()) + .build(); + + List membersSublist = restClient.fetchResult(memberUrl, searchRequest, HouseholdMemberBulkResponse.class).getHouseholdMembers(); + members.addAll(membersSublist); + } - List members = restClient.fetchResult(memberUrl, searchRequest, HouseholdMemberBulkResponse.class).getHouseholdMembers(); downsync.setHouseholdMembers(members); - return members.stream().map(HouseholdMember::getIndividualId).collect(Collectors.toSet()); + return members.stream().map(HouseholdMember::getIndividualClientReferenceId).collect(Collectors.toList()); } /** @@ -278,22 +303,32 @@ private List searchBeneficiaries(DownsyncRequest downsyncRequest, Downsy if(CollectionUtils.isEmpty(beneficiaryIds)) return Collections.emptyList(); - StringBuilder url = new StringBuilder(configs.getProjectHost()) - .append(configs.getProjectBeneficiarySearchUrl()); + List> subLists = splitList(beneficiaryIds, SEARCH_MAX_COUNT); + List beneficiaries = new ArrayList<>(); - url = appendUrlParams(url, criteria, 0, beneficiaryClientRefIds.size(),false); + for (List list : subLists) { + StringBuilder url = new StringBuilder(configs.getProjectHost()) + .append(configs.getProjectBeneficiarySearchUrl()); - ProjectBeneficiarySearch search = ProjectBeneficiarySearch.builder() - .id(beneficiaryIds) - .projectId(Collections.singletonList(downsyncRequest.getDownsyncCriteria().getProjectId())) - .build(); + url = appendUrlParams(url, criteria, 0, list.size(),false); + + ProjectBeneficiarySearch search = ProjectBeneficiarySearch.builder() + .id(list) + .projectId(Collections.singletonList(downsyncRequest.getDownsyncCriteria().getProjectId())) + .build(); + + BeneficiarySearchRequest searchRequest = BeneficiarySearchRequest.builder() + .projectBeneficiary(search) + .requestInfo(requestInfo) + .build(); + + + List beneficiariesSublist = restClient.fetchResult(url, searchRequest, BeneficiaryBulkResponse.class).getProjectBeneficiaries(); + + beneficiaries.addAll(beneficiariesSublist); + } - BeneficiarySearchRequest searchRequest = BeneficiarySearchRequest.builder() - .projectBeneficiary(search) - .requestInfo(requestInfo) - .build(); - List beneficiaries = restClient.fetchResult(url, searchRequest, BeneficiaryBulkResponse.class).getProjectBeneficiaries(); downsync.setProjectBeneficiaries(beneficiaries); return beneficiaries.stream().map(ProjectBeneficiary::getClientReferenceId).collect(Collectors.toList()); @@ -320,22 +355,29 @@ private List searchTasks(DownsyncRequest downsyncRequest, Downsync downs if(CollectionUtils.isEmpty(taskIds)) return Collections.emptyList(); - StringBuilder url = new StringBuilder(configs.getProjectHost()) - .append(configs.getProjectTaskSearchUrl()); + List> subLists = splitList(taskIds, SEARCH_MAX_COUNT); + List tasks = new ArrayList<>(); - url = appendUrlParams(url, criteria, 0, taskIds.size(), false); + for (List list : subLists) { + StringBuilder url = new StringBuilder(configs.getProjectHost()) + .append(configs.getProjectTaskSearchUrl()); - TaskSearch search = TaskSearch.builder() - .id(taskIds) - .projectId(Collections.singletonList(downsyncRequest.getDownsyncCriteria().getProjectId())) - .build(); + url = appendUrlParams(url, criteria, 0, list.size(), false); - TaskSearchRequest searchRequest = TaskSearchRequest.builder() - .task(search) - .requestInfo(requestInfo) - .build(); + TaskSearch search = TaskSearch.builder() + .id(list) + .projectId(Collections.singletonList(downsyncRequest.getDownsyncCriteria().getProjectId())) + .build(); + + TaskSearchRequest searchRequest = TaskSearchRequest.builder() + .task(search) + .requestInfo(requestInfo) + .build(); + + List tasksSublist = restClient.fetchResult(url, searchRequest, TaskBulkResponse.class).getTasks(); + tasks.addAll(tasksSublist); + } - List tasks = restClient.fetchResult(url, searchRequest, TaskBulkResponse.class).getTasks(); downsync.setTasks(tasks); return tasks.stream().map(Task::getClientReferenceId).collect(Collectors.toList()); @@ -486,4 +528,12 @@ private StringBuilder appendUrlParams(StringBuilder url, DownsyncCriteria criter return url; } + + private List> splitList(List list, int size) { + List> subLists = new ArrayList<>(); + for (int i = 0; i < list.size(); i += size) { + subLists.add(list.subList(i, Math.min(i + size, list.size()))); + } + return subLists; + } } diff --git a/health-services/referralmanagement/src/main/resources/application.properties b/health-services/referralmanagement/src/main/resources/application.properties index b7ac09656ac..a6da9ddedd7 100644 --- a/health-services/referralmanagement/src/main/resources/application.properties +++ b/health-services/referralmanagement/src/main/resources/application.properties @@ -152,6 +152,7 @@ project.document.id.verification.required=false project.mdms.module=HCM-PROJECT-TYPES egov.location.hierarchy.type=ADMIN +egov.enable.matview.search=false