From 129adb7c5a43a4b76804e3aa0ec0ddb49139bdac Mon Sep 17 00:00:00 2001 From: Avinash Ramachandruni Date: Tue, 1 Aug 2023 07:43:21 +0530 Subject: [PATCH 01/36] added new endpoints for hr analytics using org schema - syncFailures and userActivity --- .../etl/controller/ReportController.java | 75 ++++++++++++++ .../avniproject/etl/dto/UserActivityDTO.java | 98 +++++++++++++++++++ .../etl/repository/ReportRepository.java | 94 ++++++++++++++++++ .../reports/UserActivityMapper.java | 21 ++++ .../rowMappers/reports/UserCountMapper.java | 17 ++++ .../etl/service/ReportService.java | 5 + .../org/avniproject/etl/util/ReportUtil.java | 22 +++++ .../org/avniproject/etl/util/StrUtil.java | 17 ++++ 8 files changed, 349 insertions(+) create mode 100644 src/main/java/org/avniproject/etl/controller/ReportController.java create mode 100644 src/main/java/org/avniproject/etl/dto/UserActivityDTO.java create mode 100644 src/main/java/org/avniproject/etl/repository/ReportRepository.java create mode 100644 src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserActivityMapper.java create mode 100644 src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserCountMapper.java create mode 100644 src/main/java/org/avniproject/etl/service/ReportService.java create mode 100644 src/main/java/org/avniproject/etl/util/ReportUtil.java create mode 100644 src/main/java/org/avniproject/etl/util/StrUtil.java diff --git a/src/main/java/org/avniproject/etl/controller/ReportController.java b/src/main/java/org/avniproject/etl/controller/ReportController.java new file mode 100644 index 0000000..a879ab0 --- /dev/null +++ b/src/main/java/org/avniproject/etl/controller/ReportController.java @@ -0,0 +1,75 @@ +package org.avniproject.etl.controller; + +import org.avniproject.etl.domain.OrgIdentityContextHolder; +import org.avniproject.etl.dto.UserActivityDTO; +import org.avniproject.etl.repository.ReportRepository; +import org.avniproject.etl.service.ReportService; +import org.avniproject.etl.util.ReportUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +@RestController +@PreAuthorize("hasAnyAuthority('organisation_admin','admin')") +@RequestMapping("/reports") +public class ReportController { + + private final ReportService reportService; + private final ReportRepository reportRepository; + private final ReportUtil reportUtil; + + @Autowired + public ReportController(ReportService reportService, ReportRepository reportRepository, ReportUtil reportUtil) { + this.reportService = reportService; + this.reportRepository = reportRepository; + this.reportUtil = reportUtil; + } + + @GetMapping("/hr/userActivity") + public ResponseEntity getUserActivity(@RequestParam(value = "start_date", required = false) String startDate, + @RequestParam(value = "endDate", required = false) String endDate, + @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) + { + try { + List userActivity = reportRepository.getUserActivity( + OrgIdentityContextHolder.getDbSchema(), + reportUtil.getDateDynamicWhere(startDate, endDate, "registration_date"), + reportUtil.getDateDynamicWhere(startDate, endDate, "encounter_date_time"), + reportUtil.getDateDynamicWhere(startDate, endDate, "enrolment_date_time"), + reportUtil.getDynamicUserWhere(userIds, "u.id") + ); + + return ResponseEntity.ok(userActivity); + } + catch (Exception e) { + return ResponseEntity.badRequest().body(e.getMessage()); + } + } + + @GetMapping("/hr/syncFailures") + public ResponseEntity getUserWiseSyncFailures(@RequestParam(value = "start_date", required = false) String startDate, + @RequestParam(value = "endDate", required = false) String endDate, + @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds){ + try { + List userActivity = reportRepository.getUserSyncFailures( + OrgIdentityContextHolder.getDbSchema(), + reportUtil.getDateDynamicWhere(startDate, endDate, "sync_start_time"), + reportUtil.getDynamicUserWhere(userIds, "u.id") + ); + return ResponseEntity.ok(userActivity); + } + catch (Exception e) { + return ResponseEntity.badRequest().body(e.getMessage()); + } + } + + + +} + diff --git a/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java b/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java new file mode 100644 index 0000000..392f9d3 --- /dev/null +++ b/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java @@ -0,0 +1,98 @@ +package org.avniproject.etl.dto; + +import org.joda.time.DateTime; + +public class UserActivityDTO { + + private String userName; + private Long id; + private Long registrationCount; + private Long programEnrolmentCount; + private Long programEncounterCount; + private Long generalEncounterCount; + private Long count; + private String appVersion; + private String deviceModel; + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getRegistrationCount() { + return registrationCount; + } + + public void setRegistrationCount(Long registrationCount) { + this.registrationCount = registrationCount; + } + + public Long getProgramEnrolmentCount() { + return programEnrolmentCount; + } + + public void setProgramEnrolmentCount(Long programEnrolmentCount) { + this.programEnrolmentCount = programEnrolmentCount; + } + + public Long getProgramEncounterCount() { + return programEncounterCount; + } + + public void setProgramEncounterCount(Long programEncounterCount) { + this.programEncounterCount = programEncounterCount; + } + + public Long getGeneralEncounterCount() { + return generalEncounterCount; + } + + public void setGeneralEncounterCount(Long generalEncounterCount) { + this.generalEncounterCount = generalEncounterCount; + } + + public Long getCount() { + return count; + } + + public void setCount(Long count) { + this.count = count; + } + + public String getAppVersion() { + return appVersion; + } + + public void setAppVersion(String appVersion) { + this.appVersion = appVersion; + } + + public String getDeviceModel() { + return deviceModel; + } + + public void setDeviceModel(String deviceModel) { + this.deviceModel = deviceModel; + } + + public DateTime getLastSuccessfulSync() { + return lastSuccessfulSync; + } + + public void setLastSuccessfulSync(DateTime lastSuccessfulSync) { + this.lastSuccessfulSync = lastSuccessfulSync; + } + + private DateTime lastSuccessfulSync; +} diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java new file mode 100644 index 0000000..18df119 --- /dev/null +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -0,0 +1,94 @@ +package org.avniproject.etl.repository; + +import org.avniproject.etl.dto.UserActivityDTO; +import org.avniproject.etl.repository.rowMappers.reports.UserActivityMapper; +import org.avniproject.etl.repository.rowMappers.reports.UserCountMapper; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; + +import java.util.List; + +public class ReportRepository { + private final NamedParameterJdbcTemplate jdbcTemplate; + + @Autowired + public ReportRepository(NamedParameterJdbcTemplate jdbcTemplate) { + this.jdbcTemplate = jdbcTemplate; + } + + public List getUserActivity(String orgSchemaName, String subjectWhere, String encounterWhere, String enrolmentWhere, String userWhere) { + String baseQuery = "with registrations as (\n" + + " select last_modified_by_id, count(*) as registration_count\n" + + " from ${schemaName}.individual\n" + + " where is_voided = false\n" + + " ${subjectWhere}\n" + + " group by last_modified_by_id\n" + + "),\n" + + " encounters as (\n" + + " select last_modified_by_id, count(*) as encounter_count\n" + + " from ${schemaName}.encounter\n" + + " where is_voided = false\n" + + " ${encounterWhere}\n" + + " group by last_modified_by_id\n" + + " ),\n" + + " enrolments as (\n" + + " select last_modified_by_id, count(*) as enrolment_count\n" + + " from ${schemaName}.program_enrolment\n" + + " where is_voided = false\n" + + " ${enrolmentWhere}\n" + + " group by last_modified_by_id\n" + + " ),\n" + + " program_encounters as (\n" + + " select last_modified_by_id, count(*) as program_encounter_count\n" + + " from ${schemaName}.program_encounter\n" + + " where is_voided = false\n" + + " ${encounterWhere}\n" + + " group by last_modified_by_id\n" + + " )\n" + + "select u.id as id,\n" + + " coalesce(u.name, u.username) as name,\n" + + " coalesce(registration_count, 0) as registration_count,\n" + + " coalesce(encounter_count, 0) as encounter_count,\n" + + " coalesce(enrolment_count, 0) as enrolment_count,\n" + + " coalesce(program_encounter_count, 0) as program_encounter_count,\n" + + " coalesce(coalesce(registration_count, 0) + coalesce(encounter_count, 0) + coalesce(enrolment_count, 0) +\n" + + " coalesce(program_encounter_count, 0), 0) as total\n" + + "from ${schemaName}.users u\n" + + " left join ${schemaName}.registrations r on r.last_modified_by_id = u.id\n" + + " left join ${schemaName}.encounters e on e.last_modified_by_id = u.id\n" + + " left join ${schemaName}.enrolments enl on enl.last_modified_by_id = u.id\n" + + " left join ${schemaName}.program_encounters enc on enc.last_modified_by_id = u.id\n" + + "where u.is_voided = false and u.organisation_id notnull\n" + + " and coalesce(coalesce(registration_count, 0) + coalesce(encounter_count, 0) + coalesce(enrolment_count, 0) +\n" + + " coalesce(program_encounter_count, 0), 0) > 0\n" + + " ${userWhere}\n" + + "order by 7 desc\n" + + "limit 10;"; + String query = baseQuery + .replace("${schemaName}", orgSchemaName) + .replace("${subjectWhere}", subjectWhere) + .replace("${encounterWhere}", encounterWhere) + .replace("${enrolmentWhere}", enrolmentWhere) + .replace("${userWhere}", userWhere); + return jdbcTemplate.query(query, new UserActivityMapper()); + } + + public List getUserSyncFailures(String orgSchemaName, String syncTelemetryWhere, String userWhere) { + String baseQuery = "select coalesce(u.name, u.username) as name, \n" + + " count(*) as count\n" + + "from ${schemaName}.sync_telemetry st\n" + + " join ${schemaName}.users u on st.user_id = u.id\n" + + "where sync_status = 'incomplete'\n" + + "and u.is_voided = false and u.organisation_id notnull\n" + + "${syncTelemetryWhere}\n"+ + "${userWhere}\n"+ + "group by 1\n" + + "order by 2 desc\n" + + "limit 10;"; + String query = baseQuery + .replace("${schemaName}", orgSchemaName) + .replace("${syncTelemetryWhere}", syncTelemetryWhere) + .replace("${userWhere}", userWhere); + return jdbcTemplate.query(query, new UserCountMapper()); + } +} diff --git a/src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserActivityMapper.java b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserActivityMapper.java new file mode 100644 index 0000000..c2ab736 --- /dev/null +++ b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserActivityMapper.java @@ -0,0 +1,21 @@ +package org.avniproject.etl.repository.rowMappers.reports; + +import org.avniproject.etl.dto.UserActivityDTO; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class UserActivityMapper implements RowMapper { + @Override + public UserActivityDTO mapRow(ResultSet rs, int rowNum) throws SQLException { + UserActivityDTO userActivityDTO = new UserActivityDTO(); + userActivityDTO.setId(rs.getLong("id")); + userActivityDTO.setUserName(rs.getString("name")); + userActivityDTO.setRegistrationCount(rs.getLong("registration_count")); + userActivityDTO.setGeneralEncounterCount(rs.getLong("encounter_count")); + userActivityDTO.setProgramEnrolmentCount(rs.getLong("enrolment_count")); + userActivityDTO.setProgramEncounterCount(rs.getLong("program_encounter_count")); + return userActivityDTO; + } +} diff --git a/src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserCountMapper.java b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserCountMapper.java new file mode 100644 index 0000000..3ce01a9 --- /dev/null +++ b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserCountMapper.java @@ -0,0 +1,17 @@ +package org.avniproject.etl.repository.rowMappers.reports; + +import org.avniproject.etl.dto.UserActivityDTO; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class UserCountMapper implements RowMapper { + @Override + public UserActivityDTO mapRow(ResultSet rs, int rowNum) throws SQLException { + UserActivityDTO userActivityDTO = new UserActivityDTO(); + userActivityDTO.setUserName(rs.getString("name")); + userActivityDTO.setCount(rs.getLong("count")); + return userActivityDTO; + } +} diff --git a/src/main/java/org/avniproject/etl/service/ReportService.java b/src/main/java/org/avniproject/etl/service/ReportService.java new file mode 100644 index 0000000..89d789a --- /dev/null +++ b/src/main/java/org/avniproject/etl/service/ReportService.java @@ -0,0 +1,5 @@ +package org.avniproject.etl.service; + +public class ReportService { + +} diff --git a/src/main/java/org/avniproject/etl/util/ReportUtil.java b/src/main/java/org/avniproject/etl/util/ReportUtil.java new file mode 100644 index 0000000..dde2755 --- /dev/null +++ b/src/main/java/org/avniproject/etl/util/ReportUtil.java @@ -0,0 +1,22 @@ +package org.avniproject.etl.util; + +import java.util.List; + +import static java.lang.String.format; + +public class ReportUtil { + public String getDateDynamicWhere(String startDate, String endDate, String columnName) { + if (startDate != null) { + return format("and %s::date between '%s'::date and '%s'::date", columnName, startDate, endDate); + } + return ""; + } + + public String getDynamicUserWhere(List userIds, String columnName) { + if (!userIds.isEmpty()) { + return format("and %s in (%s)", columnName, StrUtil.joinLongToList(userIds)); + } + return ""; + } + +} diff --git a/src/main/java/org/avniproject/etl/util/StrUtil.java b/src/main/java/org/avniproject/etl/util/StrUtil.java new file mode 100644 index 0000000..10e759c --- /dev/null +++ b/src/main/java/org/avniproject/etl/util/StrUtil.java @@ -0,0 +1,17 @@ +package org.avniproject.etl.util; + +import java.util.List; +import java.util.stream.Collectors; + +public class StrUtil { + + public static boolean isEmpty(String string) { + return string == null || string.trim().isEmpty(); + } + + public static String joinLongToList(List lists) { + return lists.isEmpty() ? "" : lists.stream().map(String::valueOf) + .collect(Collectors.joining(",")); + } + +} From 37a9040895b0393eb212b21c8b93435ae481ef77 Mon Sep 17 00:00:00 2001 From: ak2502 Date: Tue, 1 Aug 2023 18:45:10 +0530 Subject: [PATCH 02/36] avniproject/avniproduct#1334 , AvinashRamachandruni/avni-etl | updated userSyncStatus endpoint and verified working Signed-off-by: ak2502 --- .../etl/controller/ReportController.java | 23 +++++-------------- .../etl/repository/ReportRepository.java | 9 +++++--- .../etl/service/ReportService.java | 3 +++ .../org/avniproject/etl/util/ReportUtil.java | 3 +++ 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/avniproject/etl/controller/ReportController.java b/src/main/java/org/avniproject/etl/controller/ReportController.java index a879ab0..a1e3394 100644 --- a/src/main/java/org/avniproject/etl/controller/ReportController.java +++ b/src/main/java/org/avniproject/etl/controller/ReportController.java @@ -3,21 +3,16 @@ import org.avniproject.etl.domain.OrgIdentityContextHolder; import org.avniproject.etl.dto.UserActivityDTO; import org.avniproject.etl.repository.ReportRepository; +import org.avniproject.etl.service.EtlService; import org.avniproject.etl.service.ReportService; import org.avniproject.etl.util.ReportUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.ResponseEntity; -import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import java.util.List; @RestController -@PreAuthorize("hasAnyAuthority('organisation_admin','admin')") -@RequestMapping("/reports") public class ReportController { private final ReportService reportService; @@ -31,7 +26,7 @@ public ReportController(ReportService reportService, ReportRepository reportRepo this.reportUtil = reportUtil; } - @GetMapping("/hr/userActivity") + @RequestMapping(value = "reports/hr/userActivity", method = RequestMethod.GET) public ResponseEntity getUserActivity(@RequestParam(value = "start_date", required = false) String startDate, @RequestParam(value = "endDate", required = false) String endDate, @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) @@ -52,21 +47,15 @@ public ResponseEntity getUserActivity(@RequestParam(value = "start_date", requir } } - @GetMapping("/hr/syncFailures") - public ResponseEntity getUserWiseSyncFailures(@RequestParam(value = "start_date", required = false) String startDate, + @RequestMapping(value = "/report/hr/syncFailures",method = RequestMethod.GET) + public List getUserWiseSyncFailures(@RequestParam(value = "startDate", required = false) String startDate, @RequestParam(value = "endDate", required = false) String endDate, @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds){ - try { - List userActivity = reportRepository.getUserSyncFailures( + return reportRepository.generateUserSyncFailures( OrgIdentityContextHolder.getDbSchema(), reportUtil.getDateDynamicWhere(startDate, endDate, "sync_start_time"), reportUtil.getDynamicUserWhere(userIds, "u.id") ); - return ResponseEntity.ok(userActivity); - } - catch (Exception e) { - return ResponseEntity.badRequest().body(e.getMessage()); - } } diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java index 18df119..6922a6a 100644 --- a/src/main/java/org/avniproject/etl/repository/ReportRepository.java +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -5,9 +5,11 @@ import org.avniproject.etl.repository.rowMappers.reports.UserCountMapper; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Component; import java.util.List; +@Component public class ReportRepository { private final NamedParameterJdbcTemplate jdbcTemplate; @@ -73,13 +75,14 @@ public List getUserActivity(String orgSchemaName, String subjec return jdbcTemplate.query(query, new UserActivityMapper()); } - public List getUserSyncFailures(String orgSchemaName, String syncTelemetryWhere, String userWhere) { + public List generateUserSyncFailures(String orgSchemaName, String syncTelemetryWhere, String userWhere) { String baseQuery = "select coalesce(u.name, u.username) as name, \n" + " count(*) as count\n" + "from ${schemaName}.sync_telemetry st\n" + - " join ${schemaName}.users u on st.user_id = u.id\n" + + " join ${schemaName}.user u on st.user_id = u.id\n" + "where sync_status = 'incomplete'\n" + - "and u.is_voided = false and u.organisation_id notnull\n" + + "and (u.is_voided = false or u.is_voided isnull)\n" + + "and u.organisation_id notnull\n" + "${syncTelemetryWhere}\n"+ "${userWhere}\n"+ "group by 1\n" + diff --git a/src/main/java/org/avniproject/etl/service/ReportService.java b/src/main/java/org/avniproject/etl/service/ReportService.java index 89d789a..ec5f58e 100644 --- a/src/main/java/org/avniproject/etl/service/ReportService.java +++ b/src/main/java/org/avniproject/etl/service/ReportService.java @@ -1,5 +1,8 @@ package org.avniproject.etl.service; +import org.springframework.stereotype.Service; + +@Service public class ReportService { } diff --git a/src/main/java/org/avniproject/etl/util/ReportUtil.java b/src/main/java/org/avniproject/etl/util/ReportUtil.java index dde2755..2813240 100644 --- a/src/main/java/org/avniproject/etl/util/ReportUtil.java +++ b/src/main/java/org/avniproject/etl/util/ReportUtil.java @@ -1,9 +1,12 @@ package org.avniproject.etl.util; +import org.springframework.stereotype.Service; + import java.util.List; import static java.lang.String.format; +@Service public class ReportUtil { public String getDateDynamicWhere(String startDate, String endDate, String columnName) { if (startDate != null) { From 4f762d4c3562235fbe205d69bad7cedca6a226dd Mon Sep 17 00:00:00 2001 From: ak2502 Date: Tue, 1 Aug 2023 18:45:10 +0530 Subject: [PATCH 03/36] avniproject#1334 | updated userSyncStatus endpoint and verified working Signed-off-by: ak2502 --- .../etl/controller/ReportController.java | 23 +++++-------------- .../etl/repository/ReportRepository.java | 9 +++++--- .../etl/service/ReportService.java | 3 +++ .../org/avniproject/etl/util/ReportUtil.java | 3 +++ 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/avniproject/etl/controller/ReportController.java b/src/main/java/org/avniproject/etl/controller/ReportController.java index a879ab0..a1e3394 100644 --- a/src/main/java/org/avniproject/etl/controller/ReportController.java +++ b/src/main/java/org/avniproject/etl/controller/ReportController.java @@ -3,21 +3,16 @@ import org.avniproject.etl.domain.OrgIdentityContextHolder; import org.avniproject.etl.dto.UserActivityDTO; import org.avniproject.etl.repository.ReportRepository; +import org.avniproject.etl.service.EtlService; import org.avniproject.etl.service.ReportService; import org.avniproject.etl.util.ReportUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.ResponseEntity; -import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import java.util.List; @RestController -@PreAuthorize("hasAnyAuthority('organisation_admin','admin')") -@RequestMapping("/reports") public class ReportController { private final ReportService reportService; @@ -31,7 +26,7 @@ public ReportController(ReportService reportService, ReportRepository reportRepo this.reportUtil = reportUtil; } - @GetMapping("/hr/userActivity") + @RequestMapping(value = "reports/hr/userActivity", method = RequestMethod.GET) public ResponseEntity getUserActivity(@RequestParam(value = "start_date", required = false) String startDate, @RequestParam(value = "endDate", required = false) String endDate, @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) @@ -52,21 +47,15 @@ public ResponseEntity getUserActivity(@RequestParam(value = "start_date", requir } } - @GetMapping("/hr/syncFailures") - public ResponseEntity getUserWiseSyncFailures(@RequestParam(value = "start_date", required = false) String startDate, + @RequestMapping(value = "/report/hr/syncFailures",method = RequestMethod.GET) + public List getUserWiseSyncFailures(@RequestParam(value = "startDate", required = false) String startDate, @RequestParam(value = "endDate", required = false) String endDate, @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds){ - try { - List userActivity = reportRepository.getUserSyncFailures( + return reportRepository.generateUserSyncFailures( OrgIdentityContextHolder.getDbSchema(), reportUtil.getDateDynamicWhere(startDate, endDate, "sync_start_time"), reportUtil.getDynamicUserWhere(userIds, "u.id") ); - return ResponseEntity.ok(userActivity); - } - catch (Exception e) { - return ResponseEntity.badRequest().body(e.getMessage()); - } } diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java index 18df119..6922a6a 100644 --- a/src/main/java/org/avniproject/etl/repository/ReportRepository.java +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -5,9 +5,11 @@ import org.avniproject.etl.repository.rowMappers.reports.UserCountMapper; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Component; import java.util.List; +@Component public class ReportRepository { private final NamedParameterJdbcTemplate jdbcTemplate; @@ -73,13 +75,14 @@ public List getUserActivity(String orgSchemaName, String subjec return jdbcTemplate.query(query, new UserActivityMapper()); } - public List getUserSyncFailures(String orgSchemaName, String syncTelemetryWhere, String userWhere) { + public List generateUserSyncFailures(String orgSchemaName, String syncTelemetryWhere, String userWhere) { String baseQuery = "select coalesce(u.name, u.username) as name, \n" + " count(*) as count\n" + "from ${schemaName}.sync_telemetry st\n" + - " join ${schemaName}.users u on st.user_id = u.id\n" + + " join ${schemaName}.user u on st.user_id = u.id\n" + "where sync_status = 'incomplete'\n" + - "and u.is_voided = false and u.organisation_id notnull\n" + + "and (u.is_voided = false or u.is_voided isnull)\n" + + "and u.organisation_id notnull\n" + "${syncTelemetryWhere}\n"+ "${userWhere}\n"+ "group by 1\n" + diff --git a/src/main/java/org/avniproject/etl/service/ReportService.java b/src/main/java/org/avniproject/etl/service/ReportService.java index 89d789a..ec5f58e 100644 --- a/src/main/java/org/avniproject/etl/service/ReportService.java +++ b/src/main/java/org/avniproject/etl/service/ReportService.java @@ -1,5 +1,8 @@ package org.avniproject.etl.service; +import org.springframework.stereotype.Service; + +@Service public class ReportService { } diff --git a/src/main/java/org/avniproject/etl/util/ReportUtil.java b/src/main/java/org/avniproject/etl/util/ReportUtil.java index dde2755..2813240 100644 --- a/src/main/java/org/avniproject/etl/util/ReportUtil.java +++ b/src/main/java/org/avniproject/etl/util/ReportUtil.java @@ -1,9 +1,12 @@ package org.avniproject.etl.util; +import org.springframework.stereotype.Service; + import java.util.List; import static java.lang.String.format; +@Service public class ReportUtil { public String getDateDynamicWhere(String startDate, String endDate, String columnName) { if (startDate != null) { From 911cab23a7b2d571fc1df64d498ee5a47732ad56 Mon Sep 17 00:00:00 2001 From: ak2502 Date: Tue, 1 Aug 2023 18:45:10 +0530 Subject: [PATCH 04/36] avniproject/avni-product#1334 | updated userSyncStatus endpoint and verified working Signed-off-by: ak2502 --- .../etl/controller/ReportController.java | 23 +++++-------------- .../etl/repository/ReportRepository.java | 9 +++++--- .../etl/service/ReportService.java | 3 +++ .../org/avniproject/etl/util/ReportUtil.java | 3 +++ 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/avniproject/etl/controller/ReportController.java b/src/main/java/org/avniproject/etl/controller/ReportController.java index a879ab0..a1e3394 100644 --- a/src/main/java/org/avniproject/etl/controller/ReportController.java +++ b/src/main/java/org/avniproject/etl/controller/ReportController.java @@ -3,21 +3,16 @@ import org.avniproject.etl.domain.OrgIdentityContextHolder; import org.avniproject.etl.dto.UserActivityDTO; import org.avniproject.etl.repository.ReportRepository; +import org.avniproject.etl.service.EtlService; import org.avniproject.etl.service.ReportService; import org.avniproject.etl.util.ReportUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.ResponseEntity; -import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import java.util.List; @RestController -@PreAuthorize("hasAnyAuthority('organisation_admin','admin')") -@RequestMapping("/reports") public class ReportController { private final ReportService reportService; @@ -31,7 +26,7 @@ public ReportController(ReportService reportService, ReportRepository reportRepo this.reportUtil = reportUtil; } - @GetMapping("/hr/userActivity") + @RequestMapping(value = "reports/hr/userActivity", method = RequestMethod.GET) public ResponseEntity getUserActivity(@RequestParam(value = "start_date", required = false) String startDate, @RequestParam(value = "endDate", required = false) String endDate, @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) @@ -52,21 +47,15 @@ public ResponseEntity getUserActivity(@RequestParam(value = "start_date", requir } } - @GetMapping("/hr/syncFailures") - public ResponseEntity getUserWiseSyncFailures(@RequestParam(value = "start_date", required = false) String startDate, + @RequestMapping(value = "/report/hr/syncFailures",method = RequestMethod.GET) + public List getUserWiseSyncFailures(@RequestParam(value = "startDate", required = false) String startDate, @RequestParam(value = "endDate", required = false) String endDate, @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds){ - try { - List userActivity = reportRepository.getUserSyncFailures( + return reportRepository.generateUserSyncFailures( OrgIdentityContextHolder.getDbSchema(), reportUtil.getDateDynamicWhere(startDate, endDate, "sync_start_time"), reportUtil.getDynamicUserWhere(userIds, "u.id") ); - return ResponseEntity.ok(userActivity); - } - catch (Exception e) { - return ResponseEntity.badRequest().body(e.getMessage()); - } } diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java index 18df119..6922a6a 100644 --- a/src/main/java/org/avniproject/etl/repository/ReportRepository.java +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -5,9 +5,11 @@ import org.avniproject.etl.repository.rowMappers.reports.UserCountMapper; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Component; import java.util.List; +@Component public class ReportRepository { private final NamedParameterJdbcTemplate jdbcTemplate; @@ -73,13 +75,14 @@ public List getUserActivity(String orgSchemaName, String subjec return jdbcTemplate.query(query, new UserActivityMapper()); } - public List getUserSyncFailures(String orgSchemaName, String syncTelemetryWhere, String userWhere) { + public List generateUserSyncFailures(String orgSchemaName, String syncTelemetryWhere, String userWhere) { String baseQuery = "select coalesce(u.name, u.username) as name, \n" + " count(*) as count\n" + "from ${schemaName}.sync_telemetry st\n" + - " join ${schemaName}.users u on st.user_id = u.id\n" + + " join ${schemaName}.user u on st.user_id = u.id\n" + "where sync_status = 'incomplete'\n" + - "and u.is_voided = false and u.organisation_id notnull\n" + + "and (u.is_voided = false or u.is_voided isnull)\n" + + "and u.organisation_id notnull\n" + "${syncTelemetryWhere}\n"+ "${userWhere}\n"+ "group by 1\n" + diff --git a/src/main/java/org/avniproject/etl/service/ReportService.java b/src/main/java/org/avniproject/etl/service/ReportService.java index 89d789a..ec5f58e 100644 --- a/src/main/java/org/avniproject/etl/service/ReportService.java +++ b/src/main/java/org/avniproject/etl/service/ReportService.java @@ -1,5 +1,8 @@ package org.avniproject.etl.service; +import org.springframework.stereotype.Service; + +@Service public class ReportService { } diff --git a/src/main/java/org/avniproject/etl/util/ReportUtil.java b/src/main/java/org/avniproject/etl/util/ReportUtil.java index dde2755..2813240 100644 --- a/src/main/java/org/avniproject/etl/util/ReportUtil.java +++ b/src/main/java/org/avniproject/etl/util/ReportUtil.java @@ -1,9 +1,12 @@ package org.avniproject.etl.util; +import org.springframework.stereotype.Service; + import java.util.List; import static java.lang.String.format; +@Service public class ReportUtil { public String getDateDynamicWhere(String startDate, String endDate, String columnName) { if (startDate != null) { From 96bdfa407b2f4439aa2e2aee5ab95482c51c91e7 Mon Sep 17 00:00:00 2001 From: Avinash Ramachandruni Date: Thu, 3 Aug 2023 07:42:02 +0530 Subject: [PATCH 05/36] Added 3 endpoints for hr tab analytics that can pull data from org schema - deviceModels, appVersions, userDetails --- .../etl/controller/ReportController.java | 25 +++++++ .../etl/dto/AggregateReportResult.java | 31 +++++++++ .../etl/repository/ReportRepository.java | 65 ++++++++++++++++++- .../reports/AggregateReportMapper.java | 19 ++++++ .../rowMappers/reports/UserDetailsMapper.java | 20 ++++++ 5 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/avniproject/etl/dto/AggregateReportResult.java create mode 100644 src/main/java/org/avniproject/etl/repository/rowMappers/reports/AggregateReportMapper.java create mode 100644 src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserDetailsMapper.java diff --git a/src/main/java/org/avniproject/etl/controller/ReportController.java b/src/main/java/org/avniproject/etl/controller/ReportController.java index a1e3394..3c04ebb 100644 --- a/src/main/java/org/avniproject/etl/controller/ReportController.java +++ b/src/main/java/org/avniproject/etl/controller/ReportController.java @@ -1,6 +1,7 @@ package org.avniproject.etl.controller; import org.avniproject.etl.domain.OrgIdentityContextHolder; +import org.avniproject.etl.dto.AggregateReportResult; import org.avniproject.etl.dto.UserActivityDTO; import org.avniproject.etl.repository.ReportRepository; import org.avniproject.etl.service.EtlService; @@ -58,6 +59,30 @@ public List getUserWiseSyncFailures(@RequestParam(value = "star ); } + @RequestMapping(value = "/report/hr/deviceModels", method = RequestMethod.GET) + public List getUserWiseDeviceModels(@RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) { + + return reportRepository.generateUserDeviceModels( + OrgIdentityContextHolder.getDbSchema(), + reportUtil.getDynamicUserWhere(userIds, "u.id")); + } + + @RequestMapping(value = "/report/hr/appVersions", method = RequestMethod.GET) + public List getUserWiseAppVersions(@RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) { + + return reportRepository.generateUserAppVersions( + OrgIdentityContextHolder.getDbSchema(), + reportUtil.getDynamicUserWhere(userIds, "u.id")); + } + + @RequestMapping(value = "/report/hr/userDetails", method = RequestMethod.GET) + public List getUserDetails(@RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) { + + return reportRepository.generateUserDetails( + OrgIdentityContextHolder.getDbSchema(), + reportUtil.getDynamicUserWhere(userIds, "u.id")); + } + } diff --git a/src/main/java/org/avniproject/etl/dto/AggregateReportResult.java b/src/main/java/org/avniproject/etl/dto/AggregateReportResult.java new file mode 100644 index 0000000..e684c03 --- /dev/null +++ b/src/main/java/org/avniproject/etl/dto/AggregateReportResult.java @@ -0,0 +1,31 @@ +package org.avniproject.etl.dto; + +public class AggregateReportResult { + private String label; + private Long value; + private String id; + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + + public Long getValue() { + return value; + } + + public void setValue(Long value) { + this.value = value; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } +} diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java index 6922a6a..689c5c7 100644 --- a/src/main/java/org/avniproject/etl/repository/ReportRepository.java +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -1,8 +1,11 @@ package org.avniproject.etl.repository; +import org.avniproject.etl.dto.AggregateReportResult; import org.avniproject.etl.dto.UserActivityDTO; +import org.avniproject.etl.repository.rowMappers.reports.AggregateReportMapper; import org.avniproject.etl.repository.rowMappers.reports.UserActivityMapper; import org.avniproject.etl.repository.rowMappers.reports.UserCountMapper; +import org.avniproject.etl.repository.rowMappers.reports.UserDetailsMapper; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.stereotype.Component; @@ -60,7 +63,7 @@ public List getUserActivity(String orgSchemaName, String subjec " left join ${schemaName}.encounters e on e.last_modified_by_id = u.id\n" + " left join ${schemaName}.enrolments enl on enl.last_modified_by_id = u.id\n" + " left join ${schemaName}.program_encounters enc on enc.last_modified_by_id = u.id\n" + - "where u.is_voided = false and u.organisation_id notnull\n" + + "where (u.is_voided = false or u.is_voided isnull) and u.organisation_id notnull\n" + " and coalesce(coalesce(registration_count, 0) + coalesce(encounter_count, 0) + coalesce(enrolment_count, 0) +\n" + " coalesce(program_encounter_count, 0), 0) > 0\n" + " ${userWhere}\n" + @@ -94,4 +97,64 @@ public List generateUserSyncFailures(String orgSchemaName, Stri .replace("${userWhere}", userWhere); return jdbcTemplate.query(query, new UserCountMapper()); } + + public List generateUserAppVersions(String orgSchemaName, String userWhere) { + String baseQuery = "select app_version as indicator,\n" + + " count(*) as count\n" + + "from ${schemaName}.users u\n" + + " join\n" + + " (select user_id,\n" + + " app_version,\n" + + " row_number() over (partition by user_id order by sync_start_time desc ) as rn\n" + + " from ${schemaName}.sync_telemetry) l on l.user_id = u.id and rn = 1\n" + + "where (u.is_voided = false or u.is_voided isnull) and u.organisation_id notnull\n" + + "${userWhere}\n"+ + "group by app_version;"; + String query = baseQuery + .replace("${schemaName}", orgSchemaName) + .replace("${userWhere}", userWhere); + return jdbcTemplate.query(query, new AggregateReportMapper()); + } + + public List generateUserDeviceModels(String orgSchemaName, String userWhere) { + String baseQuery = "select device_model as indicator,\n" + + " count(*) as count\n" + + "from ${schemaName}.users u\n" + + " join\n" + + " (select user_id,\n" + + " coalesce(device_info ->> 'brand', device_name) as device_model,\n" + + " row_number() over (partition by user_id order by sync_start_time desc ) as rn\n" + + " from ${schemaName}.sync_telemetry) l on l.user_id = u.id and rn = 1\n" + + "where (u.is_voided = false or u.is_voided isnull) and u.organisation_id notnull \n" + + "${userWhere}\n"+ + "group by device_model;"; + String query = baseQuery + .replace("${schemaName}", orgSchemaName) + .replace("${userWhere}", userWhere); + return jdbcTemplate.query(query, new AggregateReportMapper()); + } + + public List generateUserDetails(String orgSchemaName, String userWhere) { + String baseQuery = "select coalesce(u.name, u.username) as name,\n" + + " app_version,\n" + + " device_model,\n" + + " sync_start_time\n" + + "from ${schemaName}.users u\n" + + " join\n" + + " (select user_id,\n" + + " app_version,\n" + + " coalesce(device_info ->> 'brand', device_name) as device_model,\n" + + " sync_start_time,\n" + + " row_number() over (partition by user_id order by sync_start_time desc ) as rn\n" + + " from ${schemaName}.sync_telemetry\n" + + " where sync_status = 'complete') l on l.user_id = u.id and rn = 1\n" + + "where (u.is_voided = false or u.is_voided isnull)\n" + + " and u.organisation_id notnull\n" + + " ${userWhere}\n"+ + "order by 1 desc;"; + String query = baseQuery + .replace("${schemaName}", orgSchemaName) + .replace("${userWhere}", userWhere); + return jdbcTemplate.query(query, new UserDetailsMapper()); + } } diff --git a/src/main/java/org/avniproject/etl/repository/rowMappers/reports/AggregateReportMapper.java b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/AggregateReportMapper.java new file mode 100644 index 0000000..d214b57 --- /dev/null +++ b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/AggregateReportMapper.java @@ -0,0 +1,19 @@ +package org.avniproject.etl.repository.rowMappers.reports; + +import org.avniproject.etl.dto.AggregateReportResult; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class AggregateReportMapper implements RowMapper { + @Override + public AggregateReportResult mapRow(ResultSet rs, int rowNum) throws SQLException { + AggregateReportResult aggregateReportResult = new AggregateReportResult(); + aggregateReportResult.setLabel(rs.getString("indicator")); + aggregateReportResult.setValue(rs.getLong("count")); + aggregateReportResult.setId(rs.getString("indicator")); + + return aggregateReportResult; + } +} diff --git a/src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserDetailsMapper.java b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserDetailsMapper.java new file mode 100644 index 0000000..5f95123 --- /dev/null +++ b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserDetailsMapper.java @@ -0,0 +1,20 @@ +package org.avniproject.etl.repository.rowMappers.reports; + +import org.avniproject.etl.dto.UserActivityDTO; +import org.joda.time.DateTime; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class UserDetailsMapper implements RowMapper { + @Override + public UserActivityDTO mapRow(ResultSet rs, int rowNum) throws SQLException { + UserActivityDTO userActivityDTO = new UserActivityDTO(); + userActivityDTO.setUserName(rs.getString("name")); + userActivityDTO.setAppVersion(rs.getString("app_version")); + userActivityDTO.setDeviceModel(rs.getString("device_model")); + userActivityDTO.setLastSuccessfulSync(new DateTime(rs.getDate("sync_start_time"))); + return userActivityDTO; + } +} From 81be4928fd14e6166ef6b1ea6d2b30e58db1bd27 Mon Sep 17 00:00:00 2001 From: ak2502 Date: Thu, 3 Aug 2023 17:10:53 +0530 Subject: [PATCH 06/36] avniproject/avni-product#1334 | adding rest of the endpoints for HR tab Signed-off-by: ak2502 --- .../etl/controller/ReportController.java | 48 ++++++++--- .../etl/repository/ReportRepository.java | 86 ++++++++++++++++++- 2 files changed, 117 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/avniproject/etl/controller/ReportController.java b/src/main/java/org/avniproject/etl/controller/ReportController.java index 3c04ebb..47f4e22 100644 --- a/src/main/java/org/avniproject/etl/controller/ReportController.java +++ b/src/main/java/org/avniproject/etl/controller/ReportController.java @@ -4,11 +4,9 @@ import org.avniproject.etl.dto.AggregateReportResult; import org.avniproject.etl.dto.UserActivityDTO; import org.avniproject.etl.repository.ReportRepository; -import org.avniproject.etl.service.EtlService; import org.avniproject.etl.service.ReportService; import org.avniproject.etl.util.ReportUtil; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; import java.util.List; @@ -28,24 +26,16 @@ public ReportController(ReportService reportService, ReportRepository reportRepo } @RequestMapping(value = "reports/hr/userActivity", method = RequestMethod.GET) - public ResponseEntity getUserActivity(@RequestParam(value = "start_date", required = false) String startDate, + public List getUserActivity(@RequestParam(value = "startDate", required = false) String startDate, @RequestParam(value = "endDate", required = false) String endDate, - @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) - { - try { - List userActivity = reportRepository.getUserActivity( + @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds){ + return reportRepository.generateUserActivity( OrgIdentityContextHolder.getDbSchema(), reportUtil.getDateDynamicWhere(startDate, endDate, "registration_date"), reportUtil.getDateDynamicWhere(startDate, endDate, "encounter_date_time"), reportUtil.getDateDynamicWhere(startDate, endDate, "enrolment_date_time"), reportUtil.getDynamicUserWhere(userIds, "u.id") ); - - return ResponseEntity.ok(userActivity); - } - catch (Exception e) { - return ResponseEntity.badRequest().body(e.getMessage()); - } } @RequestMapping(value = "/report/hr/syncFailures",method = RequestMethod.GET) @@ -83,6 +73,38 @@ public List getUserDetails(@RequestParam(value = "userIds", req reportUtil.getDynamicUserWhere(userIds, "u.id")); } + @RequestMapping(value = "/report/hr/championUsers", method = RequestMethod.GET) + public List getChampionUsers(@RequestParam(value = "startDate", required = false) String startDate, + @RequestParam(value = "endDate", required = false) String endDate, + @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) { + return reportRepository.generateCompletedVisitsOnTimeByProportion( + ">= 0.8", + OrgIdentityContextHolder.getDbSchema(), + reportUtil.getDateDynamicWhere(startDate, endDate, "encounter_date_time"), + reportUtil.getDynamicUserWhere(userIds, "u.id")); + } + + @RequestMapping(value = "/report/hr/nonPerformingUsers", method = RequestMethod.GET) + public List getNonPerformingUsers(@RequestParam(value = "startDate", required = false) String startDate, + @RequestParam(value = "endDate", required = false) String endDate, + @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) { + return reportRepository.generateCompletedVisitsOnTimeByProportion( + "<= 0.5", + OrgIdentityContextHolder.getDbSchema(), + reportUtil.getDateDynamicWhere(startDate, endDate, "encounter_date_time"), + reportUtil.getDynamicUserWhere(userIds, "u.id") + ); + } + + @RequestMapping(value = "/report/hr/mostCancelled", method = RequestMethod.GET) + public List getUsersCancellingMostVisits(@RequestParam(value = "startDate", required = false) String startDate, + @RequestParam(value = "endDate", required = false) String endDate, + @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) { + return reportRepository.generateUserCancellingMostVisits( + OrgIdentityContextHolder.getDbSchema(), + reportUtil.getDateDynamicWhere(startDate, endDate, "encounter_date_time"), + reportUtil.getDynamicUserWhere(userIds, "u.id")); + } } diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java index 689c5c7..e3ce884 100644 --- a/src/main/java/org/avniproject/etl/repository/ReportRepository.java +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -1,11 +1,13 @@ package org.avniproject.etl.repository; import org.avniproject.etl.dto.AggregateReportResult; +import org.apache.log4j.Logger; import org.avniproject.etl.dto.UserActivityDTO; import org.avniproject.etl.repository.rowMappers.reports.AggregateReportMapper; import org.avniproject.etl.repository.rowMappers.reports.UserActivityMapper; import org.avniproject.etl.repository.rowMappers.reports.UserCountMapper; import org.avniproject.etl.repository.rowMappers.reports.UserDetailsMapper; +import org.avniproject.etl.service.EtlService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.stereotype.Component; @@ -15,13 +17,14 @@ @Component public class ReportRepository { private final NamedParameterJdbcTemplate jdbcTemplate; + private static final Logger log = Logger.getLogger(EtlService.class); @Autowired public ReportRepository(NamedParameterJdbcTemplate jdbcTemplate) { this.jdbcTemplate = jdbcTemplate; } - public List getUserActivity(String orgSchemaName, String subjectWhere, String encounterWhere, String enrolmentWhere, String userWhere) { + public List generateUserActivity(String orgSchemaName, String subjectWhere, String encounterWhere, String enrolmentWhere, String userWhere) { String baseQuery = "with registrations as (\n" + " select last_modified_by_id, count(*) as registration_count\n" + " from ${schemaName}.individual\n" + @@ -75,6 +78,7 @@ public List getUserActivity(String orgSchemaName, String subjec .replace("${encounterWhere}", encounterWhere) .replace("${enrolmentWhere}", enrolmentWhere) .replace("${userWhere}", userWhere); + log.info(query); return jdbcTemplate.query(query, new UserActivityMapper()); } @@ -82,7 +86,7 @@ public List generateUserSyncFailures(String orgSchemaName, Stri String baseQuery = "select coalesce(u.name, u.username) as name, \n" + " count(*) as count\n" + "from ${schemaName}.sync_telemetry st\n" + - " join ${schemaName}.user u on st.user_id = u.id\n" + + " join ${schemaName}.users u on st.user_id = u.id\n" + "where sync_status = 'incomplete'\n" + "and (u.is_voided = false or u.is_voided isnull)\n" + "and u.organisation_id notnull\n" + @@ -122,7 +126,7 @@ public List generateUserDeviceModels(String orgSchemaName "from ${schemaName}.users u\n" + " join\n" + " (select user_id,\n" + - " coalesce(device_info ->> 'brand', device_name) as device_model,\n" + + " device_name as device_model,\n" + " row_number() over (partition by user_id order by sync_start_time desc ) as rn\n" + " from ${schemaName}.sync_telemetry) l on l.user_id = u.id and rn = 1\n" + "where (u.is_voided = false or u.is_voided isnull) and u.organisation_id notnull \n" + @@ -143,7 +147,7 @@ public List generateUserDetails(String orgSchemaName, String us " join\n" + " (select user_id,\n" + " app_version,\n" + - " coalesce(device_info ->> 'brand', device_name) as device_model,\n" + + " device_name as device_model,\n" + " sync_start_time,\n" + " row_number() over (partition by user_id order by sync_start_time desc ) as rn\n" + " from ${schemaName}.sync_telemetry\n" + @@ -157,4 +161,78 @@ public List generateUserDetails(String orgSchemaName, String us .replace("${userWhere}", userWhere); return jdbcTemplate.query(query, new UserDetailsMapper()); } + + public List generateCompletedVisitsOnTimeByProportion(String proportionCondition, String orgSchemaName, String encounterWhere, String userWhere) { + String baseQuery = "with program_enc_data as (\n" + + " select last_modified_by_id,\n" + + " count(*) filter ( where encounter_date_time <= max_visit_date_time ) visits_done_on_time,\n" + + " count(*) filter ( where encounter_date_time notnull and earliest_visit_date_time notnull ) total_scheduled\n" + + " from program_encounter\n" + + " where is_voided = false\n" + + " ${encounterWhere}\n" + + " group by last_modified_by_id\n" + + "),\n" + + " general_enc_data as (\n" + + " select last_modified_by_id,\n" + + " count(*) filter ( where encounter_date_time <= max_visit_date_time ) visits_done_on_time,\n" + + " count(*)\n" + + " filter ( where encounter_date_time notnull and earliest_visit_date_time notnull ) total_scheduled\n" + + " from encounter\n" + + " where is_voided = false\n" + + " ${encounterWhere}\n" + + " group by last_modified_by_id\n" + + " )\n" + + "select coalesce(u.name, u.username) as indicator,\n" + + " coalesce(ged.visits_done_on_time, 0) + coalesce(ped.visits_done_on_time, 0) as count\n" + + "from users u\n" + + " join general_enc_data ged on ged.last_modified_by_id = u.id\n" + + " join program_enc_data ped on ped.last_modified_by_id = u.id\n" + + "where u.organisation_id notnull\n" + + " and is_voided = false\n" + + " and coalesce(ged.visits_done_on_time, 0) + coalesce(ped.visits_done_on_time, 0) > 0\n" + + " ${userWhere}\n" + + " and ((coalesce(ged.visits_done_on_time, 0.0) + coalesce(ped.visits_done_on_time, 0.0)) /\n" + + " nullif((coalesce(ged.total_scheduled, 0) + coalesce(ped.total_scheduled, 0)), 0)) ${proportion_condition}\n"; + String query = baseQuery + .replace("${proportion_condition}", proportionCondition) + .replace("${schemaName}", orgSchemaName) + .replace("${encounterWhere}", encounterWhere) + .replace("${userWhere}", userWhere); + return jdbcTemplate.query(query, new AggregateReportMapper()); + } + + public List generateUserCancellingMostVisits(String orgSchemaName, String encounterWhere, String userWhere) { + String baseQuery = "with program_enc_data as (\n" + + " select last_modified_by_id,\n" + + " count(*) filter ( where cancel_date_time notnull ) cancelled_visits\n" + + " from program_encounter\n" + + " where is_voided = false\n" + + " ${encounterWhere}\n" + + " group by last_modified_by_id\n" + + "),\n" + + " general_enc_data as (\n" + + " select last_modified_by_id,\n" + + " count(*) filter ( where cancel_date_time notnull ) cancelled_visits\n" + + " from encounter\n" + + " where is_voided = false\n" + + " ${encounterWhere}\n" + + " group by last_modified_by_id\n" + + " )\n" + + "select coalesce(u.name, u.username) as indicator,\n" + + " coalesce(ged.cancelled_visits, 0) + coalesce(ped.cancelled_visits, 0) as count\n" + + "from users u\n" + + " join general_enc_data ged on ged.last_modified_by_id = u.id\n" + + " join program_enc_data ped on ped.last_modified_by_id = u.id\n" + + "where u.organisation_id notnull\n" + + " and is_voided = false\n" + + " and coalesce(ged.cancelled_visits, 0) + coalesce(ped.cancelled_visits, 0) > 0 \n" + + " ${userWhere}\n" + + "order by coalesce(ged.cancelled_visits, 0.0) + coalesce(ped.cancelled_visits, 0.0) desc\n" + + "limit 5;"; + String query = baseQuery + .replace("${schemaName}", orgSchemaName) + .replace("${encounterWhere}", encounterWhere) + .replace("${userWhere}", userWhere); + return jdbcTemplate.query(query, new AggregateReportMapper()); + } } From 49db7a5b75fc39facdfb95d6539382b86f3f1f0b Mon Sep 17 00:00:00 2001 From: ak2502 Date: Mon, 14 Aug 2023 11:22:47 +0530 Subject: [PATCH 07/36] avniproject/avni-product#1334 | Joda issue fixed Signed-off-by: ak2502 --- build.gradle | 6 ++++++ .../etl/util/ObjectMapperSingleton.java | 17 +++++++++++++++++ src/main/resources/application.properties | 2 ++ 3 files changed, 25 insertions(+) create mode 100644 src/main/java/org/avniproject/etl/util/ObjectMapperSingleton.java diff --git a/build.gradle b/build.gradle index 254dc91..b3799d2 100644 --- a/build.gradle +++ b/build.gradle @@ -30,6 +30,12 @@ dependencies { implementation 'org.keycloak:keycloak-admin-client:21.0.2' implementation 'org.antlr:ST4:4.3.4' implementation 'org.hibernate:hibernate-validator:7.0.5.Final' + implementation "joda-time:joda-time:2.9.4" + implementation 'com.fasterxml.jackson.core:jackson-databind:2.12.6' + implementation 'com.fasterxml.jackson.core:jackson-core:2.12.6' + implementation 'com.fasterxml.jackson.core:jackson-annotations:2.12.6' + implementation 'com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.12.6' + implementation 'com.fasterxml.jackson.datatype:jackson-datatype-joda:2.12.6' } tasks.named('test') { diff --git a/src/main/java/org/avniproject/etl/util/ObjectMapperSingleton.java b/src/main/java/org/avniproject/etl/util/ObjectMapperSingleton.java new file mode 100644 index 0000000..ede7d43 --- /dev/null +++ b/src/main/java/org/avniproject/etl/util/ObjectMapperSingleton.java @@ -0,0 +1,17 @@ +package org.avniproject.etl.util; + +import com.fasterxml.jackson.datatype.joda.JodaModule; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; + +@Configuration +@PropertySource("classpath:application.properties") +public class ObjectMapperSingleton { + + @Bean + public JodaModule jodaModule() { + JodaModule module = new JodaModule(); + return module; + } +} \ No newline at end of file diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 1446e28..f4ef015 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -2,3 +2,5 @@ avni.database=${OPENCHS_DATABASE_NAME:openchs} spring.config.import=classpath:/main-application.properties debug=${ETL_DEBUG_MODE:false} avni.current.time.offset.seconds=${AVNI_CURRENT_TIME_OFFSET_SECONDS:10} +spring.jackson.serialization.write-dates-as-timestamps=false +spring.jackson.default-property-inclusion=non_null From 689c7aa6a911aec813b9a464a7d3104d0f3e0d55 Mon Sep 17 00:00:00 2001 From: ak2502 Date: Thu, 17 Aug 2023 13:33:38 +0530 Subject: [PATCH 08/36] avniproject/avni-product#1334 | generateUserActivity query updated according to ETL Signed-off-by: ak2502 --- .../etl/controller/ReportController.java | 2 +- .../etl/repository/ReportRepository.java | 131 ++++++++++++------ 2 files changed, 93 insertions(+), 40 deletions(-) diff --git a/src/main/java/org/avniproject/etl/controller/ReportController.java b/src/main/java/org/avniproject/etl/controller/ReportController.java index 47f4e22..d162a78 100644 --- a/src/main/java/org/avniproject/etl/controller/ReportController.java +++ b/src/main/java/org/avniproject/etl/controller/ReportController.java @@ -25,7 +25,7 @@ public ReportController(ReportService reportService, ReportRepository reportRepo this.reportUtil = reportUtil; } - @RequestMapping(value = "reports/hr/userActivity", method = RequestMethod.GET) + @RequestMapping(value = "report/hr/userActivity", method = RequestMethod.GET) public List getUserActivity(@RequestParam(value = "startDate", required = false) String startDate, @RequestParam(value = "endDate", required = false) String endDate, @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds){ diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java index e3ce884..cb1ce00 100644 --- a/src/main/java/org/avniproject/etl/repository/ReportRepository.java +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -1,5 +1,7 @@ package org.avniproject.etl.repository; +import org.avniproject.etl.domain.metadata.SchemaMetadata; +import org.avniproject.etl.domain.metadata.TableMetadata; import org.avniproject.etl.dto.AggregateReportResult; import org.apache.log4j.Logger; import org.avniproject.etl.dto.UserActivityDTO; @@ -12,74 +14,125 @@ import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.stereotype.Component; +import java.util.Collection; import java.util.List; +import java.util.stream.Stream; @Component public class ReportRepository { private final NamedParameterJdbcTemplate jdbcTemplate; + private final SchemaMetadataRepository schemaMetadataRepository; private static final Logger log = Logger.getLogger(EtlService.class); @Autowired - public ReportRepository(NamedParameterJdbcTemplate jdbcTemplate) { + public ReportRepository(NamedParameterJdbcTemplate jdbcTemplate, SchemaMetadataRepository schemaMetadataRepository) { this.jdbcTemplate = jdbcTemplate; + this.schemaMetadataRepository = schemaMetadataRepository; } public List generateUserActivity(String orgSchemaName, String subjectWhere, String encounterWhere, String enrolmentWhere, String userWhere) { - String baseQuery = "with registrations as (\n" + - " select last_modified_by_id, count(*) as registration_count\n" + - " from ${schemaName}.individual\n" + - " where is_voided = false\n" + - " ${subjectWhere}\n" + - " group by last_modified_by_id\n" + - "),\n" + - " encounters as (\n" + - " select last_modified_by_id, count(*) as encounter_count\n" + - " from ${schemaName}.encounter\n" + + SchemaMetadata schema = schemaMetadataRepository.getExistingSchemaMetadata(); + List subjectTables = Stream.of(schema.getAllSubjectTables()).flatMap(Collection::stream).toList(); + List encounterTables = schema.getAllEncounterTables().stream().toList(); + List programEnrolmentTables = schema.getAllProgramEnrolmentTables().stream().toList(); + List programEncounterTables = schema.getAllProgramEncounterTables().stream().toList(); + + StringBuilder query = new StringBuilder(); + + String baseQuery1 = "select last_modified_by_id, count(*) as registration_count\n" + + " from ${schemaName}.${subject}\n" + + " where is_voided = false\n" + + " ${subjectWhere}\n" + + " group by last_modified_by_id\n" ; + baseQuery1 = baseQuery1.replace("${schemaName}", orgSchemaName) + .replace("${subjectWhere}", subjectWhere); + query.append("with registrations as (\n"); + query.append(baseQuery1.replace("${subject}", subjectTables.get(0).getName())); + for (int i = 1 ; i < subjectTables.size() ; i++) { + query.append("union all\n" ); + query.append(baseQuery1.replace("${subject}", subjectTables.get(i).getName())); + } + query.append("),\n"); + + String baseQuery2 = "select last_modified_by_id, count(*) as encounter_count\n" + + " from ${schemaName}.${encounter}\n" + " where is_voided = false\n" + " ${encounterWhere}\n" + - " group by last_modified_by_id\n" + - " ),\n" + - " enrolments as (\n" + - " select last_modified_by_id, count(*) as enrolment_count\n" + - " from ${schemaName}.program_enrolment\n" + + " group by last_modified_by_id\n" ; + baseQuery2 = baseQuery2.replace("${schemaName}", orgSchemaName) + .replace("${encounterWhere}", encounterWhere); + query.append("encounters as (\n"); + query.append(baseQuery2.replace("${encounter}", encounterTables.get(0).getName())); + for (int i = 1 ; i < encounterTables.size() ; i++) { + query.append("union all\n" ); + query.append(baseQuery2.replace("${encounter}", encounterTables.get(i).getName())); + } + query.append("),\n"); + + String baseQuery3 = "select last_modified_by_id, count(*) as enrolment_count\n" + + " from ${schemaName}.${enrolment}\n" + " where is_voided = false\n" + " ${enrolmentWhere}\n" + - " group by last_modified_by_id\n" + - " ),\n" + - " program_encounters as (\n" + - " select last_modified_by_id, count(*) as program_encounter_count\n" + - " from ${schemaName}.program_encounter\n" + + " group by last_modified_by_id\n" ; + baseQuery3 = baseQuery3.replace("${schemaName}", orgSchemaName) + .replace("${enrolmentWhere}", enrolmentWhere); + query.append("enrolments as (\n"); + query.append(baseQuery3.replace("${enrolment}", programEnrolmentTables.get(0).getName())); + for (int i = 1 ; i < programEnrolmentTables.size() ; i++) { + query.append("union all\n" ); + query.append(baseQuery3.replace("${enrolment}", programEnrolmentTables.get(i).getName())); + } + query.append("),\n"); + + String baseQuery4 = "select last_modified_by_id, count(*) as program_encounter_count\n" + + " from ${schemaName}.${programEncounter}\n" + " where is_voided = false\n" + " ${encounterWhere}\n" + - " group by last_modified_by_id\n" + - " )\n" + + " group by last_modified_by_id\n" ; + baseQuery4 = baseQuery4.replace("${schemaName}", orgSchemaName) + .replace("${encounterWhere}", encounterWhere); + query.append("program_encounters as (\n"); + query.append(baseQuery4.replace("${programEncounter}", programEncounterTables.get(0).getName())); + for (int i = 1 ; i < programEncounterTables.size() ; i++) { + query.append("union all\n" ); + query.append(baseQuery4.replace("${programEncounter}", programEncounterTables.get(i).getName())); + } + query.append("),\n"); + + String baseQuery5 = "activity_table as (\n" + "select u.id as id,\n" + " coalesce(u.name, u.username) as name,\n" + " coalesce(registration_count, 0) as registration_count,\n" + " coalesce(encounter_count, 0) as encounter_count,\n" + " coalesce(enrolment_count, 0) as enrolment_count,\n" + - " coalesce(program_encounter_count, 0) as program_encounter_count,\n" + - " coalesce(coalesce(registration_count, 0) + coalesce(encounter_count, 0) + coalesce(enrolment_count, 0) +\n" + - " coalesce(program_encounter_count, 0), 0) as total\n" + + " coalesce(program_encounter_count, 0) as program_encounter_count\n" + "from ${schemaName}.users u\n" + - " left join ${schemaName}.registrations r on r.last_modified_by_id = u.id\n" + - " left join ${schemaName}.encounters e on e.last_modified_by_id = u.id\n" + - " left join ${schemaName}.enrolments enl on enl.last_modified_by_id = u.id\n" + - " left join ${schemaName}.program_encounters enc on enc.last_modified_by_id = u.id\n" + + " left join registrations r on r.last_modified_by_id = u.id\n" + + " left join encounters e on e.last_modified_by_id = u.id\n" + + " left join enrolments enl on enl.last_modified_by_id = u.id\n" + + " left join program_encounters enc on enc.last_modified_by_id = u.id\n" + "where (u.is_voided = false or u.is_voided isnull) and u.organisation_id notnull\n" + " and coalesce(coalesce(registration_count, 0) + coalesce(encounter_count, 0) + coalesce(enrolment_count, 0) +\n" + " coalesce(program_encounter_count, 0), 0) > 0\n" + - " ${userWhere}\n" + + " ${userWhere}\n)," + + " final_table as (\n" + + "select id, name,\n"+ + " sum(distinct registration_count) as registration_count, \n" + + " sum(distinct encounter_count) as encounter_count,\n" + + " sum(distinct enrolment_count) as enrolment_count,\n" + + " sum(distinct program_encounter_count) as program_encounter_count\n" + + "from activity_table \n" + + "group by id,name\n" + + ")" + + "select *, coalesce(coalesce(registration_count, 0) + coalesce(encounter_count, 0) + coalesce(enrolment_count, 0) +\n" + + " coalesce(program_encounter_count, 0), 0) as total\n" + + "from final_table\n" + "order by 7 desc\n" + "limit 10;"; - String query = baseQuery - .replace("${schemaName}", orgSchemaName) - .replace("${subjectWhere}", subjectWhere) - .replace("${encounterWhere}", encounterWhere) - .replace("${enrolmentWhere}", enrolmentWhere) - .replace("${userWhere}", userWhere); - log.info(query); - return jdbcTemplate.query(query, new UserActivityMapper()); + query.append(baseQuery5.replace("${schemaName}", orgSchemaName) + .replace("${userWhere}", userWhere)); + + return jdbcTemplate.query(query.toString(), new UserActivityMapper()); } public List generateUserSyncFailures(String orgSchemaName, String syncTelemetryWhere, String userWhere) { From 8b221c05d424c53ead895f54c84f5822e0c40829 Mon Sep 17 00:00:00 2001 From: ak2502 Date: Fri, 18 Aug 2023 14:04:41 +0530 Subject: [PATCH 09/36] avniproject/avni-product#1334 | modified generateCompletedVisitsOnTimeByProportion query according to ETL Signed-off-by: ak2502 --- .../etl/controller/ReportController.java | 2 +- .../etl/repository/ReportRepository.java | 87 ++++++++++++------- 2 files changed, 58 insertions(+), 31 deletions(-) diff --git a/src/main/java/org/avniproject/etl/controller/ReportController.java b/src/main/java/org/avniproject/etl/controller/ReportController.java index d162a78..0b914c9 100644 --- a/src/main/java/org/avniproject/etl/controller/ReportController.java +++ b/src/main/java/org/avniproject/etl/controller/ReportController.java @@ -78,7 +78,7 @@ public List getChampionUsers(@RequestParam(value = "start @RequestParam(value = "endDate", required = false) String endDate, @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) { return reportRepository.generateCompletedVisitsOnTimeByProportion( - ">= 0.8", + ">= 0.5", OrgIdentityContextHolder.getDbSchema(), reportUtil.getDateDynamicWhere(startDate, endDate, "encounter_date_time"), reportUtil.getDynamicUserWhere(userIds, "u.id")); diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java index cb1ce00..fc4e674 100644 --- a/src/main/java/org/avniproject/etl/repository/ReportRepository.java +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -216,42 +216,69 @@ public List generateUserDetails(String orgSchemaName, String us } public List generateCompletedVisitsOnTimeByProportion(String proportionCondition, String orgSchemaName, String encounterWhere, String userWhere) { - String baseQuery = "with program_enc_data as (\n" + - " select last_modified_by_id,\n" + - " count(*) filter ( where encounter_date_time <= max_visit_date_time ) visits_done_on_time,\n" + + SchemaMetadata schema = schemaMetadataRepository.getExistingSchemaMetadata(); + List encounterTables = schema.getAllEncounterTables().stream().toList(); + List programEncounterTables = schema.getAllProgramEncounterTables().stream().toList(); + + StringBuilder query = new StringBuilder(); + String baseQuery1 = "select last_modified_by_id,\n" + + " count(*) filter ( where encounter_date_time <= max_visit_date_time ) visits_done_on_time,\n" + " count(*) filter ( where encounter_date_time notnull and earliest_visit_date_time notnull ) total_scheduled\n" + - " from program_encounter\n" + + " from ${schemaName}.${programEncounter}\n" + " where is_voided = false\n" + " ${encounterWhere}\n" + - " group by last_modified_by_id\n" + - "),\n" + - " general_enc_data as (\n" + - " select last_modified_by_id,\n" + - " count(*) filter ( where encounter_date_time <= max_visit_date_time ) visits_done_on_time,\n" + - " count(*)\n" + - " filter ( where encounter_date_time notnull and earliest_visit_date_time notnull ) total_scheduled\n" + - " from encounter\n" + + " group by last_modified_by_id\n" ; + baseQuery1 = baseQuery1.replace("${schemaName}", orgSchemaName) + .replace("${encounterWhere}", encounterWhere); + query.append("with program_enc_data as (\n" ); + query.append(baseQuery1.replace("${programEncounter}", programEncounterTables.get(0).getName())); + for (int i = 1; i < programEncounterTables.size(); i++) { + query.append("union all\n"); + query.append(baseQuery1.replace("${programEncounter}", programEncounterTables.get(i).getName())); + } + query.append("),\n"); + + String baseQuery2 = "select last_modified_by_id,\n" + + " count(*) filter ( where encounter_date_time <= max_visit_date_time ) visits_done_on_time,\n" + + " count(*) filter ( where encounter_date_time notnull and earliest_visit_date_time notnull ) total_scheduled\n" + + " from ${schemaName}.${encounter}\n" + " where is_voided = false\n" + " ${encounterWhere}\n" + - " group by last_modified_by_id\n" + - " )\n" + - "select coalesce(u.name, u.username) as indicator,\n" + - " coalesce(ged.visits_done_on_time, 0) + coalesce(ped.visits_done_on_time, 0) as count\n" + - "from users u\n" + - " join general_enc_data ged on ged.last_modified_by_id = u.id\n" + - " join program_enc_data ped on ped.last_modified_by_id = u.id\n" + - "where u.organisation_id notnull\n" + - " and is_voided = false\n" + - " and coalesce(ged.visits_done_on_time, 0) + coalesce(ped.visits_done_on_time, 0) > 0\n" + - " ${userWhere}\n" + - " and ((coalesce(ged.visits_done_on_time, 0.0) + coalesce(ped.visits_done_on_time, 0.0)) /\n" + - " nullif((coalesce(ged.total_scheduled, 0) + coalesce(ped.total_scheduled, 0)), 0)) ${proportion_condition}\n"; - String query = baseQuery - .replace("${proportion_condition}", proportionCondition) + " group by last_modified_by_id\n" ; + baseQuery2 = baseQuery2.replace("${schemaName}", orgSchemaName) + .replace("${encounterWhere}", encounterWhere); + query.append("general_enc_data as (\n"); + query.append(baseQuery2.replace("${encounter}", encounterTables.get(0).getName())); + for (int i = 1 ; i < encounterTables.size() ; i++) { + query.append("union all\n" ); + query.append(baseQuery2.replace("${encounter}", encounterTables.get(i).getName())); + } + query.append("),\n"); + + String baseQuery3 = "table1 as (\n" + + " select coalesce(u.name, u.username) as indicator,\n" + + " coalesce(sum(distinct ged.visits_done_on_time), 0) as ged_visits_on_time,\n" + + " coalesce(sum(distinct ped.visits_done_on_time), 0) as ped_visits_on_time,\n" + + " coalesce(sum(distinct ged.total_scheduled), 0) as ged_total_scheduled,\n" + + " coalesce(sum(distinct ped.total_scheduled), 0) as ped_total_scheduled\n" + + " from org1_schema.users u \n" + + " left join general_enc_data ged on ged.last_modified_by_id = u.id\n" + + " left join program_enc_data ped on ped.last_modified_by_id = u.id\n" + + " where u.organisation_id notnull\n" + + " and (is_voided = false or is_voided isnull)\n" + + " ${userWhere}\n" + + " group by u.name,u.username \n" + + ")\n" + + "select indicator,\n" + + " ged_visits_on_time + ped_visits_on_time as count\n" + + "from table1\n" + + "where ged_visits_on_time + ped_visits_on_time > 0\n" + + " and ((ged_visits_on_time + ped_visits_on_time) /\n" + + "nullif(ged_total_scheduled + ped_total_scheduled, 0)) ${proportion_condition};"; + query.append(baseQuery3.replace("${proportion_condition}", proportionCondition) .replace("${schemaName}", orgSchemaName) - .replace("${encounterWhere}", encounterWhere) - .replace("${userWhere}", userWhere); - return jdbcTemplate.query(query, new AggregateReportMapper()); + .replace("${userWhere}", userWhere)); + return jdbcTemplate.query(query.toString(), new AggregateReportMapper()); } public List generateUserCancellingMostVisits(String orgSchemaName, String encounterWhere, String userWhere) { From 49f17da72d0034eee8bc037d20489b540cce965c Mon Sep 17 00:00:00 2001 From: ak2502 Date: Fri, 18 Aug 2023 17:04:24 +0530 Subject: [PATCH 10/36] avniproject/avni-product#1334 | modified generateUserCancellingMostVisits query according to ETL Signed-off-by: ak2502 --- .../etl/repository/ReportRepository.java | 80 +++++++++++++------ 1 file changed, 55 insertions(+), 25 deletions(-) diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java index fc4e674..96aaa4b 100644 --- a/src/main/java/org/avniproject/etl/repository/ReportRepository.java +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -261,7 +261,7 @@ public List generateCompletedVisitsOnTimeByProportion(Str " coalesce(sum(distinct ped.visits_done_on_time), 0) as ped_visits_on_time,\n" + " coalesce(sum(distinct ged.total_scheduled), 0) as ged_total_scheduled,\n" + " coalesce(sum(distinct ped.total_scheduled), 0) as ped_total_scheduled\n" + - " from org1_schema.users u \n" + + " from ${schemaName}.users u \n" + " left join general_enc_data ged on ged.last_modified_by_id = u.id\n" + " left join program_enc_data ped on ped.last_modified_by_id = u.id\n" + " where u.organisation_id notnull\n" + @@ -282,37 +282,67 @@ public List generateCompletedVisitsOnTimeByProportion(Str } public List generateUserCancellingMostVisits(String orgSchemaName, String encounterWhere, String userWhere) { - String baseQuery = "with program_enc_data as (\n" + - " select last_modified_by_id,\n" + - " count(*) filter ( where cancel_date_time notnull ) cancelled_visits\n" + - " from program_encounter\n" + + SchemaMetadata schema = schemaMetadataRepository.getExistingSchemaMetadata(); + List encounterTables = schema.getAllEncounterTables().stream().toList(); + List programEncounterTables = schema.getAllProgramEncounterTables().stream().toList(); + + StringBuilder query = new StringBuilder(); + String baseQuery1 = " select last_modified_by_id,\n" + + " count(*) " + + "filter ( where cancel_date_time notnull ) " + + "cancelled_visits\n" + + " from ${schemaName}.${programEncounter}\n" + " where is_voided = false\n" + " ${encounterWhere}\n" + - " group by last_modified_by_id\n" + - "),\n" + - " general_enc_data as (\n" + - " select last_modified_by_id,\n" + - " count(*) filter ( where cancel_date_time notnull ) cancelled_visits\n" + - " from encounter\n" + + " group by last_modified_by_id\n" ; + baseQuery1 = baseQuery1.replace("${schemaName}", orgSchemaName) + .replace("${encounterWhere}", encounterWhere); + query.append("with program_enc_data as (\n" ); + query.append(baseQuery1.replace("${programEncounter}", programEncounterTables.get(0).getName())); + for (int i = 1; i < programEncounterTables.size(); i++) { + query.append("union all\n"); + query.append(baseQuery1.replace("${programEncounter}", programEncounterTables.get(i).getName())); + } + query.append("),\n"); + + String baseQuery2 = " select last_modified_by_id,\n" + + " count(*) " + + "filter ( where cancel_date_time notnull )" + + " cancelled_visits\n" + + " from ${schemaName}.${encounter}\n" + " where is_voided = false\n" + " ${encounterWhere}\n" + - " group by last_modified_by_id\n" + - " )\n" + - "select coalesce(u.name, u.username) as indicator,\n" + - " coalesce(ged.cancelled_visits, 0) + coalesce(ped.cancelled_visits, 0) as count\n" + - "from users u\n" + - " join general_enc_data ged on ged.last_modified_by_id = u.id\n" + - " join program_enc_data ped on ped.last_modified_by_id = u.id\n" + + " group by last_modified_by_id\n" ; + baseQuery2 = baseQuery2.replace("${schemaName}", orgSchemaName) + .replace("${encounterWhere}", encounterWhere); + query.append("general_enc_data as (\n"); + query.append(baseQuery2.replace("${encounter}", encounterTables.get(0).getName())); + for (int i = 1 ; i < encounterTables.size() ; i++) { + query.append("union all\n" ); + query.append(baseQuery2.replace("${encounter}", encounterTables.get(i).getName())); + } + query.append("),\n"); + + String baseQuery3 ="table1 as (\n" + + "select coalesce(u.name, u.username) as indicator,\n" + + " coalesce(sum(distinct ged.cancelled_visits), 0) as ged_cancelled_visits,\n" + + " coalesce(sum(distinct ped.cancelled_visits), 0) as ped_cancelled_visits\n" + + "from ${schemaName}.users u\n" + + " left join general_enc_data ged on ged.last_modified_by_id = u.id\n" + + " left join program_enc_data ped on ped.last_modified_by_id = u.id\n" + "where u.organisation_id notnull\n" + - " and is_voided = false\n" + + " and (is_voided = false or is_voided isnull)\n" + " and coalesce(ged.cancelled_visits, 0) + coalesce(ped.cancelled_visits, 0) > 0 \n" + " ${userWhere}\n" + - "order by coalesce(ged.cancelled_visits, 0.0) + coalesce(ped.cancelled_visits, 0.0) desc\n" + + " group by u.name,u.username \n" + + ")\n" + + "select indicator,\n" + + " ged_cancelled_visits + ped_cancelled_visits as count\n" + + "from table1\n" + + "order by ged_cancelled_visits + ped_cancelled_visits desc\n" + "limit 5;"; - String query = baseQuery - .replace("${schemaName}", orgSchemaName) - .replace("${encounterWhere}", encounterWhere) - .replace("${userWhere}", userWhere); - return jdbcTemplate.query(query, new AggregateReportMapper()); + query.append(baseQuery3.replace("${schemaName}", orgSchemaName) + .replace("${userWhere}", userWhere)); + return jdbcTemplate.query(query.toString(), new AggregateReportMapper()); } } From 724ca9a546d12358f0c1058e39e8f5f02eeaec32 Mon Sep 17 00:00:00 2001 From: ak2502 Date: Mon, 21 Aug 2023 14:56:04 +0530 Subject: [PATCH 11/36] avniproject/avni-product#1334 | adding analytics_user in ReportController Signed-off-by: ak2502 --- .../org/avniproject/etl/controller/ReportController.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/org/avniproject/etl/controller/ReportController.java b/src/main/java/org/avniproject/etl/controller/ReportController.java index 0b914c9..a9d0a1e 100644 --- a/src/main/java/org/avniproject/etl/controller/ReportController.java +++ b/src/main/java/org/avniproject/etl/controller/ReportController.java @@ -7,6 +7,7 @@ import org.avniproject.etl.service.ReportService; import org.avniproject.etl.util.ReportUtil; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.*; import java.util.List; @@ -25,6 +26,7 @@ public ReportController(ReportService reportService, ReportRepository reportRepo this.reportUtil = reportUtil; } + @PreAuthorize("hasAnyAuthority('analytics_user')") @RequestMapping(value = "report/hr/userActivity", method = RequestMethod.GET) public List getUserActivity(@RequestParam(value = "startDate", required = false) String startDate, @RequestParam(value = "endDate", required = false) String endDate, @@ -38,6 +40,7 @@ public List getUserActivity(@RequestParam(value = "startDate", ); } + @PreAuthorize("hasAnyAuthority('analytics_user')") @RequestMapping(value = "/report/hr/syncFailures",method = RequestMethod.GET) public List getUserWiseSyncFailures(@RequestParam(value = "startDate", required = false) String startDate, @RequestParam(value = "endDate", required = false) String endDate, @@ -49,6 +52,7 @@ public List getUserWiseSyncFailures(@RequestParam(value = "star ); } + @PreAuthorize("hasAnyAuthority('analytics_user')") @RequestMapping(value = "/report/hr/deviceModels", method = RequestMethod.GET) public List getUserWiseDeviceModels(@RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) { @@ -57,6 +61,7 @@ public List getUserWiseDeviceModels(@RequestParam(value = reportUtil.getDynamicUserWhere(userIds, "u.id")); } + @PreAuthorize("hasAnyAuthority('analytics_user')") @RequestMapping(value = "/report/hr/appVersions", method = RequestMethod.GET) public List getUserWiseAppVersions(@RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) { @@ -65,6 +70,7 @@ public List getUserWiseAppVersions(@RequestParam(value = reportUtil.getDynamicUserWhere(userIds, "u.id")); } + @PreAuthorize("hasAnyAuthority('analytics_user')") @RequestMapping(value = "/report/hr/userDetails", method = RequestMethod.GET) public List getUserDetails(@RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) { @@ -73,6 +79,7 @@ public List getUserDetails(@RequestParam(value = "userIds", req reportUtil.getDynamicUserWhere(userIds, "u.id")); } + @PreAuthorize("hasAnyAuthority('analytics_user')") @RequestMapping(value = "/report/hr/championUsers", method = RequestMethod.GET) public List getChampionUsers(@RequestParam(value = "startDate", required = false) String startDate, @RequestParam(value = "endDate", required = false) String endDate, @@ -84,6 +91,7 @@ public List getChampionUsers(@RequestParam(value = "start reportUtil.getDynamicUserWhere(userIds, "u.id")); } + @PreAuthorize("hasAnyAuthority('analytics_user')") @RequestMapping(value = "/report/hr/nonPerformingUsers", method = RequestMethod.GET) public List getNonPerformingUsers(@RequestParam(value = "startDate", required = false) String startDate, @RequestParam(value = "endDate", required = false) String endDate, @@ -96,6 +104,7 @@ public List getNonPerformingUsers(@RequestParam(value = " ); } + @PreAuthorize("hasAnyAuthority('analytics_user')") @RequestMapping(value = "/report/hr/mostCancelled", method = RequestMethod.GET) public List getUsersCancellingMostVisits(@RequestParam(value = "startDate", required = false) String startDate, @RequestParam(value = "endDate", required = false) String endDate, From ea60a48f47cdaf44b7b040bf2b2937cd2067262c Mon Sep 17 00:00:00 2001 From: ak2502 Date: Mon, 21 Aug 2023 17:24:05 +0530 Subject: [PATCH 12/36] avniproject/avni-product#1334 | Sync telemetry - Latest Syncs endpoint added Signed-off-by: ak2502 --- .../etl/controller/ReportController.java | 11 +++++ .../avniproject/etl/dto/UserActivityDTO.java | 46 ++++++++++++++++++- .../etl/repository/ReportRepository.java | 25 +++++++--- .../rowMappers/reports/LatestSyncMapper.java | 25 ++++++++++ 4 files changed, 99 insertions(+), 8 deletions(-) create mode 100644 src/main/java/org/avniproject/etl/repository/rowMappers/reports/LatestSyncMapper.java diff --git a/src/main/java/org/avniproject/etl/controller/ReportController.java b/src/main/java/org/avniproject/etl/controller/ReportController.java index a9d0a1e..96f1784 100644 --- a/src/main/java/org/avniproject/etl/controller/ReportController.java +++ b/src/main/java/org/avniproject/etl/controller/ReportController.java @@ -79,6 +79,17 @@ public List getUserDetails(@RequestParam(value = "userIds", req reportUtil.getDynamicUserWhere(userIds, "u.id")); } + @PreAuthorize("hasAnyAuthority('analytics_user')") + @RequestMapping(value = "/report/hr/latestSyncs", method = RequestMethod.GET) + public List getLatestSyncs(@RequestParam(value = "startDate", required = false) String startDate, + @RequestParam(value = "endDate", required = false) String endDate, + @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) { + + return reportRepository.generateLatestSyncs( + OrgIdentityContextHolder.getDbSchema(), + reportUtil.getDateDynamicWhere(startDate, endDate, "sync_end_time"), + reportUtil.getDynamicUserWhere(userIds, "u.id")); + } @PreAuthorize("hasAnyAuthority('analytics_user')") @RequestMapping(value = "/report/hr/championUsers", method = RequestMethod.GET) public List getChampionUsers(@RequestParam(value = "startDate", required = false) String startDate, diff --git a/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java b/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java index 392f9d3..9362396 100644 --- a/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java +++ b/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java @@ -11,8 +11,14 @@ public class UserActivityDTO { private Long programEncounterCount; private Long generalEncounterCount; private Long count; + private String androidVersion; private String appVersion; private String deviceModel; + private String syncStatus; + private String syncSource; + private DateTime syncStart; + private DateTime syncEnd; + private DateTime lastSuccessfulSync; public String getUserName() { return userName; @@ -70,6 +76,14 @@ public void setCount(Long count) { this.count = count; } + public String getAndroidVersion() { + return androidVersion; + } + + public void setAndroidVersion(String androidVersion) { + this.androidVersion = androidVersion; + } + public String getAppVersion() { return appVersion; } @@ -94,5 +108,35 @@ public void setLastSuccessfulSync(DateTime lastSuccessfulSync) { this.lastSuccessfulSync = lastSuccessfulSync; } - private DateTime lastSuccessfulSync; + public DateTime getSyncStart() { + return syncStart; + } + + public void setSyncStart(DateTime syncStart) { + this.syncStart = syncStart; + } + + public DateTime getSyncEnd() { + return syncEnd; + } + + public void setSyncEnd(DateTime syncEnd) { + this.syncEnd = syncEnd; + } + + public String getSyncStatus() { + return syncStatus; + } + + public void setSyncStatus(String syncStatus) { + this.syncStatus = syncStatus; + } + + public String getSyncSource() { + return syncSource; + } + + public void setSyncSource(String syncSource) { + this.syncSource = syncSource; + } } diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java index 96aaa4b..80b0579 100644 --- a/src/main/java/org/avniproject/etl/repository/ReportRepository.java +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -5,18 +5,13 @@ import org.avniproject.etl.dto.AggregateReportResult; import org.apache.log4j.Logger; import org.avniproject.etl.dto.UserActivityDTO; -import org.avniproject.etl.repository.rowMappers.reports.AggregateReportMapper; -import org.avniproject.etl.repository.rowMappers.reports.UserActivityMapper; -import org.avniproject.etl.repository.rowMappers.reports.UserCountMapper; -import org.avniproject.etl.repository.rowMappers.reports.UserDetailsMapper; +import org.avniproject.etl.repository.rowMappers.reports.*; import org.avniproject.etl.service.EtlService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.stereotype.Component; -import java.util.Collection; import java.util.List; -import java.util.stream.Stream; @Component public class ReportRepository { @@ -32,7 +27,7 @@ public ReportRepository(NamedParameterJdbcTemplate jdbcTemplate, SchemaMetadataR public List generateUserActivity(String orgSchemaName, String subjectWhere, String encounterWhere, String enrolmentWhere, String userWhere) { SchemaMetadata schema = schemaMetadataRepository.getExistingSchemaMetadata(); - List subjectTables = Stream.of(schema.getAllSubjectTables()).flatMap(Collection::stream).toList(); + List subjectTables = schema.getAllSubjectTables().stream().toList(); List encounterTables = schema.getAllEncounterTables().stream().toList(); List programEnrolmentTables = schema.getAllProgramEnrolmentTables().stream().toList(); List programEncounterTables = schema.getAllProgramEncounterTables().stream().toList(); @@ -215,6 +210,22 @@ public List generateUserDetails(String orgSchemaName, String us return jdbcTemplate.query(query, new UserDetailsMapper()); } + public List generateLatestSyncs(String orgSchemaName, String syncTelemetryWhere, String userWhere) { + String baseQuery = "SELECT coalesce(u.name,u.username) as name, \n" + + " android_version, app_version, device_name, sync_start_time, sync_end_time, sync_status, sync_source\n" + + "FROM public.sync_telemetry st\n" + + "join ${schemaName}.users u on st.last_modified_by_id = u.id\n" + + "where (u.is_voided = false or u.is_voided isnull) and u.organisation_id notnull\n" + + "${syncTelemetryWhere}\n"+ + "${userWhere}\n"+ + "order by 6 desc\n" + + "limit 10;\n"; + String query = baseQuery + .replace("${schemaName}", orgSchemaName) + .replace("${syncTelemetryWhere}", syncTelemetryWhere) + .replace("${userWhere}", userWhere); + return jdbcTemplate.query(query, new LatestSyncMapper()); + } public List generateCompletedVisitsOnTimeByProportion(String proportionCondition, String orgSchemaName, String encounterWhere, String userWhere) { SchemaMetadata schema = schemaMetadataRepository.getExistingSchemaMetadata(); List encounterTables = schema.getAllEncounterTables().stream().toList(); diff --git a/src/main/java/org/avniproject/etl/repository/rowMappers/reports/LatestSyncMapper.java b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/LatestSyncMapper.java new file mode 100644 index 0000000..640d437 --- /dev/null +++ b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/LatestSyncMapper.java @@ -0,0 +1,25 @@ +package org.avniproject.etl.repository.rowMappers.reports; + +import org.avniproject.etl.dto.UserActivityDTO; +import org.joda.time.DateTime; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class LatestSyncMapper implements RowMapper { + + @Override + public UserActivityDTO mapRow(ResultSet rs, int rowNum) throws SQLException { + UserActivityDTO userActivityDTO = new UserActivityDTO(); + userActivityDTO.setUserName(rs.getString("name")); + userActivityDTO.setAndroidVersion(rs.getString("android_version")); + userActivityDTO.setAppVersion(rs.getString("app_version")); + userActivityDTO.setDeviceModel(rs.getString("device_name")); + userActivityDTO.setSyncStart(new DateTime(rs.getDate("sync_start_time"))); + userActivityDTO.setSyncEnd(new DateTime(rs.getDate("sync_end_time"))); + userActivityDTO.setSyncStatus(rs.getString("sync_status")); + userActivityDTO.setSyncSource(rs.getString("sync_source")); + return userActivityDTO; + } +} From cb0aef044e03f158a81bee57f580e62656a8c8dd Mon Sep 17 00:00:00 2001 From: ak2502 Date: Tue, 22 Aug 2023 22:16:23 +0530 Subject: [PATCH 13/36] avniproject/avni-product#1334 | modified code with string template Signed-off-by: ak2502 --- .../etl/domain/metadata/SchemaMetadata.java | 36 ++ .../etl/repository/ReportRepository.java | 369 +++++++++--------- 2 files changed, 222 insertions(+), 183 deletions(-) diff --git a/src/main/java/org/avniproject/etl/domain/metadata/SchemaMetadata.java b/src/main/java/org/avniproject/etl/domain/metadata/SchemaMetadata.java index cb80569..48c6b0e 100644 --- a/src/main/java/org/avniproject/etl/domain/metadata/SchemaMetadata.java +++ b/src/main/java/org/avniproject/etl/domain/metadata/SchemaMetadata.java @@ -42,18 +42,54 @@ public List getAllSubjectTables() { return tableMetadata.stream().filter(TableMetadata::isSubjectTable).toList(); } + public List getAllSubjectTableNames() { + List subjectTables = getAllSubjectTables(); + List subjectTableNames = new ArrayList<>(); + for(TableMetadata subject : subjectTables){ + subjectTableNames.add(subject.getName()); + } + return subjectTableNames; + } + public List getAllProgramEnrolmentTables() { return tableMetadata.stream().filter(table -> table.getType() == TableMetadata.Type.ProgramEnrolment).toList(); } + public List getAllProgramEnrolmentTableNames() { + List programEnrolmentTables = getAllProgramEnrolmentTables(); + List programEnrolmentTableNames = new ArrayList<>(); + for(TableMetadata programEnrolment : programEnrolmentTables){ + programEnrolmentTableNames.add(programEnrolment.getName()); + } + return programEnrolmentTableNames; + } + public List getAllProgramEncounterTables() { return tableMetadata.stream().filter(table -> table.getType() == TableMetadata.Type.ProgramEncounter).toList(); } + public List getAllProgramEncounterTableNames() { + List programEncounterTables = getAllProgramEncounterTables(); + List programEncounterTableNames = new ArrayList<>(); + for(TableMetadata programEncounter : programEncounterTables){ + programEncounterTableNames.add(programEncounter.getName()); + } + return programEncounterTableNames; + } + public List getAllEncounterTables() { return tableMetadata.stream().filter(table -> table.getType() == TableMetadata.Type.Encounter).toList(); } + public List getAllEncounterTableNames() { + List encounterTables = getAllEncounterTables(); + List encounterTableNames = new ArrayList<>(); + for(TableMetadata encounter : encounterTables){ + encounterTableNames.add(encounter.getName()); + } + return encounterTableNames; + } + private List findChanges(SchemaMetadata currentSchema, TableMetadata newTable) { List diffs = new ArrayList<>(); Optional optionalMatchingTable = currentSchema.findMatchingTable(newTable); diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java index 80b0579..9e566eb 100644 --- a/src/main/java/org/avniproject/etl/repository/ReportRepository.java +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -1,15 +1,13 @@ package org.avniproject.etl.repository; import org.avniproject.etl.domain.metadata.SchemaMetadata; -import org.avniproject.etl.domain.metadata.TableMetadata; import org.avniproject.etl.dto.AggregateReportResult; -import org.apache.log4j.Logger; import org.avniproject.etl.dto.UserActivityDTO; import org.avniproject.etl.repository.rowMappers.reports.*; -import org.avniproject.etl.service.EtlService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.stereotype.Component; +import org.stringtemplate.v4.ST; import java.util.List; @@ -17,7 +15,6 @@ public class ReportRepository { private final NamedParameterJdbcTemplate jdbcTemplate; private final SchemaMetadataRepository schemaMetadataRepository; - private static final Logger log = Logger.getLogger(EtlService.class); @Autowired public ReportRepository(NamedParameterJdbcTemplate jdbcTemplate, SchemaMetadataRepository schemaMetadataRepository) { @@ -27,107 +24,109 @@ public ReportRepository(NamedParameterJdbcTemplate jdbcTemplate, SchemaMetadataR public List generateUserActivity(String orgSchemaName, String subjectWhere, String encounterWhere, String enrolmentWhere, String userWhere) { SchemaMetadata schema = schemaMetadataRepository.getExistingSchemaMetadata(); - List subjectTables = schema.getAllSubjectTables().stream().toList(); - List encounterTables = schema.getAllEncounterTables().stream().toList(); - List programEnrolmentTables = schema.getAllProgramEnrolmentTables().stream().toList(); - List programEncounterTables = schema.getAllProgramEncounterTables().stream().toList(); + List subjectTableNames = schema.getAllSubjectTableNames().stream().toList(); + List encounterTableNames = schema.getAllEncounterTableNames().stream().toList(); + List programEnrolmentTableNames = schema.getAllProgramEnrolmentTableNames().stream().toList(); + List programEncounterTableNames = schema.getAllProgramEncounterTableNames().stream().toList(); - StringBuilder query = new StringBuilder(); - - String baseQuery1 = "select last_modified_by_id, count(*) as registration_count\n" + - " from ${schemaName}.${subject}\n" + + ST baseQuery = new ST("with registrations as (\n" + + " select last_modified_by_id, count(*) as registration_count\n" + + " from $schemaName.\n" + " where is_voided = false\n" + - " ${subjectWhere}\n" + - " group by last_modified_by_id\n" ; - baseQuery1 = baseQuery1.replace("${schemaName}", orgSchemaName) - .replace("${subjectWhere}", subjectWhere); - query.append("with registrations as (\n"); - query.append(baseQuery1.replace("${subject}", subjectTables.get(0).getName())); - for (int i = 1 ; i < subjectTables.size() ; i++) { - query.append("union all\n" ); - query.append(baseQuery1.replace("${subject}", subjectTables.get(i).getName())); - } - query.append("),\n"); - - String baseQuery2 = "select last_modified_by_id, count(*) as encounter_count\n" + - " from ${schemaName}.${encounter}\n" + + " $subjectWhere\n" + + " group by last_modified_by_id\n" + + " \n" + " where is_voided = false\n" + - " ${encounterWhere}\n" + - " group by last_modified_by_id\n" ; - baseQuery2 = baseQuery2.replace("${schemaName}", orgSchemaName) - .replace("${encounterWhere}", encounterWhere); - query.append("encounters as (\n"); - query.append(baseQuery2.replace("${encounter}", encounterTables.get(0).getName())); - for (int i = 1 ; i < encounterTables.size() ; i++) { - query.append("union all\n" ); - query.append(baseQuery2.replace("${encounter}", encounterTables.get(i).getName())); - } - query.append("),\n"); - - String baseQuery3 = "select last_modified_by_id, count(*) as enrolment_count\n" + - " from ${schemaName}.${enrolment}\n" + + " $subjectWhere \n" + + " group by last_modified_by_id \n}> " + + " ),\n " + + "encounters as (\n" + + " select last_modified_by_id, count(*) as encounter_count\n" + + " from $schemaName.\n" + " where is_voided = false\n" + - " ${enrolmentWhere}\n" + - " group by last_modified_by_id\n" ; - baseQuery3 = baseQuery3.replace("${schemaName}", orgSchemaName) - .replace("${enrolmentWhere}", enrolmentWhere); - query.append("enrolments as (\n"); - query.append(baseQuery3.replace("${enrolment}", programEnrolmentTables.get(0).getName())); - for (int i = 1 ; i < programEnrolmentTables.size() ; i++) { - query.append("union all\n" ); - query.append(baseQuery3.replace("${enrolment}", programEnrolmentTables.get(i).getName())); - } - query.append("),\n"); - - String baseQuery4 = "select last_modified_by_id, count(*) as program_encounter_count\n" + - " from ${schemaName}.${programEncounter}\n" + + " $encounterWhere\n" + + " group by last_modified_by_id\n" + + " \n" + " where is_voided = false\n" + - " ${encounterWhere}\n" + - " group by last_modified_by_id\n" ; - baseQuery4 = baseQuery4.replace("${schemaName}", orgSchemaName) - .replace("${encounterWhere}", encounterWhere); - query.append("program_encounters as (\n"); - query.append(baseQuery4.replace("${programEncounter}", programEncounterTables.get(0).getName())); - for (int i = 1 ; i < programEncounterTables.size() ; i++) { - query.append("union all\n" ); - query.append(baseQuery4.replace("${programEncounter}", programEncounterTables.get(i).getName())); - } - query.append("),\n"); - - String baseQuery5 = "activity_table as (\n" + - "select u.id as id,\n" + + " $encounterWhere \n" + + " group by last_modified_by_id \n}> " + + " ),\n " + + "enrolments as (\n" + + " select last_modified_by_id, count(*) as enrolment_count\n" + + " from $schemaName.\n" + + " where is_voided = false\n" + + " $enrolmentWhere\n" + + " group by last_modified_by_id\n" + + " \n" + + " where is_voided = false\n" + + " $enrolmentWhere \n" + + " group by last_modified_by_id \n}> " + + " ),\n " + + "program_encounters as (\n" + + " select last_modified_by_id, count(*) as program_encounter_count\n" + + " from $schemaName.\n" + + " where is_voided = false\n" + + " $encounterWhere\n" + + " group by last_modified_by_id\n" + + " \n" + + " where is_voided = false\n" + + " $encounterWhere \n" + + " group by last_modified_by_id \n}> " + + " ),\n " + + "activity_table as (\n" + + " select u.id as id,\n" + " coalesce(u.name, u.username) as name,\n" + " coalesce(registration_count, 0) as registration_count,\n" + " coalesce(encounter_count, 0) as encounter_count,\n" + " coalesce(enrolment_count, 0) as enrolment_count,\n" + " coalesce(program_encounter_count, 0) as program_encounter_count\n" + - "from ${schemaName}.users u\n" + + " from $schemaName.users u\n" + " left join registrations r on r.last_modified_by_id = u.id\n" + " left join encounters e on e.last_modified_by_id = u.id\n" + " left join enrolments enl on enl.last_modified_by_id = u.id\n" + " left join program_encounters enc on enc.last_modified_by_id = u.id\n" + - "where (u.is_voided = false or u.is_voided isnull) and u.organisation_id notnull\n" + + " where (u.is_voided = false or u.is_voided isnull) and u.organisation_id notnull\n" + " and coalesce(coalesce(registration_count, 0) + coalesce(encounter_count, 0) + coalesce(enrolment_count, 0) +\n" + " coalesce(program_encounter_count, 0), 0) > 0\n" + - " ${userWhere}\n)," + - " final_table as (\n" + - "select id, name,\n"+ + " $userWhere \n" + + "),\n" + + "final_table as (\n" + + " select id, name,\n"+ " sum(distinct registration_count) as registration_count, \n" + " sum(distinct encounter_count) as encounter_count,\n" + " sum(distinct enrolment_count) as enrolment_count,\n" + " sum(distinct program_encounter_count) as program_encounter_count\n" + - "from activity_table \n" + - "group by id,name\n" + - ")" + + " from activity_table \n" + + " group by id,name\n" + + ")\n" + "select *, coalesce(coalesce(registration_count, 0) + coalesce(encounter_count, 0) + coalesce(enrolment_count, 0) +\n" + " coalesce(program_encounter_count, 0), 0) as total\n" + "from final_table\n" + "order by 7 desc\n" + - "limit 10;"; - query.append(baseQuery5.replace("${schemaName}", orgSchemaName) - .replace("${userWhere}", userWhere)); - - return jdbcTemplate.query(query.toString(), new UserActivityMapper()); + "limit 10;" + ); + baseQuery.add("subjectTableNames", subjectTableNames) + .add("encounterTableNames", encounterTableNames) + .add("programEnrolmentTableNames", programEnrolmentTableNames) + .add("programEncounterTableNames", programEncounterTableNames); + String query = baseQuery.render().replace("$schemaName", orgSchemaName) + .replace("$subjectWhere", subjectWhere) + .replace("$encounterWhere", encounterWhere) + .replace("$enrolmentWhere", enrolmentWhere) + .replace("$userWhere", userWhere); + return jdbcTemplate.query(query, new UserActivityMapper()); } public List generateUserSyncFailures(String orgSchemaName, String syncTelemetryWhere, String userWhere) { @@ -228,132 +227,136 @@ public List generateLatestSyncs(String orgSchemaName, String sy } public List generateCompletedVisitsOnTimeByProportion(String proportionCondition, String orgSchemaName, String encounterWhere, String userWhere) { SchemaMetadata schema = schemaMetadataRepository.getExistingSchemaMetadata(); - List encounterTables = schema.getAllEncounterTables().stream().toList(); - List programEncounterTables = schema.getAllProgramEncounterTables().stream().toList(); + List encounterTableNames = schema.getAllEncounterTableNames().stream().toList(); + List programEncounterTableNames = schema.getAllProgramEncounterTableNames().stream().toList(); - StringBuilder query = new StringBuilder(); - String baseQuery1 = "select last_modified_by_id,\n" + - " count(*) filter ( where encounter_date_time <= max_visit_date_time ) visits_done_on_time,\n" + + ST baseQuery = new ST("with program_enc_data as (\n " + + " select last_modified_by_id, \n" + + " count(*) filter ( where encounter_date_time \\<= max_visit_date_time ) visits_done_on_time,\n" + " count(*) filter ( where encounter_date_time notnull and earliest_visit_date_time notnull ) total_scheduled\n" + - " from ${schemaName}.${programEncounter}\n" + + " from $schemaName.\n" + " where is_voided = false\n" + - " ${encounterWhere}\n" + - " group by last_modified_by_id\n" ; - baseQuery1 = baseQuery1.replace("${schemaName}", orgSchemaName) - .replace("${encounterWhere}", encounterWhere); - query.append("with program_enc_data as (\n" ); - query.append(baseQuery1.replace("${programEncounter}", programEncounterTables.get(0).getName())); - for (int i = 1; i < programEncounterTables.size(); i++) { - query.append("union all\n"); - query.append(baseQuery1.replace("${programEncounter}", programEncounterTables.get(i).getName())); - } - query.append("),\n"); - - String baseQuery2 = "select last_modified_by_id,\n" + - " count(*) filter ( where encounter_date_time <= max_visit_date_time ) visits_done_on_time,\n" + + " $encounterWhere\n" + + " group by last_modified_by_id\n" + + " \n" + + " where is_voided = false\n" + + " $encounterWhere\n" + + " group by last_modified_by_id\n }> " + + "),\n" + + "general_enc_data as (\n" + + " select last_modified_by_id,\n" + + " count(*) filter ( where encounter_date_time \\<= max_visit_date_time ) visits_done_on_time,\n" + " count(*) filter ( where encounter_date_time notnull and earliest_visit_date_time notnull ) total_scheduled\n" + - " from ${schemaName}.${encounter}\n" + + " from $schemaName.\n" + " where is_voided = false\n" + - " ${encounterWhere}\n" + - " group by last_modified_by_id\n" ; - baseQuery2 = baseQuery2.replace("${schemaName}", orgSchemaName) - .replace("${encounterWhere}", encounterWhere); - query.append("general_enc_data as (\n"); - query.append(baseQuery2.replace("${encounter}", encounterTables.get(0).getName())); - for (int i = 1 ; i < encounterTables.size() ; i++) { - query.append("union all\n" ); - query.append(baseQuery2.replace("${encounter}", encounterTables.get(i).getName())); - } - query.append("),\n"); - - String baseQuery3 = "table1 as (\n" + - " select coalesce(u.name, u.username) as indicator,\n" + - " coalesce(sum(distinct ged.visits_done_on_time), 0) as ged_visits_on_time,\n" + - " coalesce(sum(distinct ped.visits_done_on_time), 0) as ped_visits_on_time,\n" + - " coalesce(sum(distinct ged.total_scheduled), 0) as ged_total_scheduled,\n" + - " coalesce(sum(distinct ped.total_scheduled), 0) as ped_total_scheduled\n" + - " from ${schemaName}.users u \n" + - " left join general_enc_data ged on ged.last_modified_by_id = u.id\n" + - " left join program_enc_data ped on ped.last_modified_by_id = u.id\n" + - " where u.organisation_id notnull\n" + - " and (is_voided = false or is_voided isnull)\n" + - " ${userWhere}\n" + - " group by u.name,u.username \n" + - ")\n" + - "select indicator,\n" + - " ged_visits_on_time + ped_visits_on_time as count\n" + - "from table1\n" + - "where ged_visits_on_time + ped_visits_on_time > 0\n" + - " and ((ged_visits_on_time + ped_visits_on_time) /\n" + - "nullif(ged_total_scheduled + ped_total_scheduled, 0)) ${proportion_condition};"; - query.append(baseQuery3.replace("${proportion_condition}", proportionCondition) - .replace("${schemaName}", orgSchemaName) - .replace("${userWhere}", userWhere)); - return jdbcTemplate.query(query.toString(), new AggregateReportMapper()); + " $encounterWhere\n" + + " group by last_modified_by_id\n" + + "\n" + + " where is_voided = false\n" + + " $encounterWhere\n" + + " group by last_modified_by_id \n }>" + + "),\n" + + "total_visits_table as (\n" + + " select coalesce(u.name, u.username) as indicator,\n" + + " coalesce(sum(distinct ged.visits_done_on_time), 0) as ged_visits_on_time,\n" + + " coalesce(sum(distinct ped.visits_done_on_time), 0) as ped_visits_on_time,\n" + + " coalesce(sum(distinct ged.total_scheduled), 0) as ged_total_scheduled,\n" + + " coalesce(sum(distinct ped.total_scheduled), 0) as ped_total_scheduled\n" + + " from $schemaName.users u \n" + + " left join general_enc_data ged on ged.last_modified_by_id = u.id\n" + + " left join program_enc_data ped on ped.last_modified_by_id = u.id\n" + + " where u.organisation_id notnull\n" + + " and (is_voided = false or is_voided isnull)\n" + + " $userWhere\n" + + " group by u.name, u.username \n" + + ")\n" + + "select indicator,\n" + + " ged_visits_on_time + ped_visits_on_time as count\n" + + "from total_visits_table\n" + + "where ged_visits_on_time + ped_visits_on_time > 0\n" + + " and ((ged_visits_on_time + ped_visits_on_time) /\n" + + "nullif(ged_total_scheduled + ped_total_scheduled, 0)) $proportion_condition;" + ); + baseQuery.add("programEncounterTableNames", programEncounterTableNames) + .add("encounterTableNames", encounterTableNames); + String query = baseQuery.render().replace("$proportion_condition", proportionCondition) + .replace("$schemaName", orgSchemaName) + .replace("$encounterWhere", encounterWhere) + .replace("$userWhere", userWhere); + return jdbcTemplate.query(query, new AggregateReportMapper()); } public List generateUserCancellingMostVisits(String orgSchemaName, String encounterWhere, String userWhere) { SchemaMetadata schema = schemaMetadataRepository.getExistingSchemaMetadata(); - List encounterTables = schema.getAllEncounterTables().stream().toList(); - List programEncounterTables = schema.getAllProgramEncounterTables().stream().toList(); + List encounterTableNames = schema.getAllEncounterTableNames().stream().toList(); + List programEncounterTableNames = schema.getAllProgramEncounterTableNames().stream().toList(); - StringBuilder query = new StringBuilder(); - String baseQuery1 = " select last_modified_by_id,\n" + - " count(*) " + - "filter ( where cancel_date_time notnull ) " + - "cancelled_visits\n" + - " from ${schemaName}.${programEncounter}\n" + + ST baseQuery = new ST("with program_enc_data as (\n " + + " select last_modified_by_id,\n" + + " count(*) filter ( where cancel_date_time notnull ) cancelled_visits\n" + + " from $schemaName.\n" + " where is_voided = false\n" + - " ${encounterWhere}\n" + - " group by last_modified_by_id\n" ; - baseQuery1 = baseQuery1.replace("${schemaName}", orgSchemaName) - .replace("${encounterWhere}", encounterWhere); - query.append("with program_enc_data as (\n" ); - query.append(baseQuery1.replace("${programEncounter}", programEncounterTables.get(0).getName())); - for (int i = 1; i < programEncounterTables.size(); i++) { - query.append("union all\n"); - query.append(baseQuery1.replace("${programEncounter}", programEncounterTables.get(i).getName())); - } - query.append("),\n"); - - String baseQuery2 = " select last_modified_by_id,\n" + - " count(*) " + - "filter ( where cancel_date_time notnull )" + - " cancelled_visits\n" + - " from ${schemaName}.${encounter}\n" + + " $encounterWhere\n" + + " group by last_modified_by_id\n" + + "\n" + + " where is_voided = false\n" + + " $encounterWhere\n" + + " group by last_modified_by_id \n }>" + + "),\n" + + "general_enc_data as (\n" + + " select last_modified_by_id,\n" + + " count(*) filter ( where cancel_date_time notnull ) cancelled_visits\n" + + " from $schemaName.\n" + " where is_voided = false\n" + - " ${encounterWhere}\n" + - " group by last_modified_by_id\n" ; - baseQuery2 = baseQuery2.replace("${schemaName}", orgSchemaName) - .replace("${encounterWhere}", encounterWhere); - query.append("general_enc_data as (\n"); - query.append(baseQuery2.replace("${encounter}", encounterTables.get(0).getName())); - for (int i = 1 ; i < encounterTables.size() ; i++) { - query.append("union all\n" ); - query.append(baseQuery2.replace("${encounter}", encounterTables.get(i).getName())); - } - query.append("),\n"); - - String baseQuery3 ="table1 as (\n" + + " $encounterWhere\n" + + " group by last_modified_by_id\n" + + "\n" + + " where is_voided = false\n" + + " $encounterWhere\n" + + " group by last_modified_by_id \n }>" + + "),\n" + + "cancelled_visits_table as (\n" + "select coalesce(u.name, u.username) as indicator,\n" + " coalesce(sum(distinct ged.cancelled_visits), 0) as ged_cancelled_visits,\n" + " coalesce(sum(distinct ped.cancelled_visits), 0) as ped_cancelled_visits\n" + - "from ${schemaName}.users u\n" + + "from $schemaName.users u\n" + " left join general_enc_data ged on ged.last_modified_by_id = u.id\n" + " left join program_enc_data ped on ped.last_modified_by_id = u.id\n" + "where u.organisation_id notnull\n" + " and (is_voided = false or is_voided isnull)\n" + " and coalesce(ged.cancelled_visits, 0) + coalesce(ped.cancelled_visits, 0) > 0 \n" + - " ${userWhere}\n" + + " $userWhere\n" + " group by u.name,u.username \n" + ")\n" + "select indicator,\n" + " ged_cancelled_visits + ped_cancelled_visits as count\n" + - "from table1\n" + + "from cancelled_visits_table\n" + "order by ged_cancelled_visits + ped_cancelled_visits desc\n" + - "limit 5;"; - query.append(baseQuery3.replace("${schemaName}", orgSchemaName) - .replace("${userWhere}", userWhere)); - return jdbcTemplate.query(query.toString(), new AggregateReportMapper()); + "limit 5;" + ); + baseQuery.add("programEncounterTableNames", programEncounterTableNames) + .add("encounterTableNames", encounterTableNames); + String query = baseQuery.render().replace("$schemaName", orgSchemaName) + .replace("$encounterWhere", encounterWhere) + .replace("$userWhere", userWhere); + return jdbcTemplate.query(query, new AggregateReportMapper()); } } From 75be325b1f5d0175c132f12bd019ccefb50cafb9 Mon Sep 17 00:00:00 2001 From: ak2502 Date: Tue, 22 Aug 2023 22:28:06 +0530 Subject: [PATCH 14/36] avniproject/avni-product#1334 | removed empty ReportService class Signed-off-by: ak2502 --- .../org/avniproject/etl/controller/ReportController.java | 5 +---- .../java/org/avniproject/etl/service/ReportService.java | 8 -------- 2 files changed, 1 insertion(+), 12 deletions(-) delete mode 100644 src/main/java/org/avniproject/etl/service/ReportService.java diff --git a/src/main/java/org/avniproject/etl/controller/ReportController.java b/src/main/java/org/avniproject/etl/controller/ReportController.java index 96f1784..69ebcf6 100644 --- a/src/main/java/org/avniproject/etl/controller/ReportController.java +++ b/src/main/java/org/avniproject/etl/controller/ReportController.java @@ -4,7 +4,6 @@ import org.avniproject.etl.dto.AggregateReportResult; import org.avniproject.etl.dto.UserActivityDTO; import org.avniproject.etl.repository.ReportRepository; -import org.avniproject.etl.service.ReportService; import org.avniproject.etl.util.ReportUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.access.prepost.PreAuthorize; @@ -15,13 +14,11 @@ @RestController public class ReportController { - private final ReportService reportService; private final ReportRepository reportRepository; private final ReportUtil reportUtil; @Autowired - public ReportController(ReportService reportService, ReportRepository reportRepository, ReportUtil reportUtil) { - this.reportService = reportService; + public ReportController(ReportRepository reportRepository, ReportUtil reportUtil) { this.reportRepository = reportRepository; this.reportUtil = reportUtil; } diff --git a/src/main/java/org/avniproject/etl/service/ReportService.java b/src/main/java/org/avniproject/etl/service/ReportService.java deleted file mode 100644 index ec5f58e..0000000 --- a/src/main/java/org/avniproject/etl/service/ReportService.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.avniproject.etl.service; - -import org.springframework.stereotype.Service; - -@Service -public class ReportService { - -} From f0f40aa097fd74a315ff825a552ed0d961aeb424 Mon Sep 17 00:00:00 2001 From: ak2502 Date: Wed, 23 Aug 2023 17:45:36 +0530 Subject: [PATCH 15/36] avniproject/avni-product#1334 | summaryTable endpoint added in Activities tab Signed-off-by: ak2502 --- .../etl/controller/ReportController.java | 10 ++++++++++ .../avniproject/etl/dto/UserActivityDTO.java | 18 ++++++++++++++++++ .../etl/repository/ReportRepository.java | 9 +++++++++ .../rowMappers/reports/SummaryTableMapper.java | 17 +++++++++++++++++ 4 files changed, 54 insertions(+) create mode 100644 src/main/java/org/avniproject/etl/repository/rowMappers/reports/SummaryTableMapper.java diff --git a/src/main/java/org/avniproject/etl/controller/ReportController.java b/src/main/java/org/avniproject/etl/controller/ReportController.java index 69ebcf6..31f202b 100644 --- a/src/main/java/org/avniproject/etl/controller/ReportController.java +++ b/src/main/java/org/avniproject/etl/controller/ReportController.java @@ -23,6 +23,16 @@ public ReportController(ReportRepository reportRepository, ReportUtil reportUtil this.reportUtil = reportUtil; } + @PreAuthorize("hasAnyAuthority('analytics_user')") + @RequestMapping(value = "/report/aggregate/summaryTable", method = RequestMethod.GET) + public List getSummaryTable(@RequestParam(value = "startDate", required = false) String startDate, + @RequestParam(value = "endDate", required = false) String endDate, + @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds){ + return reportRepository.generateSummaryTable( + OrgIdentityContextHolder.getDbSchema() + ); + } + @PreAuthorize("hasAnyAuthority('analytics_user')") @RequestMapping(value = "report/hr/userActivity", method = RequestMethod.GET) public List getUserActivity(@RequestParam(value = "startDate", required = false) String startDate, diff --git a/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java b/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java index 9362396..1a00f8f 100644 --- a/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java +++ b/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java @@ -4,6 +4,8 @@ public class UserActivityDTO { + private String tableName; + private String tableType; private String userName; private Long id; private Long registrationCount; @@ -20,6 +22,22 @@ public class UserActivityDTO { private DateTime syncEnd; private DateTime lastSuccessfulSync; + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public String getTableType() { + return tableType; + } + + public void setTableType(String tableType) { + this.tableType = tableType; + } + public String getUserName() { return userName; } diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java index 9e566eb..d173bbf 100644 --- a/src/main/java/org/avniproject/etl/repository/ReportRepository.java +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -22,6 +22,15 @@ public ReportRepository(NamedParameterJdbcTemplate jdbcTemplate, SchemaMetadataR this.schemaMetadataRepository = schemaMetadataRepository; } + public List generateSummaryTable(String orgSchemaName){ + String baseQuery = "select name, type \n" + + "from public.table_metadata\n" + + "where schema_name = '${schemaName}'\n" + + "order by type;"; + String query= baseQuery.replace("${schemaName}", orgSchemaName); + return jdbcTemplate.query(query, new SummaryTableMapper()); + } + public List generateUserActivity(String orgSchemaName, String subjectWhere, String encounterWhere, String enrolmentWhere, String userWhere) { SchemaMetadata schema = schemaMetadataRepository.getExistingSchemaMetadata(); List subjectTableNames = schema.getAllSubjectTableNames().stream().toList(); diff --git a/src/main/java/org/avniproject/etl/repository/rowMappers/reports/SummaryTableMapper.java b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/SummaryTableMapper.java new file mode 100644 index 0000000..d882cde --- /dev/null +++ b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/SummaryTableMapper.java @@ -0,0 +1,17 @@ +package org.avniproject.etl.repository.rowMappers.reports; + +import org.avniproject.etl.dto.UserActivityDTO; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class SummaryTableMapper implements RowMapper { + @Override + public UserActivityDTO mapRow(ResultSet rs, int rowNum) throws SQLException { + UserActivityDTO userActivityDTO = new UserActivityDTO(); + userActivityDTO.setTableName(rs.getString("name")); + userActivityDTO.setTableType(rs.getString("type")); + return userActivityDTO; + } +} From 5f788587f922f54ac7b7ec34d48e317c4b380e88 Mon Sep 17 00:00:00 2001 From: ak2502 Date: Thu, 24 Aug 2023 18:08:18 +0530 Subject: [PATCH 16/36] avniproject/avni-product#1334 | added endpoint for median sync time for past week table in hr tab Signed-off-by: ak2502 --- .../etl/controller/ReportController.java | 17 +++++++++++++++-- .../avniproject/etl/dto/UserActivityDTO.java | 9 +++++++++ .../etl/repository/ReportRepository.java | 18 ++++++++++++++++++ .../rowMappers/reports/MedianSyncMapper.java | 17 +++++++++++++++++ .../org/avniproject/etl/util/ReportUtil.java | 7 +++++++ 5 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/avniproject/etl/repository/rowMappers/reports/MedianSyncMapper.java diff --git a/src/main/java/org/avniproject/etl/controller/ReportController.java b/src/main/java/org/avniproject/etl/controller/ReportController.java index 31f202b..b0fff80 100644 --- a/src/main/java/org/avniproject/etl/controller/ReportController.java +++ b/src/main/java/org/avniproject/etl/controller/ReportController.java @@ -54,7 +54,7 @@ public List getUserWiseSyncFailures(@RequestParam(value = "star @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds){ return reportRepository.generateUserSyncFailures( OrgIdentityContextHolder.getDbSchema(), - reportUtil.getDateDynamicWhere(startDate, endDate, "sync_start_time"), + reportUtil.getDateDynamicWhere(startDate, endDate, "st.sync_start_time"), reportUtil.getDynamicUserWhere(userIds, "u.id") ); } @@ -94,9 +94,22 @@ public List getLatestSyncs(@RequestParam(value = "startDate", r return reportRepository.generateLatestSyncs( OrgIdentityContextHolder.getDbSchema(), - reportUtil.getDateDynamicWhere(startDate, endDate, "sync_end_time"), + reportUtil.getDateDynamicWhere(startDate, endDate, "st.sync_end_time"), reportUtil.getDynamicUserWhere(userIds, "u.id")); } + + @PreAuthorize("hasAnyAuthority('analytics_user')") + @RequestMapping(value = "/report/hr/medianSync", method = RequestMethod.GET) + public List getMedianSync(@RequestParam(value = "startDate", required = false) String startDate, + @RequestParam(value = "endDate", required = false) String endDate, + @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) { + + return reportRepository.generateMedianSync( + OrgIdentityContextHolder.getDbSchema(), + reportUtil.getDateDynamicMedianSync(startDate, endDate, "st.sync_start_time"), + reportUtil.getDynamicUserWhere(userIds, "u.id")); + } + @PreAuthorize("hasAnyAuthority('analytics_user')") @RequestMapping(value = "/report/hr/championUsers", method = RequestMethod.GET) public List getChampionUsers(@RequestParam(value = "startDate", required = false) String startDate, diff --git a/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java b/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java index 1a00f8f..5f555e3 100644 --- a/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java +++ b/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java @@ -21,6 +21,7 @@ public class UserActivityDTO { private DateTime syncStart; private DateTime syncEnd; private DateTime lastSuccessfulSync; + private String medianSync; public String getTableName() { return tableName; @@ -157,4 +158,12 @@ public String getSyncSource() { public void setSyncSource(String syncSource) { this.syncSource = syncSource; } + + public String getMedianSync() { + return medianSync; + } + + public void setMedianSync(String medianSync) { + this.medianSync = medianSync; + } } diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java index d173bbf..c8b56e0 100644 --- a/src/main/java/org/avniproject/etl/repository/ReportRepository.java +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -234,6 +234,24 @@ public List generateLatestSyncs(String orgSchemaName, String sy .replace("${userWhere}", userWhere); return jdbcTemplate.query(query, new LatestSyncMapper()); } + + public List generateMedianSync(String orgSchemaName, String syncTelemetryWhere, String userWhere) { + String baseQuery = "select coalesce(u.name, u.username) as name,\n" + + "coalesce(percentile_cont(0.5) within group (order by (sync_end_time-sync_start_time)), '00:00:00') as median_sync_time\n" + + "from ${schemaName}.sync_telemetry st\n" + + "join ${schemaName}.users u on st.last_modified_by_id = u.id\n" + + "where (u.is_voided = false or u.is_voided isnull) and u.organisation_id notnull\n" + + "${syncTelemetryWhere}\n"+ + "${userWhere}\n"+ + "group by u.name, u.username\n" + + "order by 2 desc;"; + String query = baseQuery + .replace("${schemaName}", orgSchemaName) + .replace("${syncTelemetryWhere}", syncTelemetryWhere) + .replace("${userWhere}", userWhere); + return jdbcTemplate.query(query, new MedianSyncMapper()); + } + public List generateCompletedVisitsOnTimeByProportion(String proportionCondition, String orgSchemaName, String encounterWhere, String userWhere) { SchemaMetadata schema = schemaMetadataRepository.getExistingSchemaMetadata(); List encounterTableNames = schema.getAllEncounterTableNames().stream().toList(); diff --git a/src/main/java/org/avniproject/etl/repository/rowMappers/reports/MedianSyncMapper.java b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/MedianSyncMapper.java new file mode 100644 index 0000000..7755d3c --- /dev/null +++ b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/MedianSyncMapper.java @@ -0,0 +1,17 @@ +package org.avniproject.etl.repository.rowMappers.reports; + +import org.avniproject.etl.dto.UserActivityDTO; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class MedianSyncMapper implements RowMapper { + @Override + public UserActivityDTO mapRow(ResultSet rs, int rowNum) throws SQLException { + UserActivityDTO userActivityDTO = new UserActivityDTO(); + userActivityDTO.setUserName(rs.getString("name")); + userActivityDTO.setMedianSync(rs.getString("median_sync_time")); + return userActivityDTO; + } +} diff --git a/src/main/java/org/avniproject/etl/util/ReportUtil.java b/src/main/java/org/avniproject/etl/util/ReportUtil.java index 2813240..54c362f 100644 --- a/src/main/java/org/avniproject/etl/util/ReportUtil.java +++ b/src/main/java/org/avniproject/etl/util/ReportUtil.java @@ -15,6 +15,13 @@ public String getDateDynamicWhere(String startDate, String endDate, String colum return ""; } + public String getDateDynamicMedianSync(String startDate, String endDate, String columnName) { + if (startDate != null) { + return format("and %s::date between ( '%s'::date - interval '7 days') and ('%s'::date - interval '7 days')", columnName, startDate, endDate); + } + return "and st.sync_start_time between ( current_date at time zone 'UTC'+'5:30' - interval '7 days') and current_date"; + } + public String getDynamicUserWhere(List userIds, String columnName) { if (!userIds.isEmpty()) { return format("and %s in (%s)", columnName, StrUtil.joinLongToList(userIds)); From b532ca68c96874c2d4a28b06b3a7a9104d49164c Mon Sep 17 00:00:00 2001 From: ak2502 Date: Mon, 4 Sep 2023 20:49:37 +0530 Subject: [PATCH 17/36] avniproject/avni-product#1334 | modified 'median sync-time table' to show weekly data for past 3 months Signed-off-by: ak2502 --- .../etl/controller/ReportController.java | 3 +-- .../etl/repository/ReportRepository.java | 26 ++++++++++--------- .../rowMappers/reports/MedianSyncMapper.java | 4 ++- .../org/avniproject/etl/util/ReportUtil.java | 6 ++--- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/avniproject/etl/controller/ReportController.java b/src/main/java/org/avniproject/etl/controller/ReportController.java index b0fff80..4e9f84b 100644 --- a/src/main/java/org/avniproject/etl/controller/ReportController.java +++ b/src/main/java/org/avniproject/etl/controller/ReportController.java @@ -106,8 +106,7 @@ public List getMedianSync(@RequestParam(value = "startDate", re return reportRepository.generateMedianSync( OrgIdentityContextHolder.getDbSchema(), - reportUtil.getDateDynamicMedianSync(startDate, endDate, "st.sync_start_time"), - reportUtil.getDynamicUserWhere(userIds, "u.id")); + reportUtil.getDateSeries(startDate, endDate)); } @PreAuthorize("hasAnyAuthority('analytics_user')") diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java index c8b56e0..f14b8f9 100644 --- a/src/main/java/org/avniproject/etl/repository/ReportRepository.java +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -235,20 +235,22 @@ public List generateLatestSyncs(String orgSchemaName, String sy return jdbcTemplate.query(query, new LatestSyncMapper()); } - public List generateMedianSync(String orgSchemaName, String syncTelemetryWhere, String userWhere) { - String baseQuery = "select coalesce(u.name, u.username) as name,\n" + - "coalesce(percentile_cont(0.5) within group (order by (sync_end_time-sync_start_time)), '00:00:00') as median_sync_time\n" + - "from ${schemaName}.sync_telemetry st\n" + - "join ${schemaName}.users u on st.last_modified_by_id = u.id\n" + - "where (u.is_voided = false or u.is_voided isnull) and u.organisation_id notnull\n" + - "${syncTelemetryWhere}\n"+ - "${userWhere}\n"+ - "group by u.name, u.username\n" + - "order by 2 desc;"; + public List generateMedianSync(String orgSchemaName, String syncTelemetryWhere) { + String baseQuery = "with weeks as (\n" + + " select day::date start_date, day::date+6 end_date\n" + + " ${syncTelemetryWhere}\n" + + ")\n" + + "select w.start_date, w.end_date, \n" + + " coalesce(percentile_cont(0.5) within group (order by (st.sync_end_time-st.sync_start_time)), '00:00:00') as median_sync_time\n" + + "from weeks w\t\n" + + "left join ${schemaName}.sync_telemetry st\n" + + "on st.sync_start_time::date >= w.start_date and st.sync_end_time::date <= w.end_date\n" + +// "and st.sync_source = 'manual'\n" + + "group by 1,2;"; String query = baseQuery .replace("${schemaName}", orgSchemaName) - .replace("${syncTelemetryWhere}", syncTelemetryWhere) - .replace("${userWhere}", userWhere); + .replace("${syncTelemetryWhere}", syncTelemetryWhere); + System.out.print(query); return jdbcTemplate.query(query, new MedianSyncMapper()); } diff --git a/src/main/java/org/avniproject/etl/repository/rowMappers/reports/MedianSyncMapper.java b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/MedianSyncMapper.java index 7755d3c..4b905e9 100644 --- a/src/main/java/org/avniproject/etl/repository/rowMappers/reports/MedianSyncMapper.java +++ b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/MedianSyncMapper.java @@ -1,6 +1,7 @@ package org.avniproject.etl.repository.rowMappers.reports; import org.avniproject.etl.dto.UserActivityDTO; +import org.joda.time.DateTime; import org.springframework.jdbc.core.RowMapper; import java.sql.ResultSet; @@ -10,7 +11,8 @@ public class MedianSyncMapper implements RowMapper { @Override public UserActivityDTO mapRow(ResultSet rs, int rowNum) throws SQLException { UserActivityDTO userActivityDTO = new UserActivityDTO(); - userActivityDTO.setUserName(rs.getString("name")); + userActivityDTO.setSyncStart(new DateTime(rs.getDate("start_date"))); + userActivityDTO.setSyncEnd(new DateTime(rs.getDate("end_date"))); userActivityDTO.setMedianSync(rs.getString("median_sync_time")); return userActivityDTO; } diff --git a/src/main/java/org/avniproject/etl/util/ReportUtil.java b/src/main/java/org/avniproject/etl/util/ReportUtil.java index 54c362f..239282b 100644 --- a/src/main/java/org/avniproject/etl/util/ReportUtil.java +++ b/src/main/java/org/avniproject/etl/util/ReportUtil.java @@ -15,11 +15,11 @@ public String getDateDynamicWhere(String startDate, String endDate, String colum return ""; } - public String getDateDynamicMedianSync(String startDate, String endDate, String columnName) { + public String getDateSeries(String startDate, String endDate) { if (startDate != null) { - return format("and %s::date between ( '%s'::date - interval '7 days') and ('%s'::date - interval '7 days')", columnName, startDate, endDate); + return format("from generate_series('%s'::date - interval '3 months', '%s'::date, '7d'::interval) day", startDate, endDate); } - return "and st.sync_start_time between ( current_date at time zone 'UTC'+'5:30' - interval '7 days') and current_date"; + return "from generate_series(current_date at time zone 'UTC'+'5:30'- interval '3 months' , current_date at time zone 'UTC'+'5:30' , '7d'::interval) day"; } public String getDynamicUserWhere(List userIds, String columnName) { From fc21afd4caa2e24e5516492af11127bc827ee3a0 Mon Sep 17 00:00:00 2001 From: ak2502 Date: Mon, 4 Sep 2023 22:11:11 +0530 Subject: [PATCH 18/36] removed comment --- .../java/org/avniproject/etl/repository/ReportRepository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java index f14b8f9..4fee4e7 100644 --- a/src/main/java/org/avniproject/etl/repository/ReportRepository.java +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -245,7 +245,7 @@ public List generateMedianSync(String orgSchemaName, String syn "from weeks w\t\n" + "left join ${schemaName}.sync_telemetry st\n" + "on st.sync_start_time::date >= w.start_date and st.sync_end_time::date <= w.end_date\n" + -// "and st.sync_source = 'manual'\n" + + "and st.sync_source = 'manual'\n" + "group by 1,2;"; String query = baseQuery .replace("${schemaName}", orgSchemaName) From 0fdb0c446e23a2841b2216ad778721eb679ef578 Mon Sep 17 00:00:00 2001 From: ak2502 Date: Thu, 7 Sep 2023 14:05:48 +0530 Subject: [PATCH 19/36] log used for reference removed Signed-off-by: ak2502 --- .../java/org/avniproject/etl/repository/ReportRepository.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java index 4fee4e7..0c762b7 100644 --- a/src/main/java/org/avniproject/etl/repository/ReportRepository.java +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -250,7 +250,6 @@ public List generateMedianSync(String orgSchemaName, String syn String query = baseQuery .replace("${schemaName}", orgSchemaName) .replace("${syncTelemetryWhere}", syncTelemetryWhere); - System.out.print(query); return jdbcTemplate.query(query, new MedianSyncMapper()); } From 6f517b41caf4f7c15cc997971ae36459c4e9df95 Mon Sep 17 00:00:00 2001 From: Avinash Ramachandruni Date: Tue, 1 Aug 2023 07:43:21 +0530 Subject: [PATCH 20/36] added new endpoints for hr analytics using org schema - syncFailures and userActivity --- .../etl/controller/ReportController.java | 75 ++++++++++++++ .../avniproject/etl/dto/UserActivityDTO.java | 98 +++++++++++++++++++ .../etl/repository/ReportRepository.java | 94 ++++++++++++++++++ .../reports/UserActivityMapper.java | 21 ++++ .../rowMappers/reports/UserCountMapper.java | 17 ++++ .../etl/service/ReportService.java | 5 + .../org/avniproject/etl/util/ReportUtil.java | 22 +++++ .../org/avniproject/etl/util/StrUtil.java | 17 ++++ 8 files changed, 349 insertions(+) create mode 100644 src/main/java/org/avniproject/etl/controller/ReportController.java create mode 100644 src/main/java/org/avniproject/etl/dto/UserActivityDTO.java create mode 100644 src/main/java/org/avniproject/etl/repository/ReportRepository.java create mode 100644 src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserActivityMapper.java create mode 100644 src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserCountMapper.java create mode 100644 src/main/java/org/avniproject/etl/service/ReportService.java create mode 100644 src/main/java/org/avniproject/etl/util/ReportUtil.java create mode 100644 src/main/java/org/avniproject/etl/util/StrUtil.java diff --git a/src/main/java/org/avniproject/etl/controller/ReportController.java b/src/main/java/org/avniproject/etl/controller/ReportController.java new file mode 100644 index 0000000..a879ab0 --- /dev/null +++ b/src/main/java/org/avniproject/etl/controller/ReportController.java @@ -0,0 +1,75 @@ +package org.avniproject.etl.controller; + +import org.avniproject.etl.domain.OrgIdentityContextHolder; +import org.avniproject.etl.dto.UserActivityDTO; +import org.avniproject.etl.repository.ReportRepository; +import org.avniproject.etl.service.ReportService; +import org.avniproject.etl.util.ReportUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +@RestController +@PreAuthorize("hasAnyAuthority('organisation_admin','admin')") +@RequestMapping("/reports") +public class ReportController { + + private final ReportService reportService; + private final ReportRepository reportRepository; + private final ReportUtil reportUtil; + + @Autowired + public ReportController(ReportService reportService, ReportRepository reportRepository, ReportUtil reportUtil) { + this.reportService = reportService; + this.reportRepository = reportRepository; + this.reportUtil = reportUtil; + } + + @GetMapping("/hr/userActivity") + public ResponseEntity getUserActivity(@RequestParam(value = "start_date", required = false) String startDate, + @RequestParam(value = "endDate", required = false) String endDate, + @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) + { + try { + List userActivity = reportRepository.getUserActivity( + OrgIdentityContextHolder.getDbSchema(), + reportUtil.getDateDynamicWhere(startDate, endDate, "registration_date"), + reportUtil.getDateDynamicWhere(startDate, endDate, "encounter_date_time"), + reportUtil.getDateDynamicWhere(startDate, endDate, "enrolment_date_time"), + reportUtil.getDynamicUserWhere(userIds, "u.id") + ); + + return ResponseEntity.ok(userActivity); + } + catch (Exception e) { + return ResponseEntity.badRequest().body(e.getMessage()); + } + } + + @GetMapping("/hr/syncFailures") + public ResponseEntity getUserWiseSyncFailures(@RequestParam(value = "start_date", required = false) String startDate, + @RequestParam(value = "endDate", required = false) String endDate, + @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds){ + try { + List userActivity = reportRepository.getUserSyncFailures( + OrgIdentityContextHolder.getDbSchema(), + reportUtil.getDateDynamicWhere(startDate, endDate, "sync_start_time"), + reportUtil.getDynamicUserWhere(userIds, "u.id") + ); + return ResponseEntity.ok(userActivity); + } + catch (Exception e) { + return ResponseEntity.badRequest().body(e.getMessage()); + } + } + + + +} + diff --git a/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java b/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java new file mode 100644 index 0000000..392f9d3 --- /dev/null +++ b/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java @@ -0,0 +1,98 @@ +package org.avniproject.etl.dto; + +import org.joda.time.DateTime; + +public class UserActivityDTO { + + private String userName; + private Long id; + private Long registrationCount; + private Long programEnrolmentCount; + private Long programEncounterCount; + private Long generalEncounterCount; + private Long count; + private String appVersion; + private String deviceModel; + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getRegistrationCount() { + return registrationCount; + } + + public void setRegistrationCount(Long registrationCount) { + this.registrationCount = registrationCount; + } + + public Long getProgramEnrolmentCount() { + return programEnrolmentCount; + } + + public void setProgramEnrolmentCount(Long programEnrolmentCount) { + this.programEnrolmentCount = programEnrolmentCount; + } + + public Long getProgramEncounterCount() { + return programEncounterCount; + } + + public void setProgramEncounterCount(Long programEncounterCount) { + this.programEncounterCount = programEncounterCount; + } + + public Long getGeneralEncounterCount() { + return generalEncounterCount; + } + + public void setGeneralEncounterCount(Long generalEncounterCount) { + this.generalEncounterCount = generalEncounterCount; + } + + public Long getCount() { + return count; + } + + public void setCount(Long count) { + this.count = count; + } + + public String getAppVersion() { + return appVersion; + } + + public void setAppVersion(String appVersion) { + this.appVersion = appVersion; + } + + public String getDeviceModel() { + return deviceModel; + } + + public void setDeviceModel(String deviceModel) { + this.deviceModel = deviceModel; + } + + public DateTime getLastSuccessfulSync() { + return lastSuccessfulSync; + } + + public void setLastSuccessfulSync(DateTime lastSuccessfulSync) { + this.lastSuccessfulSync = lastSuccessfulSync; + } + + private DateTime lastSuccessfulSync; +} diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java new file mode 100644 index 0000000..18df119 --- /dev/null +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -0,0 +1,94 @@ +package org.avniproject.etl.repository; + +import org.avniproject.etl.dto.UserActivityDTO; +import org.avniproject.etl.repository.rowMappers.reports.UserActivityMapper; +import org.avniproject.etl.repository.rowMappers.reports.UserCountMapper; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; + +import java.util.List; + +public class ReportRepository { + private final NamedParameterJdbcTemplate jdbcTemplate; + + @Autowired + public ReportRepository(NamedParameterJdbcTemplate jdbcTemplate) { + this.jdbcTemplate = jdbcTemplate; + } + + public List getUserActivity(String orgSchemaName, String subjectWhere, String encounterWhere, String enrolmentWhere, String userWhere) { + String baseQuery = "with registrations as (\n" + + " select last_modified_by_id, count(*) as registration_count\n" + + " from ${schemaName}.individual\n" + + " where is_voided = false\n" + + " ${subjectWhere}\n" + + " group by last_modified_by_id\n" + + "),\n" + + " encounters as (\n" + + " select last_modified_by_id, count(*) as encounter_count\n" + + " from ${schemaName}.encounter\n" + + " where is_voided = false\n" + + " ${encounterWhere}\n" + + " group by last_modified_by_id\n" + + " ),\n" + + " enrolments as (\n" + + " select last_modified_by_id, count(*) as enrolment_count\n" + + " from ${schemaName}.program_enrolment\n" + + " where is_voided = false\n" + + " ${enrolmentWhere}\n" + + " group by last_modified_by_id\n" + + " ),\n" + + " program_encounters as (\n" + + " select last_modified_by_id, count(*) as program_encounter_count\n" + + " from ${schemaName}.program_encounter\n" + + " where is_voided = false\n" + + " ${encounterWhere}\n" + + " group by last_modified_by_id\n" + + " )\n" + + "select u.id as id,\n" + + " coalesce(u.name, u.username) as name,\n" + + " coalesce(registration_count, 0) as registration_count,\n" + + " coalesce(encounter_count, 0) as encounter_count,\n" + + " coalesce(enrolment_count, 0) as enrolment_count,\n" + + " coalesce(program_encounter_count, 0) as program_encounter_count,\n" + + " coalesce(coalesce(registration_count, 0) + coalesce(encounter_count, 0) + coalesce(enrolment_count, 0) +\n" + + " coalesce(program_encounter_count, 0), 0) as total\n" + + "from ${schemaName}.users u\n" + + " left join ${schemaName}.registrations r on r.last_modified_by_id = u.id\n" + + " left join ${schemaName}.encounters e on e.last_modified_by_id = u.id\n" + + " left join ${schemaName}.enrolments enl on enl.last_modified_by_id = u.id\n" + + " left join ${schemaName}.program_encounters enc on enc.last_modified_by_id = u.id\n" + + "where u.is_voided = false and u.organisation_id notnull\n" + + " and coalesce(coalesce(registration_count, 0) + coalesce(encounter_count, 0) + coalesce(enrolment_count, 0) +\n" + + " coalesce(program_encounter_count, 0), 0) > 0\n" + + " ${userWhere}\n" + + "order by 7 desc\n" + + "limit 10;"; + String query = baseQuery + .replace("${schemaName}", orgSchemaName) + .replace("${subjectWhere}", subjectWhere) + .replace("${encounterWhere}", encounterWhere) + .replace("${enrolmentWhere}", enrolmentWhere) + .replace("${userWhere}", userWhere); + return jdbcTemplate.query(query, new UserActivityMapper()); + } + + public List getUserSyncFailures(String orgSchemaName, String syncTelemetryWhere, String userWhere) { + String baseQuery = "select coalesce(u.name, u.username) as name, \n" + + " count(*) as count\n" + + "from ${schemaName}.sync_telemetry st\n" + + " join ${schemaName}.users u on st.user_id = u.id\n" + + "where sync_status = 'incomplete'\n" + + "and u.is_voided = false and u.organisation_id notnull\n" + + "${syncTelemetryWhere}\n"+ + "${userWhere}\n"+ + "group by 1\n" + + "order by 2 desc\n" + + "limit 10;"; + String query = baseQuery + .replace("${schemaName}", orgSchemaName) + .replace("${syncTelemetryWhere}", syncTelemetryWhere) + .replace("${userWhere}", userWhere); + return jdbcTemplate.query(query, new UserCountMapper()); + } +} diff --git a/src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserActivityMapper.java b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserActivityMapper.java new file mode 100644 index 0000000..c2ab736 --- /dev/null +++ b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserActivityMapper.java @@ -0,0 +1,21 @@ +package org.avniproject.etl.repository.rowMappers.reports; + +import org.avniproject.etl.dto.UserActivityDTO; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class UserActivityMapper implements RowMapper { + @Override + public UserActivityDTO mapRow(ResultSet rs, int rowNum) throws SQLException { + UserActivityDTO userActivityDTO = new UserActivityDTO(); + userActivityDTO.setId(rs.getLong("id")); + userActivityDTO.setUserName(rs.getString("name")); + userActivityDTO.setRegistrationCount(rs.getLong("registration_count")); + userActivityDTO.setGeneralEncounterCount(rs.getLong("encounter_count")); + userActivityDTO.setProgramEnrolmentCount(rs.getLong("enrolment_count")); + userActivityDTO.setProgramEncounterCount(rs.getLong("program_encounter_count")); + return userActivityDTO; + } +} diff --git a/src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserCountMapper.java b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserCountMapper.java new file mode 100644 index 0000000..3ce01a9 --- /dev/null +++ b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserCountMapper.java @@ -0,0 +1,17 @@ +package org.avniproject.etl.repository.rowMappers.reports; + +import org.avniproject.etl.dto.UserActivityDTO; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class UserCountMapper implements RowMapper { + @Override + public UserActivityDTO mapRow(ResultSet rs, int rowNum) throws SQLException { + UserActivityDTO userActivityDTO = new UserActivityDTO(); + userActivityDTO.setUserName(rs.getString("name")); + userActivityDTO.setCount(rs.getLong("count")); + return userActivityDTO; + } +} diff --git a/src/main/java/org/avniproject/etl/service/ReportService.java b/src/main/java/org/avniproject/etl/service/ReportService.java new file mode 100644 index 0000000..89d789a --- /dev/null +++ b/src/main/java/org/avniproject/etl/service/ReportService.java @@ -0,0 +1,5 @@ +package org.avniproject.etl.service; + +public class ReportService { + +} diff --git a/src/main/java/org/avniproject/etl/util/ReportUtil.java b/src/main/java/org/avniproject/etl/util/ReportUtil.java new file mode 100644 index 0000000..dde2755 --- /dev/null +++ b/src/main/java/org/avniproject/etl/util/ReportUtil.java @@ -0,0 +1,22 @@ +package org.avniproject.etl.util; + +import java.util.List; + +import static java.lang.String.format; + +public class ReportUtil { + public String getDateDynamicWhere(String startDate, String endDate, String columnName) { + if (startDate != null) { + return format("and %s::date between '%s'::date and '%s'::date", columnName, startDate, endDate); + } + return ""; + } + + public String getDynamicUserWhere(List userIds, String columnName) { + if (!userIds.isEmpty()) { + return format("and %s in (%s)", columnName, StrUtil.joinLongToList(userIds)); + } + return ""; + } + +} diff --git a/src/main/java/org/avniproject/etl/util/StrUtil.java b/src/main/java/org/avniproject/etl/util/StrUtil.java new file mode 100644 index 0000000..10e759c --- /dev/null +++ b/src/main/java/org/avniproject/etl/util/StrUtil.java @@ -0,0 +1,17 @@ +package org.avniproject.etl.util; + +import java.util.List; +import java.util.stream.Collectors; + +public class StrUtil { + + public static boolean isEmpty(String string) { + return string == null || string.trim().isEmpty(); + } + + public static String joinLongToList(List lists) { + return lists.isEmpty() ? "" : lists.stream().map(String::valueOf) + .collect(Collectors.joining(",")); + } + +} From 0c2d8f241c9958615683bf092b38bb554c3ac3c7 Mon Sep 17 00:00:00 2001 From: ak2502 Date: Tue, 1 Aug 2023 18:45:10 +0530 Subject: [PATCH 21/36] avniproject/avni-product#1334 | updated userSyncStatus endpoint and verified working Signed-off-by: ak2502 --- .../etl/controller/ReportController.java | 23 +++++-------------- .../etl/repository/ReportRepository.java | 9 +++++--- .../etl/service/ReportService.java | 3 +++ .../org/avniproject/etl/util/ReportUtil.java | 3 +++ 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/avniproject/etl/controller/ReportController.java b/src/main/java/org/avniproject/etl/controller/ReportController.java index a879ab0..a1e3394 100644 --- a/src/main/java/org/avniproject/etl/controller/ReportController.java +++ b/src/main/java/org/avniproject/etl/controller/ReportController.java @@ -3,21 +3,16 @@ import org.avniproject.etl.domain.OrgIdentityContextHolder; import org.avniproject.etl.dto.UserActivityDTO; import org.avniproject.etl.repository.ReportRepository; +import org.avniproject.etl.service.EtlService; import org.avniproject.etl.service.ReportService; import org.avniproject.etl.util.ReportUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.ResponseEntity; -import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import java.util.List; @RestController -@PreAuthorize("hasAnyAuthority('organisation_admin','admin')") -@RequestMapping("/reports") public class ReportController { private final ReportService reportService; @@ -31,7 +26,7 @@ public ReportController(ReportService reportService, ReportRepository reportRepo this.reportUtil = reportUtil; } - @GetMapping("/hr/userActivity") + @RequestMapping(value = "reports/hr/userActivity", method = RequestMethod.GET) public ResponseEntity getUserActivity(@RequestParam(value = "start_date", required = false) String startDate, @RequestParam(value = "endDate", required = false) String endDate, @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) @@ -52,21 +47,15 @@ public ResponseEntity getUserActivity(@RequestParam(value = "start_date", requir } } - @GetMapping("/hr/syncFailures") - public ResponseEntity getUserWiseSyncFailures(@RequestParam(value = "start_date", required = false) String startDate, + @RequestMapping(value = "/report/hr/syncFailures",method = RequestMethod.GET) + public List getUserWiseSyncFailures(@RequestParam(value = "startDate", required = false) String startDate, @RequestParam(value = "endDate", required = false) String endDate, @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds){ - try { - List userActivity = reportRepository.getUserSyncFailures( + return reportRepository.generateUserSyncFailures( OrgIdentityContextHolder.getDbSchema(), reportUtil.getDateDynamicWhere(startDate, endDate, "sync_start_time"), reportUtil.getDynamicUserWhere(userIds, "u.id") ); - return ResponseEntity.ok(userActivity); - } - catch (Exception e) { - return ResponseEntity.badRequest().body(e.getMessage()); - } } diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java index 18df119..6922a6a 100644 --- a/src/main/java/org/avniproject/etl/repository/ReportRepository.java +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -5,9 +5,11 @@ import org.avniproject.etl.repository.rowMappers.reports.UserCountMapper; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Component; import java.util.List; +@Component public class ReportRepository { private final NamedParameterJdbcTemplate jdbcTemplate; @@ -73,13 +75,14 @@ public List getUserActivity(String orgSchemaName, String subjec return jdbcTemplate.query(query, new UserActivityMapper()); } - public List getUserSyncFailures(String orgSchemaName, String syncTelemetryWhere, String userWhere) { + public List generateUserSyncFailures(String orgSchemaName, String syncTelemetryWhere, String userWhere) { String baseQuery = "select coalesce(u.name, u.username) as name, \n" + " count(*) as count\n" + "from ${schemaName}.sync_telemetry st\n" + - " join ${schemaName}.users u on st.user_id = u.id\n" + + " join ${schemaName}.user u on st.user_id = u.id\n" + "where sync_status = 'incomplete'\n" + - "and u.is_voided = false and u.organisation_id notnull\n" + + "and (u.is_voided = false or u.is_voided isnull)\n" + + "and u.organisation_id notnull\n" + "${syncTelemetryWhere}\n"+ "${userWhere}\n"+ "group by 1\n" + diff --git a/src/main/java/org/avniproject/etl/service/ReportService.java b/src/main/java/org/avniproject/etl/service/ReportService.java index 89d789a..ec5f58e 100644 --- a/src/main/java/org/avniproject/etl/service/ReportService.java +++ b/src/main/java/org/avniproject/etl/service/ReportService.java @@ -1,5 +1,8 @@ package org.avniproject.etl.service; +import org.springframework.stereotype.Service; + +@Service public class ReportService { } diff --git a/src/main/java/org/avniproject/etl/util/ReportUtil.java b/src/main/java/org/avniproject/etl/util/ReportUtil.java index dde2755..2813240 100644 --- a/src/main/java/org/avniproject/etl/util/ReportUtil.java +++ b/src/main/java/org/avniproject/etl/util/ReportUtil.java @@ -1,9 +1,12 @@ package org.avniproject.etl.util; +import org.springframework.stereotype.Service; + import java.util.List; import static java.lang.String.format; +@Service public class ReportUtil { public String getDateDynamicWhere(String startDate, String endDate, String columnName) { if (startDate != null) { From a355d0c7f88d667089ceee5a784f95a951b83e94 Mon Sep 17 00:00:00 2001 From: Avinash Ramachandruni Date: Thu, 3 Aug 2023 07:42:02 +0530 Subject: [PATCH 22/36] Added 3 endpoints for hr tab analytics that can pull data from org schema - deviceModels, appVersions, userDetails --- .../etl/controller/ReportController.java | 25 +++++++ .../etl/dto/AggregateReportResult.java | 31 +++++++++ .../etl/repository/ReportRepository.java | 65 ++++++++++++++++++- .../reports/AggregateReportMapper.java | 19 ++++++ .../rowMappers/reports/UserDetailsMapper.java | 20 ++++++ 5 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/avniproject/etl/dto/AggregateReportResult.java create mode 100644 src/main/java/org/avniproject/etl/repository/rowMappers/reports/AggregateReportMapper.java create mode 100644 src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserDetailsMapper.java diff --git a/src/main/java/org/avniproject/etl/controller/ReportController.java b/src/main/java/org/avniproject/etl/controller/ReportController.java index a1e3394..3c04ebb 100644 --- a/src/main/java/org/avniproject/etl/controller/ReportController.java +++ b/src/main/java/org/avniproject/etl/controller/ReportController.java @@ -1,6 +1,7 @@ package org.avniproject.etl.controller; import org.avniproject.etl.domain.OrgIdentityContextHolder; +import org.avniproject.etl.dto.AggregateReportResult; import org.avniproject.etl.dto.UserActivityDTO; import org.avniproject.etl.repository.ReportRepository; import org.avniproject.etl.service.EtlService; @@ -58,6 +59,30 @@ public List getUserWiseSyncFailures(@RequestParam(value = "star ); } + @RequestMapping(value = "/report/hr/deviceModels", method = RequestMethod.GET) + public List getUserWiseDeviceModels(@RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) { + + return reportRepository.generateUserDeviceModels( + OrgIdentityContextHolder.getDbSchema(), + reportUtil.getDynamicUserWhere(userIds, "u.id")); + } + + @RequestMapping(value = "/report/hr/appVersions", method = RequestMethod.GET) + public List getUserWiseAppVersions(@RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) { + + return reportRepository.generateUserAppVersions( + OrgIdentityContextHolder.getDbSchema(), + reportUtil.getDynamicUserWhere(userIds, "u.id")); + } + + @RequestMapping(value = "/report/hr/userDetails", method = RequestMethod.GET) + public List getUserDetails(@RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) { + + return reportRepository.generateUserDetails( + OrgIdentityContextHolder.getDbSchema(), + reportUtil.getDynamicUserWhere(userIds, "u.id")); + } + } diff --git a/src/main/java/org/avniproject/etl/dto/AggregateReportResult.java b/src/main/java/org/avniproject/etl/dto/AggregateReportResult.java new file mode 100644 index 0000000..e684c03 --- /dev/null +++ b/src/main/java/org/avniproject/etl/dto/AggregateReportResult.java @@ -0,0 +1,31 @@ +package org.avniproject.etl.dto; + +public class AggregateReportResult { + private String label; + private Long value; + private String id; + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + + public Long getValue() { + return value; + } + + public void setValue(Long value) { + this.value = value; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } +} diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java index 6922a6a..689c5c7 100644 --- a/src/main/java/org/avniproject/etl/repository/ReportRepository.java +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -1,8 +1,11 @@ package org.avniproject.etl.repository; +import org.avniproject.etl.dto.AggregateReportResult; import org.avniproject.etl.dto.UserActivityDTO; +import org.avniproject.etl.repository.rowMappers.reports.AggregateReportMapper; import org.avniproject.etl.repository.rowMappers.reports.UserActivityMapper; import org.avniproject.etl.repository.rowMappers.reports.UserCountMapper; +import org.avniproject.etl.repository.rowMappers.reports.UserDetailsMapper; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.stereotype.Component; @@ -60,7 +63,7 @@ public List getUserActivity(String orgSchemaName, String subjec " left join ${schemaName}.encounters e on e.last_modified_by_id = u.id\n" + " left join ${schemaName}.enrolments enl on enl.last_modified_by_id = u.id\n" + " left join ${schemaName}.program_encounters enc on enc.last_modified_by_id = u.id\n" + - "where u.is_voided = false and u.organisation_id notnull\n" + + "where (u.is_voided = false or u.is_voided isnull) and u.organisation_id notnull\n" + " and coalesce(coalesce(registration_count, 0) + coalesce(encounter_count, 0) + coalesce(enrolment_count, 0) +\n" + " coalesce(program_encounter_count, 0), 0) > 0\n" + " ${userWhere}\n" + @@ -94,4 +97,64 @@ public List generateUserSyncFailures(String orgSchemaName, Stri .replace("${userWhere}", userWhere); return jdbcTemplate.query(query, new UserCountMapper()); } + + public List generateUserAppVersions(String orgSchemaName, String userWhere) { + String baseQuery = "select app_version as indicator,\n" + + " count(*) as count\n" + + "from ${schemaName}.users u\n" + + " join\n" + + " (select user_id,\n" + + " app_version,\n" + + " row_number() over (partition by user_id order by sync_start_time desc ) as rn\n" + + " from ${schemaName}.sync_telemetry) l on l.user_id = u.id and rn = 1\n" + + "where (u.is_voided = false or u.is_voided isnull) and u.organisation_id notnull\n" + + "${userWhere}\n"+ + "group by app_version;"; + String query = baseQuery + .replace("${schemaName}", orgSchemaName) + .replace("${userWhere}", userWhere); + return jdbcTemplate.query(query, new AggregateReportMapper()); + } + + public List generateUserDeviceModels(String orgSchemaName, String userWhere) { + String baseQuery = "select device_model as indicator,\n" + + " count(*) as count\n" + + "from ${schemaName}.users u\n" + + " join\n" + + " (select user_id,\n" + + " coalesce(device_info ->> 'brand', device_name) as device_model,\n" + + " row_number() over (partition by user_id order by sync_start_time desc ) as rn\n" + + " from ${schemaName}.sync_telemetry) l on l.user_id = u.id and rn = 1\n" + + "where (u.is_voided = false or u.is_voided isnull) and u.organisation_id notnull \n" + + "${userWhere}\n"+ + "group by device_model;"; + String query = baseQuery + .replace("${schemaName}", orgSchemaName) + .replace("${userWhere}", userWhere); + return jdbcTemplate.query(query, new AggregateReportMapper()); + } + + public List generateUserDetails(String orgSchemaName, String userWhere) { + String baseQuery = "select coalesce(u.name, u.username) as name,\n" + + " app_version,\n" + + " device_model,\n" + + " sync_start_time\n" + + "from ${schemaName}.users u\n" + + " join\n" + + " (select user_id,\n" + + " app_version,\n" + + " coalesce(device_info ->> 'brand', device_name) as device_model,\n" + + " sync_start_time,\n" + + " row_number() over (partition by user_id order by sync_start_time desc ) as rn\n" + + " from ${schemaName}.sync_telemetry\n" + + " where sync_status = 'complete') l on l.user_id = u.id and rn = 1\n" + + "where (u.is_voided = false or u.is_voided isnull)\n" + + " and u.organisation_id notnull\n" + + " ${userWhere}\n"+ + "order by 1 desc;"; + String query = baseQuery + .replace("${schemaName}", orgSchemaName) + .replace("${userWhere}", userWhere); + return jdbcTemplate.query(query, new UserDetailsMapper()); + } } diff --git a/src/main/java/org/avniproject/etl/repository/rowMappers/reports/AggregateReportMapper.java b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/AggregateReportMapper.java new file mode 100644 index 0000000..d214b57 --- /dev/null +++ b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/AggregateReportMapper.java @@ -0,0 +1,19 @@ +package org.avniproject.etl.repository.rowMappers.reports; + +import org.avniproject.etl.dto.AggregateReportResult; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class AggregateReportMapper implements RowMapper { + @Override + public AggregateReportResult mapRow(ResultSet rs, int rowNum) throws SQLException { + AggregateReportResult aggregateReportResult = new AggregateReportResult(); + aggregateReportResult.setLabel(rs.getString("indicator")); + aggregateReportResult.setValue(rs.getLong("count")); + aggregateReportResult.setId(rs.getString("indicator")); + + return aggregateReportResult; + } +} diff --git a/src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserDetailsMapper.java b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserDetailsMapper.java new file mode 100644 index 0000000..5f95123 --- /dev/null +++ b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserDetailsMapper.java @@ -0,0 +1,20 @@ +package org.avniproject.etl.repository.rowMappers.reports; + +import org.avniproject.etl.dto.UserActivityDTO; +import org.joda.time.DateTime; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class UserDetailsMapper implements RowMapper { + @Override + public UserActivityDTO mapRow(ResultSet rs, int rowNum) throws SQLException { + UserActivityDTO userActivityDTO = new UserActivityDTO(); + userActivityDTO.setUserName(rs.getString("name")); + userActivityDTO.setAppVersion(rs.getString("app_version")); + userActivityDTO.setDeviceModel(rs.getString("device_model")); + userActivityDTO.setLastSuccessfulSync(new DateTime(rs.getDate("sync_start_time"))); + return userActivityDTO; + } +} From 433fd0be15acf8231151801e5c4d2ba62f7a7a45 Mon Sep 17 00:00:00 2001 From: ak2502 Date: Thu, 3 Aug 2023 17:10:53 +0530 Subject: [PATCH 23/36] avniproject/avni-product#1334 | adding rest of the endpoints for HR tab Signed-off-by: ak2502 --- .../etl/controller/ReportController.java | 48 ++++++++--- .../etl/repository/ReportRepository.java | 86 ++++++++++++++++++- 2 files changed, 117 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/avniproject/etl/controller/ReportController.java b/src/main/java/org/avniproject/etl/controller/ReportController.java index 3c04ebb..47f4e22 100644 --- a/src/main/java/org/avniproject/etl/controller/ReportController.java +++ b/src/main/java/org/avniproject/etl/controller/ReportController.java @@ -4,11 +4,9 @@ import org.avniproject.etl.dto.AggregateReportResult; import org.avniproject.etl.dto.UserActivityDTO; import org.avniproject.etl.repository.ReportRepository; -import org.avniproject.etl.service.EtlService; import org.avniproject.etl.service.ReportService; import org.avniproject.etl.util.ReportUtil; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; import java.util.List; @@ -28,24 +26,16 @@ public ReportController(ReportService reportService, ReportRepository reportRepo } @RequestMapping(value = "reports/hr/userActivity", method = RequestMethod.GET) - public ResponseEntity getUserActivity(@RequestParam(value = "start_date", required = false) String startDate, + public List getUserActivity(@RequestParam(value = "startDate", required = false) String startDate, @RequestParam(value = "endDate", required = false) String endDate, - @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) - { - try { - List userActivity = reportRepository.getUserActivity( + @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds){ + return reportRepository.generateUserActivity( OrgIdentityContextHolder.getDbSchema(), reportUtil.getDateDynamicWhere(startDate, endDate, "registration_date"), reportUtil.getDateDynamicWhere(startDate, endDate, "encounter_date_time"), reportUtil.getDateDynamicWhere(startDate, endDate, "enrolment_date_time"), reportUtil.getDynamicUserWhere(userIds, "u.id") ); - - return ResponseEntity.ok(userActivity); - } - catch (Exception e) { - return ResponseEntity.badRequest().body(e.getMessage()); - } } @RequestMapping(value = "/report/hr/syncFailures",method = RequestMethod.GET) @@ -83,6 +73,38 @@ public List getUserDetails(@RequestParam(value = "userIds", req reportUtil.getDynamicUserWhere(userIds, "u.id")); } + @RequestMapping(value = "/report/hr/championUsers", method = RequestMethod.GET) + public List getChampionUsers(@RequestParam(value = "startDate", required = false) String startDate, + @RequestParam(value = "endDate", required = false) String endDate, + @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) { + return reportRepository.generateCompletedVisitsOnTimeByProportion( + ">= 0.8", + OrgIdentityContextHolder.getDbSchema(), + reportUtil.getDateDynamicWhere(startDate, endDate, "encounter_date_time"), + reportUtil.getDynamicUserWhere(userIds, "u.id")); + } + + @RequestMapping(value = "/report/hr/nonPerformingUsers", method = RequestMethod.GET) + public List getNonPerformingUsers(@RequestParam(value = "startDate", required = false) String startDate, + @RequestParam(value = "endDate", required = false) String endDate, + @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) { + return reportRepository.generateCompletedVisitsOnTimeByProportion( + "<= 0.5", + OrgIdentityContextHolder.getDbSchema(), + reportUtil.getDateDynamicWhere(startDate, endDate, "encounter_date_time"), + reportUtil.getDynamicUserWhere(userIds, "u.id") + ); + } + + @RequestMapping(value = "/report/hr/mostCancelled", method = RequestMethod.GET) + public List getUsersCancellingMostVisits(@RequestParam(value = "startDate", required = false) String startDate, + @RequestParam(value = "endDate", required = false) String endDate, + @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) { + return reportRepository.generateUserCancellingMostVisits( + OrgIdentityContextHolder.getDbSchema(), + reportUtil.getDateDynamicWhere(startDate, endDate, "encounter_date_time"), + reportUtil.getDynamicUserWhere(userIds, "u.id")); + } } diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java index 689c5c7..e3ce884 100644 --- a/src/main/java/org/avniproject/etl/repository/ReportRepository.java +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -1,11 +1,13 @@ package org.avniproject.etl.repository; import org.avniproject.etl.dto.AggregateReportResult; +import org.apache.log4j.Logger; import org.avniproject.etl.dto.UserActivityDTO; import org.avniproject.etl.repository.rowMappers.reports.AggregateReportMapper; import org.avniproject.etl.repository.rowMappers.reports.UserActivityMapper; import org.avniproject.etl.repository.rowMappers.reports.UserCountMapper; import org.avniproject.etl.repository.rowMappers.reports.UserDetailsMapper; +import org.avniproject.etl.service.EtlService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.stereotype.Component; @@ -15,13 +17,14 @@ @Component public class ReportRepository { private final NamedParameterJdbcTemplate jdbcTemplate; + private static final Logger log = Logger.getLogger(EtlService.class); @Autowired public ReportRepository(NamedParameterJdbcTemplate jdbcTemplate) { this.jdbcTemplate = jdbcTemplate; } - public List getUserActivity(String orgSchemaName, String subjectWhere, String encounterWhere, String enrolmentWhere, String userWhere) { + public List generateUserActivity(String orgSchemaName, String subjectWhere, String encounterWhere, String enrolmentWhere, String userWhere) { String baseQuery = "with registrations as (\n" + " select last_modified_by_id, count(*) as registration_count\n" + " from ${schemaName}.individual\n" + @@ -75,6 +78,7 @@ public List getUserActivity(String orgSchemaName, String subjec .replace("${encounterWhere}", encounterWhere) .replace("${enrolmentWhere}", enrolmentWhere) .replace("${userWhere}", userWhere); + log.info(query); return jdbcTemplate.query(query, new UserActivityMapper()); } @@ -82,7 +86,7 @@ public List generateUserSyncFailures(String orgSchemaName, Stri String baseQuery = "select coalesce(u.name, u.username) as name, \n" + " count(*) as count\n" + "from ${schemaName}.sync_telemetry st\n" + - " join ${schemaName}.user u on st.user_id = u.id\n" + + " join ${schemaName}.users u on st.user_id = u.id\n" + "where sync_status = 'incomplete'\n" + "and (u.is_voided = false or u.is_voided isnull)\n" + "and u.organisation_id notnull\n" + @@ -122,7 +126,7 @@ public List generateUserDeviceModels(String orgSchemaName "from ${schemaName}.users u\n" + " join\n" + " (select user_id,\n" + - " coalesce(device_info ->> 'brand', device_name) as device_model,\n" + + " device_name as device_model,\n" + " row_number() over (partition by user_id order by sync_start_time desc ) as rn\n" + " from ${schemaName}.sync_telemetry) l on l.user_id = u.id and rn = 1\n" + "where (u.is_voided = false or u.is_voided isnull) and u.organisation_id notnull \n" + @@ -143,7 +147,7 @@ public List generateUserDetails(String orgSchemaName, String us " join\n" + " (select user_id,\n" + " app_version,\n" + - " coalesce(device_info ->> 'brand', device_name) as device_model,\n" + + " device_name as device_model,\n" + " sync_start_time,\n" + " row_number() over (partition by user_id order by sync_start_time desc ) as rn\n" + " from ${schemaName}.sync_telemetry\n" + @@ -157,4 +161,78 @@ public List generateUserDetails(String orgSchemaName, String us .replace("${userWhere}", userWhere); return jdbcTemplate.query(query, new UserDetailsMapper()); } + + public List generateCompletedVisitsOnTimeByProportion(String proportionCondition, String orgSchemaName, String encounterWhere, String userWhere) { + String baseQuery = "with program_enc_data as (\n" + + " select last_modified_by_id,\n" + + " count(*) filter ( where encounter_date_time <= max_visit_date_time ) visits_done_on_time,\n" + + " count(*) filter ( where encounter_date_time notnull and earliest_visit_date_time notnull ) total_scheduled\n" + + " from program_encounter\n" + + " where is_voided = false\n" + + " ${encounterWhere}\n" + + " group by last_modified_by_id\n" + + "),\n" + + " general_enc_data as (\n" + + " select last_modified_by_id,\n" + + " count(*) filter ( where encounter_date_time <= max_visit_date_time ) visits_done_on_time,\n" + + " count(*)\n" + + " filter ( where encounter_date_time notnull and earliest_visit_date_time notnull ) total_scheduled\n" + + " from encounter\n" + + " where is_voided = false\n" + + " ${encounterWhere}\n" + + " group by last_modified_by_id\n" + + " )\n" + + "select coalesce(u.name, u.username) as indicator,\n" + + " coalesce(ged.visits_done_on_time, 0) + coalesce(ped.visits_done_on_time, 0) as count\n" + + "from users u\n" + + " join general_enc_data ged on ged.last_modified_by_id = u.id\n" + + " join program_enc_data ped on ped.last_modified_by_id = u.id\n" + + "where u.organisation_id notnull\n" + + " and is_voided = false\n" + + " and coalesce(ged.visits_done_on_time, 0) + coalesce(ped.visits_done_on_time, 0) > 0\n" + + " ${userWhere}\n" + + " and ((coalesce(ged.visits_done_on_time, 0.0) + coalesce(ped.visits_done_on_time, 0.0)) /\n" + + " nullif((coalesce(ged.total_scheduled, 0) + coalesce(ped.total_scheduled, 0)), 0)) ${proportion_condition}\n"; + String query = baseQuery + .replace("${proportion_condition}", proportionCondition) + .replace("${schemaName}", orgSchemaName) + .replace("${encounterWhere}", encounterWhere) + .replace("${userWhere}", userWhere); + return jdbcTemplate.query(query, new AggregateReportMapper()); + } + + public List generateUserCancellingMostVisits(String orgSchemaName, String encounterWhere, String userWhere) { + String baseQuery = "with program_enc_data as (\n" + + " select last_modified_by_id,\n" + + " count(*) filter ( where cancel_date_time notnull ) cancelled_visits\n" + + " from program_encounter\n" + + " where is_voided = false\n" + + " ${encounterWhere}\n" + + " group by last_modified_by_id\n" + + "),\n" + + " general_enc_data as (\n" + + " select last_modified_by_id,\n" + + " count(*) filter ( where cancel_date_time notnull ) cancelled_visits\n" + + " from encounter\n" + + " where is_voided = false\n" + + " ${encounterWhere}\n" + + " group by last_modified_by_id\n" + + " )\n" + + "select coalesce(u.name, u.username) as indicator,\n" + + " coalesce(ged.cancelled_visits, 0) + coalesce(ped.cancelled_visits, 0) as count\n" + + "from users u\n" + + " join general_enc_data ged on ged.last_modified_by_id = u.id\n" + + " join program_enc_data ped on ped.last_modified_by_id = u.id\n" + + "where u.organisation_id notnull\n" + + " and is_voided = false\n" + + " and coalesce(ged.cancelled_visits, 0) + coalesce(ped.cancelled_visits, 0) > 0 \n" + + " ${userWhere}\n" + + "order by coalesce(ged.cancelled_visits, 0.0) + coalesce(ped.cancelled_visits, 0.0) desc\n" + + "limit 5;"; + String query = baseQuery + .replace("${schemaName}", orgSchemaName) + .replace("${encounterWhere}", encounterWhere) + .replace("${userWhere}", userWhere); + return jdbcTemplate.query(query, new AggregateReportMapper()); + } } From 3493ae3483107ee2cbdff5595e2cbf54d7f24532 Mon Sep 17 00:00:00 2001 From: ak2502 Date: Mon, 14 Aug 2023 11:22:47 +0530 Subject: [PATCH 24/36] avniproject/avni-product#1334 | Joda issue fixed Signed-off-by: ak2502 --- build.gradle | 6 ++++++ .../etl/util/ObjectMapperSingleton.java | 17 +++++++++++++++++ src/main/resources/application.properties | 2 ++ 3 files changed, 25 insertions(+) create mode 100644 src/main/java/org/avniproject/etl/util/ObjectMapperSingleton.java diff --git a/build.gradle b/build.gradle index 254dc91..b3799d2 100644 --- a/build.gradle +++ b/build.gradle @@ -30,6 +30,12 @@ dependencies { implementation 'org.keycloak:keycloak-admin-client:21.0.2' implementation 'org.antlr:ST4:4.3.4' implementation 'org.hibernate:hibernate-validator:7.0.5.Final' + implementation "joda-time:joda-time:2.9.4" + implementation 'com.fasterxml.jackson.core:jackson-databind:2.12.6' + implementation 'com.fasterxml.jackson.core:jackson-core:2.12.6' + implementation 'com.fasterxml.jackson.core:jackson-annotations:2.12.6' + implementation 'com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.12.6' + implementation 'com.fasterxml.jackson.datatype:jackson-datatype-joda:2.12.6' } tasks.named('test') { diff --git a/src/main/java/org/avniproject/etl/util/ObjectMapperSingleton.java b/src/main/java/org/avniproject/etl/util/ObjectMapperSingleton.java new file mode 100644 index 0000000..ede7d43 --- /dev/null +++ b/src/main/java/org/avniproject/etl/util/ObjectMapperSingleton.java @@ -0,0 +1,17 @@ +package org.avniproject.etl.util; + +import com.fasterxml.jackson.datatype.joda.JodaModule; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; + +@Configuration +@PropertySource("classpath:application.properties") +public class ObjectMapperSingleton { + + @Bean + public JodaModule jodaModule() { + JodaModule module = new JodaModule(); + return module; + } +} \ No newline at end of file diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 1446e28..f4ef015 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -2,3 +2,5 @@ avni.database=${OPENCHS_DATABASE_NAME:openchs} spring.config.import=classpath:/main-application.properties debug=${ETL_DEBUG_MODE:false} avni.current.time.offset.seconds=${AVNI_CURRENT_TIME_OFFSET_SECONDS:10} +spring.jackson.serialization.write-dates-as-timestamps=false +spring.jackson.default-property-inclusion=non_null From a3e562211f503935983fb69a62b6c5ecaf4f179a Mon Sep 17 00:00:00 2001 From: ak2502 Date: Thu, 17 Aug 2023 13:33:38 +0530 Subject: [PATCH 25/36] avniproject/avni-product#1334 | generateUserActivity query updated according to ETL Signed-off-by: ak2502 --- .../etl/controller/ReportController.java | 2 +- .../etl/repository/ReportRepository.java | 131 ++++++++++++------ 2 files changed, 93 insertions(+), 40 deletions(-) diff --git a/src/main/java/org/avniproject/etl/controller/ReportController.java b/src/main/java/org/avniproject/etl/controller/ReportController.java index 47f4e22..d162a78 100644 --- a/src/main/java/org/avniproject/etl/controller/ReportController.java +++ b/src/main/java/org/avniproject/etl/controller/ReportController.java @@ -25,7 +25,7 @@ public ReportController(ReportService reportService, ReportRepository reportRepo this.reportUtil = reportUtil; } - @RequestMapping(value = "reports/hr/userActivity", method = RequestMethod.GET) + @RequestMapping(value = "report/hr/userActivity", method = RequestMethod.GET) public List getUserActivity(@RequestParam(value = "startDate", required = false) String startDate, @RequestParam(value = "endDate", required = false) String endDate, @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds){ diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java index e3ce884..cb1ce00 100644 --- a/src/main/java/org/avniproject/etl/repository/ReportRepository.java +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -1,5 +1,7 @@ package org.avniproject.etl.repository; +import org.avniproject.etl.domain.metadata.SchemaMetadata; +import org.avniproject.etl.domain.metadata.TableMetadata; import org.avniproject.etl.dto.AggregateReportResult; import org.apache.log4j.Logger; import org.avniproject.etl.dto.UserActivityDTO; @@ -12,74 +14,125 @@ import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.stereotype.Component; +import java.util.Collection; import java.util.List; +import java.util.stream.Stream; @Component public class ReportRepository { private final NamedParameterJdbcTemplate jdbcTemplate; + private final SchemaMetadataRepository schemaMetadataRepository; private static final Logger log = Logger.getLogger(EtlService.class); @Autowired - public ReportRepository(NamedParameterJdbcTemplate jdbcTemplate) { + public ReportRepository(NamedParameterJdbcTemplate jdbcTemplate, SchemaMetadataRepository schemaMetadataRepository) { this.jdbcTemplate = jdbcTemplate; + this.schemaMetadataRepository = schemaMetadataRepository; } public List generateUserActivity(String orgSchemaName, String subjectWhere, String encounterWhere, String enrolmentWhere, String userWhere) { - String baseQuery = "with registrations as (\n" + - " select last_modified_by_id, count(*) as registration_count\n" + - " from ${schemaName}.individual\n" + - " where is_voided = false\n" + - " ${subjectWhere}\n" + - " group by last_modified_by_id\n" + - "),\n" + - " encounters as (\n" + - " select last_modified_by_id, count(*) as encounter_count\n" + - " from ${schemaName}.encounter\n" + + SchemaMetadata schema = schemaMetadataRepository.getExistingSchemaMetadata(); + List subjectTables = Stream.of(schema.getAllSubjectTables()).flatMap(Collection::stream).toList(); + List encounterTables = schema.getAllEncounterTables().stream().toList(); + List programEnrolmentTables = schema.getAllProgramEnrolmentTables().stream().toList(); + List programEncounterTables = schema.getAllProgramEncounterTables().stream().toList(); + + StringBuilder query = new StringBuilder(); + + String baseQuery1 = "select last_modified_by_id, count(*) as registration_count\n" + + " from ${schemaName}.${subject}\n" + + " where is_voided = false\n" + + " ${subjectWhere}\n" + + " group by last_modified_by_id\n" ; + baseQuery1 = baseQuery1.replace("${schemaName}", orgSchemaName) + .replace("${subjectWhere}", subjectWhere); + query.append("with registrations as (\n"); + query.append(baseQuery1.replace("${subject}", subjectTables.get(0).getName())); + for (int i = 1 ; i < subjectTables.size() ; i++) { + query.append("union all\n" ); + query.append(baseQuery1.replace("${subject}", subjectTables.get(i).getName())); + } + query.append("),\n"); + + String baseQuery2 = "select last_modified_by_id, count(*) as encounter_count\n" + + " from ${schemaName}.${encounter}\n" + " where is_voided = false\n" + " ${encounterWhere}\n" + - " group by last_modified_by_id\n" + - " ),\n" + - " enrolments as (\n" + - " select last_modified_by_id, count(*) as enrolment_count\n" + - " from ${schemaName}.program_enrolment\n" + + " group by last_modified_by_id\n" ; + baseQuery2 = baseQuery2.replace("${schemaName}", orgSchemaName) + .replace("${encounterWhere}", encounterWhere); + query.append("encounters as (\n"); + query.append(baseQuery2.replace("${encounter}", encounterTables.get(0).getName())); + for (int i = 1 ; i < encounterTables.size() ; i++) { + query.append("union all\n" ); + query.append(baseQuery2.replace("${encounter}", encounterTables.get(i).getName())); + } + query.append("),\n"); + + String baseQuery3 = "select last_modified_by_id, count(*) as enrolment_count\n" + + " from ${schemaName}.${enrolment}\n" + " where is_voided = false\n" + " ${enrolmentWhere}\n" + - " group by last_modified_by_id\n" + - " ),\n" + - " program_encounters as (\n" + - " select last_modified_by_id, count(*) as program_encounter_count\n" + - " from ${schemaName}.program_encounter\n" + + " group by last_modified_by_id\n" ; + baseQuery3 = baseQuery3.replace("${schemaName}", orgSchemaName) + .replace("${enrolmentWhere}", enrolmentWhere); + query.append("enrolments as (\n"); + query.append(baseQuery3.replace("${enrolment}", programEnrolmentTables.get(0).getName())); + for (int i = 1 ; i < programEnrolmentTables.size() ; i++) { + query.append("union all\n" ); + query.append(baseQuery3.replace("${enrolment}", programEnrolmentTables.get(i).getName())); + } + query.append("),\n"); + + String baseQuery4 = "select last_modified_by_id, count(*) as program_encounter_count\n" + + " from ${schemaName}.${programEncounter}\n" + " where is_voided = false\n" + " ${encounterWhere}\n" + - " group by last_modified_by_id\n" + - " )\n" + + " group by last_modified_by_id\n" ; + baseQuery4 = baseQuery4.replace("${schemaName}", orgSchemaName) + .replace("${encounterWhere}", encounterWhere); + query.append("program_encounters as (\n"); + query.append(baseQuery4.replace("${programEncounter}", programEncounterTables.get(0).getName())); + for (int i = 1 ; i < programEncounterTables.size() ; i++) { + query.append("union all\n" ); + query.append(baseQuery4.replace("${programEncounter}", programEncounterTables.get(i).getName())); + } + query.append("),\n"); + + String baseQuery5 = "activity_table as (\n" + "select u.id as id,\n" + " coalesce(u.name, u.username) as name,\n" + " coalesce(registration_count, 0) as registration_count,\n" + " coalesce(encounter_count, 0) as encounter_count,\n" + " coalesce(enrolment_count, 0) as enrolment_count,\n" + - " coalesce(program_encounter_count, 0) as program_encounter_count,\n" + - " coalesce(coalesce(registration_count, 0) + coalesce(encounter_count, 0) + coalesce(enrolment_count, 0) +\n" + - " coalesce(program_encounter_count, 0), 0) as total\n" + + " coalesce(program_encounter_count, 0) as program_encounter_count\n" + "from ${schemaName}.users u\n" + - " left join ${schemaName}.registrations r on r.last_modified_by_id = u.id\n" + - " left join ${schemaName}.encounters e on e.last_modified_by_id = u.id\n" + - " left join ${schemaName}.enrolments enl on enl.last_modified_by_id = u.id\n" + - " left join ${schemaName}.program_encounters enc on enc.last_modified_by_id = u.id\n" + + " left join registrations r on r.last_modified_by_id = u.id\n" + + " left join encounters e on e.last_modified_by_id = u.id\n" + + " left join enrolments enl on enl.last_modified_by_id = u.id\n" + + " left join program_encounters enc on enc.last_modified_by_id = u.id\n" + "where (u.is_voided = false or u.is_voided isnull) and u.organisation_id notnull\n" + " and coalesce(coalesce(registration_count, 0) + coalesce(encounter_count, 0) + coalesce(enrolment_count, 0) +\n" + " coalesce(program_encounter_count, 0), 0) > 0\n" + - " ${userWhere}\n" + + " ${userWhere}\n)," + + " final_table as (\n" + + "select id, name,\n"+ + " sum(distinct registration_count) as registration_count, \n" + + " sum(distinct encounter_count) as encounter_count,\n" + + " sum(distinct enrolment_count) as enrolment_count,\n" + + " sum(distinct program_encounter_count) as program_encounter_count\n" + + "from activity_table \n" + + "group by id,name\n" + + ")" + + "select *, coalesce(coalesce(registration_count, 0) + coalesce(encounter_count, 0) + coalesce(enrolment_count, 0) +\n" + + " coalesce(program_encounter_count, 0), 0) as total\n" + + "from final_table\n" + "order by 7 desc\n" + "limit 10;"; - String query = baseQuery - .replace("${schemaName}", orgSchemaName) - .replace("${subjectWhere}", subjectWhere) - .replace("${encounterWhere}", encounterWhere) - .replace("${enrolmentWhere}", enrolmentWhere) - .replace("${userWhere}", userWhere); - log.info(query); - return jdbcTemplate.query(query, new UserActivityMapper()); + query.append(baseQuery5.replace("${schemaName}", orgSchemaName) + .replace("${userWhere}", userWhere)); + + return jdbcTemplate.query(query.toString(), new UserActivityMapper()); } public List generateUserSyncFailures(String orgSchemaName, String syncTelemetryWhere, String userWhere) { From e0918799dc004e08185d1f4d1ebc6ad5a615acbb Mon Sep 17 00:00:00 2001 From: ak2502 Date: Fri, 18 Aug 2023 14:04:41 +0530 Subject: [PATCH 26/36] avniproject/avni-product#1334 | modified generateCompletedVisitsOnTimeByProportion query according to ETL Signed-off-by: ak2502 --- .../etl/controller/ReportController.java | 2 +- .../etl/repository/ReportRepository.java | 87 ++++++++++++------- 2 files changed, 58 insertions(+), 31 deletions(-) diff --git a/src/main/java/org/avniproject/etl/controller/ReportController.java b/src/main/java/org/avniproject/etl/controller/ReportController.java index d162a78..0b914c9 100644 --- a/src/main/java/org/avniproject/etl/controller/ReportController.java +++ b/src/main/java/org/avniproject/etl/controller/ReportController.java @@ -78,7 +78,7 @@ public List getChampionUsers(@RequestParam(value = "start @RequestParam(value = "endDate", required = false) String endDate, @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) { return reportRepository.generateCompletedVisitsOnTimeByProportion( - ">= 0.8", + ">= 0.5", OrgIdentityContextHolder.getDbSchema(), reportUtil.getDateDynamicWhere(startDate, endDate, "encounter_date_time"), reportUtil.getDynamicUserWhere(userIds, "u.id")); diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java index cb1ce00..fc4e674 100644 --- a/src/main/java/org/avniproject/etl/repository/ReportRepository.java +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -216,42 +216,69 @@ public List generateUserDetails(String orgSchemaName, String us } public List generateCompletedVisitsOnTimeByProportion(String proportionCondition, String orgSchemaName, String encounterWhere, String userWhere) { - String baseQuery = "with program_enc_data as (\n" + - " select last_modified_by_id,\n" + - " count(*) filter ( where encounter_date_time <= max_visit_date_time ) visits_done_on_time,\n" + + SchemaMetadata schema = schemaMetadataRepository.getExistingSchemaMetadata(); + List encounterTables = schema.getAllEncounterTables().stream().toList(); + List programEncounterTables = schema.getAllProgramEncounterTables().stream().toList(); + + StringBuilder query = new StringBuilder(); + String baseQuery1 = "select last_modified_by_id,\n" + + " count(*) filter ( where encounter_date_time <= max_visit_date_time ) visits_done_on_time,\n" + " count(*) filter ( where encounter_date_time notnull and earliest_visit_date_time notnull ) total_scheduled\n" + - " from program_encounter\n" + + " from ${schemaName}.${programEncounter}\n" + " where is_voided = false\n" + " ${encounterWhere}\n" + - " group by last_modified_by_id\n" + - "),\n" + - " general_enc_data as (\n" + - " select last_modified_by_id,\n" + - " count(*) filter ( where encounter_date_time <= max_visit_date_time ) visits_done_on_time,\n" + - " count(*)\n" + - " filter ( where encounter_date_time notnull and earliest_visit_date_time notnull ) total_scheduled\n" + - " from encounter\n" + + " group by last_modified_by_id\n" ; + baseQuery1 = baseQuery1.replace("${schemaName}", orgSchemaName) + .replace("${encounterWhere}", encounterWhere); + query.append("with program_enc_data as (\n" ); + query.append(baseQuery1.replace("${programEncounter}", programEncounterTables.get(0).getName())); + for (int i = 1; i < programEncounterTables.size(); i++) { + query.append("union all\n"); + query.append(baseQuery1.replace("${programEncounter}", programEncounterTables.get(i).getName())); + } + query.append("),\n"); + + String baseQuery2 = "select last_modified_by_id,\n" + + " count(*) filter ( where encounter_date_time <= max_visit_date_time ) visits_done_on_time,\n" + + " count(*) filter ( where encounter_date_time notnull and earliest_visit_date_time notnull ) total_scheduled\n" + + " from ${schemaName}.${encounter}\n" + " where is_voided = false\n" + " ${encounterWhere}\n" + - " group by last_modified_by_id\n" + - " )\n" + - "select coalesce(u.name, u.username) as indicator,\n" + - " coalesce(ged.visits_done_on_time, 0) + coalesce(ped.visits_done_on_time, 0) as count\n" + - "from users u\n" + - " join general_enc_data ged on ged.last_modified_by_id = u.id\n" + - " join program_enc_data ped on ped.last_modified_by_id = u.id\n" + - "where u.organisation_id notnull\n" + - " and is_voided = false\n" + - " and coalesce(ged.visits_done_on_time, 0) + coalesce(ped.visits_done_on_time, 0) > 0\n" + - " ${userWhere}\n" + - " and ((coalesce(ged.visits_done_on_time, 0.0) + coalesce(ped.visits_done_on_time, 0.0)) /\n" + - " nullif((coalesce(ged.total_scheduled, 0) + coalesce(ped.total_scheduled, 0)), 0)) ${proportion_condition}\n"; - String query = baseQuery - .replace("${proportion_condition}", proportionCondition) + " group by last_modified_by_id\n" ; + baseQuery2 = baseQuery2.replace("${schemaName}", orgSchemaName) + .replace("${encounterWhere}", encounterWhere); + query.append("general_enc_data as (\n"); + query.append(baseQuery2.replace("${encounter}", encounterTables.get(0).getName())); + for (int i = 1 ; i < encounterTables.size() ; i++) { + query.append("union all\n" ); + query.append(baseQuery2.replace("${encounter}", encounterTables.get(i).getName())); + } + query.append("),\n"); + + String baseQuery3 = "table1 as (\n" + + " select coalesce(u.name, u.username) as indicator,\n" + + " coalesce(sum(distinct ged.visits_done_on_time), 0) as ged_visits_on_time,\n" + + " coalesce(sum(distinct ped.visits_done_on_time), 0) as ped_visits_on_time,\n" + + " coalesce(sum(distinct ged.total_scheduled), 0) as ged_total_scheduled,\n" + + " coalesce(sum(distinct ped.total_scheduled), 0) as ped_total_scheduled\n" + + " from org1_schema.users u \n" + + " left join general_enc_data ged on ged.last_modified_by_id = u.id\n" + + " left join program_enc_data ped on ped.last_modified_by_id = u.id\n" + + " where u.organisation_id notnull\n" + + " and (is_voided = false or is_voided isnull)\n" + + " ${userWhere}\n" + + " group by u.name,u.username \n" + + ")\n" + + "select indicator,\n" + + " ged_visits_on_time + ped_visits_on_time as count\n" + + "from table1\n" + + "where ged_visits_on_time + ped_visits_on_time > 0\n" + + " and ((ged_visits_on_time + ped_visits_on_time) /\n" + + "nullif(ged_total_scheduled + ped_total_scheduled, 0)) ${proportion_condition};"; + query.append(baseQuery3.replace("${proportion_condition}", proportionCondition) .replace("${schemaName}", orgSchemaName) - .replace("${encounterWhere}", encounterWhere) - .replace("${userWhere}", userWhere); - return jdbcTemplate.query(query, new AggregateReportMapper()); + .replace("${userWhere}", userWhere)); + return jdbcTemplate.query(query.toString(), new AggregateReportMapper()); } public List generateUserCancellingMostVisits(String orgSchemaName, String encounterWhere, String userWhere) { From 20d0ab98413b956c68f5f2243d52ba826b759956 Mon Sep 17 00:00:00 2001 From: ak2502 Date: Fri, 18 Aug 2023 17:04:24 +0530 Subject: [PATCH 27/36] avniproject/avni-product#1334 | modified generateUserCancellingMostVisits query according to ETL Signed-off-by: ak2502 --- .../etl/repository/ReportRepository.java | 80 +++++++++++++------ 1 file changed, 55 insertions(+), 25 deletions(-) diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java index fc4e674..96aaa4b 100644 --- a/src/main/java/org/avniproject/etl/repository/ReportRepository.java +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -261,7 +261,7 @@ public List generateCompletedVisitsOnTimeByProportion(Str " coalesce(sum(distinct ped.visits_done_on_time), 0) as ped_visits_on_time,\n" + " coalesce(sum(distinct ged.total_scheduled), 0) as ged_total_scheduled,\n" + " coalesce(sum(distinct ped.total_scheduled), 0) as ped_total_scheduled\n" + - " from org1_schema.users u \n" + + " from ${schemaName}.users u \n" + " left join general_enc_data ged on ged.last_modified_by_id = u.id\n" + " left join program_enc_data ped on ped.last_modified_by_id = u.id\n" + " where u.organisation_id notnull\n" + @@ -282,37 +282,67 @@ public List generateCompletedVisitsOnTimeByProportion(Str } public List generateUserCancellingMostVisits(String orgSchemaName, String encounterWhere, String userWhere) { - String baseQuery = "with program_enc_data as (\n" + - " select last_modified_by_id,\n" + - " count(*) filter ( where cancel_date_time notnull ) cancelled_visits\n" + - " from program_encounter\n" + + SchemaMetadata schema = schemaMetadataRepository.getExistingSchemaMetadata(); + List encounterTables = schema.getAllEncounterTables().stream().toList(); + List programEncounterTables = schema.getAllProgramEncounterTables().stream().toList(); + + StringBuilder query = new StringBuilder(); + String baseQuery1 = " select last_modified_by_id,\n" + + " count(*) " + + "filter ( where cancel_date_time notnull ) " + + "cancelled_visits\n" + + " from ${schemaName}.${programEncounter}\n" + " where is_voided = false\n" + " ${encounterWhere}\n" + - " group by last_modified_by_id\n" + - "),\n" + - " general_enc_data as (\n" + - " select last_modified_by_id,\n" + - " count(*) filter ( where cancel_date_time notnull ) cancelled_visits\n" + - " from encounter\n" + + " group by last_modified_by_id\n" ; + baseQuery1 = baseQuery1.replace("${schemaName}", orgSchemaName) + .replace("${encounterWhere}", encounterWhere); + query.append("with program_enc_data as (\n" ); + query.append(baseQuery1.replace("${programEncounter}", programEncounterTables.get(0).getName())); + for (int i = 1; i < programEncounterTables.size(); i++) { + query.append("union all\n"); + query.append(baseQuery1.replace("${programEncounter}", programEncounterTables.get(i).getName())); + } + query.append("),\n"); + + String baseQuery2 = " select last_modified_by_id,\n" + + " count(*) " + + "filter ( where cancel_date_time notnull )" + + " cancelled_visits\n" + + " from ${schemaName}.${encounter}\n" + " where is_voided = false\n" + " ${encounterWhere}\n" + - " group by last_modified_by_id\n" + - " )\n" + - "select coalesce(u.name, u.username) as indicator,\n" + - " coalesce(ged.cancelled_visits, 0) + coalesce(ped.cancelled_visits, 0) as count\n" + - "from users u\n" + - " join general_enc_data ged on ged.last_modified_by_id = u.id\n" + - " join program_enc_data ped on ped.last_modified_by_id = u.id\n" + + " group by last_modified_by_id\n" ; + baseQuery2 = baseQuery2.replace("${schemaName}", orgSchemaName) + .replace("${encounterWhere}", encounterWhere); + query.append("general_enc_data as (\n"); + query.append(baseQuery2.replace("${encounter}", encounterTables.get(0).getName())); + for (int i = 1 ; i < encounterTables.size() ; i++) { + query.append("union all\n" ); + query.append(baseQuery2.replace("${encounter}", encounterTables.get(i).getName())); + } + query.append("),\n"); + + String baseQuery3 ="table1 as (\n" + + "select coalesce(u.name, u.username) as indicator,\n" + + " coalesce(sum(distinct ged.cancelled_visits), 0) as ged_cancelled_visits,\n" + + " coalesce(sum(distinct ped.cancelled_visits), 0) as ped_cancelled_visits\n" + + "from ${schemaName}.users u\n" + + " left join general_enc_data ged on ged.last_modified_by_id = u.id\n" + + " left join program_enc_data ped on ped.last_modified_by_id = u.id\n" + "where u.organisation_id notnull\n" + - " and is_voided = false\n" + + " and (is_voided = false or is_voided isnull)\n" + " and coalesce(ged.cancelled_visits, 0) + coalesce(ped.cancelled_visits, 0) > 0 \n" + " ${userWhere}\n" + - "order by coalesce(ged.cancelled_visits, 0.0) + coalesce(ped.cancelled_visits, 0.0) desc\n" + + " group by u.name,u.username \n" + + ")\n" + + "select indicator,\n" + + " ged_cancelled_visits + ped_cancelled_visits as count\n" + + "from table1\n" + + "order by ged_cancelled_visits + ped_cancelled_visits desc\n" + "limit 5;"; - String query = baseQuery - .replace("${schemaName}", orgSchemaName) - .replace("${encounterWhere}", encounterWhere) - .replace("${userWhere}", userWhere); - return jdbcTemplate.query(query, new AggregateReportMapper()); + query.append(baseQuery3.replace("${schemaName}", orgSchemaName) + .replace("${userWhere}", userWhere)); + return jdbcTemplate.query(query.toString(), new AggregateReportMapper()); } } From 7e7ddd206818896c638399983e51523c4fbd5a62 Mon Sep 17 00:00:00 2001 From: ak2502 Date: Mon, 21 Aug 2023 14:56:04 +0530 Subject: [PATCH 28/36] avniproject/avni-product#1334 | adding analytics_user in ReportController Signed-off-by: ak2502 --- .../org/avniproject/etl/controller/ReportController.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/org/avniproject/etl/controller/ReportController.java b/src/main/java/org/avniproject/etl/controller/ReportController.java index 0b914c9..a9d0a1e 100644 --- a/src/main/java/org/avniproject/etl/controller/ReportController.java +++ b/src/main/java/org/avniproject/etl/controller/ReportController.java @@ -7,6 +7,7 @@ import org.avniproject.etl.service.ReportService; import org.avniproject.etl.util.ReportUtil; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.*; import java.util.List; @@ -25,6 +26,7 @@ public ReportController(ReportService reportService, ReportRepository reportRepo this.reportUtil = reportUtil; } + @PreAuthorize("hasAnyAuthority('analytics_user')") @RequestMapping(value = "report/hr/userActivity", method = RequestMethod.GET) public List getUserActivity(@RequestParam(value = "startDate", required = false) String startDate, @RequestParam(value = "endDate", required = false) String endDate, @@ -38,6 +40,7 @@ public List getUserActivity(@RequestParam(value = "startDate", ); } + @PreAuthorize("hasAnyAuthority('analytics_user')") @RequestMapping(value = "/report/hr/syncFailures",method = RequestMethod.GET) public List getUserWiseSyncFailures(@RequestParam(value = "startDate", required = false) String startDate, @RequestParam(value = "endDate", required = false) String endDate, @@ -49,6 +52,7 @@ public List getUserWiseSyncFailures(@RequestParam(value = "star ); } + @PreAuthorize("hasAnyAuthority('analytics_user')") @RequestMapping(value = "/report/hr/deviceModels", method = RequestMethod.GET) public List getUserWiseDeviceModels(@RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) { @@ -57,6 +61,7 @@ public List getUserWiseDeviceModels(@RequestParam(value = reportUtil.getDynamicUserWhere(userIds, "u.id")); } + @PreAuthorize("hasAnyAuthority('analytics_user')") @RequestMapping(value = "/report/hr/appVersions", method = RequestMethod.GET) public List getUserWiseAppVersions(@RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) { @@ -65,6 +70,7 @@ public List getUserWiseAppVersions(@RequestParam(value = reportUtil.getDynamicUserWhere(userIds, "u.id")); } + @PreAuthorize("hasAnyAuthority('analytics_user')") @RequestMapping(value = "/report/hr/userDetails", method = RequestMethod.GET) public List getUserDetails(@RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) { @@ -73,6 +79,7 @@ public List getUserDetails(@RequestParam(value = "userIds", req reportUtil.getDynamicUserWhere(userIds, "u.id")); } + @PreAuthorize("hasAnyAuthority('analytics_user')") @RequestMapping(value = "/report/hr/championUsers", method = RequestMethod.GET) public List getChampionUsers(@RequestParam(value = "startDate", required = false) String startDate, @RequestParam(value = "endDate", required = false) String endDate, @@ -84,6 +91,7 @@ public List getChampionUsers(@RequestParam(value = "start reportUtil.getDynamicUserWhere(userIds, "u.id")); } + @PreAuthorize("hasAnyAuthority('analytics_user')") @RequestMapping(value = "/report/hr/nonPerformingUsers", method = RequestMethod.GET) public List getNonPerformingUsers(@RequestParam(value = "startDate", required = false) String startDate, @RequestParam(value = "endDate", required = false) String endDate, @@ -96,6 +104,7 @@ public List getNonPerformingUsers(@RequestParam(value = " ); } + @PreAuthorize("hasAnyAuthority('analytics_user')") @RequestMapping(value = "/report/hr/mostCancelled", method = RequestMethod.GET) public List getUsersCancellingMostVisits(@RequestParam(value = "startDate", required = false) String startDate, @RequestParam(value = "endDate", required = false) String endDate, From dc2304040b034da5ac9fabe23233eae24a033dc3 Mon Sep 17 00:00:00 2001 From: ak2502 Date: Mon, 21 Aug 2023 17:24:05 +0530 Subject: [PATCH 29/36] avniproject/avni-product#1334 | Sync telemetry - Latest Syncs endpoint added Signed-off-by: ak2502 --- .../etl/controller/ReportController.java | 11 +++++ .../avniproject/etl/dto/UserActivityDTO.java | 46 ++++++++++++++++++- .../etl/repository/ReportRepository.java | 25 +++++++--- .../rowMappers/reports/LatestSyncMapper.java | 25 ++++++++++ 4 files changed, 99 insertions(+), 8 deletions(-) create mode 100644 src/main/java/org/avniproject/etl/repository/rowMappers/reports/LatestSyncMapper.java diff --git a/src/main/java/org/avniproject/etl/controller/ReportController.java b/src/main/java/org/avniproject/etl/controller/ReportController.java index a9d0a1e..96f1784 100644 --- a/src/main/java/org/avniproject/etl/controller/ReportController.java +++ b/src/main/java/org/avniproject/etl/controller/ReportController.java @@ -79,6 +79,17 @@ public List getUserDetails(@RequestParam(value = "userIds", req reportUtil.getDynamicUserWhere(userIds, "u.id")); } + @PreAuthorize("hasAnyAuthority('analytics_user')") + @RequestMapping(value = "/report/hr/latestSyncs", method = RequestMethod.GET) + public List getLatestSyncs(@RequestParam(value = "startDate", required = false) String startDate, + @RequestParam(value = "endDate", required = false) String endDate, + @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) { + + return reportRepository.generateLatestSyncs( + OrgIdentityContextHolder.getDbSchema(), + reportUtil.getDateDynamicWhere(startDate, endDate, "sync_end_time"), + reportUtil.getDynamicUserWhere(userIds, "u.id")); + } @PreAuthorize("hasAnyAuthority('analytics_user')") @RequestMapping(value = "/report/hr/championUsers", method = RequestMethod.GET) public List getChampionUsers(@RequestParam(value = "startDate", required = false) String startDate, diff --git a/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java b/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java index 392f9d3..9362396 100644 --- a/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java +++ b/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java @@ -11,8 +11,14 @@ public class UserActivityDTO { private Long programEncounterCount; private Long generalEncounterCount; private Long count; + private String androidVersion; private String appVersion; private String deviceModel; + private String syncStatus; + private String syncSource; + private DateTime syncStart; + private DateTime syncEnd; + private DateTime lastSuccessfulSync; public String getUserName() { return userName; @@ -70,6 +76,14 @@ public void setCount(Long count) { this.count = count; } + public String getAndroidVersion() { + return androidVersion; + } + + public void setAndroidVersion(String androidVersion) { + this.androidVersion = androidVersion; + } + public String getAppVersion() { return appVersion; } @@ -94,5 +108,35 @@ public void setLastSuccessfulSync(DateTime lastSuccessfulSync) { this.lastSuccessfulSync = lastSuccessfulSync; } - private DateTime lastSuccessfulSync; + public DateTime getSyncStart() { + return syncStart; + } + + public void setSyncStart(DateTime syncStart) { + this.syncStart = syncStart; + } + + public DateTime getSyncEnd() { + return syncEnd; + } + + public void setSyncEnd(DateTime syncEnd) { + this.syncEnd = syncEnd; + } + + public String getSyncStatus() { + return syncStatus; + } + + public void setSyncStatus(String syncStatus) { + this.syncStatus = syncStatus; + } + + public String getSyncSource() { + return syncSource; + } + + public void setSyncSource(String syncSource) { + this.syncSource = syncSource; + } } diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java index 96aaa4b..80b0579 100644 --- a/src/main/java/org/avniproject/etl/repository/ReportRepository.java +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -5,18 +5,13 @@ import org.avniproject.etl.dto.AggregateReportResult; import org.apache.log4j.Logger; import org.avniproject.etl.dto.UserActivityDTO; -import org.avniproject.etl.repository.rowMappers.reports.AggregateReportMapper; -import org.avniproject.etl.repository.rowMappers.reports.UserActivityMapper; -import org.avniproject.etl.repository.rowMappers.reports.UserCountMapper; -import org.avniproject.etl.repository.rowMappers.reports.UserDetailsMapper; +import org.avniproject.etl.repository.rowMappers.reports.*; import org.avniproject.etl.service.EtlService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.stereotype.Component; -import java.util.Collection; import java.util.List; -import java.util.stream.Stream; @Component public class ReportRepository { @@ -32,7 +27,7 @@ public ReportRepository(NamedParameterJdbcTemplate jdbcTemplate, SchemaMetadataR public List generateUserActivity(String orgSchemaName, String subjectWhere, String encounterWhere, String enrolmentWhere, String userWhere) { SchemaMetadata schema = schemaMetadataRepository.getExistingSchemaMetadata(); - List subjectTables = Stream.of(schema.getAllSubjectTables()).flatMap(Collection::stream).toList(); + List subjectTables = schema.getAllSubjectTables().stream().toList(); List encounterTables = schema.getAllEncounterTables().stream().toList(); List programEnrolmentTables = schema.getAllProgramEnrolmentTables().stream().toList(); List programEncounterTables = schema.getAllProgramEncounterTables().stream().toList(); @@ -215,6 +210,22 @@ public List generateUserDetails(String orgSchemaName, String us return jdbcTemplate.query(query, new UserDetailsMapper()); } + public List generateLatestSyncs(String orgSchemaName, String syncTelemetryWhere, String userWhere) { + String baseQuery = "SELECT coalesce(u.name,u.username) as name, \n" + + " android_version, app_version, device_name, sync_start_time, sync_end_time, sync_status, sync_source\n" + + "FROM public.sync_telemetry st\n" + + "join ${schemaName}.users u on st.last_modified_by_id = u.id\n" + + "where (u.is_voided = false or u.is_voided isnull) and u.organisation_id notnull\n" + + "${syncTelemetryWhere}\n"+ + "${userWhere}\n"+ + "order by 6 desc\n" + + "limit 10;\n"; + String query = baseQuery + .replace("${schemaName}", orgSchemaName) + .replace("${syncTelemetryWhere}", syncTelemetryWhere) + .replace("${userWhere}", userWhere); + return jdbcTemplate.query(query, new LatestSyncMapper()); + } public List generateCompletedVisitsOnTimeByProportion(String proportionCondition, String orgSchemaName, String encounterWhere, String userWhere) { SchemaMetadata schema = schemaMetadataRepository.getExistingSchemaMetadata(); List encounterTables = schema.getAllEncounterTables().stream().toList(); diff --git a/src/main/java/org/avniproject/etl/repository/rowMappers/reports/LatestSyncMapper.java b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/LatestSyncMapper.java new file mode 100644 index 0000000..640d437 --- /dev/null +++ b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/LatestSyncMapper.java @@ -0,0 +1,25 @@ +package org.avniproject.etl.repository.rowMappers.reports; + +import org.avniproject.etl.dto.UserActivityDTO; +import org.joda.time.DateTime; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class LatestSyncMapper implements RowMapper { + + @Override + public UserActivityDTO mapRow(ResultSet rs, int rowNum) throws SQLException { + UserActivityDTO userActivityDTO = new UserActivityDTO(); + userActivityDTO.setUserName(rs.getString("name")); + userActivityDTO.setAndroidVersion(rs.getString("android_version")); + userActivityDTO.setAppVersion(rs.getString("app_version")); + userActivityDTO.setDeviceModel(rs.getString("device_name")); + userActivityDTO.setSyncStart(new DateTime(rs.getDate("sync_start_time"))); + userActivityDTO.setSyncEnd(new DateTime(rs.getDate("sync_end_time"))); + userActivityDTO.setSyncStatus(rs.getString("sync_status")); + userActivityDTO.setSyncSource(rs.getString("sync_source")); + return userActivityDTO; + } +} From dfe8708531a8ae7d30b6208a0ae283a86d41a5b7 Mon Sep 17 00:00:00 2001 From: ak2502 Date: Tue, 22 Aug 2023 22:16:23 +0530 Subject: [PATCH 30/36] avniproject/avni-product#1334 | modified code with string template Signed-off-by: ak2502 --- .../etl/domain/metadata/SchemaMetadata.java | 36 ++ .../etl/repository/ReportRepository.java | 369 +++++++++--------- 2 files changed, 222 insertions(+), 183 deletions(-) diff --git a/src/main/java/org/avniproject/etl/domain/metadata/SchemaMetadata.java b/src/main/java/org/avniproject/etl/domain/metadata/SchemaMetadata.java index cb80569..48c6b0e 100644 --- a/src/main/java/org/avniproject/etl/domain/metadata/SchemaMetadata.java +++ b/src/main/java/org/avniproject/etl/domain/metadata/SchemaMetadata.java @@ -42,18 +42,54 @@ public List getAllSubjectTables() { return tableMetadata.stream().filter(TableMetadata::isSubjectTable).toList(); } + public List getAllSubjectTableNames() { + List subjectTables = getAllSubjectTables(); + List subjectTableNames = new ArrayList<>(); + for(TableMetadata subject : subjectTables){ + subjectTableNames.add(subject.getName()); + } + return subjectTableNames; + } + public List getAllProgramEnrolmentTables() { return tableMetadata.stream().filter(table -> table.getType() == TableMetadata.Type.ProgramEnrolment).toList(); } + public List getAllProgramEnrolmentTableNames() { + List programEnrolmentTables = getAllProgramEnrolmentTables(); + List programEnrolmentTableNames = new ArrayList<>(); + for(TableMetadata programEnrolment : programEnrolmentTables){ + programEnrolmentTableNames.add(programEnrolment.getName()); + } + return programEnrolmentTableNames; + } + public List getAllProgramEncounterTables() { return tableMetadata.stream().filter(table -> table.getType() == TableMetadata.Type.ProgramEncounter).toList(); } + public List getAllProgramEncounterTableNames() { + List programEncounterTables = getAllProgramEncounterTables(); + List programEncounterTableNames = new ArrayList<>(); + for(TableMetadata programEncounter : programEncounterTables){ + programEncounterTableNames.add(programEncounter.getName()); + } + return programEncounterTableNames; + } + public List getAllEncounterTables() { return tableMetadata.stream().filter(table -> table.getType() == TableMetadata.Type.Encounter).toList(); } + public List getAllEncounterTableNames() { + List encounterTables = getAllEncounterTables(); + List encounterTableNames = new ArrayList<>(); + for(TableMetadata encounter : encounterTables){ + encounterTableNames.add(encounter.getName()); + } + return encounterTableNames; + } + private List findChanges(SchemaMetadata currentSchema, TableMetadata newTable) { List diffs = new ArrayList<>(); Optional optionalMatchingTable = currentSchema.findMatchingTable(newTable); diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java index 80b0579..9e566eb 100644 --- a/src/main/java/org/avniproject/etl/repository/ReportRepository.java +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -1,15 +1,13 @@ package org.avniproject.etl.repository; import org.avniproject.etl.domain.metadata.SchemaMetadata; -import org.avniproject.etl.domain.metadata.TableMetadata; import org.avniproject.etl.dto.AggregateReportResult; -import org.apache.log4j.Logger; import org.avniproject.etl.dto.UserActivityDTO; import org.avniproject.etl.repository.rowMappers.reports.*; -import org.avniproject.etl.service.EtlService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.stereotype.Component; +import org.stringtemplate.v4.ST; import java.util.List; @@ -17,7 +15,6 @@ public class ReportRepository { private final NamedParameterJdbcTemplate jdbcTemplate; private final SchemaMetadataRepository schemaMetadataRepository; - private static final Logger log = Logger.getLogger(EtlService.class); @Autowired public ReportRepository(NamedParameterJdbcTemplate jdbcTemplate, SchemaMetadataRepository schemaMetadataRepository) { @@ -27,107 +24,109 @@ public ReportRepository(NamedParameterJdbcTemplate jdbcTemplate, SchemaMetadataR public List generateUserActivity(String orgSchemaName, String subjectWhere, String encounterWhere, String enrolmentWhere, String userWhere) { SchemaMetadata schema = schemaMetadataRepository.getExistingSchemaMetadata(); - List subjectTables = schema.getAllSubjectTables().stream().toList(); - List encounterTables = schema.getAllEncounterTables().stream().toList(); - List programEnrolmentTables = schema.getAllProgramEnrolmentTables().stream().toList(); - List programEncounterTables = schema.getAllProgramEncounterTables().stream().toList(); + List subjectTableNames = schema.getAllSubjectTableNames().stream().toList(); + List encounterTableNames = schema.getAllEncounterTableNames().stream().toList(); + List programEnrolmentTableNames = schema.getAllProgramEnrolmentTableNames().stream().toList(); + List programEncounterTableNames = schema.getAllProgramEncounterTableNames().stream().toList(); - StringBuilder query = new StringBuilder(); - - String baseQuery1 = "select last_modified_by_id, count(*) as registration_count\n" + - " from ${schemaName}.${subject}\n" + + ST baseQuery = new ST("with registrations as (\n" + + " select last_modified_by_id, count(*) as registration_count\n" + + " from $schemaName.\n" + " where is_voided = false\n" + - " ${subjectWhere}\n" + - " group by last_modified_by_id\n" ; - baseQuery1 = baseQuery1.replace("${schemaName}", orgSchemaName) - .replace("${subjectWhere}", subjectWhere); - query.append("with registrations as (\n"); - query.append(baseQuery1.replace("${subject}", subjectTables.get(0).getName())); - for (int i = 1 ; i < subjectTables.size() ; i++) { - query.append("union all\n" ); - query.append(baseQuery1.replace("${subject}", subjectTables.get(i).getName())); - } - query.append("),\n"); - - String baseQuery2 = "select last_modified_by_id, count(*) as encounter_count\n" + - " from ${schemaName}.${encounter}\n" + + " $subjectWhere\n" + + " group by last_modified_by_id\n" + + " \n" + " where is_voided = false\n" + - " ${encounterWhere}\n" + - " group by last_modified_by_id\n" ; - baseQuery2 = baseQuery2.replace("${schemaName}", orgSchemaName) - .replace("${encounterWhere}", encounterWhere); - query.append("encounters as (\n"); - query.append(baseQuery2.replace("${encounter}", encounterTables.get(0).getName())); - for (int i = 1 ; i < encounterTables.size() ; i++) { - query.append("union all\n" ); - query.append(baseQuery2.replace("${encounter}", encounterTables.get(i).getName())); - } - query.append("),\n"); - - String baseQuery3 = "select last_modified_by_id, count(*) as enrolment_count\n" + - " from ${schemaName}.${enrolment}\n" + + " $subjectWhere \n" + + " group by last_modified_by_id \n}> " + + " ),\n " + + "encounters as (\n" + + " select last_modified_by_id, count(*) as encounter_count\n" + + " from $schemaName.\n" + " where is_voided = false\n" + - " ${enrolmentWhere}\n" + - " group by last_modified_by_id\n" ; - baseQuery3 = baseQuery3.replace("${schemaName}", orgSchemaName) - .replace("${enrolmentWhere}", enrolmentWhere); - query.append("enrolments as (\n"); - query.append(baseQuery3.replace("${enrolment}", programEnrolmentTables.get(0).getName())); - for (int i = 1 ; i < programEnrolmentTables.size() ; i++) { - query.append("union all\n" ); - query.append(baseQuery3.replace("${enrolment}", programEnrolmentTables.get(i).getName())); - } - query.append("),\n"); - - String baseQuery4 = "select last_modified_by_id, count(*) as program_encounter_count\n" + - " from ${schemaName}.${programEncounter}\n" + + " $encounterWhere\n" + + " group by last_modified_by_id\n" + + " \n" + " where is_voided = false\n" + - " ${encounterWhere}\n" + - " group by last_modified_by_id\n" ; - baseQuery4 = baseQuery4.replace("${schemaName}", orgSchemaName) - .replace("${encounterWhere}", encounterWhere); - query.append("program_encounters as (\n"); - query.append(baseQuery4.replace("${programEncounter}", programEncounterTables.get(0).getName())); - for (int i = 1 ; i < programEncounterTables.size() ; i++) { - query.append("union all\n" ); - query.append(baseQuery4.replace("${programEncounter}", programEncounterTables.get(i).getName())); - } - query.append("),\n"); - - String baseQuery5 = "activity_table as (\n" + - "select u.id as id,\n" + + " $encounterWhere \n" + + " group by last_modified_by_id \n}> " + + " ),\n " + + "enrolments as (\n" + + " select last_modified_by_id, count(*) as enrolment_count\n" + + " from $schemaName.\n" + + " where is_voided = false\n" + + " $enrolmentWhere\n" + + " group by last_modified_by_id\n" + + " \n" + + " where is_voided = false\n" + + " $enrolmentWhere \n" + + " group by last_modified_by_id \n}> " + + " ),\n " + + "program_encounters as (\n" + + " select last_modified_by_id, count(*) as program_encounter_count\n" + + " from $schemaName.\n" + + " where is_voided = false\n" + + " $encounterWhere\n" + + " group by last_modified_by_id\n" + + " \n" + + " where is_voided = false\n" + + " $encounterWhere \n" + + " group by last_modified_by_id \n}> " + + " ),\n " + + "activity_table as (\n" + + " select u.id as id,\n" + " coalesce(u.name, u.username) as name,\n" + " coalesce(registration_count, 0) as registration_count,\n" + " coalesce(encounter_count, 0) as encounter_count,\n" + " coalesce(enrolment_count, 0) as enrolment_count,\n" + " coalesce(program_encounter_count, 0) as program_encounter_count\n" + - "from ${schemaName}.users u\n" + + " from $schemaName.users u\n" + " left join registrations r on r.last_modified_by_id = u.id\n" + " left join encounters e on e.last_modified_by_id = u.id\n" + " left join enrolments enl on enl.last_modified_by_id = u.id\n" + " left join program_encounters enc on enc.last_modified_by_id = u.id\n" + - "where (u.is_voided = false or u.is_voided isnull) and u.organisation_id notnull\n" + + " where (u.is_voided = false or u.is_voided isnull) and u.organisation_id notnull\n" + " and coalesce(coalesce(registration_count, 0) + coalesce(encounter_count, 0) + coalesce(enrolment_count, 0) +\n" + " coalesce(program_encounter_count, 0), 0) > 0\n" + - " ${userWhere}\n)," + - " final_table as (\n" + - "select id, name,\n"+ + " $userWhere \n" + + "),\n" + + "final_table as (\n" + + " select id, name,\n"+ " sum(distinct registration_count) as registration_count, \n" + " sum(distinct encounter_count) as encounter_count,\n" + " sum(distinct enrolment_count) as enrolment_count,\n" + " sum(distinct program_encounter_count) as program_encounter_count\n" + - "from activity_table \n" + - "group by id,name\n" + - ")" + + " from activity_table \n" + + " group by id,name\n" + + ")\n" + "select *, coalesce(coalesce(registration_count, 0) + coalesce(encounter_count, 0) + coalesce(enrolment_count, 0) +\n" + " coalesce(program_encounter_count, 0), 0) as total\n" + "from final_table\n" + "order by 7 desc\n" + - "limit 10;"; - query.append(baseQuery5.replace("${schemaName}", orgSchemaName) - .replace("${userWhere}", userWhere)); - - return jdbcTemplate.query(query.toString(), new UserActivityMapper()); + "limit 10;" + ); + baseQuery.add("subjectTableNames", subjectTableNames) + .add("encounterTableNames", encounterTableNames) + .add("programEnrolmentTableNames", programEnrolmentTableNames) + .add("programEncounterTableNames", programEncounterTableNames); + String query = baseQuery.render().replace("$schemaName", orgSchemaName) + .replace("$subjectWhere", subjectWhere) + .replace("$encounterWhere", encounterWhere) + .replace("$enrolmentWhere", enrolmentWhere) + .replace("$userWhere", userWhere); + return jdbcTemplate.query(query, new UserActivityMapper()); } public List generateUserSyncFailures(String orgSchemaName, String syncTelemetryWhere, String userWhere) { @@ -228,132 +227,136 @@ public List generateLatestSyncs(String orgSchemaName, String sy } public List generateCompletedVisitsOnTimeByProportion(String proportionCondition, String orgSchemaName, String encounterWhere, String userWhere) { SchemaMetadata schema = schemaMetadataRepository.getExistingSchemaMetadata(); - List encounterTables = schema.getAllEncounterTables().stream().toList(); - List programEncounterTables = schema.getAllProgramEncounterTables().stream().toList(); + List encounterTableNames = schema.getAllEncounterTableNames().stream().toList(); + List programEncounterTableNames = schema.getAllProgramEncounterTableNames().stream().toList(); - StringBuilder query = new StringBuilder(); - String baseQuery1 = "select last_modified_by_id,\n" + - " count(*) filter ( where encounter_date_time <= max_visit_date_time ) visits_done_on_time,\n" + + ST baseQuery = new ST("with program_enc_data as (\n " + + " select last_modified_by_id, \n" + + " count(*) filter ( where encounter_date_time \\<= max_visit_date_time ) visits_done_on_time,\n" + " count(*) filter ( where encounter_date_time notnull and earliest_visit_date_time notnull ) total_scheduled\n" + - " from ${schemaName}.${programEncounter}\n" + + " from $schemaName.\n" + " where is_voided = false\n" + - " ${encounterWhere}\n" + - " group by last_modified_by_id\n" ; - baseQuery1 = baseQuery1.replace("${schemaName}", orgSchemaName) - .replace("${encounterWhere}", encounterWhere); - query.append("with program_enc_data as (\n" ); - query.append(baseQuery1.replace("${programEncounter}", programEncounterTables.get(0).getName())); - for (int i = 1; i < programEncounterTables.size(); i++) { - query.append("union all\n"); - query.append(baseQuery1.replace("${programEncounter}", programEncounterTables.get(i).getName())); - } - query.append("),\n"); - - String baseQuery2 = "select last_modified_by_id,\n" + - " count(*) filter ( where encounter_date_time <= max_visit_date_time ) visits_done_on_time,\n" + + " $encounterWhere\n" + + " group by last_modified_by_id\n" + + " \n" + + " where is_voided = false\n" + + " $encounterWhere\n" + + " group by last_modified_by_id\n }> " + + "),\n" + + "general_enc_data as (\n" + + " select last_modified_by_id,\n" + + " count(*) filter ( where encounter_date_time \\<= max_visit_date_time ) visits_done_on_time,\n" + " count(*) filter ( where encounter_date_time notnull and earliest_visit_date_time notnull ) total_scheduled\n" + - " from ${schemaName}.${encounter}\n" + + " from $schemaName.\n" + " where is_voided = false\n" + - " ${encounterWhere}\n" + - " group by last_modified_by_id\n" ; - baseQuery2 = baseQuery2.replace("${schemaName}", orgSchemaName) - .replace("${encounterWhere}", encounterWhere); - query.append("general_enc_data as (\n"); - query.append(baseQuery2.replace("${encounter}", encounterTables.get(0).getName())); - for (int i = 1 ; i < encounterTables.size() ; i++) { - query.append("union all\n" ); - query.append(baseQuery2.replace("${encounter}", encounterTables.get(i).getName())); - } - query.append("),\n"); - - String baseQuery3 = "table1 as (\n" + - " select coalesce(u.name, u.username) as indicator,\n" + - " coalesce(sum(distinct ged.visits_done_on_time), 0) as ged_visits_on_time,\n" + - " coalesce(sum(distinct ped.visits_done_on_time), 0) as ped_visits_on_time,\n" + - " coalesce(sum(distinct ged.total_scheduled), 0) as ged_total_scheduled,\n" + - " coalesce(sum(distinct ped.total_scheduled), 0) as ped_total_scheduled\n" + - " from ${schemaName}.users u \n" + - " left join general_enc_data ged on ged.last_modified_by_id = u.id\n" + - " left join program_enc_data ped on ped.last_modified_by_id = u.id\n" + - " where u.organisation_id notnull\n" + - " and (is_voided = false or is_voided isnull)\n" + - " ${userWhere}\n" + - " group by u.name,u.username \n" + - ")\n" + - "select indicator,\n" + - " ged_visits_on_time + ped_visits_on_time as count\n" + - "from table1\n" + - "where ged_visits_on_time + ped_visits_on_time > 0\n" + - " and ((ged_visits_on_time + ped_visits_on_time) /\n" + - "nullif(ged_total_scheduled + ped_total_scheduled, 0)) ${proportion_condition};"; - query.append(baseQuery3.replace("${proportion_condition}", proportionCondition) - .replace("${schemaName}", orgSchemaName) - .replace("${userWhere}", userWhere)); - return jdbcTemplate.query(query.toString(), new AggregateReportMapper()); + " $encounterWhere\n" + + " group by last_modified_by_id\n" + + "\n" + + " where is_voided = false\n" + + " $encounterWhere\n" + + " group by last_modified_by_id \n }>" + + "),\n" + + "total_visits_table as (\n" + + " select coalesce(u.name, u.username) as indicator,\n" + + " coalesce(sum(distinct ged.visits_done_on_time), 0) as ged_visits_on_time,\n" + + " coalesce(sum(distinct ped.visits_done_on_time), 0) as ped_visits_on_time,\n" + + " coalesce(sum(distinct ged.total_scheduled), 0) as ged_total_scheduled,\n" + + " coalesce(sum(distinct ped.total_scheduled), 0) as ped_total_scheduled\n" + + " from $schemaName.users u \n" + + " left join general_enc_data ged on ged.last_modified_by_id = u.id\n" + + " left join program_enc_data ped on ped.last_modified_by_id = u.id\n" + + " where u.organisation_id notnull\n" + + " and (is_voided = false or is_voided isnull)\n" + + " $userWhere\n" + + " group by u.name, u.username \n" + + ")\n" + + "select indicator,\n" + + " ged_visits_on_time + ped_visits_on_time as count\n" + + "from total_visits_table\n" + + "where ged_visits_on_time + ped_visits_on_time > 0\n" + + " and ((ged_visits_on_time + ped_visits_on_time) /\n" + + "nullif(ged_total_scheduled + ped_total_scheduled, 0)) $proportion_condition;" + ); + baseQuery.add("programEncounterTableNames", programEncounterTableNames) + .add("encounterTableNames", encounterTableNames); + String query = baseQuery.render().replace("$proportion_condition", proportionCondition) + .replace("$schemaName", orgSchemaName) + .replace("$encounterWhere", encounterWhere) + .replace("$userWhere", userWhere); + return jdbcTemplate.query(query, new AggregateReportMapper()); } public List generateUserCancellingMostVisits(String orgSchemaName, String encounterWhere, String userWhere) { SchemaMetadata schema = schemaMetadataRepository.getExistingSchemaMetadata(); - List encounterTables = schema.getAllEncounterTables().stream().toList(); - List programEncounterTables = schema.getAllProgramEncounterTables().stream().toList(); + List encounterTableNames = schema.getAllEncounterTableNames().stream().toList(); + List programEncounterTableNames = schema.getAllProgramEncounterTableNames().stream().toList(); - StringBuilder query = new StringBuilder(); - String baseQuery1 = " select last_modified_by_id,\n" + - " count(*) " + - "filter ( where cancel_date_time notnull ) " + - "cancelled_visits\n" + - " from ${schemaName}.${programEncounter}\n" + + ST baseQuery = new ST("with program_enc_data as (\n " + + " select last_modified_by_id,\n" + + " count(*) filter ( where cancel_date_time notnull ) cancelled_visits\n" + + " from $schemaName.\n" + " where is_voided = false\n" + - " ${encounterWhere}\n" + - " group by last_modified_by_id\n" ; - baseQuery1 = baseQuery1.replace("${schemaName}", orgSchemaName) - .replace("${encounterWhere}", encounterWhere); - query.append("with program_enc_data as (\n" ); - query.append(baseQuery1.replace("${programEncounter}", programEncounterTables.get(0).getName())); - for (int i = 1; i < programEncounterTables.size(); i++) { - query.append("union all\n"); - query.append(baseQuery1.replace("${programEncounter}", programEncounterTables.get(i).getName())); - } - query.append("),\n"); - - String baseQuery2 = " select last_modified_by_id,\n" + - " count(*) " + - "filter ( where cancel_date_time notnull )" + - " cancelled_visits\n" + - " from ${schemaName}.${encounter}\n" + + " $encounterWhere\n" + + " group by last_modified_by_id\n" + + "\n" + + " where is_voided = false\n" + + " $encounterWhere\n" + + " group by last_modified_by_id \n }>" + + "),\n" + + "general_enc_data as (\n" + + " select last_modified_by_id,\n" + + " count(*) filter ( where cancel_date_time notnull ) cancelled_visits\n" + + " from $schemaName.\n" + " where is_voided = false\n" + - " ${encounterWhere}\n" + - " group by last_modified_by_id\n" ; - baseQuery2 = baseQuery2.replace("${schemaName}", orgSchemaName) - .replace("${encounterWhere}", encounterWhere); - query.append("general_enc_data as (\n"); - query.append(baseQuery2.replace("${encounter}", encounterTables.get(0).getName())); - for (int i = 1 ; i < encounterTables.size() ; i++) { - query.append("union all\n" ); - query.append(baseQuery2.replace("${encounter}", encounterTables.get(i).getName())); - } - query.append("),\n"); - - String baseQuery3 ="table1 as (\n" + + " $encounterWhere\n" + + " group by last_modified_by_id\n" + + "\n" + + " where is_voided = false\n" + + " $encounterWhere\n" + + " group by last_modified_by_id \n }>" + + "),\n" + + "cancelled_visits_table as (\n" + "select coalesce(u.name, u.username) as indicator,\n" + " coalesce(sum(distinct ged.cancelled_visits), 0) as ged_cancelled_visits,\n" + " coalesce(sum(distinct ped.cancelled_visits), 0) as ped_cancelled_visits\n" + - "from ${schemaName}.users u\n" + + "from $schemaName.users u\n" + " left join general_enc_data ged on ged.last_modified_by_id = u.id\n" + " left join program_enc_data ped on ped.last_modified_by_id = u.id\n" + "where u.organisation_id notnull\n" + " and (is_voided = false or is_voided isnull)\n" + " and coalesce(ged.cancelled_visits, 0) + coalesce(ped.cancelled_visits, 0) > 0 \n" + - " ${userWhere}\n" + + " $userWhere\n" + " group by u.name,u.username \n" + ")\n" + "select indicator,\n" + " ged_cancelled_visits + ped_cancelled_visits as count\n" + - "from table1\n" + + "from cancelled_visits_table\n" + "order by ged_cancelled_visits + ped_cancelled_visits desc\n" + - "limit 5;"; - query.append(baseQuery3.replace("${schemaName}", orgSchemaName) - .replace("${userWhere}", userWhere)); - return jdbcTemplate.query(query.toString(), new AggregateReportMapper()); + "limit 5;" + ); + baseQuery.add("programEncounterTableNames", programEncounterTableNames) + .add("encounterTableNames", encounterTableNames); + String query = baseQuery.render().replace("$schemaName", orgSchemaName) + .replace("$encounterWhere", encounterWhere) + .replace("$userWhere", userWhere); + return jdbcTemplate.query(query, new AggregateReportMapper()); } } From ca432dbf61e27cd5938e55fe498326897ec8f522 Mon Sep 17 00:00:00 2001 From: ak2502 Date: Tue, 22 Aug 2023 22:28:06 +0530 Subject: [PATCH 31/36] avniproject/avni-product#1334 | removed empty ReportService class Signed-off-by: ak2502 --- .../org/avniproject/etl/controller/ReportController.java | 5 +---- .../java/org/avniproject/etl/service/ReportService.java | 8 -------- 2 files changed, 1 insertion(+), 12 deletions(-) delete mode 100644 src/main/java/org/avniproject/etl/service/ReportService.java diff --git a/src/main/java/org/avniproject/etl/controller/ReportController.java b/src/main/java/org/avniproject/etl/controller/ReportController.java index 96f1784..69ebcf6 100644 --- a/src/main/java/org/avniproject/etl/controller/ReportController.java +++ b/src/main/java/org/avniproject/etl/controller/ReportController.java @@ -4,7 +4,6 @@ import org.avniproject.etl.dto.AggregateReportResult; import org.avniproject.etl.dto.UserActivityDTO; import org.avniproject.etl.repository.ReportRepository; -import org.avniproject.etl.service.ReportService; import org.avniproject.etl.util.ReportUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.access.prepost.PreAuthorize; @@ -15,13 +14,11 @@ @RestController public class ReportController { - private final ReportService reportService; private final ReportRepository reportRepository; private final ReportUtil reportUtil; @Autowired - public ReportController(ReportService reportService, ReportRepository reportRepository, ReportUtil reportUtil) { - this.reportService = reportService; + public ReportController(ReportRepository reportRepository, ReportUtil reportUtil) { this.reportRepository = reportRepository; this.reportUtil = reportUtil; } diff --git a/src/main/java/org/avniproject/etl/service/ReportService.java b/src/main/java/org/avniproject/etl/service/ReportService.java deleted file mode 100644 index ec5f58e..0000000 --- a/src/main/java/org/avniproject/etl/service/ReportService.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.avniproject.etl.service; - -import org.springframework.stereotype.Service; - -@Service -public class ReportService { - -} From fb0796f2505a9deb9a2607646526e653794bbb6a Mon Sep 17 00:00:00 2001 From: ak2502 Date: Wed, 23 Aug 2023 17:45:36 +0530 Subject: [PATCH 32/36] avniproject/avni-product#1334 | summaryTable endpoint added in Activities tab Signed-off-by: ak2502 --- .../etl/controller/ReportController.java | 10 ++++++++++ .../avniproject/etl/dto/UserActivityDTO.java | 18 ++++++++++++++++++ .../etl/repository/ReportRepository.java | 9 +++++++++ .../rowMappers/reports/SummaryTableMapper.java | 17 +++++++++++++++++ 4 files changed, 54 insertions(+) create mode 100644 src/main/java/org/avniproject/etl/repository/rowMappers/reports/SummaryTableMapper.java diff --git a/src/main/java/org/avniproject/etl/controller/ReportController.java b/src/main/java/org/avniproject/etl/controller/ReportController.java index 69ebcf6..31f202b 100644 --- a/src/main/java/org/avniproject/etl/controller/ReportController.java +++ b/src/main/java/org/avniproject/etl/controller/ReportController.java @@ -23,6 +23,16 @@ public ReportController(ReportRepository reportRepository, ReportUtil reportUtil this.reportUtil = reportUtil; } + @PreAuthorize("hasAnyAuthority('analytics_user')") + @RequestMapping(value = "/report/aggregate/summaryTable", method = RequestMethod.GET) + public List getSummaryTable(@RequestParam(value = "startDate", required = false) String startDate, + @RequestParam(value = "endDate", required = false) String endDate, + @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds){ + return reportRepository.generateSummaryTable( + OrgIdentityContextHolder.getDbSchema() + ); + } + @PreAuthorize("hasAnyAuthority('analytics_user')") @RequestMapping(value = "report/hr/userActivity", method = RequestMethod.GET) public List getUserActivity(@RequestParam(value = "startDate", required = false) String startDate, diff --git a/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java b/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java index 9362396..1a00f8f 100644 --- a/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java +++ b/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java @@ -4,6 +4,8 @@ public class UserActivityDTO { + private String tableName; + private String tableType; private String userName; private Long id; private Long registrationCount; @@ -20,6 +22,22 @@ public class UserActivityDTO { private DateTime syncEnd; private DateTime lastSuccessfulSync; + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public String getTableType() { + return tableType; + } + + public void setTableType(String tableType) { + this.tableType = tableType; + } + public String getUserName() { return userName; } diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java index 9e566eb..d173bbf 100644 --- a/src/main/java/org/avniproject/etl/repository/ReportRepository.java +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -22,6 +22,15 @@ public ReportRepository(NamedParameterJdbcTemplate jdbcTemplate, SchemaMetadataR this.schemaMetadataRepository = schemaMetadataRepository; } + public List generateSummaryTable(String orgSchemaName){ + String baseQuery = "select name, type \n" + + "from public.table_metadata\n" + + "where schema_name = '${schemaName}'\n" + + "order by type;"; + String query= baseQuery.replace("${schemaName}", orgSchemaName); + return jdbcTemplate.query(query, new SummaryTableMapper()); + } + public List generateUserActivity(String orgSchemaName, String subjectWhere, String encounterWhere, String enrolmentWhere, String userWhere) { SchemaMetadata schema = schemaMetadataRepository.getExistingSchemaMetadata(); List subjectTableNames = schema.getAllSubjectTableNames().stream().toList(); diff --git a/src/main/java/org/avniproject/etl/repository/rowMappers/reports/SummaryTableMapper.java b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/SummaryTableMapper.java new file mode 100644 index 0000000..d882cde --- /dev/null +++ b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/SummaryTableMapper.java @@ -0,0 +1,17 @@ +package org.avniproject.etl.repository.rowMappers.reports; + +import org.avniproject.etl.dto.UserActivityDTO; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class SummaryTableMapper implements RowMapper { + @Override + public UserActivityDTO mapRow(ResultSet rs, int rowNum) throws SQLException { + UserActivityDTO userActivityDTO = new UserActivityDTO(); + userActivityDTO.setTableName(rs.getString("name")); + userActivityDTO.setTableType(rs.getString("type")); + return userActivityDTO; + } +} From b5c7e1bda086683601b32e15717383e44f9aab9b Mon Sep 17 00:00:00 2001 From: ak2502 Date: Thu, 24 Aug 2023 18:08:18 +0530 Subject: [PATCH 33/36] avniproject/avni-product#1334 | added endpoint for median sync time for past week table in hr tab Signed-off-by: ak2502 --- .../etl/controller/ReportController.java | 17 +++++++++++++++-- .../avniproject/etl/dto/UserActivityDTO.java | 9 +++++++++ .../etl/repository/ReportRepository.java | 18 ++++++++++++++++++ .../rowMappers/reports/MedianSyncMapper.java | 17 +++++++++++++++++ .../org/avniproject/etl/util/ReportUtil.java | 7 +++++++ 5 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/avniproject/etl/repository/rowMappers/reports/MedianSyncMapper.java diff --git a/src/main/java/org/avniproject/etl/controller/ReportController.java b/src/main/java/org/avniproject/etl/controller/ReportController.java index 31f202b..b0fff80 100644 --- a/src/main/java/org/avniproject/etl/controller/ReportController.java +++ b/src/main/java/org/avniproject/etl/controller/ReportController.java @@ -54,7 +54,7 @@ public List getUserWiseSyncFailures(@RequestParam(value = "star @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds){ return reportRepository.generateUserSyncFailures( OrgIdentityContextHolder.getDbSchema(), - reportUtil.getDateDynamicWhere(startDate, endDate, "sync_start_time"), + reportUtil.getDateDynamicWhere(startDate, endDate, "st.sync_start_time"), reportUtil.getDynamicUserWhere(userIds, "u.id") ); } @@ -94,9 +94,22 @@ public List getLatestSyncs(@RequestParam(value = "startDate", r return reportRepository.generateLatestSyncs( OrgIdentityContextHolder.getDbSchema(), - reportUtil.getDateDynamicWhere(startDate, endDate, "sync_end_time"), + reportUtil.getDateDynamicWhere(startDate, endDate, "st.sync_end_time"), reportUtil.getDynamicUserWhere(userIds, "u.id")); } + + @PreAuthorize("hasAnyAuthority('analytics_user')") + @RequestMapping(value = "/report/hr/medianSync", method = RequestMethod.GET) + public List getMedianSync(@RequestParam(value = "startDate", required = false) String startDate, + @RequestParam(value = "endDate", required = false) String endDate, + @RequestParam(value = "userIds", required = false, defaultValue = "") List userIds) { + + return reportRepository.generateMedianSync( + OrgIdentityContextHolder.getDbSchema(), + reportUtil.getDateDynamicMedianSync(startDate, endDate, "st.sync_start_time"), + reportUtil.getDynamicUserWhere(userIds, "u.id")); + } + @PreAuthorize("hasAnyAuthority('analytics_user')") @RequestMapping(value = "/report/hr/championUsers", method = RequestMethod.GET) public List getChampionUsers(@RequestParam(value = "startDate", required = false) String startDate, diff --git a/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java b/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java index 1a00f8f..5f555e3 100644 --- a/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java +++ b/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java @@ -21,6 +21,7 @@ public class UserActivityDTO { private DateTime syncStart; private DateTime syncEnd; private DateTime lastSuccessfulSync; + private String medianSync; public String getTableName() { return tableName; @@ -157,4 +158,12 @@ public String getSyncSource() { public void setSyncSource(String syncSource) { this.syncSource = syncSource; } + + public String getMedianSync() { + return medianSync; + } + + public void setMedianSync(String medianSync) { + this.medianSync = medianSync; + } } diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java index d173bbf..c8b56e0 100644 --- a/src/main/java/org/avniproject/etl/repository/ReportRepository.java +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -234,6 +234,24 @@ public List generateLatestSyncs(String orgSchemaName, String sy .replace("${userWhere}", userWhere); return jdbcTemplate.query(query, new LatestSyncMapper()); } + + public List generateMedianSync(String orgSchemaName, String syncTelemetryWhere, String userWhere) { + String baseQuery = "select coalesce(u.name, u.username) as name,\n" + + "coalesce(percentile_cont(0.5) within group (order by (sync_end_time-sync_start_time)), '00:00:00') as median_sync_time\n" + + "from ${schemaName}.sync_telemetry st\n" + + "join ${schemaName}.users u on st.last_modified_by_id = u.id\n" + + "where (u.is_voided = false or u.is_voided isnull) and u.organisation_id notnull\n" + + "${syncTelemetryWhere}\n"+ + "${userWhere}\n"+ + "group by u.name, u.username\n" + + "order by 2 desc;"; + String query = baseQuery + .replace("${schemaName}", orgSchemaName) + .replace("${syncTelemetryWhere}", syncTelemetryWhere) + .replace("${userWhere}", userWhere); + return jdbcTemplate.query(query, new MedianSyncMapper()); + } + public List generateCompletedVisitsOnTimeByProportion(String proportionCondition, String orgSchemaName, String encounterWhere, String userWhere) { SchemaMetadata schema = schemaMetadataRepository.getExistingSchemaMetadata(); List encounterTableNames = schema.getAllEncounterTableNames().stream().toList(); diff --git a/src/main/java/org/avniproject/etl/repository/rowMappers/reports/MedianSyncMapper.java b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/MedianSyncMapper.java new file mode 100644 index 0000000..7755d3c --- /dev/null +++ b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/MedianSyncMapper.java @@ -0,0 +1,17 @@ +package org.avniproject.etl.repository.rowMappers.reports; + +import org.avniproject.etl.dto.UserActivityDTO; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class MedianSyncMapper implements RowMapper { + @Override + public UserActivityDTO mapRow(ResultSet rs, int rowNum) throws SQLException { + UserActivityDTO userActivityDTO = new UserActivityDTO(); + userActivityDTO.setUserName(rs.getString("name")); + userActivityDTO.setMedianSync(rs.getString("median_sync_time")); + return userActivityDTO; + } +} diff --git a/src/main/java/org/avniproject/etl/util/ReportUtil.java b/src/main/java/org/avniproject/etl/util/ReportUtil.java index 2813240..54c362f 100644 --- a/src/main/java/org/avniproject/etl/util/ReportUtil.java +++ b/src/main/java/org/avniproject/etl/util/ReportUtil.java @@ -15,6 +15,13 @@ public String getDateDynamicWhere(String startDate, String endDate, String colum return ""; } + public String getDateDynamicMedianSync(String startDate, String endDate, String columnName) { + if (startDate != null) { + return format("and %s::date between ( '%s'::date - interval '7 days') and ('%s'::date - interval '7 days')", columnName, startDate, endDate); + } + return "and st.sync_start_time between ( current_date at time zone 'UTC'+'5:30' - interval '7 days') and current_date"; + } + public String getDynamicUserWhere(List userIds, String columnName) { if (!userIds.isEmpty()) { return format("and %s in (%s)", columnName, StrUtil.joinLongToList(userIds)); From 48ece8ba6faeaac36e9a1507a2c8aa69f2b44258 Mon Sep 17 00:00:00 2001 From: ak2502 Date: Mon, 4 Sep 2023 20:49:37 +0530 Subject: [PATCH 34/36] avniproject/avni-product#1334 | modified 'median sync-time table' to show weekly data for past 3 months Signed-off-by: ak2502 --- .../etl/controller/ReportController.java | 3 +-- .../etl/repository/ReportRepository.java | 26 ++++++++++--------- .../rowMappers/reports/MedianSyncMapper.java | 4 ++- .../org/avniproject/etl/util/ReportUtil.java | 6 ++--- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/avniproject/etl/controller/ReportController.java b/src/main/java/org/avniproject/etl/controller/ReportController.java index b0fff80..4e9f84b 100644 --- a/src/main/java/org/avniproject/etl/controller/ReportController.java +++ b/src/main/java/org/avniproject/etl/controller/ReportController.java @@ -106,8 +106,7 @@ public List getMedianSync(@RequestParam(value = "startDate", re return reportRepository.generateMedianSync( OrgIdentityContextHolder.getDbSchema(), - reportUtil.getDateDynamicMedianSync(startDate, endDate, "st.sync_start_time"), - reportUtil.getDynamicUserWhere(userIds, "u.id")); + reportUtil.getDateSeries(startDate, endDate)); } @PreAuthorize("hasAnyAuthority('analytics_user')") diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java index c8b56e0..f14b8f9 100644 --- a/src/main/java/org/avniproject/etl/repository/ReportRepository.java +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -235,20 +235,22 @@ public List generateLatestSyncs(String orgSchemaName, String sy return jdbcTemplate.query(query, new LatestSyncMapper()); } - public List generateMedianSync(String orgSchemaName, String syncTelemetryWhere, String userWhere) { - String baseQuery = "select coalesce(u.name, u.username) as name,\n" + - "coalesce(percentile_cont(0.5) within group (order by (sync_end_time-sync_start_time)), '00:00:00') as median_sync_time\n" + - "from ${schemaName}.sync_telemetry st\n" + - "join ${schemaName}.users u on st.last_modified_by_id = u.id\n" + - "where (u.is_voided = false or u.is_voided isnull) and u.organisation_id notnull\n" + - "${syncTelemetryWhere}\n"+ - "${userWhere}\n"+ - "group by u.name, u.username\n" + - "order by 2 desc;"; + public List generateMedianSync(String orgSchemaName, String syncTelemetryWhere) { + String baseQuery = "with weeks as (\n" + + " select day::date start_date, day::date+6 end_date\n" + + " ${syncTelemetryWhere}\n" + + ")\n" + + "select w.start_date, w.end_date, \n" + + " coalesce(percentile_cont(0.5) within group (order by (st.sync_end_time-st.sync_start_time)), '00:00:00') as median_sync_time\n" + + "from weeks w\t\n" + + "left join ${schemaName}.sync_telemetry st\n" + + "on st.sync_start_time::date >= w.start_date and st.sync_end_time::date <= w.end_date\n" + +// "and st.sync_source = 'manual'\n" + + "group by 1,2;"; String query = baseQuery .replace("${schemaName}", orgSchemaName) - .replace("${syncTelemetryWhere}", syncTelemetryWhere) - .replace("${userWhere}", userWhere); + .replace("${syncTelemetryWhere}", syncTelemetryWhere); + System.out.print(query); return jdbcTemplate.query(query, new MedianSyncMapper()); } diff --git a/src/main/java/org/avniproject/etl/repository/rowMappers/reports/MedianSyncMapper.java b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/MedianSyncMapper.java index 7755d3c..4b905e9 100644 --- a/src/main/java/org/avniproject/etl/repository/rowMappers/reports/MedianSyncMapper.java +++ b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/MedianSyncMapper.java @@ -1,6 +1,7 @@ package org.avniproject.etl.repository.rowMappers.reports; import org.avniproject.etl.dto.UserActivityDTO; +import org.joda.time.DateTime; import org.springframework.jdbc.core.RowMapper; import java.sql.ResultSet; @@ -10,7 +11,8 @@ public class MedianSyncMapper implements RowMapper { @Override public UserActivityDTO mapRow(ResultSet rs, int rowNum) throws SQLException { UserActivityDTO userActivityDTO = new UserActivityDTO(); - userActivityDTO.setUserName(rs.getString("name")); + userActivityDTO.setSyncStart(new DateTime(rs.getDate("start_date"))); + userActivityDTO.setSyncEnd(new DateTime(rs.getDate("end_date"))); userActivityDTO.setMedianSync(rs.getString("median_sync_time")); return userActivityDTO; } diff --git a/src/main/java/org/avniproject/etl/util/ReportUtil.java b/src/main/java/org/avniproject/etl/util/ReportUtil.java index 54c362f..239282b 100644 --- a/src/main/java/org/avniproject/etl/util/ReportUtil.java +++ b/src/main/java/org/avniproject/etl/util/ReportUtil.java @@ -15,11 +15,11 @@ public String getDateDynamicWhere(String startDate, String endDate, String colum return ""; } - public String getDateDynamicMedianSync(String startDate, String endDate, String columnName) { + public String getDateSeries(String startDate, String endDate) { if (startDate != null) { - return format("and %s::date between ( '%s'::date - interval '7 days') and ('%s'::date - interval '7 days')", columnName, startDate, endDate); + return format("from generate_series('%s'::date - interval '3 months', '%s'::date, '7d'::interval) day", startDate, endDate); } - return "and st.sync_start_time between ( current_date at time zone 'UTC'+'5:30' - interval '7 days') and current_date"; + return "from generate_series(current_date at time zone 'UTC'+'5:30'- interval '3 months' , current_date at time zone 'UTC'+'5:30' , '7d'::interval) day"; } public String getDynamicUserWhere(List userIds, String columnName) { From b5cf691df6667345d8bb1f06b03e75eeec405aaa Mon Sep 17 00:00:00 2001 From: ak2502 Date: Mon, 4 Sep 2023 22:11:11 +0530 Subject: [PATCH 35/36] removed comment --- .../java/org/avniproject/etl/repository/ReportRepository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java index f14b8f9..4fee4e7 100644 --- a/src/main/java/org/avniproject/etl/repository/ReportRepository.java +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -245,7 +245,7 @@ public List generateMedianSync(String orgSchemaName, String syn "from weeks w\t\n" + "left join ${schemaName}.sync_telemetry st\n" + "on st.sync_start_time::date >= w.start_date and st.sync_end_time::date <= w.end_date\n" + -// "and st.sync_source = 'manual'\n" + + "and st.sync_source = 'manual'\n" + "group by 1,2;"; String query = baseQuery .replace("${schemaName}", orgSchemaName) From e5de0df69c5c0acfde0e191fab5889ad0953b824 Mon Sep 17 00:00:00 2001 From: ak2502 Date: Thu, 7 Sep 2023 14:05:48 +0530 Subject: [PATCH 36/36] log used for reference removed Signed-off-by: ak2502 --- .../java/org/avniproject/etl/repository/ReportRepository.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java index 4fee4e7..0c762b7 100644 --- a/src/main/java/org/avniproject/etl/repository/ReportRepository.java +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -250,7 +250,6 @@ public List generateMedianSync(String orgSchemaName, String syn String query = baseQuery .replace("${schemaName}", orgSchemaName) .replace("${syncTelemetryWhere}", syncTelemetryWhere); - System.out.print(query); return jdbcTemplate.query(query, new MedianSyncMapper()); }