Skip to content

Commit

Permalink
Add caching for statistics (#115)
Browse files Browse the repository at this point in the history 
* Allow to cache statistics

* Fix caching inverted behavior

* Change caching time to 6 hours
  • Loading branch information
@campos20
campos20 authored Jul 22, 2024
1 parent 12fbeac commit 53edbfe
Showing 1 changed file with 87 additions and 51 deletions.
138 changes: 87 additions & 51 deletions ...src/main/java/org/worldcubeassociation/statistics/service/impl/StatisticsServiceImpl.java
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,28 @@
package org.worldcubeassociation.statistics.service.impl;

import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
import java.io.InputStream;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
import javax.validation.Valid;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.Resource;
import org.springframework.core.io.ResourceLoader;
import org.springframework.core.io.support.ResourcePatternUtils;
import org.springframework.data.util.Pair;
import org.springframework.stereotype.Service;
import org.worldcubeassociation.statistics.dto.ControlItemDTO;
import org.worldcubeassociation.statistics.dto.DatabaseQueryBaseDTO;
Expand All @@ -25,23 +41,10 @@
import org.worldcubeassociation.statistics.service.StatisticsService;
import org.yaml.snakeyaml.Yaml;

import java.io.IOException;
import java.io.InputStream;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import javax.validation.Valid;

@Slf4j
@Service
public class StatisticsServiceImpl implements StatisticsService {

@Autowired
private DatabaseQueryService databaseQueryService;

Expand All @@ -56,6 +59,12 @@ public class StatisticsServiceImpl implements StatisticsService {

private static final Yaml YAML = new Yaml();

// Website loading is taking 1min. This is a temporary solution to cache the statistics
// We should move to redis to cache the statistics instead
private static final Map<String, Pair<LocalDateTime, Object>> CACHE = new HashMap<>();
private static final String MAIN_LIST_CACHE = "MAIN_LIST_CACHE";
private static final int CACHING_TIME = 6;

@Override
public StatisticsResponseDTO sqlToStatistics(StatisticsRequestDTO statisticsRequestDTO) {
log.info("SQL to statistics for {}", statisticsRequestDTO);
Expand All @@ -70,8 +79,9 @@ public StatisticsResponseDTO sqlToStatistics(StatisticsRequestDTO statisticsRequ
}

statisticsDTO
.setDisplayMode(
Optional.ofNullable(statisticsRequestDTO.getDisplayMode()).orElse(DisplayModeEnum.DEFAULT));
.setDisplayMode(
Optional.ofNullable(statisticsRequestDTO.getDisplayMode())
.orElse(DisplayModeEnum.DEFAULT));
statisticsDTO.setExplanation(statisticsRequestDTO.getExplanation());
statisticsDTO.setTitle(statisticsRequestDTO.getTitle());
statisticsDTO.setGroupName(statisticsRequestDTO.getGroupName());
Expand All @@ -80,9 +90,10 @@ public StatisticsResponseDTO sqlToStatistics(StatisticsRequestDTO statisticsRequ
}

private void buildStatistics(StatisticsDTO statisticsDTO, StatisticsGroupRequestDTO query,
DatabaseQueryBaseDTO sqlResult) {
DatabaseQueryBaseDTO sqlResult) {
if (query.getKeyColumnIndex() == null) {
addResult(query, statisticsDTO, query.getKeys(), sqlResult.getContent(), sqlResult.getHeaders());
addResult(query, statisticsDTO, query.getKeys(), sqlResult.getContent(),
sqlResult.getHeaders());
} else {
var map = new LinkedHashMap<String, List<List<String>>>();

Expand All @@ -100,14 +111,16 @@ private void buildStatistics(StatisticsDTO statisticsDTO, StatisticsGroupRequest
}

for (var entries : map.entrySet()) {
addResult(query, statisticsDTO, List.of(entries.getKey().split(",")), entries.getValue(),
sqlResult.getHeaders());
addResult(query, statisticsDTO, List.of(entries.getKey().split(",")),
entries.getValue(),
sqlResult.getHeaders());
}
}
}

private void addResult(StatisticsGroupRequestDTO query, StatisticsDTO statisticsDTO, List<String> key,
List<List<String>> content, List<String> headers) {
private void addResult(StatisticsGroupRequestDTO query, StatisticsDTO statisticsDTO,
List<String> key,
List<List<String>> content, List<String> headers) {
StatisticsGroupResponseDTO statisticsGroupResponseDTO = new StatisticsGroupResponseDTO();
statisticsGroupResponseDTO.setKeys(key);
statisticsGroupResponseDTO.setContent(content);
Expand All @@ -116,10 +129,10 @@ private void addResult(StatisticsGroupRequestDTO query, StatisticsDTO statistics
statisticsGroupResponseDTO.setExplanation(query.getExplanation());
statisticsGroupResponseDTO.setSqlQueryCustom(query.getSqlQueryCustom());
statisticsGroupResponseDTO.setHeaders(
// First option is the headers provided in this key
Optional.ofNullable(query.getHeaders())
// Then, the one provided by the query
.orElse(headers));
// First option is the headers provided in this key
Optional.ofNullable(query.getHeaders())
// Then, the one provided by the query
.orElse(headers));

statisticsDTO.getStatistics().add(statisticsGroupResponseDTO);
}
Expand All @@ -129,8 +142,8 @@ public void generateAllFromSql() throws IOException {
log.info("Find all statistics");

List<Resource> resources =
Arrays.asList(ResourcePatternUtils.getResourcePatternResolver(resourceLoader)
.getResources("classpath:statistics-request-list/*.yml"));
Arrays.asList(ResourcePatternUtils.getResourcePatternResolver(resourceLoader)
.getResources("classpath:statistics-request-list/*.yml"));

resourcesToStatistics(resources);
log.info("Generated");
Expand All @@ -141,8 +154,8 @@ public void generateFromSql(String filename) throws IOException {
log.info("Generate statistics from the file {}", filename);

List<Resource> resources =
Arrays.asList(ResourcePatternUtils.getResourcePatternResolver(resourceLoader)
.getResources(String.format("classpath:statistics-request-list/%s.yml", filename)));
Arrays.asList(ResourcePatternUtils.getResourcePatternResolver(resourceLoader)
.getResources(String.format("classpath:statistics-request-list/%s.yml", filename)));

if (resources.isEmpty()) {
throw new NotFoundException(String.format("Resource %s not found", filename));
Expand Down Expand Up @@ -184,55 +197,78 @@ public StatisticsListDTO list(String term) {
order by
group_name */

if (StringUtils.isBlank(term)) {
var cache = CACHE.get(MAIN_LIST_CACHE);
if (cache != null && cache.getFirst().plusHours(CACHING_TIME)
.isAfter(LocalDateTime.now())) {
return (StatisticsListDTO) cache.getSecond();
}
}

List<ControlItemDTO> controlList = statisticsRepository.list(term);

List<StatisticsGroupDTO> list =
controlList.stream().collect(Collectors.groupingBy(it -> it.getGroupName(), Collectors.toList()))
.entrySet()
.stream()
.map((k) -> StatisticsGroupDTO.builder().group(k.getKey())
.statistics(k.getValue()
// Sort inner statistics based on title
.stream().sorted(Comparator.comparing(ControlItemDTO::getTitle))
.collect(Collectors.toList())).build())
// Sorts groups based on group name
.sorted(Comparator.comparing(StatisticsGroupDTO::getGroup))
.collect(Collectors.toList());
controlList.stream()
.collect(Collectors.groupingBy(ControlItemDTO::getGroupName, Collectors.toList()))
.entrySet()
.stream()
.map(k -> StatisticsGroupDTO.builder().group(k.getKey())
.statistics(k.getValue()
// Sort inner statistics based on title
.stream().sorted(Comparator.comparing(ControlItemDTO::getTitle))
.collect(Collectors.toList())).build())
// Sorts groups based on group name
.sorted(Comparator.comparing(StatisticsGroupDTO::getGroup))
.collect(Collectors.toList());

StatisticsListDTO statisticsListDTO = new StatisticsListDTO();
statisticsListDTO.setList(list);

if (StringUtils.isBlank(term)) {
CACHE.put(MAIN_LIST_CACHE, Pair.of(LocalDateTime.now(), statisticsListDTO));
}

return statisticsListDTO;
}

@Override
public StatisticsResponseDTO getStatistic(String path) {
var cache = CACHE.get(path);
if (cache != null && cache.getFirst().plusHours(CACHING_TIME)
.isAfter(LocalDateTime.now())) {
return (StatisticsResponseDTO) cache.getSecond();
}

Statistics statistics = statisticsRepository.findById(path)
.orElseThrow(() -> new NotFoundException(String.format("Statistic %s does not exists", path)));
return objectMapper.convertValue(statistics, StatisticsResponseDTO.class);
.orElseThrow(
() -> new NotFoundException(String.format("Statistic %s does not exists", path)));
var toReturn = objectMapper.convertValue(statistics, StatisticsResponseDTO.class);
CACHE.put(path, Pair.of(LocalDateTime.now(), toReturn));
return toReturn;
}

@Override
public StatisticsResponseDTO create(@Valid StatisticsDTO statisticsDTO) {
log.info("Create statistics from {}", statisticsDTO);

statisticsDTO
.setDisplayMode(Optional.ofNullable(statisticsDTO.getDisplayMode()).orElse(DisplayModeEnum.DEFAULT));
.setDisplayMode(Optional.ofNullable(statisticsDTO.getDisplayMode())
.orElse(DisplayModeEnum.DEFAULT));

StatisticsResponseDTO statisticsResponseDTO =
objectMapper.convertValue(statisticsDTO, StatisticsResponseDTO.class);
objectMapper.convertValue(statisticsDTO, StatisticsResponseDTO.class);

String path = String.join("-",
StringUtils.stripAccents(statisticsDTO.getTitle().replaceAll("[^a-zA-Z0-9 ]", "")).split(" "))
.toLowerCase();
StringUtils.stripAccents(statisticsDTO.getTitle().replaceAll("[^a-zA-Z0-9 ]", ""))
.split(" "))
.toLowerCase();

statisticsResponseDTO.setPath(path);
statisticsResponseDTO.setGroupName(statisticsDTO.getGroupName());

statisticsResponseDTO.getStatistics().forEach(stat -> {
Optional.ofNullable(stat.getSqlQueryCustom()).ifPresent(q -> stat.setSqlQueryCustom(
URLEncoder.encode(q, StandardCharsets.UTF_8)));
});
statisticsResponseDTO.getStatistics().forEach(
stat -> Optional.ofNullable(stat.getSqlQueryCustom()).ifPresent(
q -> stat.setSqlQueryCustom(URLEncoder.encode(q, StandardCharsets.UTF_8))));

saveStatistics(statisticsResponseDTO);

Expand Down

0 comments on commit 53edbfe

Please sign in to comment.