Skip to content

Commit

Permalink
Merge pull request #22 from pagopa/PRDP-267-add-time-trigger-function…
Browse files Browse the repository at this point in the history
…-for-recover-failed

[PRDP-267] feat: Add timer trigger function for recovering failed receipt
  • Loading branch information
pasqualespica authored Dec 6, 2023
2 parents 5eba0d7 + db1c77d commit 32f4cb5
Show file tree
Hide file tree
Showing 13 changed files with 400 additions and 38 deletions.
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ then replace env variables with correct values
| `PDV_TOKENIZER_MAX_RETRIES` | PDV Tokenizer max request retry | 3 |
| `TOKENIZER_APIM_HEADER_KEY` | Tokenizer APIM header key | x-api-key |
| `MAX_DATE_DIFF_MILLIS` | Difference in millis between the current time and the date from witch the<br/> receipts will be fetched in massive recover operation | 360000 |
| `RECOVER_FAILED_CRON` | CRON expression for timer trigger function that recover failed receipt | |

> to doc details about AZ fn config
> see [here](https://stackoverflow.com/questions/62669672/azure-functions-what-is-the-purpose-of-having-host-json-and-local-settings-jso)
Expand Down
1 change: 1 addition & 0 deletions helm/values-dev.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ microservice-chart:
MAX_DATE_DIFF_MILLIS: "360000"
MAX_DATE_DIFF_NOTIFY_MILLIS: "360000"
TRIGGER_NOTIFY_REC_SCHEDULE: "0 0 */6 * * *"
RECOVER_FAILED_CRON: "0 0 /12 * * *"
AZURE_FUNCTIONS_MESH_JAVA_OPTS: "-javaagent:/home/site/wwwroot/jmx_prometheus_javaagent-0.19.0.jar=12345:/home/site/wwwroot/config.yaml -javaagent:/home/site/wwwroot/opentelemetry-javaagent.jar -Xmx768m -XX:+UseG1GC"
envFieldRef:
APP_NAME: "metadata.labels['app.kubernetes.io/instance']"
Expand Down
1 change: 1 addition & 0 deletions helm/values-prod.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ microservice-chart:
MAX_DATE_DIFF_MILLIS: "360000"
MAX_DATE_DIFF_NOTIFY_MILLIS: "360000"
TRIGGER_NOTIFY_REC_SCHEDULE: "0 0 */6 * * *"
RECOVER_FAILED_CRON: "0 0 /12 * * *"
AZURE_FUNCTIONS_MESH_JAVA_OPTS: "-javaagent:/home/site/wwwroot/jmx_prometheus_javaagent-0.19.0.jar=12345:/home/site/wwwroot/config.yaml -javaagent:/home/site/wwwroot/opentelemetry-javaagent.jar -Xmx768m -XX:+UseG1GC"
envFieldRef:
APP_NAME: "metadata.labels['app.kubernetes.io/instance']"
Expand Down
1 change: 1 addition & 0 deletions helm/values-uat.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ microservice-chart:
MAX_DATE_DIFF_MILLIS: "360000"
MAX_DATE_DIFF_NOTIFY_MILLIS: "360000"
TRIGGER_NOTIFY_REC_SCHEDULE: "0 0 */6 * * *"
RECOVER_FAILED_CRON: "0 0 /12 * * *"
AZURE_FUNCTIONS_MESH_JAVA_OPTS: "-javaagent:/home/site/wwwroot/jmx_prometheus_javaagent-0.19.0.jar=12345:/home/site/wwwroot/config.yaml -javaagent:/home/site/wwwroot/opentelemetry-javaagent.jar -Xmx768m -XX:+UseG1GC"
envFieldRef:
APP_NAME: "metadata.labels['app.kubernetes.io/instance']"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import it.gov.pagopa.receipt.pdf.helpdesk.client.BizEventCosmosClient;
import it.gov.pagopa.receipt.pdf.helpdesk.client.impl.BizEventCosmosClientImpl;
import it.gov.pagopa.receipt.pdf.helpdesk.entity.receipt.Receipt;
import it.gov.pagopa.receipt.pdf.helpdesk.entity.receipt.enumeration.ReceiptStatusType;
import it.gov.pagopa.receipt.pdf.helpdesk.exception.BizEventNotFoundException;
import it.gov.pagopa.receipt.pdf.helpdesk.exception.PDVTokenizerException;
import it.gov.pagopa.receipt.pdf.helpdesk.model.ProblemJson;
Expand Down Expand Up @@ -55,9 +56,16 @@ public RecoverFailedReceipt(){
}

/**
* This function will be invoked when a Http Trigger occurs
* This function will be invoked when a Http Trigger occurs.
* <p>
* It recovers the receipt with the specified biz event id that has the following status:
* - ({@link ReceiptStatusType#INSERTED})
* - ({@link ReceiptStatusType#FAILED})
* - ({@link ReceiptStatusType#NOT_QUEUE_SENT})
* <p>
* It creates the receipts if not exist and send on queue the event in order to proceed with the receipt generation.
*
* @return response with HttpStatus.OK
* @return response with {@link HttpStatus#OK} if the operation succeeded
*/
@FunctionName("RecoverFailedReceipt")
public HttpResponseMessage run (
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package it.gov.pagopa.receipt.pdf.helpdesk;

import com.azure.cosmos.models.FeedResponse;
import com.microsoft.azure.functions.ExecutionContext;
import com.microsoft.azure.functions.HttpMethod;
import com.microsoft.azure.functions.HttpRequestMessage;
Expand All @@ -15,6 +14,7 @@
import it.gov.pagopa.receipt.pdf.helpdesk.client.impl.BizEventCosmosClientImpl;
import it.gov.pagopa.receipt.pdf.helpdesk.entity.receipt.Receipt;
import it.gov.pagopa.receipt.pdf.helpdesk.entity.receipt.enumeration.ReceiptStatusType;
import it.gov.pagopa.receipt.pdf.helpdesk.model.MassiveRecoverResult;
import it.gov.pagopa.receipt.pdf.helpdesk.model.ProblemJson;
import it.gov.pagopa.receipt.pdf.helpdesk.service.BizEventToReceiptService;
import it.gov.pagopa.receipt.pdf.helpdesk.service.ReceiptCosmosService;
Expand All @@ -24,12 +24,11 @@
import org.slf4j.LoggerFactory;

import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Optional;

import static it.gov.pagopa.receipt.pdf.helpdesk.utils.BizEventToReceiptUtils.getEvent;
import static it.gov.pagopa.receipt.pdf.helpdesk.utils.BizEventToReceiptUtils.massiveRecoverByStatus;

/**
* Azure Functions with Azure Http trigger.
Expand All @@ -42,27 +41,34 @@ public class RecoverFailedReceiptMassive {
private final BizEventCosmosClient bizEventCosmosClient;
private final ReceiptCosmosService receiptCosmosService;

public RecoverFailedReceiptMassive(){
public RecoverFailedReceiptMassive() {
this.bizEventToReceiptService = new BizEventToReceiptServiceImpl();
this.receiptCosmosService = new ReceiptCosmosServiceImpl();
this.bizEventCosmosClient = BizEventCosmosClientImpl.getInstance();
}

RecoverFailedReceiptMassive(BizEventToReceiptService bizEventToReceiptService,
BizEventCosmosClient bizEventCosmosClient,
ReceiptCosmosService receiptCosmosService){
ReceiptCosmosService receiptCosmosService) {
this.bizEventToReceiptService = bizEventToReceiptService;
this.bizEventCosmosClient = bizEventCosmosClient;
this.receiptCosmosService = receiptCosmosService;
}

/**
* This function will be invoked when a Http Trigger occurs
* This function will be invoked when a Http Trigger occurs.
* <p>
* It recovers all the receipts with the specified status that has to be one of:
* - ({@link ReceiptStatusType#INSERTED})
* - ({@link ReceiptStatusType#FAILED})
* - ({@link ReceiptStatusType#NOT_QUEUE_SENT})
* <p>
* It creates the receipts if not exist and send on queue the event in order to proceed with the receipt generation.
*
* @return response with HttpStatus.OK
* @return response with {@link HttpStatus#OK} if the operation succeeded
*/
@FunctionName("RecoverFailedReceiptMassive")
public HttpResponseMessage run (
public HttpResponseMessage run(
@HttpTrigger(name = "RecoverFailedReceiptMassiveTrigger",
methods = {HttpMethod.POST},
route = "receipts/recover-failed",
Expand Down Expand Up @@ -104,30 +110,12 @@ public HttpResponseMessage run (
.build();
}

List<Receipt> receiptList = new ArrayList<>();
String continuationToken = null;
int errorCounter = 0;
MassiveRecoverResult recoverResult;
try {
do {
Iterable<FeedResponse<Receipt>> feedResponseIterator =
this.receiptCosmosService.getFailedReceiptByStatus(continuationToken, 100, statusType);

for (FeedResponse<Receipt> page : feedResponseIterator) {
for (Receipt receipt : page.getResults()) {
try {
Receipt restored = getEvent(receipt.getEventId(), context, this.bizEventToReceiptService,
this.bizEventCosmosClient, this.receiptCosmosService, receipt, logger);
receiptList.add(restored);
} catch (Exception e) {
logger.error(e.getMessage(), e);
errorCounter++;
}
}
continuationToken = page.getContinuationToken();
}
} while (continuationToken != null);
recoverResult = massiveRecoverByStatus(
context, bizEventToReceiptService, bizEventCosmosClient, receiptCosmosService, logger, statusType);
} catch (NoSuchElementException e) {
logger.error(e.getMessage(), e);
logger.error("[{}] Unexpected error during recover of failed receipt", context.getFunctionName(), e);
return request
.createResponseBuilder(HttpStatus.INTERNAL_SERVER_ERROR)
.body(ProblemJson.builder()
Expand All @@ -137,7 +125,9 @@ public HttpResponseMessage run (
.build())
.build();
}

List<Receipt> receiptList = recoverResult.getReceiptList();
int errorCounter = recoverResult.getErrorCounter();

documentdb.setValue(receiptList);
if (errorCounter > 0) {
String msg = String.format("Recovered %s receipts but %s encountered an error.", receiptList.size(), errorCounter);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
package it.gov.pagopa.receipt.pdf.helpdesk;

import com.microsoft.azure.functions.ExecutionContext;
import com.microsoft.azure.functions.OutputBinding;
import com.microsoft.azure.functions.annotation.CosmosDBOutput;
import com.microsoft.azure.functions.annotation.FunctionName;
import com.microsoft.azure.functions.annotation.TimerTrigger;
import it.gov.pagopa.receipt.pdf.helpdesk.client.BizEventCosmosClient;
import it.gov.pagopa.receipt.pdf.helpdesk.client.impl.BizEventCosmosClientImpl;
import it.gov.pagopa.receipt.pdf.helpdesk.entity.receipt.Receipt;
import it.gov.pagopa.receipt.pdf.helpdesk.entity.receipt.enumeration.ReceiptStatusType;
import it.gov.pagopa.receipt.pdf.helpdesk.model.MassiveRecoverResult;
import it.gov.pagopa.receipt.pdf.helpdesk.service.BizEventToReceiptService;
import it.gov.pagopa.receipt.pdf.helpdesk.service.ReceiptCosmosService;
import it.gov.pagopa.receipt.pdf.helpdesk.service.impl.BizEventToReceiptServiceImpl;
import it.gov.pagopa.receipt.pdf.helpdesk.service.impl.ReceiptCosmosServiceImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.NoSuchElementException;

import static it.gov.pagopa.receipt.pdf.helpdesk.utils.BizEventToReceiptUtils.massiveRecoverByStatus;

/**
* Azure Functions with Timer trigger.
*/
public class RecoverFailedReceiptScheduled {

private final Logger logger = LoggerFactory.getLogger(RecoverFailedReceiptScheduled.class);

private final boolean isEnabled = Boolean.parseBoolean(System.getenv().getOrDefault("FAILED_AUTORECOVER_ENABLED", "true"));

private final BizEventToReceiptService bizEventToReceiptService;
private final BizEventCosmosClient bizEventCosmosClient;
private final ReceiptCosmosService receiptCosmosService;

public RecoverFailedReceiptScheduled() {
this.bizEventToReceiptService = new BizEventToReceiptServiceImpl();
this.receiptCosmosService = new ReceiptCosmosServiceImpl();
this.bizEventCosmosClient = BizEventCosmosClientImpl.getInstance();
}

RecoverFailedReceiptScheduled(BizEventToReceiptService bizEventToReceiptService,
BizEventCosmosClient bizEventCosmosClient,
ReceiptCosmosService receiptCosmosService) {
this.bizEventToReceiptService = bizEventToReceiptService;
this.bizEventCosmosClient = bizEventCosmosClient;
this.receiptCosmosService = receiptCosmosService;
}

/**
* This function will be invoked periodically according to the specified schedule.
* <p>
* It recovers all the receipts with the following status:
* - ({@link ReceiptStatusType#INSERTED})
* - ({@link ReceiptStatusType#FAILED})
* - ({@link ReceiptStatusType#NOT_QUEUE_SENT})
* <p>
* It creates the receipts if not exist and send on queue the event in order to proceed with the receipt generation.
*/
@FunctionName("RecoverFailedReceiptScheduled")
public void run(
@TimerTrigger(name = "timerInfo", schedule = "%RECOVER_FAILED_CRON%") String timerInfo,
@CosmosDBOutput(
name = "ReceiptDatastore",
databaseName = "db",
collectionName = "receipts",
connectionStringSetting = "COSMOS_RECEIPTS_CONN_STRING")
OutputBinding<List<Receipt>> documentdb,
final ExecutionContext context
) {
if (isEnabled) {
logger.info("[{}] function called at {}", context.getFunctionName(), LocalDateTime.now());
List<Receipt> receiptList = new ArrayList<>();

receiptList.addAll(recover(context, ReceiptStatusType.INSERTED));
receiptList.addAll(recover(context, ReceiptStatusType.FAILED));
receiptList.addAll(recover(context, ReceiptStatusType.NOT_QUEUE_SENT));

documentdb.setValue(receiptList);
}
}

private List<Receipt> recover(ExecutionContext context, ReceiptStatusType statusType) {
try {
MassiveRecoverResult recoverResult = massiveRecoverByStatus(
context, bizEventToReceiptService, bizEventCosmosClient, receiptCosmosService, logger, statusType);
if (recoverResult.getErrorCounter() > 0) {
logger.error("[{}] Error recovering {} failed receipts for status {}",
context.getFunctionName(), recoverResult.getErrorCounter(), statusType);
}
return recoverResult.getReceiptList();
} catch (NoSuchElementException e) {
logger.error("[{}] Unexpected error during recover of failed receipt for status {}",
context.getFunctionName(), statusType, e);
return Collections.emptyList();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public RecoverNotNotifiedReceipt() {
* not triggered ({@link ReceiptStatusType#GENERATED} by clearing the errors and update the status to the
* previous step ({@link ReceiptStatusType#GENERATED}).
*
* @return response with {@link HttpStatus#OK} if the notification succeeded
* @return response with {@link HttpStatus#OK} if the operation succeeded
*/
@FunctionName("RecoverNotNotifiedReceipt")
public HttpResponseMessage run(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ public RecoverNotNotifiedReceiptMassive() {
* not triggered ({@link ReceiptStatusType#GENERATED} by clearing the errors and update the status to the
* previous step ({@link ReceiptStatusType#GENERATED}).
*
* @return response with {@link HttpStatus#OK} if the notification succeeded
* @return response with {@link HttpStatus#OK} if the operation succeeded
*/
@FunctionName("RecoverNotNotifiedReceiptMassive")
public HttpResponseMessage run(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package it.gov.pagopa.receipt.pdf.helpdesk.model;

import it.gov.pagopa.receipt.pdf.helpdesk.entity.receipt.Receipt;
import lombok.Builder;
import lombok.Data;

import java.util.List;

@Data
@Builder
public class MassiveRecoverResult {

private List<Receipt> receiptList;
private int errorCounter;
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package it.gov.pagopa.receipt.pdf.helpdesk.utils;

import com.azure.cosmos.models.FeedResponse;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.microsoft.azure.functions.ExecutionContext;
import it.gov.pagopa.receipt.pdf.helpdesk.client.BizEventCosmosClient;
Expand All @@ -13,10 +14,12 @@
import it.gov.pagopa.receipt.pdf.helpdesk.exception.BizEventNotFoundException;
import it.gov.pagopa.receipt.pdf.helpdesk.exception.PDVTokenizerException;
import it.gov.pagopa.receipt.pdf.helpdesk.exception.ReceiptNotFoundException;
import it.gov.pagopa.receipt.pdf.helpdesk.model.MassiveRecoverResult;
import it.gov.pagopa.receipt.pdf.helpdesk.service.BizEventToReceiptService;
import it.gov.pagopa.receipt.pdf.helpdesk.service.ReceiptCosmosService;
import org.slf4j.Logger;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
Expand Down Expand Up @@ -59,9 +62,9 @@ public static Receipt getEvent(
if (receipt.getEventData() == null || receipt.getEventData().getDebtorFiscalCode() == null) {
tokenizeReceipt(bizEventToReceiptService, bizEvent, receipt);
}
receipt.setStatus(ReceiptStatusType.INSERTED);
bizEventToReceiptService.handleSendMessageToQueue(bizEvent, receipt);
if(receipt.getStatus() != ReceiptStatusType.NOT_QUEUE_SENT){
receipt.setStatus(ReceiptStatusType.INSERTED);
if (receipt.getStatus() != ReceiptStatusType.NOT_QUEUE_SENT) {
receipt.setInsertedAt(System.currentTimeMillis());
receipt.setReasonErr(null);
receipt.setReasonErrPayer(null);
Expand All @@ -71,6 +74,40 @@ public static Receipt getEvent(
return null;
}

public static MassiveRecoverResult massiveRecoverByStatus(
ExecutionContext context,
BizEventToReceiptService bizEventToReceiptService,
BizEventCosmosClient bizEventCosmosClient,
ReceiptCosmosService receiptCosmosService,
Logger logger,
ReceiptStatusType statusType) {
int errorCounter = 0;
List<Receipt> receiptList = new ArrayList<>();
String continuationToken = null;
do {
Iterable<FeedResponse<Receipt>> feedResponseIterator =
receiptCosmosService.getFailedReceiptByStatus(continuationToken, 100, statusType);

for (FeedResponse<Receipt> page : feedResponseIterator) {
for (Receipt receipt : page.getResults()) {
try {
Receipt restored = getEvent(receipt.getEventId(), context, bizEventToReceiptService,
bizEventCosmosClient, receiptCosmosService, receipt, logger);
receiptList.add(restored);
} catch (Exception e) {
logger.error(e.getMessage(), e);
errorCounter++;
}
}
continuationToken = page.getContinuationToken();
}
} while (continuationToken != null);
return MassiveRecoverResult.builder()
.receiptList(receiptList)
.errorCounter(errorCounter)
.build();
}

/**
* Creates a new instance of Receipt, using the tokenizer service to mask the PII, based on
* the provided BizEvent
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ void shouldReturnMissingReceiptErrorMessageOnNonExistingErrorinCosmos() throws R

// test assertion
assertNotNull(response);
assertEquals(HttpStatus.OK, response.getStatus());
assertEquals(HttpStatus.NOT_FOUND, response.getStatus());
assertEquals(ProblemJson.class, response.getBody().getClass());

ProblemJson responseData = (ProblemJson) response.getBody();
Expand Down
Loading

0 comments on commit 32f4cb5

Please sign in to comment.