Skip to content

Commit

Permalink
reset alert converge reduce cache when restored alert trigger (#1281)
Browse files Browse the repository at this point in the history
  • Loading branch information
tomsun28 authored Oct 11, 2023
1 parent ba9b3d8 commit cdfd797
Show file tree
Hide file tree
Showing 3 changed files with 26 additions and 9 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,6 @@ private void handleRecoveredAlert(long currentTimeMilli, long monitorId, String
if (notResolvedAlert != null) {
// Sending an alarm Restore
Map<String, String> tags = notResolvedAlert.getTags();
tags.put(CommonConstants.TAG_ALARM_TYPE, "recover");
String content = this.bundle.getString("alerter.alarm.recover") + " : " + expr;
Alert resumeAlert = Alert.builder()
.tags(tags)
Expand Down Expand Up @@ -262,6 +261,7 @@ private void afterThresholdRuleMatch(long currentTimeMilli, long monitorId, Stri
Map<String, String> tags = new HashMap<>(6);
tags.put(CommonConstants.TAG_MONITOR_ID, String.valueOf(monitorId));
tags.put(CommonConstants.TAG_MONITOR_APP, app);
tags.put(CommonConstants.TAG_POLICY_ID, String.valueOf(define.getId()));
if (!CollectionUtils.isEmpty(define.getTags())) {
for (TagItem tagItem : define.getTags()) {
fieldValueMap.put(tagItem.getName(), tagItem.getValue());
Expand Down Expand Up @@ -320,6 +320,7 @@ private void handlerAvailableMetrics(long monitorId, String app, CollectRep.Metr
Map<String, String> tags = new HashMap<>(6);
tags.put(CommonConstants.TAG_MONITOR_ID, String.valueOf(monitorId));
tags.put(CommonConstants.TAG_MONITOR_APP, app);
tags.put(CommonConstants.TAG_POLICY_ID, String.valueOf(avaAlertDefine.getId()));
tags.put("metrics", CommonConstants.AVAILABILITY);
tags.put("code", metricsData.getCode().name());
Map<String, Object> valueMap = tags.entrySet().stream()
Expand All @@ -334,7 +335,7 @@ private void handlerAvailableMetrics(long monitorId, String app, CollectRep.Metr
if (preAlert == null) {
Alert.AlertBuilder alertBuilder = Alert.builder()
.tags(tags)
.priority(CommonConstants.ALERT_PRIORITY_CODE_EMERGENCY)
.priority(avaAlertDefine.getPriority())
.status(ALERT_STATUS_CODE_PENDING)
.target(CommonConstants.AVAILABILITY)
.content(AlertTemplateUtil.render(avaAlertDefine.getTemplate(), valueMap))
Expand Down Expand Up @@ -377,16 +378,10 @@ private void handlerAvailableMetrics(long monitorId, String app, CollectRep.Metr
Alert notResolvedAlert = notRecoveredAlertMap.remove(notResolvedAlertKey);
if (notResolvedAlert != null) {
// Sending an alarm Restore
Map<String, String> tags = new HashMap<>(6);
tags.put(CommonConstants.TAG_MONITOR_ID, String.valueOf(monitorId));
tags.put(CommonConstants.TAG_MONITOR_APP, app);
tags.put(CommonConstants.TAG_ALARM_TYPE, "recover");
Map<String, String> tags = notResolvedAlert.getTags();
if (!avaAlertDefine.isRecoverNotice()) {
tags.put(CommonConstants.IGNORE, CommonConstants.IGNORE);
}
if (notResolvedAlert.getTags() != null) {
tags.putAll(notResolvedAlert.getTags());
}
String content = this.bundle.getString("alerter.availability.recover");
Alert resumeAlert = Alert.builder()
.tags(tags)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,22 @@ public boolean filterConverge(Alert currentAlert) {
if (currentAlert.getTags() != null && currentAlert.getTags().containsKey(CommonConstants.IGNORE)) {
return true;
}
if (currentAlert.getStatus() == CommonConstants.ALERT_STATUS_CODE_RESTORED) {
// restored alert
int alertHash = Objects.hash(CommonConstants.ALERT_PRIORITY_CODE_CRITICAL)
+ Arrays.hashCode(currentAlert.getTags().keySet().toArray(new String[0]))
+ Arrays.hashCode(currentAlert.getTags().values().toArray(new String[0]));
converageAlertMap.remove(alertHash);
alertHash = Objects.hash(CommonConstants.ALERT_PRIORITY_CODE_EMERGENCY)
+ Arrays.hashCode(currentAlert.getTags().keySet().toArray(new String[0]))
+ Arrays.hashCode(currentAlert.getTags().values().toArray(new String[0]));
converageAlertMap.remove(alertHash);
alertHash = Objects.hash(CommonConstants.ALERT_PRIORITY_CODE_WARNING)
+ Arrays.hashCode(currentAlert.getTags().keySet().toArray(new String[0]))
+ Arrays.hashCode(currentAlert.getTags().values().toArray(new String[0]));
converageAlertMap.remove(alertHash);
return true;
}
ICacheService<String, Object> convergeCache = CacheFactory.getAlertConvergeCache();
List<AlertConverge> alertConvergeList = (List<AlertConverge>) convergeCache.get(CommonConstants.CACHE_ALERT_CONVERGE);
if (alertConvergeList == null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,11 @@ public interface CommonConstants {
*/
String TAG_MONITOR_NAME = "monitorName";

/**
* 内有标签: policyId 告警阈值规则ID
*/
String TAG_POLICY_ID = "policyId";

/**
* 内有标签: app 监控类型
*/
Expand Down Expand Up @@ -304,6 +309,7 @@ public interface CommonConstants {

/**
* ignore label
* 处理未配置恢复告警,但需要使用恢复告警变更监控状态的情况
*/
String IGNORE = "ignore";

Expand Down

0 comments on commit cdfd797

Please sign in to comment.