From 350f2261d2af6e30de77f676862abf3310c49fd1 Mon Sep 17 00:00:00 2001 From: tomsun28 Date: Sun, 17 Dec 2023 19:53:31 +0800 Subject: [PATCH] update label structure store in victoria metrics, make it prometheus like (#1426) Signed-off-by: tomsun28 --- .../common/constants/CommonConstants.java | 10 +++ home/i18n/en/code.json | 2 +- .../scheduler/CollectorAndJobScheduler.java | 3 + .../manager/scheduler/SchedulerInit.java | 3 + .../hertzbeat/manager/service/AppService.java | 9 +++ .../manager/service/impl/AppServiceImpl.java | 9 +++ .../service/impl/MonitorServiceImpl.java | 27 ++++++-- .../manager/service/MonitorServiceTest.java | 2 +- .../HistoryVictoriaMetricsDataStorage.java | 64 +++++++++++-------- 9 files changed, 98 insertions(+), 31 deletions(-) diff --git a/common/src/main/java/org/dromara/hertzbeat/common/constants/CommonConstants.java b/common/src/main/java/org/dromara/hertzbeat/common/constants/CommonConstants.java index ff7c84e7c77..82a9f9d23c6 100644 --- a/common/src/main/java/org/dromara/hertzbeat/common/constants/CommonConstants.java +++ b/common/src/main/java/org/dromara/hertzbeat/common/constants/CommonConstants.java @@ -319,4 +319,14 @@ public interface CommonConstants { * collector auth failed message */ String COLLECTOR_AUTH_FAILED = "Auth Failed"; + + /** + * for prometheus task name prefix + */ + String PROMETHEUS_APP_PREFIX = "_prometheus_"; + + /** + * prometheus + */ + String PROMETHEUS = "prometheus"; } diff --git a/home/i18n/en/code.json b/home/i18n/en/code.json index 42f17828c4a..b1c7653459e 100644 --- a/home/i18n/en/code.json +++ b/home/i18n/en/code.json @@ -266,7 +266,7 @@ "message": "HertzBeat's stand-alone and cluster version code are all open source, unlimited and anyone who are interested in it are very welcome to contribute. No matter how small the contribution is, whether it is a document code or a typo, we respect everyone and learn and grow together. {br} We hope that the code we participate in can be deployed to thousands of machines, leaving our code footprints in this world.{br}{github} {gitee}" }, "slogan": { - "message": "Open Source Real-time Monitoring Tool" + "message": "Open Source Real-time Monitoring System" }, "Who uses HertzBeat?": { "message": "Who uses HertzBeat?" diff --git a/manager/src/main/java/org/dromara/hertzbeat/manager/scheduler/CollectorAndJobScheduler.java b/manager/src/main/java/org/dromara/hertzbeat/manager/scheduler/CollectorAndJobScheduler.java index 2c7280d0145..e29eaf12715 100644 --- a/manager/src/main/java/org/dromara/hertzbeat/manager/scheduler/CollectorAndJobScheduler.java +++ b/manager/src/main/java/org/dromara/hertzbeat/manager/scheduler/CollectorAndJobScheduler.java @@ -116,6 +116,9 @@ public void collectorGoOnline(String identity, CollectorInfo collectorInfo) { try { // 构造采集任务Job实体 Job appDefine = appService.getAppDefine(monitor.getApp()); + if (CommonConstants.PROMETHEUS.equals(monitor.getApp())) { + appDefine.setApp(CommonConstants.PROMETHEUS_APP_PREFIX + monitor.getName()); + } appDefine.setMonitorId(monitor.getId()); appDefine.setInterval(monitor.getIntervals()); appDefine.setCyclic(true); diff --git a/manager/src/main/java/org/dromara/hertzbeat/manager/scheduler/SchedulerInit.java b/manager/src/main/java/org/dromara/hertzbeat/manager/scheduler/SchedulerInit.java index 5633e5a0c33..4f18c9edd09 100644 --- a/manager/src/main/java/org/dromara/hertzbeat/manager/scheduler/SchedulerInit.java +++ b/manager/src/main/java/org/dromara/hertzbeat/manager/scheduler/SchedulerInit.java @@ -81,6 +81,9 @@ public void run(String... args) throws Exception { try { // 构造采集任务Job实体 Job appDefine = appService.getAppDefine(monitor.getApp()); + if (CommonConstants.PROMETHEUS.equals(monitor.getApp())) { + appDefine.setApp(CommonConstants.PROMETHEUS_APP_PREFIX + monitor.getName()); + } appDefine.setId(monitor.getJobId()); appDefine.setMonitorId(monitor.getId()); appDefine.setInterval(monitor.getIntervals()); diff --git a/manager/src/main/java/org/dromara/hertzbeat/manager/service/AppService.java b/manager/src/main/java/org/dromara/hertzbeat/manager/service/AppService.java index 63500bbe6a2..d8048122c38 100644 --- a/manager/src/main/java/org/dromara/hertzbeat/manager/service/AppService.java +++ b/manager/src/main/java/org/dromara/hertzbeat/manager/service/AppService.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; +import java.util.Optional; /** * Monitoring Type Management Interface @@ -62,6 +63,14 @@ public interface AppService { */ Job getAppDefine(String app) throws IllegalArgumentException; + /** + * Get monitor structure definition based on monitor type name + * + * @param app Monitoring type name + * @return Monitoring Structure Definition Optional + */ + Optional getAppDefineOption(String app); + /** * 获取app定义的指标 * diff --git a/manager/src/main/java/org/dromara/hertzbeat/manager/service/impl/AppServiceImpl.java b/manager/src/main/java/org/dromara/hertzbeat/manager/service/impl/AppServiceImpl.java index 237953e1480..61fa2bcb908 100644 --- a/manager/src/main/java/org/dromara/hertzbeat/manager/service/impl/AppServiceImpl.java +++ b/manager/src/main/java/org/dromara/hertzbeat/manager/service/impl/AppServiceImpl.java @@ -172,6 +172,15 @@ public Job getAppDefine(String app) throws IllegalArgumentException { return appDefine.clone(); } + @Override + public Optional getAppDefineOption(String app) { + if (StringUtils.hasText(app)) { + Job appDefine = appDefines.get(app.toLowerCase()); + return Optional.ofNullable(appDefine); + } + return Optional.empty(); + } + @Override public List getAppDefineMetricNames(String app) { List metricNames = new ArrayList<>(16); diff --git a/manager/src/main/java/org/dromara/hertzbeat/manager/service/impl/MonitorServiceImpl.java b/manager/src/main/java/org/dromara/hertzbeat/manager/service/impl/MonitorServiceImpl.java index 001daa41340..04f85a3db64 100644 --- a/manager/src/main/java/org/dromara/hertzbeat/manager/service/impl/MonitorServiceImpl.java +++ b/manager/src/main/java/org/dromara/hertzbeat/manager/service/impl/MonitorServiceImpl.java @@ -134,6 +134,9 @@ public void detectMonitor(Monitor monitor, List params, String collector) monitorId = MONITOR_ID_TMP; } Job appDefine = appService.getAppDefine(monitor.getApp()); + if (CommonConstants.PROMETHEUS.equals(monitor.getApp())) { + appDefine.setApp(CommonConstants.PROMETHEUS_APP_PREFIX + monitor.getName()); + } appDefine.setMonitorId(monitorId); appDefine.setCyclic(false); appDefine.setTimestamp(System.currentTimeMillis()); @@ -172,8 +175,11 @@ public void addMonitor(Monitor monitor, List params, String collector) th } tags.add(Tag.builder().name(CommonConstants.TAG_MONITOR_ID).value(String.valueOf(monitorId)).type((byte) 0).build()); tags.add(Tag.builder().name(CommonConstants.TAG_MONITOR_NAME).value(String.valueOf(monitor.getName())).type((byte) 0).build()); - // Construct the collection task Job entity 构造采集任务Job实体 + // Construct the collection task Job entity Job appDefine = appService.getAppDefine(monitor.getApp()); + if (CommonConstants.PROMETHEUS.equals(monitor.getApp())) { + appDefine.setApp(CommonConstants.PROMETHEUS_APP_PREFIX + monitor.getName()); + } appDefine.setMonitorId(monitorId); appDefine.setInterval(monitor.getIntervals()); appDefine.setCyclic(true); @@ -317,17 +323,21 @@ public void validate(MonitorDto monitorDto, Boolean isModify) throws IllegalArgu param.setValue(value); }) .collect(Collectors.toMap(Param::getField, param -> param)); - // Check name uniqueness 校验名称唯一性 + // Check name uniqueness and can not equal app type if (isModify != null) { + Optional defineOptional = appService.getAppDefineOption(monitor.getName()); + if (defineOptional.isPresent()) { + throw new IllegalArgumentException("Monitoring name cannot be the existed monitoring type name!"); + } Optional monitorOptional = monitorDao.findMonitorByNameEquals(monitor.getName()); if (monitorOptional.isPresent()) { Monitor existMonitor = monitorOptional.get(); if (isModify) { if (!existMonitor.getId().equals(monitor.getId())) { - throw new IllegalArgumentException("Monitoring name cannot be repeated!"); + throw new IllegalArgumentException("Monitoring name already exists!"); } } else { - throw new IllegalArgumentException("Monitoring name cannot be repeated!"); + throw new IllegalArgumentException("Monitoring name already exists!"); } } } @@ -509,6 +519,9 @@ public void modifyMonitor(Monitor monitor, List params, String collector) // Construct the collection task Job entity // 构造采集任务Job实体 Job appDefine = appService.getAppDefine(monitor.getApp()); + if (CommonConstants.PROMETHEUS.equals(monitor.getApp())) { + appDefine.setApp(CommonConstants.PROMETHEUS_APP_PREFIX + monitor.getName()); + } appDefine.setId(preMonitor.getJobId()); appDefine.setMonitorId(monitorId); appDefine.setInterval(monitor.getIntervals()); @@ -659,6 +672,9 @@ public void enableManageMonitors(HashSet ids) { // Construct the collection task Job entity // 构造采集任务Job实体 Job appDefine = appService.getAppDefine(monitor.getApp()); + if (CommonConstants.PROMETHEUS.equals(monitor.getApp())) { + appDefine.setApp(CommonConstants.PROMETHEUS_APP_PREFIX + monitor.getName()); + } appDefine.setMonitorId(monitor.getId()); appDefine.setInterval(monitor.getIntervals()); appDefine.setCyclic(true); @@ -765,6 +781,9 @@ public void updateAppCollectJob(Job job) { log.error("update monitor job error when template modify, define | id | jobId is null. continue"); continue; } + if (CommonConstants.PROMETHEUS.equals(monitor.getApp())) { + appDefine.setApp(CommonConstants.PROMETHEUS_APP_PREFIX + monitor.getName()); + } appDefine.setId(monitor.getJobId()); appDefine.setMonitorId(monitor.getId()); appDefine.setInterval(monitor.getIntervals()); diff --git a/manager/src/test/java/org/dromara/hertzbeat/manager/service/MonitorServiceTest.java b/manager/src/test/java/org/dromara/hertzbeat/manager/service/MonitorServiceTest.java index 57134e1cced..e4dc9232e3a 100644 --- a/manager/src/test/java/org/dromara/hertzbeat/manager/service/MonitorServiceTest.java +++ b/manager/src/test/java/org/dromara/hertzbeat/manager/service/MonitorServiceTest.java @@ -200,7 +200,7 @@ void validateMonitorName() { try { monitorService.validate(dto, isModify); } catch (IllegalArgumentException e) { - assertEquals("Monitoring name cannot be repeated!", e.getMessage()); + assertEquals("Monitoring name already exists!", e.getMessage()); } } diff --git a/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryVictoriaMetricsDataStorage.java b/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryVictoriaMetricsDataStorage.java index f4980f4f8c0..599fc9ee2a1 100644 --- a/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryVictoriaMetricsDataStorage.java +++ b/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryVictoriaMetricsDataStorage.java @@ -73,13 +73,13 @@ public class HistoryVictoriaMetricsDataStorage extends AbstractHistoryDataStorag private static final String STATUS_PATH = "/api/v1/status/tsdb"; private static final String STATUS = "status"; private static final String STATUS_SUCCESS = "success"; - private static final String NAME_KEY = "__name__"; + private static final String LABEL_KEY_NAME = "__name__"; + private static final String LABEL_KEY_JOB = "job"; + private static final String LABEL_KEY_INSTANCE = "instance"; private static final String SPILT = "_"; private static final String BASIC = "Basic"; - private static final String MONITOR_TYPE_KEY = "__app__"; private static final String MONITOR_METRICS_KEY = "__metrics__"; private static final String MONITOR_METRIC_KEY = "__metric__"; - private static final String MONITOR_ID_KEY = "__monitorId__"; private final WarehouseProperties.StoreProperties.VictoriaMetricsProperties victoriaMetricsProp; @@ -125,9 +125,16 @@ public void saveData(CollectRep.MetricsData metricsData) { return; } Map defaultLabels = new HashMap<>(8); - defaultLabels.put(MONITOR_ID_KEY, String.valueOf(metricsData.getId())); - defaultLabels.put(MONITOR_TYPE_KEY, metricsData.getApp()); defaultLabels.put(MONITOR_METRICS_KEY, metricsData.getMetrics()); + boolean isPrometheusAuto = false; + if (metricsData.getApp().startsWith(CommonConstants.PROMETHEUS_APP_PREFIX)) { + isPrometheusAuto = true; + defaultLabels.put(LABEL_KEY_JOB, metricsData.getApp() + .substring(CommonConstants.PROMETHEUS_APP_PREFIX.length())); + } else { + defaultLabels.put(LABEL_KEY_JOB, metricsData.getApp()); + } + defaultLabels.put(LABEL_KEY_INSTANCE, String.valueOf(metricsData.getId())); List fields = metricsData.getFieldsList(); Long[] timestamp = new Long[]{metricsData.getTime()}; @@ -154,7 +161,9 @@ public void saveData(CollectRep.MetricsData metricsData) { if (entry.getKey() != null && entry.getValue() != null) { try { labels.putAll(defaultLabels); - labels.put(NAME_KEY, metricsData.getApp() + SPILT + metricsData.getMetrics() + SPILT + entry.getKey()); + String labelName = isPrometheusAuto ? metricsData.getMetrics() + : metricsData.getMetrics() + SPILT + entry.getKey(); + labels.put(LABEL_KEY_NAME, labelName); labels.put(MONITOR_METRIC_KEY, entry.getKey()); VictoriaMetricsContent content = VictoriaMetricsContent.builder() .metric(labels) @@ -191,8 +200,13 @@ public void destroy() {} @Override public Map> getHistoryMetricData(Long monitorId, String app, String metrics, String metric, String label, String history) { - String timeSeriesSelector = NAME_KEY + "=\"" + app + SPILT + metrics + SPILT + metric + "\"" + - "," + MONITOR_ID_KEY + "=\"" + monitorId + "\""; + String labelName = metrics + SPILT + metric; + if (CommonConstants.PROMETHEUS.equals(app)) { + labelName = metrics; + } + String timeSeriesSelector = LABEL_KEY_NAME + "=\"" + labelName + "\"" + + "," + LABEL_KEY_INSTANCE + "=\"" + monitorId + "\"" + + "," + MONITOR_METRIC_KEY + "=\"" + metric + "\""; Map> instanceValuesMap = new HashMap<>(8); try { HttpHeaders headers = new HttpHeaders(); @@ -219,9 +233,9 @@ public Map> getHistoryMetricData(Long monitorId, String app, ).collect(Collectors.toList()); for (VictoriaMetricsContent content : contents) { Map labels = content.getMetric(); - labels.remove(NAME_KEY); - labels.remove(MONITOR_ID_KEY); - labels.remove(MONITOR_TYPE_KEY); + labels.remove(LABEL_KEY_NAME); + labels.remove(LABEL_KEY_JOB); + labels.remove(LABEL_KEY_INSTANCE); labels.remove(MONITOR_METRICS_KEY); labels.remove(MONITOR_METRIC_KEY); String labelStr = JsonUtil.toJson(labels); @@ -275,8 +289,8 @@ public Map> getHistoryIntervalMetricData(Long monitorId, Str ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofHours(6)); startTime = dateTime.toEpochSecond(); } - String timeSeriesSelector = NAME_KEY + "=\"" + app + SPILT + metrics + SPILT + metric + "\"" + - "," + MONITOR_ID_KEY + "=\"" + monitorId + "\""; + String timeSeriesSelector = LABEL_KEY_NAME + "=\"" + app + SPILT + metrics + SPILT + metric + "\"" + + "," + LABEL_KEY_INSTANCE + "=\"" + monitorId + "\""; Map> instanceValuesMap = new HashMap<>(8); try { HttpHeaders headers = new HttpHeaders(); @@ -304,9 +318,9 @@ public Map> getHistoryIntervalMetricData(Long monitorId, Str List contents = responseEntity.getBody().getData().getResult(); for (VictoriaMetricsQueryContent.ContentData.Content content : contents) { Map labels = content.getMetric(); - labels.remove(NAME_KEY); - labels.remove(MONITOR_ID_KEY); - labels.remove(MONITOR_TYPE_KEY); + labels.remove(LABEL_KEY_NAME); + labels.remove(LABEL_KEY_JOB); + labels.remove(LABEL_KEY_INSTANCE); labels.remove(MONITOR_METRICS_KEY); labels.remove(MONITOR_METRIC_KEY); String labelStr = JsonUtil.toJson(labels); @@ -339,9 +353,9 @@ public Map> getHistoryIntervalMetricData(Long monitorId, Str List contents = responseEntity.getBody().getData().getResult(); for (VictoriaMetricsQueryContent.ContentData.Content content : contents) { Map labels = content.getMetric(); - labels.remove(NAME_KEY); - labels.remove(MONITOR_ID_KEY); - labels.remove(MONITOR_TYPE_KEY); + labels.remove(LABEL_KEY_NAME); + labels.remove(LABEL_KEY_JOB); + labels.remove(LABEL_KEY_INSTANCE); labels.remove(MONITOR_METRICS_KEY); labels.remove(MONITOR_METRIC_KEY); String labelStr = JsonUtil.toJson(labels); @@ -373,9 +387,9 @@ public Map> getHistoryIntervalMetricData(Long monitorId, Str List contents = responseEntity.getBody().getData().getResult(); for (VictoriaMetricsQueryContent.ContentData.Content content : contents) { Map labels = content.getMetric(); - labels.remove(NAME_KEY); - labels.remove(MONITOR_ID_KEY); - labels.remove(MONITOR_TYPE_KEY); + labels.remove(LABEL_KEY_NAME); + labels.remove(LABEL_KEY_JOB); + labels.remove(LABEL_KEY_INSTANCE); labels.remove(MONITOR_METRICS_KEY); labels.remove(MONITOR_METRIC_KEY); String labelStr = JsonUtil.toJson(labels); @@ -407,9 +421,9 @@ public Map> getHistoryIntervalMetricData(Long monitorId, Str List contents = responseEntity.getBody().getData().getResult(); for (VictoriaMetricsQueryContent.ContentData.Content content : contents) { Map labels = content.getMetric(); - labels.remove(NAME_KEY); - labels.remove(MONITOR_ID_KEY); - labels.remove(MONITOR_TYPE_KEY); + labels.remove(LABEL_KEY_NAME); + labels.remove(LABEL_KEY_JOB); + labels.remove(LABEL_KEY_INSTANCE); labels.remove(MONITOR_METRICS_KEY); labels.remove(MONITOR_METRIC_KEY); String labelStr = JsonUtil.toJson(labels);