support monitoring NebulaGraph metrics and add help doc (#1441)

Co-authored-by: 东风 <[email protected]>

collector/src/main/java/org/dromara/hertzbeat/collector/collect/nebulagraph/NebulaGraphCollectImpl.java

@@ -0,0 +1,196 @@
+package org.dromara.hertzbeat.collector.collect.nebulagraph;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.http.HttpHeaders;
+import org.apache.http.HttpHost;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.client.methods.RequestBuilder;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.protocol.HttpContext;
+import org.apache.http.util.EntityUtils;
+import org.dromara.hertzbeat.collector.collect.AbstractCollect;
+import org.dromara.hertzbeat.collector.collect.common.http.CommonHttpClient;
+import org.dromara.hertzbeat.collector.dispatch.DispatchConstants;
+import org.dromara.hertzbeat.collector.util.CollectUtil;
+import org.dromara.hertzbeat.common.constants.CollectorConstants;
+import org.dromara.hertzbeat.common.constants.CommonConstants;
+import org.dromara.hertzbeat.common.entity.job.Metrics;
+import org.dromara.hertzbeat.common.entity.job.protocol.NebulaGraphProtocol;
+import org.dromara.hertzbeat.common.entity.message.CollectRep;
+import org.dromara.hertzbeat.common.util.CommonUtil;
+import org.dromara.hertzbeat.common.util.IpDomainUtil;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Objects;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+
+/**
+ * @author dongfeng
+ */
+@Slf4j
+public class NebulaGraphCollectImpl extends AbstractCollect {
+    private final static int SUCCESS_CODE = 200;
+
+    private final static String[] TIME_RANGE = new String[]{"5", "60", "600", "3600"};
+
+    private final static String REGEX = "\\.%s\\=";
+
+    private final static String STR_SPLIT = "\n";
+
+    private final static String STORAGE_SPLIT_KEY_VALUE = "=";
+
+    private final static String GRAPH_API = "/stats";
+
+    private final static String STORAGE_API = "/rocksdb_stats";
+
+
+    @Override
+    public void collect(CollectRep.MetricsData.Builder builder, long monitorId, String app, Metrics metrics) {
+        long startTime = System.currentTimeMillis();
+        if (metrics == null || metrics.getNebulaGraph() == null) {
+            builder.setCode(CollectRep.Code.FAIL);
+            builder.setMsg("NebulaGraph collect must has NebulaGraph params");
+            return;
+        }
+        NebulaGraphProtocol nebulaGraph = metrics.getNebulaGraph();
+        String timePeriod = nebulaGraph.getTimePeriod();
+
+        if (!Objects.isNull(nebulaGraph.getTimePeriod())&&!Arrays.asList(TIME_RANGE).contains(timePeriod)) {
+            builder.setCode(CollectRep.Code.FAIL);
+            builder.setMsg("The time range for metric statistics, currently supporting 5 seconds, 60 seconds, 600 seconds, and 3600 seconds.");
+            return;
+        }
+
+        if (nebulaGraph.getHost() == null || nebulaGraph.getHost().isEmpty()) {
+            builder.setCode(CollectRep.Code.FAIL);
+            builder.setMsg("The host of NebulaGraph must be set");
+            return;
+        }
+
+        String resp;
+        long responseTime;
+        HashMap<String, String> resultMap = new HashMap<>(64);
+        CloseableHttpResponse response;
+        HttpContext httpContext = createHttpContext(nebulaGraph.getHost(), nebulaGraph.getPort());
+        HttpUriRequest request = createHttpRequest(nebulaGraph.getHost(), nebulaGraph.getPort(),
+                nebulaGraph.getUrl(), nebulaGraph.getTimeout());
+        try {
+            // 发起http请求，获取响应数据
+            response = CommonHttpClient.getHttpClient().execute(request, httpContext);
+            int statusCode = response.getStatusLine().getStatusCode();
+            if (statusCode != SUCCESS_CODE) {
+                builder.setCode(CollectRep.Code.FAIL);
+                builder.setMsg("StatusCode " + statusCode);
+                return;
+            }
+            resp = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
+            responseTime = System.currentTimeMillis() - startTime;
+            resultMap.put(CollectorConstants.RESPONSE_TIME, Long.toString(responseTime));
+            // 根据API进行不同解析
+            if (GRAPH_API.equals(nebulaGraph.getUrl())) {
+                parseStatsResponse(resp, nebulaGraph.getTimePeriod(), resultMap);
+            } else if (STORAGE_API.equals(nebulaGraph.getUrl())) {
+                parseStorageResponse(resp, resultMap);
+            }
+            List<String> aliasFields = metrics.getAliasFields();
+            CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
+            for (String field : aliasFields) {
+                String fieldValue = resultMap.get(field);
+                valueRowBuilder.addColumns(Objects.requireNonNullElse(fieldValue, CommonConstants.NULL_VALUE));
+            }
+            builder.addValues(valueRowBuilder.build());
+        } catch (IOException e) {
+            String errorMsg = CommonUtil.getMessageFromThrowable(e);
+            log.info(errorMsg);
+            builder.setCode(CollectRep.Code.FAIL);
+            builder.setMsg(errorMsg);
+        }
+    }
+
+
+    @Override
+    public String supportProtocol() {
+        return DispatchConstants.PROTOCOL_NEBULAGRAPH;
+    }
+
+    private HttpContext createHttpContext(String host, String port) {
+        HttpHost httpHost = new HttpHost(host, Integer.parseInt(port));
+        HttpClientContext httpClientContext = new HttpClientContext();
+        httpClientContext.setTargetHost(httpHost);
+        return httpClientContext;
+    }
+
+    private HttpUriRequest createHttpRequest(String host, String port, String url, String timeoutStr) {
+        RequestBuilder requestBuilder = RequestBuilder.get();
+        // uri
+        String uri = CollectUtil.replaceUriSpecialChar(url);
+        if (IpDomainUtil.isHasSchema(host)) {
+            requestBuilder.setUri(host + ":" + port + uri);
+        } else {
+            String ipAddressType = IpDomainUtil.checkIpAddressType(host);
+            String baseUri = CollectorConstants.IPV6.equals(ipAddressType)
+                    ? String.format("[%s]:%s", host, port + uri)
+                    : String.format("%s:%s", host, port + uri);
+
+            requestBuilder.setUri(CollectorConstants.HTTP_HEADER + baseUri);
+        }
+
+        requestBuilder.addHeader(HttpHeaders.CONNECTION, "keep-alive");
+        requestBuilder.addHeader(HttpHeaders.USER_AGENT, "Mozilla/5.0 (Windows NT 6.1; WOW64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.76 Safari/537.36");
+
+        requestBuilder.addHeader(HttpHeaders.ACCEPT, "text/plain");
+
+        int timeout = Integer.parseInt(timeoutStr);
+        if (timeout > 0) {
+            RequestConfig requestConfig = RequestConfig.custom()
+                    .setConnectTimeout(timeout)
+                    .setSocketTimeout(timeout)
+                    .setRedirectsEnabled(true)
+                    .build();
+            requestBuilder.setConfig(requestConfig);
+        }
+        return requestBuilder.build();
+    }
+
+    /**
+     * 解析Stats响应通过时间间隔进行筛选
+     *
+     * @param responseBody 响应体
+     * @param timePeriod   时间间隔
+     */
+    private void parseStatsResponse(String responseBody, String timePeriod, HashMap<String, String> resultMap) {
+        // 设置正则匹配
+        String timeRegex = String.format(REGEX, timePeriod);
+        Pattern pattern = Pattern.compile(timeRegex);
+        String[] strArray = responseBody.split(STR_SPLIT);
+        for (String str : strArray) {
+            Matcher matcher = pattern.matcher(str);
+            if (matcher.find()) {
+                String[] split = str.split(timeRegex);
+                resultMap.put(split[0], split[1]);
+            }
+        }
+    }
+
+
+    /**
+     * 解析Storage响应通过时间间隔进行筛选
+     *
+     * @param responseBody 响应体
+     */
+    private void parseStorageResponse(String responseBody, HashMap<String, String> resultMap) {
+        String[] strArray = responseBody.split(STR_SPLIT);
+        for (String str : strArray) {
+            String[] split = str.split(STORAGE_SPLIT_KEY_VALUE);
+            resultMap.put(split[0], split[1]);
+        }
+    }
+}

collector/src/main/java/org/dromara/hertzbeat/collector/dispatch/DispatchConstants.java

@@ -55,6 +55,10 @@ public interface DispatchConstants {
      * protocol memcached
      */
     String PROTOCOL_MEMCACHED = "memcached";
+    /**
+     * protocol nebulagraph
+     */
+    String PROTOCOL_NEBULAGRAPH = "nebulaGraph";
     /**
      * protocol udp
      */

collector/src/main/resources/META-INF/services/org.dromara.hertzbeat.collector.collect.AbstractCollect

@@ -18,4 +18,5 @@ org.dromara.hertzbeat.collector.collect.push.PushCollectImpl
 org.dromara.hertzbeat.collector.collect.dns.DnsCollectImpl
 org.dromara.hertzbeat.collector.collect.nginx.NginxCollectImpl
 org.dromara.hertzbeat.collector.collect.memcached.MemcachedCollectImpl
+org.dromara.hertzbeat.collector.collect.nebulagraph.NebulaGraphCollectImpl
 org.dromara.hertzbeat.collector.collect.pop3.Pop3CollectImpl

common/src/main/java/org/dromara/hertzbeat/common/entity/job/Metrics.java

@@ -135,6 +135,11 @@ public class Metrics {
      * 使用memcached的监控配置信息
      */
     private MemcachedProtocol memcached;
+    /**
+     * Monitoring configuration information using the nebulaGraph protocol
+     * 使用nebulaGraph的监控配置信息
+     */
+    private NebulaGraphProtocol nebulaGraph;
     /**
      * Use udp implemented by socket for service port detection configuration information
      * 使用socket实现的udp进行服务端口探测配置信息

common/src/main/java/org/dromara/hertzbeat/common/entity/job/protocol/NebulaGraphProtocol.java

@@ -0,0 +1,43 @@
+package org.dromara.hertzbeat.common.entity.job.protocol;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+/**
+ * @author dongfeng
+ */
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+public class NebulaGraphProtocol {
+    /**
+     * NebulaGraph 主机ip或域名
+     */
+    private String host;
+
+    /**
+     * NebulaGraph Graph 服务端口默认为 19669
+     * NebulaGraph Storage 服务端口默认为 19779
+     */
+    private String port;
+
+    /**
+     * NebulaGraph Graph 服务监控API为/stats
+     * NebulaGraph Storage 服务监控API为/rocksdb_stats
+     */
+    private String url;
+
+    /**
+     * NebulaGraph 监控时间间隔
+     */
+    private String timePeriod;
+
+    /**
+     * 超时时间
+     */
+    private String timeout;
+
+}

home/docs/help/nebulagraph.md

@@ -0,0 +1,74 @@
+---
+id: nebulaGraph
+title: Monitoring NebulaGraph
+sidebar_label: NebulaGraph Monitor
+keywords: [ open source monitoring tool, open source NebulaGraph monitoring tool, monitoring NebulaGraph metrics ]
+---
+
+> Collect and monitor the general performance Metrics of nebulaGraph.
+
+**Protocol Use：nebulaGraph**
+
+```text
+The monitoring has two parts,nebulaGraph_stats and rocksdb_stats.
+nebulaGraph_stats is nebulaGraph's statistics, and rocksdb_stats is rocksdb's statistics.
+```
+
+###           
+
+**1、Obtain available parameters through the stats and rocksdb stats interfaces.**
+
+1.1、 If you only need to get nebulaGraph_stats, you need to ensure that you have access to stats, or you'll get errors.
+
+The default port is 19669 and the access address is http://ip:19669/stats
+
+1.2、If you need to obtain additional parameters for rocksdb stats, you need to ensure that you have access to rocksdb
+stats, otherwise an error will be reported.
+
+Once you connect to NebulaGraph for the first time, you must first register your Storage service in order to properly
+query your data.
+
+**There is help_doc: https://docs.nebula-graph.com.cn/3.4.3/4.deployment-and-installation/connect-to-nebula-graph/**
+
+**https://docs.nebula-graph.com.cn/3.4.3/2.quick-start/3.quick-start-on-premise/3.1add-storage-hosts/**
+
+The default port is 19779 and the access address is:http://ip:19779/rocksdb_stats
+
+### Configuration parameter
+
+| Parameter name      | Parameter help description                                                                                                                                                |
+|---------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Monitoring Host     | Monitored IPV4, IPV6 or domain name. Note⚠️Without protocol header (eg: https://, http://)                                                                                |
+| Monitoring name     | Identify the name of this monitoring. The name needs to be unique                                                                                                         |
+| graphPort           | Port of the Graph service provided by Nebula Graph                                                                                                                        |
+| timePeriod          | The value can be 5 seconds, 60 seconds, 600 seconds, or 3600 seconds, indicating the last 5 seconds, last 1 minute, last 10 minutes, and last 1 hour, respectively.       |
+| storagePort         | Port of the storage service provided by Nebula Graph                                                                                                                      |
+| Timeout             | Allow collection response time                                                                                                                                            |
+| Collection interval | Interval time of monitor periodic data collection, unit: second, and the minimum interval that can be set is 30 seconds                                                   |
+| Whether to detect   | Whether to detect and check the availability of monitoring before adding monitoring. Adding and modifying operations will continue only after the detection is successful |
+| Description remarks | For more information about identifying and describing this monitoring, users can note information here                                                                    |
+
+### Collection Metrics
+
+#### Metrics Set：nebulaGraph_stats
+
+Too many indicators, related links are as follows
+**https://docs.nebula-graph.com.cn/3.4.3/6.monitor-and-metrics/1.query-performance-metrics/**
+
+| Metric name                           | Metric unit | Metric help description                                      |
+|---------------------------------------|-------------|--------------------------------------------------------------|
+| num_queries_hit_memory_watermark_rate |             | The rate of statements that reached the memory watermark.    |
+| num_queries_hit_memory_watermark_sum  |             | The sum of statements that reached the memory watermark.     |
+| num_reclaimed_expired_sessions_sum    |             | Number of expired sessions actively reclaimed by the server. |
+| ...                                   |             | ...                                                          |
+
+#### Metrics Set：rocksdb_stats
+
+Too many indicators, related links are as follows
+**https://docs.nebula-graph.com.cn/3.4.3/6.monitor-and-metrics/2.rocksdb-statistics/**
+
+| Metric name                | Metric unit | Metric help description                                     |
+|----------------------------|-------------|-------------------------------------------------------------|
+| rocksdb.backup.read.bytes  |             | Number of bytes read during the RocksDB database backup.    |
+| rocksdb.backup.write.bytes |             | Number of bytes written during the RocksDB database backup. |
+| ...                        |             | ...                                                         |