Skip to content

Commit

Permalink
support monitoring NebulaGraph metrics and add help doc (#1441)
Browse files Browse the repository at this point in the history
Co-authored-by: 东风 <[email protected]>
  • Loading branch information
ZY945 and 东风 authored Dec 24, 2023
1 parent 862785a commit 8b2da91
Show file tree
Hide file tree
Showing 8 changed files with 1,222 additions and 0 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,196 @@
package org.dromara.hertzbeat.collector.collect.nebulagraph;

import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.dromara.hertzbeat.collector.collect.AbstractCollect;
import org.dromara.hertzbeat.collector.collect.common.http.CommonHttpClient;
import org.dromara.hertzbeat.collector.dispatch.DispatchConstants;
import org.dromara.hertzbeat.collector.util.CollectUtil;
import org.dromara.hertzbeat.common.constants.CollectorConstants;
import org.dromara.hertzbeat.common.constants.CommonConstants;
import org.dromara.hertzbeat.common.entity.job.Metrics;
import org.dromara.hertzbeat.common.entity.job.protocol.NebulaGraphProtocol;
import org.dromara.hertzbeat.common.entity.message.CollectRep;
import org.dromara.hertzbeat.common.util.CommonUtil;
import org.dromara.hertzbeat.common.util.IpDomainUtil;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;


/**
*
*/
@Slf4j
public class NebulaGraphCollectImpl extends AbstractCollect {
private final static int SUCCESS_CODE = 200;

private final static String[] TIME_RANGE = new String[]{"5", "60", "600", "3600"};

private final static String REGEX = "\\.%s\\=";

private final static String STR_SPLIT = "\n";

private final static String STORAGE_SPLIT_KEY_VALUE = "=";

private final static String GRAPH_API = "/stats";

private final static String STORAGE_API = "/rocksdb_stats";


@Override
public void collect(CollectRep.MetricsData.Builder builder, long monitorId, String app, Metrics metrics) {
long startTime = System.currentTimeMillis();
if (metrics == null || metrics.getNebulaGraph() == null) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("NebulaGraph collect must has NebulaGraph params");
return;
}
NebulaGraphProtocol nebulaGraph = metrics.getNebulaGraph();
String timePeriod = nebulaGraph.getTimePeriod();

if (!Objects.isNull(nebulaGraph.getTimePeriod())&&!Arrays.asList(TIME_RANGE).contains(timePeriod)) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("The time range for metric statistics, currently supporting 5 seconds, 60 seconds, 600 seconds, and 3600 seconds.");
return;
}

if (nebulaGraph.getHost() == null || nebulaGraph.getHost().isEmpty()) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("The host of NebulaGraph must be set");
return;
}

String resp;
long responseTime;
HashMap<String, String> resultMap = new HashMap<>(64);
CloseableHttpResponse response;
HttpContext httpContext = createHttpContext(nebulaGraph.getHost(), nebulaGraph.getPort());
HttpUriRequest request = createHttpRequest(nebulaGraph.getHost(), nebulaGraph.getPort(),
nebulaGraph.getUrl(), nebulaGraph.getTimeout());
try {
// 发起http请求,获取响应数据
response = CommonHttpClient.getHttpClient().execute(request, httpContext);
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != SUCCESS_CODE) {
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("StatusCode " + statusCode);
return;
}
resp = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
responseTime = System.currentTimeMillis() - startTime;
resultMap.put(CollectorConstants.RESPONSE_TIME, Long.toString(responseTime));
// 根据API进行不同解析
if (GRAPH_API.equals(nebulaGraph.getUrl())) {
parseStatsResponse(resp, nebulaGraph.getTimePeriod(), resultMap);
} else if (STORAGE_API.equals(nebulaGraph.getUrl())) {
parseStorageResponse(resp, resultMap);
}
List<String> aliasFields = metrics.getAliasFields();
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String field : aliasFields) {
String fieldValue = resultMap.get(field);
valueRowBuilder.addColumns(Objects.requireNonNullElse(fieldValue, CommonConstants.NULL_VALUE));
}
builder.addValues(valueRowBuilder.build());
} catch (IOException e) {
String errorMsg = CommonUtil.getMessageFromThrowable(e);
log.info(errorMsg);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
}
}


@Override
public String supportProtocol() {
return DispatchConstants.PROTOCOL_NEBULAGRAPH;
}

private HttpContext createHttpContext(String host, String port) {
HttpHost httpHost = new HttpHost(host, Integer.parseInt(port));
HttpClientContext httpClientContext = new HttpClientContext();
httpClientContext.setTargetHost(httpHost);
return httpClientContext;
}

private HttpUriRequest createHttpRequest(String host, String port, String url, String timeoutStr) {
RequestBuilder requestBuilder = RequestBuilder.get();
// uri
String uri = CollectUtil.replaceUriSpecialChar(url);
if (IpDomainUtil.isHasSchema(host)) {
requestBuilder.setUri(host + ":" + port + uri);
} else {
String ipAddressType = IpDomainUtil.checkIpAddressType(host);
String baseUri = CollectorConstants.IPV6.equals(ipAddressType)
? String.format("[%s]:%s", host, port + uri)
: String.format("%s:%s", host, port + uri);

requestBuilder.setUri(CollectorConstants.HTTP_HEADER + baseUri);
}

requestBuilder.addHeader(HttpHeaders.CONNECTION, "keep-alive");
requestBuilder.addHeader(HttpHeaders.USER_AGENT, "Mozilla/5.0 (Windows NT 6.1; WOW64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.76 Safari/537.36");

requestBuilder.addHeader(HttpHeaders.ACCEPT, "text/plain");

int timeout = Integer.parseInt(timeoutStr);
if (timeout > 0) {
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(timeout)
.setSocketTimeout(timeout)
.setRedirectsEnabled(true)
.build();
requestBuilder.setConfig(requestConfig);
}
return requestBuilder.build();
}

/**
* 解析Stats响应通过时间间隔进行筛选
*
* @param responseBody 响应体
* @param timePeriod 时间间隔
*/
private void parseStatsResponse(String responseBody, String timePeriod, HashMap<String, String> resultMap) {
// 设置正则匹配
String timeRegex = String.format(REGEX, timePeriod);
Pattern pattern = Pattern.compile(timeRegex);
String[] strArray = responseBody.split(STR_SPLIT);
for (String str : strArray) {
Matcher matcher = pattern.matcher(str);
if (matcher.find()) {
String[] split = str.split(timeRegex);
resultMap.put(split[0], split[1]);
}
}
}


/**
* 解析Storage响应通过时间间隔进行筛选
*
* @param responseBody 响应体
*/
private void parseStorageResponse(String responseBody, HashMap<String, String> resultMap) {
String[] strArray = responseBody.split(STR_SPLIT);
for (String str : strArray) {
String[] split = str.split(STORAGE_SPLIT_KEY_VALUE);
resultMap.put(split[0], split[1]);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,10 @@ public interface DispatchConstants {
* protocol memcached
*/
String PROTOCOL_MEMCACHED = "memcached";
/**
* protocol nebulagraph
*/
String PROTOCOL_NEBULAGRAPH = "nebulaGraph";
/**
* protocol udp
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,5 @@ org.dromara.hertzbeat.collector.collect.push.PushCollectImpl
org.dromara.hertzbeat.collector.collect.dns.DnsCollectImpl
org.dromara.hertzbeat.collector.collect.nginx.NginxCollectImpl
org.dromara.hertzbeat.collector.collect.memcached.MemcachedCollectImpl
org.dromara.hertzbeat.collector.collect.nebulagraph.NebulaGraphCollectImpl
org.dromara.hertzbeat.collector.collect.pop3.Pop3CollectImpl
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,11 @@ public class Metrics {
* 使用memcached的监控配置信息
*/
private MemcachedProtocol memcached;
/**
* Monitoring configuration information using the nebulaGraph protocol
* 使用nebulaGraph的监控配置信息
*/
private NebulaGraphProtocol nebulaGraph;
/**
* Use udp implemented by socket for service port detection configuration information
* 使用socket实现的udp进行服务端口探测配置信息
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package org.dromara.hertzbeat.common.entity.job.protocol;

import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;

/**
*
*/
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class NebulaGraphProtocol {
/**
* NebulaGraph 主机ip或域名
*/
private String host;

/**
* NebulaGraph Graph 服务端口默认为 19669
* NebulaGraph Storage 服务端口默认为 19779
*/
private String port;

/**
* NebulaGraph Graph 服务监控API为/stats
* NebulaGraph Storage 服务监控API为/rocksdb_stats
*/
private String url;

/**
* NebulaGraph 监控时间间隔
*/
private String timePeriod;

/**
* 超时时间
*/
private String timeout;

}
74 changes: 74 additions & 0 deletions home/docs/help/nebulagraph.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
---
id: nebulaGraph
title: Monitoring NebulaGraph
sidebar_label: NebulaGraph Monitor
keywords: [ open source monitoring tool, open source NebulaGraph monitoring tool, monitoring NebulaGraph metrics ]
---

> Collect and monitor the general performance Metrics of nebulaGraph.
**Protocol Use:nebulaGraph**

```text
The monitoring has two parts,nebulaGraph_stats and rocksdb_stats.
nebulaGraph_stats is nebulaGraph's statistics, and rocksdb_stats is rocksdb's statistics.
```

###

**1、Obtain available parameters through the stats and rocksdb stats interfaces.**

1.1、 If you only need to get nebulaGraph_stats, you need to ensure that you have access to stats, or you'll get errors.

The default port is 19669 and the access address is http://ip:19669/stats

1.2、If you need to obtain additional parameters for rocksdb stats, you need to ensure that you have access to rocksdb
stats, otherwise an error will be reported.

Once you connect to NebulaGraph for the first time, you must first register your Storage service in order to properly
query your data.

**There is help_doc: https://docs.nebula-graph.com.cn/3.4.3/4.deployment-and-installation/connect-to-nebula-graph/**

**https://docs.nebula-graph.com.cn/3.4.3/2.quick-start/3.quick-start-on-premise/3.1add-storage-hosts/**

The default port is 19779 and the access address is:http://ip:19779/rocksdb_stats

### Configuration parameter

| Parameter name | Parameter help description |
|---------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| Monitoring Host | Monitored IPV4, IPV6 or domain name. Note⚠️Without protocol header (eg: https://, http://) |
| Monitoring name | Identify the name of this monitoring. The name needs to be unique |
| graphPort | Port of the Graph service provided by Nebula Graph |
| timePeriod | The value can be 5 seconds, 60 seconds, 600 seconds, or 3600 seconds, indicating the last 5 seconds, last 1 minute, last 10 minutes, and last 1 hour, respectively. |
| storagePort | Port of the storage service provided by Nebula Graph |
| Timeout | Allow collection response time |
| Collection interval | Interval time of monitor periodic data collection, unit: second, and the minimum interval that can be set is 30 seconds |
| Whether to detect | Whether to detect and check the availability of monitoring before adding monitoring. Adding and modifying operations will continue only after the detection is successful |
| Description remarks | For more information about identifying and describing this monitoring, users can note information here |

### Collection Metrics

#### Metrics Set:nebulaGraph_stats

Too many indicators, related links are as follows
**https://docs.nebula-graph.com.cn/3.4.3/6.monitor-and-metrics/1.query-performance-metrics/**

| Metric name | Metric unit | Metric help description |
|---------------------------------------|-------------|--------------------------------------------------------------|
| num_queries_hit_memory_watermark_rate | | The rate of statements that reached the memory watermark. |
| num_queries_hit_memory_watermark_sum | | The sum of statements that reached the memory watermark. |
| num_reclaimed_expired_sessions_sum | | Number of expired sessions actively reclaimed by the server. |
| ... | | ... |

#### Metrics Set:rocksdb_stats

Too many indicators, related links are as follows
**https://docs.nebula-graph.com.cn/3.4.3/6.monitor-and-metrics/2.rocksdb-statistics/**

| Metric name | Metric unit | Metric help description |
|----------------------------|-------------|-------------------------------------------------------------|
| rocksdb.backup.read.bytes | | Number of bytes read during the RocksDB database backup. |
| rocksdb.backup.write.bytes | | Number of bytes written during the RocksDB database backup. |
| ... | | ... |
Loading

0 comments on commit 8b2da91

Please sign in to comment.