etcd_cluster_version metric is missing
#18805
Comments
|
and here is the all available metrics |
|
I couldn't reproduce this issue in 3.5.16. Please try the latest version (3.5.16) and provide the exact steps how to reproduce it. |
|
Discussed during sig-etcd triage meeting, @ugur99 we'll need more info on how to reproduce this. We could discuss in sig-etcd slack or would you be able to provide a precise step by step here on how to recreate your scenario? At the moment there is too much ambiguity to be able to recreate your setup. |
|
Yep you are totally right @jmhbnz. I think I can reproduce the issue on my local with the following steps:
# download etcd binaries
mkdir -p /tmp/etcd-download
curl -L https://storage.googleapis.com/etcd/v3.5.12/etcd-v3.5.12-linux-arm64.tar.gz -o /tmp/etcd-v3.5.12-linux-arm64.tar.gz
tar xzvf /tmp/etcd-v3.5.12-linux-arm64.tar.gz -C /tmp/etcd-download --strip-components=1
cp /tmp/etcd-download/etcd /usr/bin/etcd
cp /tmp/etcd-download/etcdctl /usr/bin/etcdctl
cp /tmp/etcd-download/etcdutl /usr/bin/etcdutl# cat /usr/lib/systemd/system/etcd.service
[Unit]
Description=etcd
[Service]
Type=notify
User=root
EnvironmentFile=/etc/etcd.env
ExecStart=/usr/bin/etcd
NotifyAccess=all
Restart=always
RestartSec=10s
LimitNOFILE=40000
[Install]
WantedBy=multi-user.target# cat /etc/etcd.env
# Environment file for etcd v3.5.12
ETCD_DATA_DIR=/var/lib/etcd
ETCD_ADVERTISE_CLIENT_URLS=https://<node2_host_ip>:2379
ETCD_INITIAL_ADVERTISE_PEER_URLS=https://<node2_host_ip>:2380
ETCD_INITIAL_CLUSTER_STATE=existing
ETCD_METRICS=basic
ETCD_LISTEN_METRICS_URLS=http://<node2_host_ip>:2381,http://127.0.0.1:2381
ETCD_LISTEN_CLIENT_URLS=https://<node2_host_ip>:2379,https://127.0.0.1:2379
ETCD_ELECTION_TIMEOUT=5000
ETCD_HEARTBEAT_INTERVAL=250
ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd
ETCD_LISTEN_PEER_URLS=https://<node2_host_ip>:2380
ETCD_NAME=etcd2
ETCD_PROXY=off
ETCD_INITIAL_CLUSTER=etcd1=https://<node1_host_ip>:2380,etcd2=https://<node2_host_ip>:2380
ETCD_AUTO_COMPACTION_RETENTION=8
ETCD_SNAPSHOT_COUNT=10000
ETCD_QUOTA_BACKEND_BYTES=6147483648
# Flannel need etcd v2 API
ETCD_ENABLE_V2=true
# TLS settings
ETCD_TRUSTED_CA_FILE=/etc/ssl/etcd/ssl/ca.pem
ETCD_CERT_FILE=/etc/ssl/etcd/ssl/member-node2.pem
ETCD_KEY_FILE=/etc/ssl/etcd/ssl/member-node2-key.pem
ETCD_CLIENT_CERT_AUTH=true
ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/etcd/ssl/ca.pem
ETCD_PEER_CERT_FILE=/etc/ssl/etcd/ssl/member-node2.pem
ETCD_PEER_KEY_FILE=/etc/ssl/etcd/ssl/member-node2-key.pem
ETCD_PEER_CLIENT_CERT_AUTH=True
# CLI settings
ETCDCTL_ENDPOINTS=https://127.0.0.1:2379
ETCDCTL_CACERT=/etc/ssl/etcd/ssl/ca.pem
ETCDCTL_KEY=/etc/ssl/etcd/ssl/admin-node2-key.pem
ETCDCTL_CERT=/etc/ssl/etcd/ssl/admin-node2.pem
# ETCD 3.5.x issue
# https://groups.google.com/a/kubernetes.io/g/dev/c/B7gJs88XtQc/m/rSgNOzV2BwAJ?utm_medium=email&utm_source=footer
ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK=True
# ETCD tracing configuration
ETCD_EXPERIMENTAL_ENABLE_DISTRIBUTED_TRACING=true
ETCD_EXPERIMENTAL_DISTRIBUTED_TRACING_SAMPLING_RATE=100
ETCD_EXPERIMENTAL_DISTRIBUTED_TRACING_ADDRESS=0.0.0.0:4317
ETCD_EXPERIMENTAL_DISTRIBUTED_TRACING_SERVICE_NAME=etcd
ETCD_EXPERIMENTAL_DISTRIBUTED_TRACING_INSTANCE_ID=etcd2# create first helper script for certificate generation
# cat make-ssl-etcd.sh
set -o errexit
set -o pipefail
MASTERS=(node1 node2)
HOSTS=(node1 node2)
CONFIG="/root/openssl.conf"
SSLDIR="/etc/ssl/etcd/ssl"
tmpdir=$(mktemp -d letcd_cacert.XXXXXX)
trap 'rm -rf "${tmpdir}"' EXIT
cd "${tmpdir}"
# Root CA
if [ -e "$SSLDIR/ca-key.pem" ]; then
# Reuse existing CA
cp $SSLDIR/{ca.pem,ca-key.pem} .
else
openssl genrsa -out ca-key.pem 2048 > /dev/null 2>&1
openssl req -x509 -new -nodes -key ca-key.pem -days 36500 -out ca.pem -subj "/CN=etcd-ca" > /dev/null 2>&1
fi
# ETCD member
if [ -n "$MASTERS" ]; then
echo "Creating certs for masters"
for host in "${MASTERS[@]}"; do
echo "Creating certs for $host"
cn="${host%%.*}"
# Member key
openssl genrsa -out member-${host}-key.pem 2048 > /dev/null 2>&1
openssl req -new -key member-${host}-key.pem -out member-${host}.csr -subj "/CN=etcd-member-${cn}" -config ${CONFIG} > /dev/null 2>&1
openssl x509 -req -in member-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out member-${host}.pem -days 36500 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
# Admin key
openssl genrsa -out admin-${host}-key.pem 2048 > /dev/null 2>&1
openssl req -new -key admin-${host}-key.pem -out admin-${host}.csr -subj "/CN=etcd-admin-${cn}" > /dev/null 2>&1
openssl x509 -req -in admin-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out admin-${host}.pem -days 36500 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
done
fi
# Node keys
if [ -n "$HOSTS" ]; then
echo "Creating certs for hosts"
for host in "${HOSTS[@]}"; do
echo "Creating certs for $host"
cn="${host%%.*}"
openssl genrsa -out node-${host}-key.pem 2048 > /dev/null 2>&1
openssl req -new -key node-${host}-key.pem -out node-${host}.csr -subj "/CN=etcd-node-${cn}" > /dev/null 2>&1
openssl x509 -req -in node-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out node-${host}.pem -days 36500 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
done
fi
# Install certs
if [ -e "$SSLDIR/ca-key.pem" ]; then
# No pass existing CA
rm -f ca.pem ca-key.pem
fi
if [ -n "$(ls -A *.pem)" ]; then
mv *.pem ${SSLDIR}/
fichmod +x make-ssl-etcd.sh# create openssl configuration file
# cat /root/openssl.conf
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names
[ ssl_client ]
extendedKeyUsage = clientAuth, serverAuth
basicConstraints = CA:FALSE
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid,issuer
subjectAltName = @alt_names
[ v3_ca ]
basicConstraints = CA:TRUE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names
authorityKeyIdentifier=keyid:always,issuer
[alt_names]
DNS.1 = localhost
DNS.2 = etcd.kube-system.svc.cluster.local
DNS.3 = etcd.kube-system.svc
DNS.4 = etcd.kube-system
DNS.5 = etcd
DNS.6 = node1
DNS.7 = node2
DNS.8 = lb-apiserver.kubernetes.local
IP.1 = 127.0.0.1
IP.2 = <node1_host_ip>
IP.3 = <node2_host_ip># create certificates directory on the second node
mkdir -p /etc/ssl/etcd/ssl
# copy etcd root CA and key to the second node (for simplicity I will skip controlplane joining steps; but in our environments we first join the new controlplane to the cluster and it fetches etcd CA from the secret in the cluster and we have another script that fetches etcd CA.key from the vault)
scp /etc/ssl/etcd/ssl/ca.pem /etc/ssl/etcd/ssl/ca-key.pem root@<node2_host_ip>:/etc/ssl/etcd/ssl/
# run the certificate generation script on the second node
./make-ssl-etcd.sh# create second helper script for etcd member joining
# cat join-etcd-member.sh
#!/bin/bash
export ETCDCTL_API=3
ETCD_CLUSTER=(<node1_host_ip> <node2_host_ip>)
PEER_URL="https://<node2_host_ip>:2380"
KEY="/etc/ssl/etcd/ssl/admin-node2-key.pem"
CERT="/etc/ssl/etcd/ssl/admin-node2.pem"
CACERT="/etc/ssl/etcd/ssl/ca.pem"
LOG_FILE="/var/log/etcd_join_member.log"
# Function to log messages with timestamps
log() {
echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a $LOG_FILE
}
# Function to check etcd instance status
check_etcd_status() {
local endpoint=$1
etcdctl --endpoints=https://$endpoint:2379 --key=$KEY --cert=$CERT --cacert=$CACERT endpoint status > /dev/null 2>&1
return $?
}
# Function to add new etcd member
add_etcd_member() {
local endpoint=$1
etcdctl --endpoints=https://$endpoint:2379 --key=$KEY --cert=$CERT --cacert=$CACERT member add etcd2 --peer-urls=$PEER_URL
return $?
}
# Retry mechanism
RETRIES=2
SLEEP_INTERVAL=2
for endpoint in "${ETCD_CLUSTER[@]}"; do
for ((i=1; i<=RETRIES; i++)); do
check_etcd_status $endpoint
if [ $? -eq 0 ]; then
log "etcd instance $endpoint is reachable"
add_etcd_member $endpoint
if [ $? -eq 0 ]; then
log "Successfully added new etcd member using instance $endpoint"
exit 0
else
log "Failed to add new etcd member using instance $endpoint"
fi
break
else
log "etcd instance $endpoint is not reachable (Attempt $i/$RETRIES)"
sleep $SLEEP_INTERVAL
fi
done
done
log "Failed to add new etcd member after checking all instances"
exit 1# run the etcd member joining script on the second node
./join-etcd-member.sh
# start the etcd service
systemctl start etcd
In the flatcar ignition template, we're following almost the same steps. After the new etcd/controlplane node is provisioned, these helper scripts act to set up the etcd service; download the binary; retrieve etcd ca.key from the vault (etcd root CA comes with the kubeadm join controlplane step); generate the certificate and finally join the new etcd member to the etcd cluster. So with this setup we are able to easily perform scaling operations for controlplanes running in a bare-metal environment. |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Bug report criteria
What happened?
new etcd members do not generate
etcd_cluster_versionmetrics; this is the only metrics being missing as of we've noticed. According to the etcd logs instance knows the cluster version; but this metrics is still missing.What did you expect to happen?
etcd_cluster_versionshould be availableHow can we reproduce it (as minimally and precisely as possible)?
Deploy a cluster using kubespray: deploy etcds as a systemd service on each controlplane node. Then remove the controlplanes (and etcds) one by one from the cluster by reprovisioning the new controlplanes from the ignition template (we have baked all the controlplane and etcd setup/join processes into the ignition template). We have exactly the same configuration, same etcd member names; and followed similar steps for the etcd member join/remove process.
Anything else we need to know?
No response
Etcd version (please run commands below)
Etcd configuration (command line flags or environment variables)
Environment file for etcd v3.5.12
ETCD_DATA_DIR=/var/lib/etcd
ETCD_ADVERTISE_CLIENT_URLS=https://12.60.87.25:2379
ETCD_INITIAL_ADVERTISE_PEER_URLS=https://12.60.87.25:2380
ETCD_INITIAL_CLUSTER_STATE=existing
ETCD_METRICS=basic
ETCD_LISTEN_METRICS_URLS=http://12.60.87.25:2381,http://127.0.0.1:2381
ETCD_LISTEN_CLIENT_URLS=https://12.60.87.25:2379,https://127.0.0.1:2379
ETCD_ELECTION_TIMEOUT=5000
ETCD_HEARTBEAT_INTERVAL=250
ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd
ETCD_LISTEN_PEER_URLS=https://12.60.87.25:2380
ETCD_NAME=etcd1
ETCD_PROXY=off
ETCD_INITIAL_CLUSTER=etcd1=https://12.60.87.25:2380,etcd2=https://12.60.87.90:2380,etcd3=https://12.60.87.152:2380
ETCD_AUTO_COMPACTION_RETENTION=8
ETCD_SNAPSHOT_COUNT=10000
ETCD_QUOTA_BACKEND_BYTES=6147483648
Flannel need etcd v2 API
ETCD_ENABLE_V2=true
TLS settings
ETCD_TRUSTED_CA_FILE=/etc/ssl/etcd/ssl/ca.pem
ETCD_CERT_FILE=/etc/ssl/etcd/ssl/member-dev-k8s-control1.pem
ETCD_KEY_FILE=/etc/ssl/etcd/ssl/member-dev-k8s-control1-key.pem
ETCD_CLIENT_CERT_AUTH=true
ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/etcd/ssl/ca.pem
ETCD_PEER_CERT_FILE=/etc/ssl/etcd/ssl/member-dev-k8s-control1.pem
ETCD_PEER_KEY_FILE=/etc/ssl/etcd/ssl/member-dev-k8s-control1-key.pem
ETCD_PEER_CLIENT_CERT_AUTH=True
CLI settings
ETCDCTL_ENDPOINTS=https://127.0.0.1:2379
ETCDCTL_CACERT=/etc/ssl/etcd/ssl/ca.pem
ETCDCTL_KEY=/etc/ssl/etcd/ssl/admin-dev-k8s-control1-key.pem
ETCDCTL_CERT=/etc/ssl/etcd/ssl/admin-dev-k8s-control1.pem
ETCD 3.5.x issue
https://groups.google.com/a/kubernetes.io/g/dev/c/B7gJs88XtQc/m/rSgNOzV2BwAJ?utm_medium=email&utm_source=footer
ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK=True
ETCD tracing configuration
ETCD_EXPERIMENTAL_ENABLE_DISTRIBUTED_TRACING=true
ETCD_EXPERIMENTAL_DISTRIBUTED_TRACING_SAMPLING_RATE=100
ETCD_EXPERIMENTAL_DISTRIBUTED_TRACING_ADDRESS=0.0.0.0:4317
ETCD_EXPERIMENTAL_DISTRIBUTED_TRACING_SERVICE_NAME=etcd
ETCD_EXPERIMENTAL_DISTRIBUTED_TRACING_INSTANCE_ID=etcd1
Etcd debug information (please run commands below, feel free to obfuscate the IP address or FQDN in the output)
Relevant log output
{"level":"info","ts":"2024-10-28T13:02:57.044182Z","caller":"rafthttp/stream.go:249","msg":"set message encoder","from":"335c9823932a1698","to":"5e66e8bfa5f5d5fb","stream-type":"stream Message"} {"level":"info","ts":"2024-10-28T13:02:57.044216Z","caller":"rafthttp/stream.go:274","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"335c9823932a1698","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:57.036097Z","caller":"rafthttp/stream.go:249","msg":"set message encoder","from":"335c9823932a1698","to":"5e66e8bfa5f5d5fb","stream-type":"stream MsgApp v2"} {"level":"info","ts":"2024-10-28T13:02:57.036152Z","caller":"rafthttp/stream.go:274","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"335c9823932a1698","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:57.027078Z","caller":"rafthttp/stream.go:412","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"335c9823932a1698","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:57.027352Z","caller":"rafthttp/stream.go:412","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"335c9823932a1698","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:57.023048Z","caller":"rafthttp/stream.go:412","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"335c9823932a1698","remote-peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:57.022501Z","caller":"rafthttp/stream.go:249","msg":"set message encoder","from":"335c9823932a1698","to":"f736583c8dc56a31","stream-type":"stream Message"} {"level":"info","ts":"2024-10-28T13:02:57.02253Z","caller":"rafthttp/stream.go:274","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"335c9823932a1698","remote-peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:57.021749Z","caller":"rafthttp/stream.go:249","msg":"set message encoder","from":"335c9823932a1698","to":"f736583c8dc56a31","stream-type":"stream MsgApp v2"} {"level":"info","ts":"2024-10-28T13:02:57.021771Z","caller":"rafthttp/peer_status.go:53","msg":"peer became active","peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:57.021782Z","caller":"rafthttp/stream.go:274","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"335c9823932a1698","remote-peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:57.021889Z","caller":"rafthttp/stream.go:412","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"335c9823932a1698","remote-peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:57.019182Z","caller":"rafthttp/peer_status.go:53","msg":"peer became active","peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:57.012975Z","caller":"rafthttp/stream.go:395","msg":"started stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"335c9823932a1698","remote-peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:57.01299Z","caller":"rafthttp/transport.go:317","msg":"added remote peer","local-member-id":"335c9823932a1698","remote-peer-id":"f736583c8dc56a31","remote-peer-urls":["https://12.60.87.152:2380"]} {"level":"info","ts":"2024-10-28T13:02:57.013073Z","caller":"etcdserver/server.go:1361","msg":"added peers from new cluster configuration"} {"level":"info","ts":"2024-10-28T13:02:57.013089Z","caller":"etcdserver/server.go:1240","msg":"applied snapshot","current-snapshot-index":1573588337,"current-applied-index":1573588337,"incoming-leader-snapshot-index":1573588337,"incoming-leader-snapshot-term":182} {"level":"info","ts":"2024-10-28T13:02:57.012781Z","caller":"rafthttp/stream.go:169","msg":"started stream writer with remote peer","local-member-id":"335c9823932a1698","remote-peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:57.012958Z","caller":"rafthttp/peer.go:137","msg":"started remote peer","remote-peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:57.012966Z","caller":"rafthttp/stream.go:395","msg":"started stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"335c9823932a1698","remote-peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:57.012056Z","caller":"rafthttp/peer.go:137","msg":"started remote peer","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:57.012082Z","caller":"rafthttp/transport.go:317","msg":"added remote peer","local-member-id":"335c9823932a1698","remote-peer-id":"5e66e8bfa5f5d5fb","remote-peer-urls":["https://12.60.87.90:2380"]} {"level":"info","ts":"2024-10-28T13:02:57.012097Z","caller":"rafthttp/peer.go:133","msg":"starting remote peer","remote-peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:57.012078Z","caller":"rafthttp/stream.go:395","msg":"started stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"335c9823932a1698","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:57.0121Z","caller":"rafthttp/stream.go:395","msg":"started stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"335c9823932a1698","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:57.012119Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"335c9823932a1698","remote-peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:57.012311Z","caller":"rafthttp/peer_status.go:53","msg":"peer became active","peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:57.012711Z","caller":"rafthttp/stream.go:169","msg":"started stream writer with remote peer","local-member-id":"335c9823932a1698","remote-peer-id":"f736583c8dc56a31"} {"level":"warn","ts":"2024-10-28T13:02:57.011045Z","caller":"rafthttp/stream.go:286","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream Message","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:57.011084Z","caller":"rafthttp/stream.go:294","msg":"stopped TCP streaming connection with remote peer","stream-writer-type":"stream Message","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:57.011125Z","caller":"rafthttp/pipeline.go:85","msg":"stopped HTTP pipelining with remote peer","local-member-id":"335c9823932a1698","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"warn","ts":"2024-10-28T13:02:57.01124Z","caller":"rafthttp/stream.go:421","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"335c9823932a1698","remote-peer-id":"5e66e8bfa5f5d5fb","error":"context canceled"} {"level":"warn","ts":"2024-10-28T13:02:57.011264Z","caller":"rafthttp/peer_status.go:66","msg":"peer became inactive (message send to peer failed)","peer-id":"5e66e8bfa5f5d5fb","error":"failed to read 5e66e8bfa5f5d5fb on stream MsgApp v2 (context canceled)"} {"level":"info","ts":"2024-10-28T13:02:57.011285Z","caller":"rafthttp/stream.go:442","msg":"stopped stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"335c9823932a1698","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"warn","ts":"2024-10-28T13:02:57.011347Z","caller":"rafthttp/stream.go:421","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"335c9823932a1698","remote-peer-id":"5e66e8bfa5f5d5fb","error":"context canceled"} {"level":"info","ts":"2024-10-28T13:02:57.011361Z","caller":"rafthttp/stream.go:442","msg":"stopped stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"335c9823932a1698","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:57.011375Z","caller":"rafthttp/peer.go:335","msg":"stopped remote peer","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:57.011389Z","caller":"rafthttp/transport.go:355","msg":"removed remote peer","local-member-id":"335c9823932a1698","removed-remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:57.011398Z","caller":"etcdserver/server.go:1351","msg":"removed old peers from network"} {"level":"info","ts":"2024-10-28T13:02:57.011409Z","caller":"etcdserver/server.go:1352","msg":"adding peers from new cluster configuration"} {"level":"info","ts":"2024-10-28T13:02:57.011436Z","caller":"rafthttp/peer.go:133","msg":"starting remote peer","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:57.011456Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"335c9823932a1698","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:57.011615Z","caller":"etcdserver/server.go:1302","msg":"closed old backend file"} {"level":"info","ts":"2024-10-28T13:02:57.011706Z","caller":"rafthttp/stream.go:169","msg":"started stream writer with remote peer","local-member-id":"335c9823932a1698","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:57.01178Z","caller":"rafthttp/stream.go:169","msg":"started stream writer with remote peer","local-member-id":"335c9823932a1698","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:57.010068Z","caller":"membership/cluster.go:287","msg":"set cluster version from store","cluster-version":"3.5"} {"level":"info","ts":"2024-10-28T13:02:57.009973Z","caller":"etcdserver/server.go:1341","msg":"restoring cluster configuration"} {"level":"info","ts":"2024-10-28T13:02:57.010031Z","caller":"api/capability.go:75","msg":"enabled capabilities for version","cluster-version":"3.5"} {"level":"info","ts":"2024-10-28T13:02:57.010043Z","caller":"membership/cluster.go:278","msg":"recovered/added member from store","cluster-id":"377f8845ceff7871","local-member-id":"335c9823932a1698","recovered-remote-peer-id":"5e66e8bfa5f5d5fb","recovered-remote-peer-urls":["https://12.60.87.90:2380"]} {"level":"info","ts":"2024-10-28T13:02:57.010054Z","caller":"membership/cluster.go:278","msg":"recovered/added member from store","cluster-id":"377f8845ceff7871","local-member-id":"335c9823932a1698","recovered-remote-peer-id":"f736583c8dc56a31","recovered-remote-peer-urls":["https://12.60.87.152:2380"]} {"level":"info","ts":"2024-10-28T13:02:57.010061Z","caller":"membership/cluster.go:278","msg":"recovered/added member from store","cluster-id":"377f8845ceff7871","local-member-id":"335c9823932a1698","recovered-remote-peer-id":"335c9823932a1698","recovered-remote-peer-urls":["https://12.60.87.25:2380"]} {"level":"info","ts":"2024-10-28T13:02:57.010075Z","caller":"etcdserver/server.go:1345","msg":"restored cluster configuration"} {"level":"info","ts":"2024-10-28T13:02:57.010084Z","caller":"etcdserver/server.go:1346","msg":"removing old peers from network"} {"level":"info","ts":"2024-10-28T13:02:57.010092Z","caller":"rafthttp/peer.go:330","msg":"stopping remote peer","remote-peer-id":"f736583c8dc56a31"} {"level":"warn","ts":"2024-10-28T13:02:57.010225Z","caller":"rafthttp/stream.go:286","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","remote-peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:57.010248Z","caller":"rafthttp/stream.go:294","msg":"stopped TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","remote-peer-id":"f736583c8dc56a31"} {"level":"warn","ts":"2024-10-28T13:02:57.010381Z","caller":"rafthttp/stream.go:286","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream Message","remote-peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:57.010403Z","caller":"rafthttp/stream.go:294","msg":"stopped TCP streaming connection with remote peer","stream-writer-type":"stream Message","remote-peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:57.010449Z","caller":"rafthttp/pipeline.go:85","msg":"stopped HTTP pipelining with remote peer","local-member-id":"335c9823932a1698","remote-peer-id":"f736583c8dc56a31"} {"level":"warn","ts":"2024-10-28T13:02:57.01056Z","caller":"rafthttp/stream.go:421","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"335c9823932a1698","remote-peer-id":"f736583c8dc56a31","error":"context canceled"} {"level":"warn","ts":"2024-10-28T13:02:57.010608Z","caller":"rafthttp/peer_status.go:66","msg":"peer became inactive (message send to peer failed)","peer-id":"f736583c8dc56a31","error":"failed to read f736583c8dc56a31 on stream MsgApp v2 (context canceled)"} {"level":"info","ts":"2024-10-28T13:02:57.010625Z","caller":"rafthttp/stream.go:442","msg":"stopped stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"335c9823932a1698","remote-peer-id":"f736583c8dc56a31"} {"level":"warn","ts":"2024-10-28T13:02:57.010685Z","caller":"rafthttp/stream.go:421","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"335c9823932a1698","remote-peer-id":"f736583c8dc56a31","error":"context canceled"} {"level":"info","ts":"2024-10-28T13:02:57.010706Z","caller":"rafthttp/stream.go:442","msg":"stopped stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"335c9823932a1698","remote-peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:57.010719Z","caller":"rafthttp/peer.go:335","msg":"stopped remote peer","remote-peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:57.010733Z","caller":"rafthttp/transport.go:355","msg":"removed remote peer","local-member-id":"335c9823932a1698","removed-remote-peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:57.010742Z","caller":"rafthttp/peer.go:330","msg":"stopping remote peer","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"warn","ts":"2024-10-28T13:02:57.010918Z","caller":"rafthttp/stream.go:286","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:57.010935Z","caller":"rafthttp/stream.go:294","msg":"stopped TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:57.009953Z","caller":"etcdserver/server.go:1337","msg":"restored v2 store"} {"level":"info","ts":"2024-10-28T13:02:57.009041Z","caller":"etcdserver/server.go:1328","msg":"restoring v2 store"} {"level":"info","ts":"2024-10-28T13:02:57.009033Z","caller":"etcdserver/server.go:1325","msg":"restored auth store"} {"level":"info","ts":"2024-10-28T13:02:57.00902Z","caller":"etcdserver/server.go:1321","msg":"restoring auth store"} {"level":"info","ts":"2024-10-28T13:02:57.009002Z","caller":"etcdserver/server.go:1318","msg":"restored alarm store"} {"level":"info","ts":"2024-10-28T13:02:57.00849Z","caller":"etcdserver/server.go:1300","msg":"closing old backend file"} {"level":"info","ts":"2024-10-28T13:02:57.008435Z","caller":"etcdserver/server.go:1312","msg":"restoring alarm store"} {"level":"info","ts":"2024-10-28T13:02:57.008419Z","caller":"etcdserver/server.go:1292","msg":"restored mvcc store","consistent-index":1573588337} {"level":"info","ts":"2024-10-28T13:02:57.008341Z","caller":"mvcc/kvstore.go:407","msg":"kvstore restored","current-rev":1490850186} {"level":"info","ts":"2024-10-28T13:02:56.47553Z","caller":"mvcc/kvstore.go:341","msg":"restored last compact revision","meta-bucket-name":"meta","meta-bucket-name-key":"finishedCompactRev","restored-compact-revision":1490814913} {"level":"info","ts":"2024-10-28T13:02:56.4755Z","caller":"etcdserver/server.go:1285","msg":"restoring mvcc store"} {"level":"info","ts":"2024-10-28T13:02:56.475488Z","caller":"etcdserver/server.go:1282","msg":"restored lease store"} {"level":"info","ts":"2024-10-28T13:02:56.474916Z","caller":"etcdserver/server.go:1278","msg":"restoring lease store"} {"level":"info","ts":"2024-10-28T13:02:56.474867Z","caller":"etcdserver/backend.go:81","msg":"opened backend db","path":"/var/lib/etcd/member/snap/db","took":"388.662852ms"} {"level":"info","ts":"2024-10-28T13:02:56.086084Z","caller":"etcdserver/raft.go:277","msg":"applied incoming Raft snapshot","snapshot-index":1573588337} {"level":"info","ts":"2024-10-28T13:02:56.085512Z","caller":"etcdserver/server.go:1232","msg":"applying snapshot","current-snapshot-index":0,"current-applied-index":0,"incoming-leader-snapshot-index":1573588337,"incoming-leader-snapshot-term":182} {"level":"info","ts":"2024-10-28T13:02:56.085425Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"335c9823932a1698 [commit: 1573588337] restored snapshot [index: 1573588337, term: 182]"} {"level":"info","ts":"2024-10-28T13:02:56.085418Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"335c9823932a1698 [commit: 1573588337, lastindex: 1573588337, lastterm: 182] restored snapshot [index: 1573588337, term: 182]"} {"level":"info","ts":"2024-10-28T13:02:56.085399Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"335c9823932a1698 switched to configuration voters=(3701000272351991448 6802380197010462203 17813522393210382897)"} {"level":"info","ts":"2024-10-28T13:02:56.085328Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 1573588337, term: 182]"} {"level":"info","ts":"2024-10-28T13:02:56.085172Z","caller":"rafthttp/http.go:286","msg":"received and saved database snapshot","local-member-id":"335c9823932a1698","remote-snapshot-sender-id":"5e66e8bfa5f5d5fb","incoming-snapshot-index":1573588337,"incoming-snapshot-size-bytes":1925492736,"incoming-snapshot-size":"1.9 GB","download-took":"5.781117634s"} {"level":"info","ts":"2024-10-28T13:02:56.085115Z","caller":"snap/db.go:66","msg":"saved database snapshot to disk","path":"/var/lib/etcd/member/snap/000000005dcb0d71.snap.db","bytes":1925492736,"size":"1.9 GB"} 2024/10/28 13:02:55 WARNING: [core] [Channel #1 SubChannel #2] grpc: addrConn.createTransport failed to connect to {Addr: "0.0.0.0:4317", ServerName: "0.0.0.0:4317", }. Err: connection error: desc = "transport: Error while dialing: dial tcp 0.0.0.0:4317: connect: connection refused" 2024/10/28 13:02:52 WARNING: [core] [Channel #1 SubChannel #2] grpc: addrConn.createTransport failed to connect to {Addr: "0.0.0.0:4317", ServerName: "0.0.0.0:4317", }. Err: connection error: desc = "transport: Error while dialing: dial tcp 0.0.0.0:4317: connect: connection refused" 2024/10/28 13:02:51 WARNING: [core] [Channel #1 SubChannel #2] grpc: addrConn.createTransport failed to connect to {Addr: "0.0.0.0:4317", ServerName: "0.0.0.0:4317", }. Err: connection error: desc = "transport: Error while dialing: dial tcp 0.0.0.0:4317: connect: connection refused" {"level":"info","ts":"2024-10-28T13:02:50.332892Z","caller":"etcdserver/server.go:783","msg":"initialized peer connections; fast-forwarding election ticks","local-member-id":"335c9823932a1698","forward-ticks":18,"forward-duration":"4.5s","election-ticks":20,"election-timeout":"5s","active-remote-members":2} {"level":"info","ts":"2024-10-28T13:02:50.305519Z","caller":"rafthttp/http.go:257","msg":"receiving database snapshot","local-member-id":"335c9823932a1698","remote-snapshot-sender-id":"5e66e8bfa5f5d5fb","incoming-snapshot-index":1573588337,"incoming-snapshot-message-size-bytes":37480,"incoming-snapshot-message-size":"38 kB"} {"level":"info","ts":"2024-10-28T13:02:50.301065Z","caller":"rafthttp/stream.go:274","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"335c9823932a1698","remote-peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:50.301035Z","caller":"rafthttp/stream.go:249","msg":"set message encoder","from":"335c9823932a1698","to":"f736583c8dc56a31","stream-type":"stream Message"} {"level":"info","ts":"2024-10-28T13:02:50.297754Z","caller":"rafthttp/stream.go:274","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"335c9823932a1698","remote-peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:50.297722Z","caller":"rafthttp/stream.go:249","msg":"set message encoder","from":"335c9823932a1698","to":"f736583c8dc56a31","stream-type":"stream MsgApp v2"} {"level":"info","ts":"2024-10-28T13:02:50.297371Z","caller":"rafthttp/stream.go:412","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"335c9823932a1698","remote-peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:50.296737Z","caller":"rafthttp/stream.go:274","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"335c9823932a1698","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:50.29671Z","caller":"rafthttp/stream.go:249","msg":"set message encoder","from":"335c9823932a1698","to":"5e66e8bfa5f5d5fb","stream-type":"stream MsgApp v2"} {"level":"info","ts":"2024-10-28T13:02:50.295942Z","caller":"rafthttp/stream.go:274","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"335c9823932a1698","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:50.295908Z","caller":"rafthttp/stream.go:249","msg":"set message encoder","from":"335c9823932a1698","to":"5e66e8bfa5f5d5fb","stream-type":"stream Message"} {"level":"info","ts":"2024-10-28T13:02:50.295405Z","caller":"rafthttp/stream.go:412","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"335c9823932a1698","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:50.295373Z","caller":"rafthttp/stream.go:412","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"335c9823932a1698","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:50.29531Z","caller":"rafthttp/peer_status.go:53","msg":"peer became active","peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:50.292626Z","caller":"rafthttp/stream.go:412","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"335c9823932a1698","remote-peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:50.292571Z","caller":"rafthttp/peer_status.go:53","msg":"peer became active","peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:50.286034Z","caller":"embed/etcd.go:857","msg":"serving metrics","address":"http://127.0.0.1:2381"} {"level":"info","ts":"2024-10-28T13:02:50.286011Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"raft.node: 335c9823932a1698 elected leader 5e66e8bfa5f5d5fb at term 182"} {"level":"info","ts":"2024-10-28T13:02:50.286008Z","caller":"embed/etcd.go:277","msg":"now serving peer/client/metrics","local-member-id":"335c9823932a1698","initial-advertise-peer-urls":["https://12.60.87.25:2380"],"listen-peer-urls":["https://12.60.87.25:2380"],"advertise-client-urls":["https://12.60.87.25:2379"],"listen-client-urls":["https://12.60.87.25:2379","https://127.0.0.1:2379"],"listen-metrics-urls":["http://12.60.87.25:2381","http://127.0.0.1:2381"]} {"level":"info","ts":"2024-10-28T13:02:50.285991Z","caller":"embed/etcd.go:857","msg":"serving metrics","address":"http://12.60.87.25:2381"} {"level":"info","ts":"2024-10-28T13:02:50.285995Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"335c9823932a1698 became follower at term 182"} {"level":"info","ts":"2024-10-28T13:02:50.285972Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"335c9823932a1698 [term: 0] received a MsgHeartbeat message with higher term from 5e66e8bfa5f5d5fb [term: 182]"} {"level":"info","ts":"2024-10-28T13:02:50.28574Z","caller":"embed/etcd.go:569","msg":"cmux::serve","address":"12.60.87.25:2380"} {"level":"info","ts":"2024-10-28T13:02:50.285715Z","caller":"embed/etcd.go:597","msg":"serving peer traffic","address":"12.60.87.25:2380"} {"level":"warn","ts":"2024-10-28T13:02:50.285646Z","caller":"embed/etcd.go:739","msg":"Flag `enable-v2` is deprecated and will get removed in etcd 3.6."} {"level":"info","ts":"2024-10-28T13:02:50.285623Z","caller":"embed/etcd.go:726","msg":"starting with client TLS","tls-info":"cert = /etc/ssl/etcd/ssl/member-dev-k8s-control1.pem, key = /etc/ssl/etcd/ssl/member-dev-k8s-control1-key.pem, client-cert=, client-key=, trusted-ca = /etc/ssl/etcd/ssl/ca.pem, client-cert-auth = true, crl-file = ","cipher-suites":[]} {"level":"info","ts":"2024-10-28T13:02:50.282909Z","caller":"fileutil/purge.go:50","msg":"started to purge file","dir":"/var/lib/etcd/member/wal","suffix":"wal","max":5,"interval":"30s"} {"level":"info","ts":"2024-10-28T13:02:50.282896Z","caller":"fileutil/purge.go:50","msg":"started to purge file","dir":"/var/lib/etcd/member/snap","suffix":"snap","max":5,"interval":"30s"} {"level":"info","ts":"2024-10-28T13:02:50.282815Z","caller":"fileutil/purge.go:50","msg":"started to purge file","dir":"/var/lib/etcd/member/snap","suffix":"snap.db","max":5,"interval":"30s"} {"level":"info","ts":"2024-10-28T13:02:50.282825Z","caller":"etcdserver/server.go:760","msg":"starting initial election tick advance","election-ticks":20} {"level":"info","ts":"2024-10-28T13:02:50.282732Z","caller":"etcdserver/server.go:860","msg":"starting etcd server","local-member-id":"335c9823932a1698","local-server-version":"3.5.12","cluster-version":"to_be_decided"} {"level":"info","ts":"2024-10-28T13:02:50.282637Z","caller":"rafthttp/stream.go:395","msg":"started stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"335c9823932a1698","remote-peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:50.282683Z","caller":"rafthttp/transport.go:317","msg":"added remote peer","local-member-id":"335c9823932a1698","remote-peer-id":"f736583c8dc56a31","remote-peer-urls":["https://12.60.87.152:2380"]} {"level":"info","ts":"2024-10-28T13:02:50.282661Z","caller":"rafthttp/stream.go:395","msg":"started stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"335c9823932a1698","remote-peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:50.282634Z","caller":"rafthttp/peer.go:137","msg":"started remote peer","remote-peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:50.282473Z","caller":"rafthttp/stream.go:169","msg":"started stream writer with remote peer","local-member-id":"335c9823932a1698","remote-peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:50.281891Z","caller":"rafthttp/stream.go:169","msg":"started stream writer with remote peer","local-member-id":"335c9823932a1698","remote-peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:50.281518Z","caller":"rafthttp/stream.go:395","msg":"started stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"335c9823932a1698","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:50.28158Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"335c9823932a1698","remote-peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:50.281564Z","caller":"rafthttp/peer.go:133","msg":"starting remote peer","remote-peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:50.281531Z","caller":"rafthttp/stream.go:395","msg":"started stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"335c9823932a1698","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:50.28154Z","caller":"rafthttp/transport.go:317","msg":"added remote peer","local-member-id":"335c9823932a1698","remote-peer-id":"5e66e8bfa5f5d5fb","remote-peer-urls":["https://12.60.87.90:2380"]} {"level":"info","ts":"2024-10-28T13:02:50.2815Z","caller":"rafthttp/peer.go:137","msg":"started remote peer","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:50.28144Z","caller":"rafthttp/stream.go:169","msg":"started stream writer with remote peer","local-member-id":"335c9823932a1698","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:50.280026Z","caller":"rafthttp/stream.go:169","msg":"started stream writer with remote peer","local-member-id":"335c9823932a1698","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:50.279737Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"335c9823932a1698","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:50.279726Z","caller":"rafthttp/peer.go:133","msg":"starting remote peer","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:50.279715Z","caller":"rafthttp/transport.go:286","msg":"added new remote peer","local-member-id":"335c9823932a1698","remote-peer-id":"f736583c8dc56a31","remote-peer-urls":["https://12.60.87.152:2380"]} {"level":"info","ts":"2024-10-28T13:02:50.279707Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"335c9823932a1698","remote-peer-id":"f736583c8dc56a31"} {"level":"info","ts":"2024-10-28T13:02:50.279681Z","caller":"rafthttp/transport.go:286","msg":"added new remote peer","local-member-id":"335c9823932a1698","remote-peer-id":"5e66e8bfa5f5d5fb","remote-peer-urls":["https://12.60.87.90:2380"]} {"level":"info","ts":"2024-10-28T13:02:50.279659Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"335c9823932a1698","remote-peer-id":"5e66e8bfa5f5d5fb"} {"level":"info","ts":"2024-10-28T13:02:50.278735Z","caller":"etcdserver/quota.go:117","msg":"enabled backend quota","quota-name":"v3-applier","quota-size-bytes":6147483648,"quota-size":"6.1 GB"} {"level":"info","ts":"2024-10-28T13:02:50.278557Z","caller":"mvcc/kvstore.go:407","msg":"kvstore restored","current-rev":1} {"level":"warn","ts":"2024-10-28T13:02:50.278083Z","caller":"auth/store.go:1241","msg":"simple token is not cryptographically signed"} {"level":"info","ts":"2024-10-28T13:02:50.27781Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"newRaft 335c9823932a1698 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]"} {"level":"info","ts":"2024-10-28T13:02:50.277797Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"335c9823932a1698 became follower at term 0"} {"level":"info","ts":"2024-10-28T13:02:50.277779Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"335c9823932a1698 switched to configuration voters=()"} {"level":"info","ts":"2024-10-28T13:02:50.277706Z","caller":"etcdserver/raft.go:495","msg":"starting local member","local-member-id":"335c9823932a1698","cluster-id":"377f8845ceff7871"} {"level":"info","ts":"2024-10-28T13:02:50.264092Z","caller":"netutil/netutil.go:112","msg":"resolved URL Host","url":"https://12.60.87.25:2380","host":"12.60.87.25:2380","resolved-addr":"12.60.87.25:2380"} {"level":"info","ts":"2024-10-28T13:02:50.264082Z","caller":"netutil/netutil.go:112","msg":"resolved URL Host","url":"https://12.60.87.152:2380","host":"12.60.87.152:2380","resolved-addr":"12.60.87.152:2380"} {"level":"info","ts":"2024-10-28T13:02:50.264067Z","caller":"netutil/netutil.go:112","msg":"resolved URL Host","url":"https://12.60.87.152:2380","host":"12.60.87.152:2380","resolved-addr":"12.60.87.152:2380"} {"level":"info","ts":"2024-10-28T13:02:50.264057Z","caller":"netutil/netutil.go:112","msg":"resolved URL Host","url":"https://12.60.87.90:2380","host":"12.60.87.90:2380","resolved-addr":"12.60.87.90:2380"} {"level":"info","ts":"2024-10-28T13:02:50.264037Z","caller":"netutil/netutil.go:112","msg":"resolved URL Host","url":"https://12.60.87.25:2380","host":"12.60.87.25:2380","resolved-addr":"12.60.87.25:2380"} {"level":"info","ts":"2024-10-28T13:02:50.263995Z","caller":"netutil/netutil.go:112","msg":"resolved URL Host","url":"https://12.60.87.90:2380","host":"12.60.87.90:2380","resolved-addr":"12.60.87.90:2380"} {"level":"info","ts":"2024-10-28T13:02:50.254535Z","caller":"etcdserver/backend.go:81","msg":"opened backend db","path":"/var/lib/etcd/member/snap/db","took":"2.800391ms"} 2024/10/28 13:02:50 WARNING: [core] [Channel #1 SubChannel #2] grpc: addrConn.createTransport failed to connect to {Addr: "0.0.0.0:4317", ServerName: "0.0.0.0:4317", }. Err: connection error: desc = "transport: Error while dialing: dial tcp 0.0.0.0:4317: connect: connection refused" {"level":"info","ts":"2024-10-28T13:02:50.251249Z","caller":"embed/etcd.go:308","msg":"starting an etcd server","etcd-version":"3.5.12","git-sha":"e7b3bb6cc","go-version":"go1.20.13","go-os":"linux","go-arch":"amd64","max-cpu-set":16,"max-cpu-available":16,"member-initialized":false,"name":"etcd1","data-dir":"/var/lib/etcd","wal-dir":"","wal-dir-dedicated":"","member-dir":"/var/lib/etcd/member","force-new-cluster":false,"heartbeat-interval":"250ms","election-timeout":"5s","initial-election-tick-advance":true,"snapshot-count":10000,"max-wals":5,"max-snapshots":5,"snapshot-catchup-entries":5000,"initial-advertise-peer-urls":["https://12.60.87.25:2380"],"listen-peer-urls":["https://12.60.87.25:2380"],"advertise-client-urls":["https://12.60.87.25:2379"],"listen-client-urls":["https://12.60.87.25:2379","https://127.0.0.1:2379"],"listen-metrics-urls":["http://12.60.87.25:2381","http://127.0.0.1:2381"],"cors":["*"],"host-whitelist":["*"],"initial-cluster":"etcd1=https://12.60.87.25:2380,etcd2=https://12.60.87.90:2380,etcd3=https://12.60.87.152:2380","initial-cluster-state":"existing","initial-cluster-token":"k8s_etcd","quota-backend-bytes":6147483648,"max-request-bytes":1572864,"max-concurrent-streams":4294967295,"pre-vote":true,"initial-corrupt-check":true,"corrupt-check-time-interval":"0s","compact-check-time-enabled":false,"compact-check-time-interval":"1m0s","auto-compaction-mode":"periodic","auto-compaction-retention":"8h0m0s","auto-compaction-interval":"8h0m0s","discovery-url":"","discovery-proxy":"","downgrade-check-interval":"5s"} {"level":"info","ts":"2024-10-28T13:02:50.251209Z","caller":"embed/etcd.go:240","msg":"distributed tracing setup enabled"} {"level":"info","ts":"2024-10-28T13:02:50.250809Z","caller":"embed/etcd.go:135","msg":"configuring client listeners","listen-client-urls":["https://12.60.87.25:2379","https://127.0.0.1:2379"]} {"level":"info","ts":"2024-10-28T13:02:50.250171Z","caller":"embed/etcd.go:494","msg":"starting with peer TLS","tls-info":"cert = /etc/ssl/etcd/ssl/member-dev-k8s-control1.pem, key = /etc/ssl/etcd/ssl/member-dev-k8s-control1-key.pem, client-cert=, client-key=, trusted-ca = /etc/ssl/etcd/ssl/ca.pem, client-cert-auth = true, crl-file = ","cipher-suites":[]} {"level":"info","ts":"2024-10-28T13:02:50.250148Z","caller":"embed/etcd.go:127","msg":"configuring peer listeners","listen-peer-urls":["https://12.60.87.25:2380"]} {"level":"warn","ts":"2024-10-28T13:02:50.250134Z","caller":"embed/config.go:679","msg":"Running http and grpc server on single port. This is not recommended for production."} {"level":"info","ts":"2024-10-28T13:02:50.250118Z","caller":"etcdmain/etcd.go:116","msg":"server has been already initialized","data-dir":"/var/lib/etcd","dir-type":"member"} {"level":"info","ts":"2024-10-28T13:02:50.250048Z","caller":"etcdmain/etcd.go:73","msg":"Running: ","args":["/opt/bin/etcd"]} {"level":"warn","ts":"2024-10-28T13:02:50.250008Z","caller":"embed/config.go:679","msg":"Running http and grpc server on single port. This is not recommended for production."} {"level":"info","ts":"2024-10-28T13:02:50.249701Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_CERT_FILE","variable-value":"/etc/ssl/etcd/ssl/member-dev-k8s-control1.pem"} {"level":"info","ts":"2024-10-28T13:02:50.249684Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_AUTO_COMPACTION_RETENTION","variable-value":"8"} {"level":"info","ts":"2024-10-28T13:02:50.249605Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_ADVERTISE_CLIENT_URLS","variable-value":"https://12.60.87.25:2379"} {"level":"fatal","ts":"2024-10-28T13:02:40.032197Z","caller":"etcdmain/etcd.go:204","msg":"discovery failed","error":"error validating peerURLs {ClusterID:377f8845ceff7871 Members:[&{ID:5e66e8bfa5f5d5fb RaftAttributes:{PeerURLs:[https://12.60.87.90:2380] IsLearner:false} Attributes:{Name:etcd2 ClientURLs:[https://12.60.87.90:2379]}} &{ID:f736583c8dc56a31 RaftAttributes:{PeerURLs:[https://12.60.87.152:2380] IsLearner:false} Attributes:{Name:etcd3 ClientURLs:[https://12.60.87.152:2379]}}] RemovedMemberIDs:[]}: member count is unequal","stacktrace":"go.etcd.io/etcd/server/v3/etcdmain.startEtcdOrProxyV2\n\tgo.etcd.io/etcd/server/v3/etcdmain/etcd.go:204\ngo.etcd.io/etcd/server/v3/etcdmain.Main\n\tgo.etcd.io/etcd/server/v3/etcdmain/main.go:40\nmain.main\n\tgo.etcd.io/etcd/server/v3/main.go:31\nruntime.main\n\truntime/proc.go:250"} {"level":"info","ts":"2024-10-28T13:02:40.032177Z","caller":"embed/etcd.go:377","msg":"closed etcd server","name":"etcd1","data-dir":"/var/lib/etcd","advertise-peer-urls":["https://12.60.87.25:2380"],"advertise-client-urls":["https://12.60.87.25:2379"]} {"level":"info","ts":"2024-10-28T13:02:40.031953Z","caller":"embed/etcd.go:375","msg":"closing etcd server","name":"etcd1","data-dir":"/var/lib/etcd","advertise-peer-urls":["https://12.60.87.25:2380"],"advertise-client-urls":["https://12.60.87.25:2379"]} {"level":"info","ts":"2024-10-28T13:02:40.017112Z","caller":"etcdserver/backend.go:81","msg":"opened backend db","path":"/var/lib/etcd/member/snap/db","took":"8.017932ms"} 2024/10/28 13:02:40 WARNING: [core] [Channel #1 SubChannel #2] grpc: addrConn.createTransport failed to connect to {Addr: "0.0.0.0:4317", ServerName: "0.0.0.0:4317", }. Err: connection error: desc = "transport: Error while dialing: dial tcp 0.0.0.0:4317: connect: connection refused" {"level":"info","ts":"2024-10-28T13:02:40.00888Z","caller":"embed/etcd.go:308","msg":"starting an etcd server","etcd-version":"3.5.12","git-sha":"e7b3bb6cc","go-version":"go1.20.13","go-os":"linux","go-arch":"amd64","max-cpu-set":16,"max-cpu-available":16,"member-initialized":false,"name":"etcd1","data-dir":"/var/lib/etcd","wal-dir":"","wal-dir-dedicated":"","member-dir":"/var/lib/etcd/member","force-new-cluster":false,"heartbeat-interval":"250ms","election-timeout":"5s","initial-election-tick-advance":true,"snapshot-count":10000,"max-wals":5,"max-snapshots":5,"snapshot-catchup-entries":5000,"initial-advertise-peer-urls":["https://12.60.87.25:2380"],"listen-peer-urls":["https://12.60.87.25:2380"],"advertise-client-urls":["https://12.60.87.25:2379"],"listen-client-urls":["https://12.60.87.25:2379","https://127.0.0.1:2379"],"listen-metrics-urls":["http://12.60.87.25:2381","http://127.0.0.1:2381"],"cors":["*"],"host-whitelist":["*"],"initial-cluster":"etcd1=https://12.60.87.25:2380,etcd2=https://12.60.87.90:2380,etcd3=https://12.60.87.152:2380","initial-cluster-state":"existing","initial-cluster-token":"k8s_etcd","quota-backend-bytes":6147483648,"max-request-bytes":1572864,"max-concurrent-streams":4294967295,"pre-vote":true,"initial-corrupt-check":true,"corrupt-check-time-interval":"0s","compact-check-time-enabled":false,"compact-check-time-interval":"1m0s","auto-compaction-mode":"periodic","auto-compaction-retention":"8h0m0s","auto-compaction-interval":"8h0m0s","discovery-url":"","discovery-proxy":"","downgrade-check-interval":"5s"} {"level":"info","ts":"2024-10-28T13:02:40.008828Z","caller":"embed/etcd.go:240","msg":"distributed tracing setup enabled"} {"level":"info","ts":"2024-10-28T13:02:40.008333Z","caller":"embed/etcd.go:135","msg":"configuring client listeners","listen-client-urls":["https://12.60.87.25:2379","https://127.0.0.1:2379"]} {"level":"info","ts":"2024-10-28T13:02:40.007637Z","caller":"embed/etcd.go:494","msg":"starting with peer TLS","tls-info":"cert = /etc/ssl/etcd/ssl/member-dev-k8s-control1.pem, key = /etc/ssl/etcd/ssl/member-dev-k8s-control1-key.pem, client-cert=, client-key=, trusted-ca = /etc/ssl/etcd/ssl/ca.pem, client-cert-auth = true, crl-file = ","cipher-suites":[]} {"level":"info","ts":"2024-10-28T13:02:40.007611Z","caller":"embed/etcd.go:127","msg":"configuring peer listeners","listen-peer-urls":["https://12.60.87.25:2380"]} {"level":"warn","ts":"2024-10-28T13:02:40.007604Z","caller":"embed/config.go:679","msg":"Running http and grpc server on single port. This is not recommended for production."} {"level":"info","ts":"2024-10-28T13:02:40.007588Z","caller":"etcdmain/etcd.go:116","msg":"server has been already initialized","data-dir":"/var/lib/etcd","dir-type":"member"} {"level":"info","ts":"2024-10-28T13:02:40.00752Z","caller":"etcdmain/etcd.go:73","msg":"Running: ","args":["/opt/bin/etcd"]} {"level":"warn","ts":"2024-10-28T13:02:40.007489Z","caller":"embed/config.go:679","msg":"Running http and grpc server on single port. This is not recommended for production."} {"level":"info","ts":"2024-10-28T13:02:40.007409Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_TRUSTED_CA_FILE","variable-value":"/etc/ssl/etcd/ssl/ca.pem"} {"level":"info","ts":"2024-10-28T13:02:40.007401Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_SNAPSHOT_COUNT","variable-value":"10000"} {"level":"info","ts":"2024-10-28T13:02:40.007391Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_QUOTA_BACKEND_BYTES","variable-value":"6147483648"} {"level":"info","ts":"2024-10-28T13:02:40.007383Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_PROXY","variable-value":"off"} {"level":"info","ts":"2024-10-28T13:02:40.007376Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_PEER_TRUSTED_CA_FILE","variable-value":"/etc/ssl/etcd/ssl/ca.pem"} {"level":"info","ts":"2024-10-28T13:02:40.007368Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_PEER_KEY_FILE","variable-value":"/etc/ssl/etcd/ssl/member-dev-k8s-control1-key.pem"} {"level":"info","ts":"2024-10-28T13:02:40.007358Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_PEER_CLIENT_CERT_AUTH","variable-value":"True"} {"level":"info","ts":"2024-10-28T13:02:40.007351Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_PEER_CERT_FILE","variable-value":"/etc/ssl/etcd/ssl/member-dev-k8s-control1.pem"} {"level":"info","ts":"2024-10-28T13:02:40.007343Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_NAME","variable-value":"etcd1"} {"level":"info","ts":"2024-10-28T13:02:40.007337Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_METRICS","variable-value":"basic"} {"level":"info","ts":"2024-10-28T13:02:40.007327Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_LISTEN_PEER_URLS","variable-value":"https://12.60.87.25:2380"} {"level":"info","ts":"2024-10-28T13:02:40.007319Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_LISTEN_METRICS_URLS","variable-value":"http://12.60.87.25:2381,http://127.0.0.1:2381"} {"level":"info","ts":"2024-10-28T13:02:40.00731Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_LISTEN_CLIENT_URLS","variable-value":"https://12.60.87.25:2379,https://127.0.0.1:2379"} {"level":"info","ts":"2024-10-28T13:02:40.0073Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_KEY_FILE","variable-value":"/etc/ssl/etcd/ssl/member-dev-k8s-control1-key.pem"} {"level":"info","ts":"2024-10-28T13:02:40.007293Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_INITIAL_CLUSTER_TOKEN","variable-value":"k8s_etcd"} {"level":"info","ts":"2024-10-28T13:02:40.007287Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_INITIAL_CLUSTER_STATE","variable-value":"existing"} {"level":"info","ts":"2024-10-28T13:02:40.00728Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_INITIAL_CLUSTER","variable-value":"etcd1=https://12.60.87.25:2380,etcd2=https://12.60.87.90:2380,etcd3=https://12.60.87.152:2380"} {"level":"info","ts":"2024-10-28T13:02:40.007274Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_INITIAL_ADVERTISE_PEER_URLS","variable-value":"https://12.60.87.25:2380"} {"level":"info","ts":"2024-10-28T13:02:40.007261Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_HEARTBEAT_INTERVAL","variable-value":"250"} {"level":"info","ts":"2024-10-28T13:02:40.007248Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK","variable-value":"True"} {"level":"info","ts":"2024-10-28T13:02:40.00724Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_EXPERIMENTAL_ENABLE_DISTRIBUTED_TRACING","variable-value":"true"} {"level":"info","ts":"2024-10-28T13:02:40.007233Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_EXPERIMENTAL_DISTRIBUTED_TRACING_SERVICE_NAME","variable-value":"etcd"} {"level":"info","ts":"2024-10-28T13:02:40.007226Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_EXPERIMENTAL_DISTRIBUTED_TRACING_SAMPLING_RATE","variable-value":"100"} {"level":"info","ts":"2024-10-28T13:02:40.007219Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_EXPERIMENTAL_DISTRIBUTED_TRACING_INSTANCE_ID","variable-value":"etcd1"} {"level":"info","ts":"2024-10-28T13:02:40.007212Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_EXPERIMENTAL_DISTRIBUTED_TRACING_ADDRESS","variable-value":"0.0.0.0:4317"} {"level":"info","ts":"2024-10-28T13:02:40.007202Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_ENABLE_V2","variable-value":"true"} {"level":"info","ts":"2024-10-28T13:02:40.007194Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_ELECTION_TIMEOUT","variable-value":"5000"} {"level":"info","ts":"2024-10-28T13:02:40.007186Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_DATA_DIR","variable-value":"/var/lib/etcd"} {"level":"info","ts":"2024-10-28T13:02:40.007176Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_CLIENT_CERT_AUTH","variable-value":"true"} {"level":"info","ts":"2024-10-28T13:02:40.007167Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_CERT_FILE","variable-value":"/etc/ssl/etcd/ssl/member-dev-k8s-control1.pem"} {"level":"info","ts":"2024-10-28T13:02:40.007157Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_AUTO_COMPACTION_RETENTION","variable-value":"8"} {"level":"info","ts":"2024-10-28T13:02:40.007056Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_ADVERTISE_CLIENT_URLS","variable-value":"https://12.60.87.25:2379"}The text was updated successfully, but these errors were encountered: