diff --git a/.gitignore b/.gitignore index 7dbf4eb..ecfb85f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ node_modules .devspace/ build +.idea # Ignore mkchain generated files *_values.yaml diff --git a/charts/tezos/scripts/evm-observer.sh b/charts/tezos/scripts/evm-observer.sh new file mode 100644 index 0000000..ad62bc7 --- /dev/null +++ b/charts/tezos/scripts/evm-observer.sh @@ -0,0 +1,49 @@ +set -ex + +TEZ_VAR=/var/tezos +TEZ_BIN=/usr/local/bin +ROLLUP_DATA_DIR="${TEZ_VAR}/rollup" +EVM_DATA_DIR="${TEZ_VAR}/evm" +EVM_CONFIG_FILE="${EVM_DATA_DIR}/config.json" + +EVM_CONFIGMAP_DIR="${TEZ_VAR}/evm-config" +EVM_CONFIGMAP_CONFIG_FILE="${EVM_CONFIGMAP_DIR}/config.json" + +# Wait till rollup node to be fully synchronized to run init from rollup command +WAIT_LIMIT=30 +COUNTER=1 +while [ "$(curl -s http://localhost:8932/local/synchronized | sed 's/^"//;s/"$//')" != "synchronized" ] && [ $COUNTER -lt $WAIT_LIMIT ]; do + COUNTER=$((COUNTER + 1)) + echo "polling ${COUNTER}/${WAIT_LIMIT}: rollup node not synchronized yet" + sleep 60 +done + +if [ "$(curl -s http://localhost:8932/local/synchronized | sed 's/^"//;s/"$//')" != "synchronized" ]; then + echo "ERROR: rollup node not synchronized within wait limit, exiting...." + exit 1 +fi + +if [ ! -e "${EVM_DATA_DIR}/store.sqlite" ]; then + $TEZ_BIN/octez-evm-node init from rollup node ${ROLLUP_DATA_DIR} --data-dir ${EVM_DATA_DIR} +fi + +# Provide basic config if there is no config specified in values.yaml +if [ ! -e "${EVM_CONFIGMAP_CONFIG_FILE}" ]; then + cat > "${EVM_CONFIG_FILE}" << EOF + { "rpc-addr": "0.0.0.0", "cors_origins": [ "*" ], + "cors_headers": [ "*" ], + "rollup_node_endpoint": "http://127.0.0.1:8932", "verbose": "notice", + "max_active_connections": 8000 } +EOF +else + cp -vf ${EVM_CONFIGMAP_CONFIG_FILE} ${EVM_CONFIG_FILE} +fi + +CMD="$TEZ_BIN/octez-evm-node run observer \ + --evm-node-endpoint ${EVM_NODE_ENDPOINT} \ + --rollup-node-endpoint http://127.0.0.1:8932 \ + --data-dir ${EVM_DATA_DIR} \ + --preimages-endpoint ${ROLLUP_PREIMAGES_ENDPOINT} \ + --rpc-addr 0.0.0.0 --rpc-port 8545" + +exec $CMD \ No newline at end of file diff --git a/charts/tezos/scripts/evm-sequencer.sh b/charts/tezos/scripts/evm-sequencer.sh new file mode 100644 index 0000000..e9d86a3 --- /dev/null +++ b/charts/tezos/scripts/evm-sequencer.sh @@ -0,0 +1,22 @@ +set -e + +TEZ_VAR=/var/tezos +TEZ_BIN=/usr/local/bin +ROLLUP_DATA_DIR="$TEZ_VAR/rollup" +EVM_DATA_DIR="$TEZ_VAR/evm" + +set -x + +if [ ! -e "${EVM_DATA_DIR}/store/store.dict" ]; then + $TEZ_BIN/octez-evm-node init from rollup node ${ROLLUP_DATA_DIR} --data-dir ${EVM_DATA_DIR} +fi + + +CMD="$TEZ_BIN/octez-evm-node run sequencer \ + with endpoint http://127.0.0.1:8932 \ + signing with edsk3rw6fcwjPe5xkGWAbSquLDQALKP8XyMhy4c6PQGr7qQKTYa8rX \ + --data-dir ${EVM_DATA_DIR} \ + --time-between-blocks 6 \ + --rpc-addr 0.0.0.0" + +exec $CMD diff --git a/charts/tezos/scripts/smart-rollup-observer.sh b/charts/tezos/scripts/smart-rollup-observer.sh new file mode 100644 index 0000000..bf27ea0 --- /dev/null +++ b/charts/tezos/scripts/smart-rollup-observer.sh @@ -0,0 +1,23 @@ +set -ex + +TEZ_VAR=/var/tezos +TEZ_BIN=/usr/local/bin +ROLLUP_DATA_DIR="${TEZ_VAR}/rollup" + +$TEZ_BIN/octez-smart-rollup-node init observer \ + config for "${ROLLUP_ADDRESS}" \ + with operators \ + --history-mode archive \ + --data-dir "${ROLLUP_DATA_DIR}" \ + --rpc-addr 0.0.0.0 \ + --pre-images-endpoint "${ROLLUP_PREIMAGES_ENDPOINT}" \ + --force + +CMD="$TEZ_BIN/octez-smart-rollup-node \ + --endpoint http://tezos-node-rpc:8732 \ + run \ + --data-dir ${ROLLUP_DATA_DIR} \ + --acl-override allow-all \ + --log-kernel-debug" + +exec $CMD diff --git a/charts/tezos/scripts/smart-rollup-node.sh b/charts/tezos/scripts/smart-rollup-operator.sh similarity index 68% rename from charts/tezos/scripts/smart-rollup-node.sh rename to charts/tezos/scripts/smart-rollup-operator.sh index d29e9d0..f9bb373 100644 --- a/charts/tezos/scripts/smart-rollup-node.sh +++ b/charts/tezos/scripts/smart-rollup-operator.sh @@ -1,4 +1,4 @@ -set -ex +set -e TEZ_VAR=/var/tezos TEZ_BIN=/usr/local/bin @@ -9,12 +9,20 @@ ROLLUP_DATA_DIR_PREIMAGES="$ROLLUP_DATA_DIR/wasm_2_0_0" xxd -p -c 0 /usr/local/share/tezos/evm_kernel/evm_installer.wasm | tr -d '\n' > /var/tezos/smart-rollup-boot-sector mkdir -p "$ROLLUP_DATA_DIR_PREIMAGES" cp /usr/local/share/tezos/evm_kernel/* "$ROLLUP_DATA_DIR_PREIMAGES" -CMD="$TEZ_BIN/octez-smart-rollup-node \ + +set -x +$TEZ_BIN/octez-smart-rollup-node \ --endpoint http://tezos-node-rpc:8732 \ -d $CLIENT_DIR \ - run operator for ${ROLLUP_ADDRESS} with operators ${OPERATORS_PARAMS} \ + init operator config for ${ROLLUP_ADDRESS} with operators ${OPERATORS_PARAMS} \ --data-dir ${ROLLUP_DATA_DIR} \ --boot-sector-file /var/tezos/smart-rollup-boot-sector \ - --rpc-addr 0.0.0.0" + --rpc-addr 0.0.0.0 \ + --force +CMD="$TEZ_BIN/octez-smart-rollup-node \ + --endpoint http://tezos-node-rpc:8732 \ + -d $CLIENT_DIR \ + run \ + --data-dir ${ROLLUP_DATA_DIR}" exec $CMD diff --git a/charts/tezos/scripts/smart-rollup-snapshot-downloader.sh b/charts/tezos/scripts/smart-rollup-snapshot-downloader.sh new file mode 100644 index 0000000..5ef99aa --- /dev/null +++ b/charts/tezos/scripts/smart-rollup-snapshot-downloader.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +set -e + +data_dir="/var/tezos" +rollup_data_dir="$data_dir/rollup" +snapshot_file="$rollup_data_dir/rollup.snapshot" + +if [ ! -d "$data_dir" ]; then + echo "ERROR: /var/tezos doesn't exist. There should be a volume mounted." + exit 1 +fi + +if [ -e "$rollup_data_dir/storage/l2_blocks/data" ]; then + echo "Smart rollup snapshot has already been imported. Exiting." + exit 0 +fi + +# if [ -e "${snapshot_file}" ]; then +# echo "Temporary - snapshot file already dl'd, not doing it again" +# exit 0 +# fi + +echo "Did not find a pre-existing smart rollup snapshot." + +echo "Downloading ${SNAPSHOT_URL}" +mkdir -p "$rollup_data_dir" +curl -LfsS ${SNAPSHOT_URL} | tee >(sha256sum > ${snapshot_file}.sha256sum) > "$snapshot_file" + +chown -R 1000:1000 "$data_dir" \ No newline at end of file diff --git a/charts/tezos/scripts/smart-rollup-snapshot-importer.sh b/charts/tezos/scripts/smart-rollup-snapshot-importer.sh new file mode 100644 index 0000000..fb6c07a --- /dev/null +++ b/charts/tezos/scripts/smart-rollup-snapshot-importer.sh @@ -0,0 +1,22 @@ +set -ex + +bin_dir="/usr/local/bin" +data_dir="/var/tezos" +rollup_data_dir="$data_dir/rollup" +smart_rollup_node="$bin_dir/octez-smart-rollup-node" +snapshot_file=${rollup_data_dir}/rollup.snapshot + +if [ ! -f ${snapshot_file} ]; then + echo "No snapshot to import." + exit 0 +fi + +if [ -e "$rollup_data_dir/storage/l2_blocks/data" ]; then + echo "Smart rollup snapshot has already been imported. Exiting." + exit 0 +fi + +${smart_rollup_node} --endpoint ${NODE_RPC_URL} snapshot import ${snapshot_file} --data-dir ${rollup_data_dir} --no-check +find ${node_dir} + +rm -rvf ${snapshot_file} diff --git a/charts/tezos/templates/octez-rollup-node.yaml b/charts/tezos/templates/octez-rollup-node.yaml index b6c6a95..e806948 100644 --- a/charts/tezos/templates/octez-rollup-node.yaml +++ b/charts/tezos/templates/octez-rollup-node.yaml @@ -1,10 +1,10 @@ {{- range $k, $v := .Values.smartRollupNodes }} -{{- if $v.evm_proxy | default false }} +{{- if $v.evm }} apiVersion: v1 kind: Service metadata: - name: evm-proxy-{{ $k }} + name: evm-{{ $v.evm.mode }}-{{ $k }} namespace: {{ $.Release.Namespace }} spec: type: NodePort @@ -12,89 +12,17 @@ spec: - port: 8545 name: evm-proxy selector: - app: evm-proxy-{{ $k }} + app: rollup-{{ $k }} +{{- if $v.evm.config }} --- -apiVersion: apps/v1 -kind: Deployment +apiVersion: v1 +kind: ConfigMap metadata: - name: evm-proxy-{{ $k }} + name: evm-{{ $v.evm.mode }}-{{ $k }}-configmap namespace: {{ $.Release.Namespace }} -{{- if $v.evm_proxy.annotations | default false }} -{{- with $v.evm_proxy.annotations }} - annotations: - {{- toYaml . | nindent 4 }} -{{- end }} -{{- end }} -spec: - podManagementPolicy: Parallel - replicas: 1 - serviceName: evm-proxy-{{ $k }} - selector: - matchLabels: - app: evm-proxy-{{ $k }} - template: - metadata: - labels: - app: evm-proxy-{{ $k }} - spec: - containers: - - name: octez-evm-proxy - image: "{{ $.Values.images.octez }}" - imagePullPolicy: {{ $.Values.images_pull_policy }} - ports: - - containerPort: 8545 - name: evm-proxy - command: - - /bin/sh - args: - - "-c" - - | -{{ tpl ($.Files.Get "scripts/evm-proxy.sh") $ | indent 12 }} - env: - - name: MY_POD_NAME - value: {{ $k }} - securityContext: - fsGroup: 1000 ---- -{{- if $v.evm_proxy.ingress | default false }} -{{- if $v.evm_proxy.ingress.enabled | default false }} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: evm-proxy-{{ $k }} - namespace: {{ $.Release.Namespace }} -{{- with $v.evm_proxy.ingress.labels }} - labels: - {{- toYaml . | nindent 4 }} -{{- end }} -{{- with $v.evm_proxy.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} -{{- end }} -spec: - ingressClassName: {{ $v.evm_proxy.ingress.className }} - {{- if $v.evm_proxy.ingress.tls }} - tls: - {{- range $v.evm_proxy.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - - host: {{ $v.evm_proxy.ingress.host }} - http: - paths: - - pathType: Prefix - path: / - backend: - service: - name: evm-proxy-{{ $k }} - port: - name: evm-proxy -{{- end }} +data: + config.json: | + {{ $v.evm.config | nindent 4 }} {{- end }} {{- end }} --- @@ -116,7 +44,7 @@ kind: StatefulSet metadata: name: rollup-{{ $k }} namespace: {{ $.Release.Namespace }} -{{- if $v.annotations | default false }} +{{- if $v.annotations }} {{- with $v.annotations }} annotations: {{- toYaml . | nindent 4 }} @@ -149,17 +77,108 @@ spec: args: - "-c" - | -{{ tpl ($.Files.Get "scripts/smart-rollup-node.sh") $ | indent 12 }} +{{- if eq $v.mode "observer" }} +{{ tpl ($.Files.Get "scripts/smart-rollup-observer.sh") $ | indent 12 }} +{{- else if eq $v.mode "operator" }} +{{ tpl ($.Files.Get "scripts/smart-rollup-operator.sh") $ | indent 12 }} +{{- end }} env: - name: ROLLUP_ADDRESS value: {{ $v.rollup_address }} + - name: ROLLUP_PREIMAGES_ENDPOINT + value: {{ $v.preimages_endpoint }} - name: OPERATORS_PARAMS value: > {{- range $key, $value := $v.operators }} {{- $cleanKey := regexReplaceAll "[0-9]+$" $key "" }} {{ $cleanKey }}:{{ $value }} {{- end }} +{{- if $v.resources }} +{{- with $v.resources }} + resources: + {{- toYaml . | nindent 10 }} +{{- end }} +{{- end }} +{{- if $v.evm }} + - name: octez-evm-{{ $v.evm.mode }} + image: "{{ $v.evm.image.name | default $.Values.images.octez }}" + imagePullPolicy: {{ $v.evm.image.pullPolicy | default $.Values.images_pull_policy }} + ports: + - containerPort: 8545 + name: evm + command: + - /bin/sh + args: + - "-c" + - | +{{- if eq $v.evm.mode "observer" }} +{{ tpl ($.Files.Get "scripts/evm-observer.sh") $ | indent 12 }} +{{- else if eq $v.evm.mode "sequencer" }} +{{ tpl ($.Files.Get "scripts/evm-sequencer.sh") $ | indent 12 }} +{{- end }} + volumeMounts: + - mountPath: /var/tezos + name: var-volume +{{- if $v.evm.config }} + - mountPath: /var/tezos/evm-config + name: evm-config-volume + subpathsubPath: config.json +{{- end }} + env: + - name: MY_POD_NAME + value: {{ $k }} + - name: ROLLUP_PREIMAGES_ENDPOINT + value: {{ $v.preimages_endpoint }} + {{- if eq $v.evm.mode "observer" }} + - name: EVM_NODE_ENDPOINT + value: {{ $v.evm.evm_endpoint }} + {{- end }} +{{- if $v.evm.resources }} +{{- with $v.evm.resources }} + resources: + {{- toYaml . | nindent 10 }} +{{- end }} +{{- end }} + - name: sidecar + image: "{{ $.Values.tezos_k8s_images.utils }}" + imagePullPolicy: {{ $.Values.tezos_k8s_images_pull_policy }} + args: + - sidecar-evm + env: + - name: MY_POD_NAME + value: {{ $k }} +{{- end }} initContainers: + - name: snapshot-downloader + image: "{{ $.Values.tezos_k8s_images.utils }}" + imagePullPolicy: {{ $.Values.tezos_k8s_images_pull_policy }} + command: + - /bin/sh + volumeMounts: + - mountPath: /var/tezos + name: var-volume + args: + - "-c" + - | +{{ tpl ($.Files.Get "scripts/smart-rollup-snapshot-downloader.sh") $ | indent 12 }} + env: + - name: SNAPSHOT_URL + value: {{ $v.snapshot_url }} + - name: snapshot-importer + image: "{{ $.Values.images.octez }}" + imagePullPolicy: {{ $.Values.tezos_k8s_images_pull_policy }} + command: + - /bin/sh + volumeMounts: + - mountPath: /var/tezos + name: var-volume + args: + - "-c" + - | +{{ tpl ($.Files.Get "scripts/smart-rollup-snapshot-importer.sh") $ | indent 12 }} + env: + - name: NODE_RPC_URL + value: {{ $v.node_rpc_url }} - image: {{ $.Values.tezos_k8s_images.utils }} imagePullPolicy: {{ $.Values.tezos_k8s_images_pull_policy }} name: config-generator @@ -180,12 +199,19 @@ spec: name: tezos-accounts securityContext: fsGroup: 1000 + nodeSelector: + {{- toYaml $v.nodeSelector | nindent 8 }} volumes: - volume: var-volume name: var-volume - name: tezos-accounts secret: secretName: tezos-secret +{{- if $v.evm.config }} + - name: evm-config-volume + configMap: + name: evm-{{ $v.evm.mode }}-{{ $k }}-configmap +{{- end }} volumeClaimTemplates: - metadata: name: var-volume @@ -194,7 +220,10 @@ spec: - ReadWriteOnce resources: requests: - storage: "15Gi" + storage: "400Gi" + {{- if $v.storageClassName }} + storageClassName: {{ $v.storageClassName }} + {{- end }} --- {{- if $v.ingress | default false }} {{- if $v.ingress.enabled | default false }} @@ -236,4 +265,4 @@ spec: name: rollup {{- end }} {{- end }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/tezos/values.yaml b/charts/tezos/values.yaml index f756a4a..d3bb0ae 100644 --- a/charts/tezos/values.yaml +++ b/charts/tezos/values.yaml @@ -393,6 +393,7 @@ smartRollupNodes: {} # ``` # smartRollupNodes: # rollup-node-0: +# mode: operator # operators: # operating: archive-baking-node-0 # cementing: archive-baking-node-0 @@ -402,6 +403,9 @@ smartRollupNodes: {} # batching3: batcher3-account # batching4: batcher4-account # rollup_address: sr1RYurGZtN8KNSpkMcCt9CgWeUaNkzsAfXf +# snapshot_url: https://snapshots.us.tzinit.org/etherlink-ghostnet/eth-ghostnet.snapshot +# node_rpc_url: http://rolling-node-0.rolling-node:8732 +# preimages_endpoint: https://snapshots.eu.tzinit.org/etherlink-mainnet/wasm_2_0_0 # annotations: {} # ingress: # enabled: false @@ -411,7 +415,45 @@ smartRollupNodes: {} # labels: {} # pathType: Prefix # tls: [] -# evm_proxy: +# evm: +# mode: observer +# image: +# name: tezos/tezos:master +# pullPolicy: IfNotPresent +# annotations: {} +# ingress: +# enabled: false +# host: "" +# annotations: {} +# className: "" +# labels: {} +# pathType: Prefix +# tls: [] +# rollup-node-1: +# mode: observer +# rollup_address: sr1RYurGZtN8KNSpkMcCt9CgWeUaNkzsAfXf +# preimages_endpoint: https://snapshots.eu.tzinit.org/etherlink-mainnet/wasm_2_0_0 +# config: | +# { "smart-rollup-address": "sr1RYurGZtN8KNSpkMcCt9CgWeUaNkzsAfXf", +# "smart-rollup-node-operator": {}, "rpc-addr": "0.0.0.0", +# "fee-parameters": {}, "mode": "observer", +# "pre-images-endpoint": +# "https://snapshots.eu.tzinit.org/etherlink-ghostnet/wasm_2_0_0", +# "history-mode": "archive" } +# annotations: {} +# ingress: +# enabled: false +# host: "" +# annotations: {} +# className: "" +# labels: {} +# pathType: Prefix +# tls: [] +# evm: +# mode: observer +# image: +# name: tezos/tezos:master +# pullPolicy: IfNotPresent # annotations: {} # ingress: # enabled: false diff --git a/utils/Dockerfile b/utils/Dockerfile index 082a297..d493743 100644 --- a/utils/Dockerfile +++ b/utils/Dockerfile @@ -29,6 +29,7 @@ COPY config-generator.sh / COPY entrypoint.sh / COPY logger.sh / COPY sidecar.py / +COPY sidecar-evm.py / COPY snapshot-downloader.sh / COPY wait-for-dns.sh / ENTRYPOINT ["/entrypoint.sh"] diff --git a/utils/entrypoint.sh b/utils/entrypoint.sh index 2b13396..aef525c 100755 --- a/utils/entrypoint.sh +++ b/utils/entrypoint.sh @@ -7,6 +7,7 @@ case "$CMD" in config-generator) exec /config-generator.sh "$@" ;; logger) exec /logger.sh "$@" ;; sidecar) exec /sidecar.py "$@" ;; + sidecar-evm) exec /sidecar-evm.py "$@" ;; snapshot-downloader) exec /snapshot-downloader.sh "$@" ;; wait-for-dns) exec /wait-for-dns.sh "$@" ;; esac diff --git a/utils/sidecar-evm.py b/utils/sidecar-evm.py new file mode 100755 index 0000000..fd6eb44 --- /dev/null +++ b/utils/sidecar-evm.py @@ -0,0 +1,57 @@ +#! /usr/bin/env python +from flask import Flask +import requests +import time +import logging + +log = logging.getLogger("werkzeug") +log.setLevel(logging.ERROR) + +application = Flask(__name__) + +AGE_LIMIT_IN_SECS = 600 +# https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ +# Default readiness probe timeoutSeconds is 1s, timeout sync request before that and return a +# connect timeout error if necessary +NODE_CONNECT_TIMEOUT = 0.9 + +@application.route("/health") +def health(): + try: + payload = { + "jsonrpc": "2.0", + "method": "eth_getBlockByNumber", + "params": ["latest", False], + "id": 1 + } + resp = requests.post("http://127.0.0.1:8545", json=payload, timeout=NODE_CONNECT_TIMEOUT) + resp = resp.json() + except ConnectTimeout as e: + err = "Timeout connect to node, %s" % repr(e), 500 + application.logger.error(err) + return err + except ReadTimeout as e: + err = "Timeout read from node, %s" % repr(e), 500 + application.logger.error(err) + return err + except RequestException as e: + err = "Could not connect to node, %s" % repr(e), 500 + application.logger.error(err) + return err + + latest_block = resp['timestamp'] + current_timestamp = int(time.time()) + time_diff = current_timestamp - latest_block_timestamp + if time_diff > AGE_LIMIT_IN_SECS: + err = ( + "Error: Chain head is %s secs old, older than %s" + % (time_diff, AGE_LIMIT_IN_SECS), + 500, + ) + application.logger.error(err) + return err + return "evm rpc is healthy" + + +if __name__ == "__main__": + application.run(host="0.0.0.0", port=32767, debug=False) \ No newline at end of file