-
Notifications
You must be signed in to change notification settings - Fork 1
/
Makefile
199 lines (153 loc) · 7.03 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
SHELL=/usr/bin/env bash
ifndef BIN_PATH
BIN_PATH=~/bin
endif
ifndef SOURCE_PATH
SOURCE_PATH=~/libra
endif
ifndef ARCHIVE_PATH
ARCHIVE_PATH=~/epoch-archive
endif
ifndef DATA_PATH
DATA_PATH=~/.0L
endif
ifndef DB_PATH
DB_PATH=${DATA_PATH}/db
endif
ifndef URL
URL=http://localhost
endif
ifndef EPOCH
EPOCH=$(shell expr ${EPOCH_NOW} - 1)
endif
ifndef EPOCH_LEN
EPOCH_LEN = 1
endif
ifndef TRANS_LEN
TRANS_LEN = 1
endif
ifndef EPOCH_NOW
EPOCH_NOW := $(shell db-backup one-shot query node-state | cut -d ":" -d "," -f 1 | cut -d ":" -f 2| xargs)
endif
ifndef DB_VERSION
DB_VERSION := $(shell db-backup one-shot query node-state | cut -d ":" -d "," -f 2 | cut -d ":" -f 2| xargs)
endif
ifndef LATEST_BACKUP
LATEST_BACKUP = $(shell ls -a ~/epoch-archive/ | sort -n | tail -1 | tr -dc '0-9')
endif
ifndef NEXT_BACKUP
NEXT_BACKUP = $(shell echo "${LATEST_BACKUP} + 1" | bc)
endif
ifndef END_EPOCH
END_EPOCH = $(shell expr ${EPOCH} + ${EPOCH_LEN})
endif
# TODO: Use actual epoch waypoint instead of what is in repo
#EPOCH_WAYPOINT := $(shell ol query --epoch | cut -d ":" -f 2-3| xargs)
ifndef EPOCH_WAYPOINT
EPOCH_WAYPOINT = $(shell jq -r ".waypoints[0]" ${ARCHIVE_PATH}/${EPOCH}/ep*/epoch_ending.manifest)
endif
ifndef EPOCH_HEIGHT
EPOCH_HEIGHT = $(shell echo ${EPOCH_WAYPOINT} | cut -d ":" -f 1)
endif
ifndef VERSION
VERSION = ${DB_VERSION}
endif
check:
@if test -z ${EPOCH}; then \
echo "Must provide EPOCH in environment" 1>&2; \
exit 1; \
fi
@echo data-path: ${DATA_PATH}
@echo target-db: ${DB_PATH}
@echo backup-service-url: ${URL}
@echo start-epoch: ${EPOCH}
@echo epoch-now: ${EPOCH_NOW}
@echo end-epoch: ${END_EPOCH}
@echo epoch-height: ${EPOCH_HEIGHT}
@echo db-version: ${DB_VERSION}
@echo env-versions: ${VERSION}
@echo latest-backup: ${LATEST_BACKUP}
@echo next-backup: ${NEXT_BACKUP}
wipe:
sudo rm -rf ${DB_PATH}
create-folder: check
@if test ! -d ${ARCHIVE_PATH}/${EPOCH}; then \
mkdir ${ARCHIVE_PATH}/${EPOCH}; \
fi
create-version-folder: check
@if test -z ${VERSION}; then \
echo "Must provide VERSION in environment" 1>&2; \
exit 1; \
fi
@if test ! -d ${ARCHIVE_PATH}/${EPOCH}/${VERSION}; then \
mkdir ${ARCHIVE_PATH}/${EPOCH}/${VERSION}; \
fi
bins:
cd ${SOURCE_PATH} && cargo build -p backup-cli --release
cp -f ${SOURCE_PATH}/target/release/db-restore /usr/local/bin/db-restore
cp -f ${SOURCE_PATH}/target/release/db-backup /usr/local/bin/db-backup
commit:
# save to epoch archive repo for testing
git add -A && git commit -a -m "epoch archive ${EPOCH} - ${EPOCH_WAYPOINT} - ${VERSION}" && git push
zip:
# zip -r ${EPOCH}.zip ${EPOCH}
tar -czvf ${EPOCH}.tar.gz ${EPOCH}
epoch:
@echo ${EPOCH_NOW}
restore-all: restore-epoch restore-transaction restore-snapshot restore-waypoint restore-yaml
# Destructive command. node.yaml, and db will be wiped.
restore-latest:
export EPOCH=$(shell ls | sort -n | tail -1) && make restore-all restore-yaml
backup-all: backup-epoch backup-transaction backup-snapshot
backup-epoch: create-folder
# IMPORTANT: The db-restore tool assumes you are running this from the location of your backups (likely the epoch-archive git project)
# The manifest file includes OS paths to chunks. Those paths are relative and fail if this is run outside of epoch-archive
${BIN_PATH}/db-backup one-shot backup --backup-service-address ${URL}:6186 epoch-ending --start-epoch ${EPOCH} --end-epoch ${END_EPOCH} local-fs --dir ${ARCHIVE_PATH}/${EPOCH}
backup-transaction: create-folder
${BIN_PATH}/db-backup one-shot backup --backup-service-address ${URL}:6186 transaction --num_transactions ${TRANS_LEN} --start-version ${EPOCH_HEIGHT} local-fs --dir ${ARCHIVE_PATH}/${EPOCH}
backup-snapshot: create-folder
${BIN_PATH}/db-backup one-shot backup --backup-service-address ${URL}:6186 state-snapshot --state-version ${EPOCH_HEIGHT} local-fs --dir ${ARCHIVE_PATH}/${EPOCH}
backup-version: create-version-folder
# IMPORTANT: this assumes that EPOCH is already backed up
${BIN_PATH}/db-backup one-shot backup --backup-service-address ${URL}:6186 transaction --num_transactions 1 --start-version ${VERSION} local-fs --dir ${ARCHIVE_PATH}/${EPOCH}/${VERSION}
${BIN_PATH}/db-backup one-shot backup --backup-service-address ${URL}:6186 state-snapshot --state-version ${VERSION} local-fs --dir ${ARCHIVE_PATH}/${EPOCH}/${VERSION}
restore-epoch:
${BIN_PATH}/db-restore --target-db-dir ${DB_PATH} epoch-ending --epoch-ending-manifest ${ARCHIVE_PATH}/${EPOCH}/epoch_ending_${EPOCH}*/epoch_ending.manifest local-fs --dir ${ARCHIVE_PATH}/${EPOCH}
restore-transaction:
${BIN_PATH}/db-restore --target-db-dir ${DB_PATH} transaction --transaction-manifest ${ARCHIVE_PATH}/${EPOCH}/transaction_${EPOCH_HEIGHT}*/transaction.manifest local-fs --dir ${ARCHIVE_PATH}/${EPOCH}
restore-snapshot:
${BIN_PATH}/db-restore --target-db-dir ${DB_PATH} state-snapshot --state-manifest ${ARCHIVE_PATH}/${EPOCH}/state_ver_${EPOCH_HEIGHT}*/state.manifest --state-into-version ${EPOCH_HEIGHT} local-fs --dir ${ARCHIVE_PATH}/${EPOCH}
restore-waypoint:
@echo ${EPOCH_WAYPOINT} > ${DATA_PATH}/restore_waypoint
restore-yaml:
@if test ! -d ${DATA_PATH}; then \
mkdir ${DATA_PATH}; \
fi
cp ${ARCHIVE_PATH}/fullnode_template.node.yaml ${DATA_PATH}/node.yaml
sed 's/THE_WAYPOINT/${EPOCH_WAYPOINT}/g' ${DATA_PATH}/node.yaml
restore-version: restore-all
# ${BIN_PATH}/db-restore --target-db-dir ${DB_PATH} epoch-ending --epoch-ending-manifest ${ARCHIVE_PATH}/${EPOCH}/epoch_ending_${EPOCH}*/epoch_ending.manifest local-fs --dir ${ARCHIVE_PATH}/${EPOCH}
# ${BIN_PATH}/db-restore --target-db-dir ${DB_PATH} state-snapshot --state-manifest ${ARCHIVE_PATH}/${EPOCH}/state_ver_${EPOCH_HEIGHT}*/state.manifest --state-into-version ${EPOCH_HEIGHT} local-fs --dir ${ARCHIVE_PATH}/${EPOCH}
# ${BIN_PATH}/db-restore --target-db-dir ${DB_PATH} transaction --transaction-manifest ${ARCHIVE_PATH}/${EPOCH}/transaction_${EPOCH_HEIGHT}*/transaction.manifest local-fs --dir ${ARCHIVE_PATH}/${EPOCH}
${BIN_PATH}/db-restore --target-db-dir ${DB_PATH} state-snapshot --state-manifest ${ARCHIVE_PATH}/${EPOCH}/${VERSION}/state_ver_${VERSION}*/state.manifest --state-into-version ${VERSION} local-fs --dir ${ARCHIVE_PATH}/${EPOCH}/${VERSION}
${BIN_PATH}/db-restore --target-db-dir ${DB_PATH} transaction --transaction-manifest ${ARCHIVE_PATH}/${EPOCH}/${VERSION}/transaction_${VERSION}*/transaction.manifest local-fs --dir ${ARCHIVE_PATH}/${EPOCH}/${VERSION}
cron:
echo "NOW: ${EPOCH_NOW} NEXT: ${NEXT_BACKUP}"
@if [ ${EPOCH_NOW} -gt ${NEXT_BACKUP} ]; then \
cd ~/epoch-archive/ && git pull && EPOCH=${NEXT_BACKUP} make backup-all zip commit; \
else \
echo "Newer EPOCH boundary is unavailable."; \
exit 1; \
fi
cron-nogit:
echo "NOW: ${EPOCH_NOW} NEXT: ${NEXT_BACKUP}"
@if [ ${EPOCH_NOW} -gt ${NEXT_BACKUP} ]; then \
cd ~/epoch-archive/ && EPOCH=${NEXT_BACKUP} make backup-all zip; \
else \
echo "Newer EPOCH boundary is unavailable."; \
exit 1; \
fi
cron-hourly:
cd ~/epoch-archive/ && git pull && EPOCH=${LATEST_BACKUP} VERSION=${DB_VERSION} make backup-version zip commit
cron-hourly-nogit:
cd ~/epoch-archive/ && EPOCH=${LATEST_BACKUP} VERSION=${DB_VERSION} make backup-version zip