From 76f379b81b78442ceb1edf1b83d1f2de428f1519 Mon Sep 17 00:00:00 2001 From: Andrey Samokhvalov Date: Thu, 5 Oct 2017 12:25:43 +0000 Subject: [PATCH 01/38] gitignore: update ignore to exclude binaries --- .gitignore | 7 +++++++ accesshttp/restart.sh | 4 ++-- accessws/restart.sh | 4 ++-- alertcenter/restart.sh | 4 ++-- marketprice/restart.sh | 4 ++-- matchengine/restart.sh | 4 ++-- 6 files changed, 17 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 9a0ca4b9..1410591e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,10 @@ +test/matchengine/cli +accesshttp/accesshttp +accessws/accessws +alertcenter/alertcenter +marketprice/marketprice +readhistroy/readhistory + # Compiled Object files *.slo *.lo diff --git a/accesshttp/restart.sh b/accesshttp/restart.sh index e04b3094..8504529c 100755 --- a/accesshttp/restart.sh +++ b/accesshttp/restart.sh @@ -1,5 +1,5 @@ #!/bin/bash -killall -s SIGQUIT accesshttp.exe +killall -s SIGQUIT accesshttp sleep 1 -./accesshttp.exe config.json +./accesshttp config.json diff --git a/accessws/restart.sh b/accessws/restart.sh index ecb38ee9..b247daba 100755 --- a/accessws/restart.sh +++ b/accessws/restart.sh @@ -1,5 +1,5 @@ #!/bin/bash -killall -s SIGQUIT accessws.exe +killall -s SIGQUIT accessws sleep 1 -./accessws.exe config.json +./accessws config.json diff --git a/alertcenter/restart.sh b/alertcenter/restart.sh index 7ddd1bff..3ec16e46 100755 --- a/alertcenter/restart.sh +++ b/alertcenter/restart.sh @@ -1,5 +1,5 @@ #!/bin/bash -killall -s SIGQUIT alertcenter.exe +killall -s SIGQUIT alertcenter sleep 1 -./alertcenter.exe config.json +./alertcenter config.json diff --git a/marketprice/restart.sh b/marketprice/restart.sh index ee30da5d..fac97734 100755 --- a/marketprice/restart.sh +++ b/marketprice/restart.sh @@ -1,5 +1,5 @@ #!/bin/bash -killall -s SIGQUIT marketprice.exe +killall -s SIGQUIT marketprice sleep 1 -./marketprice.exe config.json +./marketprice config.json diff --git a/matchengine/restart.sh b/matchengine/restart.sh index 98d87305..20f9d905 100755 --- a/matchengine/restart.sh +++ b/matchengine/restart.sh @@ -1,5 +1,5 @@ #!/bin/bash -killall -s SIGQUIT matchengine.exe +killall -s SIGQUIT matchengine sleep 1 -./matchengine.exe config.json +./matchengine config.json From 0a23589df220263fc4852b19c52938fe9d63b075 Mon Sep 17 00:00:00 2001 From: Andrey Samokhvalov Date: Thu, 5 Oct 2017 12:27:53 +0000 Subject: [PATCH 02/38] makefile: change on linux installation --- accesshttp/makefile | 2 +- accessws/makefile | 4 ++-- alertcenter/makefile | 4 ++-- makefile.inc | 7 +++---- marketprice/makefile | 2 +- matchengine/makefile | 4 ++-- readhistory/makefile | 4 ++-- readhistory/restart.sh | 4 ++-- test/marketprice/makefile | 2 +- test/matchengine/makefile | 2 +- 10 files changed, 17 insertions(+), 18 deletions(-) diff --git a/accesshttp/makefile b/accesshttp/makefile index 91586d19..66eba96f 100644 --- a/accesshttp/makefile +++ b/accesshttp/makefile @@ -1,4 +1,4 @@ -TARGET := accesshttp.exe +TARGET := accesshttp INCS = -I ../network -I ../utils LIBS = -L ../utils -lutils -L ../network -lnetwork -Wl,-Bstatic -lev -ljansson -lmpdec -lrdkafka -lz -lssl -lcrypto -lhiredis -Wl,-Bdynamic -lm -lpthread -ldl include ../makefile.inc diff --git a/accessws/makefile b/accessws/makefile index 0829af85..4ebfb9d0 100644 --- a/accessws/makefile +++ b/accessws/makefile @@ -1,4 +1,4 @@ -TARGET := accessws.exe +TARGET := accessws INCS = -I ../network -I ../utils -LIBS = -L ../utils -lutils -L ../network -lnetwork -Wl,-Bstatic -lev -ljansson -lmpdec -lrdkafka -lz -lssl -lcrypto -lhiredis -lcurl -Wl,-Bdynamic -lm -lpthread -ldl +LIBS = -L ../utils -lutils -L ../network -lnetwork -Wl,-Bstatic -lev -ljansson -lmpdec -lrdkafka -lz -lssl -lcrypto -lhiredis -Wl,-Bdynamic -lm -lpthread -ldl -lcurl include ../makefile.inc diff --git a/alertcenter/makefile b/alertcenter/makefile index 04f8b1ef..c292648e 100644 --- a/alertcenter/makefile +++ b/alertcenter/makefile @@ -1,4 +1,4 @@ -TARGET := alertcenter.exe +TARGET := alertcenter INCS = -I ../network -I ../utils -LIBS = -L ../utils -lutils -L ../network -lnetwork -Wl,-Bstatic -lev -lhiredis -ljansson -lmpdec -lcurl -lssl -lcrypto -Wl,-Bdynamic -lm +LIBS = -L ../utils -lutils -L ../network -lnetwork -Wl,-Bstatic -lev -lhiredis -ljansson -lmpdec -lssl -lcrypto -Wl,-Bdynamic -lm -lcurl include ../makefile.inc diff --git a/makefile.inc b/makefile.inc index be6147e1..bf732823 100644 --- a/makefile.inc +++ b/makefile.inc @@ -4,7 +4,6 @@ CC := gcc CFLAGS := -Wall -Wno-strict-aliasing -Wno-uninitialized -g -rdynamic -std=gnu99 LFLAGS := -g -rdynamic - .PHONY : all clean install all : $(TARGET) @@ -12,10 +11,10 @@ all : $(TARGET) clean : rm -rf *.d *.o $(TARGET) -$(TARGET) : $(OBJS) +$(TARGET): $(OBJS) $(CC) $(LFLAGS) -o $@ $(OBJS) $(LIBS) -.c.o : + +.c.o: $(CC) $(CFLAGS) -c -o $@ $< $(INCS) install : - diff --git a/marketprice/makefile b/marketprice/makefile index 9135cf51..3ca4a637 100644 --- a/marketprice/makefile +++ b/marketprice/makefile @@ -1,4 +1,4 @@ -TARGET := marketprice.exe +TARGET := marketprice INCS = -I ../network -I ../utils LIBS = -L ../utils -lutils -L ../network -lnetwork -Wl,-Bstatic -lev -ljansson -lmpdec -lrdkafka -lz -lssl -lcrypto -lhiredis -lcurl -Wl,-Bdynamic -lm -lpthread -ldl include ../makefile.inc diff --git a/matchengine/makefile b/matchengine/makefile index d396061c..e1a060d6 100644 --- a/matchengine/makefile +++ b/matchengine/makefile @@ -1,4 +1,4 @@ -TARGET := matchengine.exe +TARGET := matchengine INCS = -I ../network -I ../utils -LIBS = -L ../utils -lutils -L ../network -lnetwork -Wl,-Bstatic -lev -ljansson -lmpdec -lmysqlclient -lz -lrdkafka -lssl -lcrypto -lhiredis -Wl,-Bdynamic -lm -lpthread -ldl +LIBS = -L ../utils -lutils -L ../network -lnetwork -Wl,-Bstatic -lev -ljansson -lmpdec -lrdkafka -lssl -lcrypto -lhiredis -lz -Wl,-Bdynamic -lm -lpthread -ldl -lmysqlclient include ../makefile.inc diff --git a/readhistory/makefile b/readhistory/makefile index 05a2eb76..7784b870 100644 --- a/readhistory/makefile +++ b/readhistory/makefile @@ -1,4 +1,4 @@ -TARGET := readhistory.exe +TARGET := readhistory INCS = -I ../network -I ../utils -LIBS = -L ../utils -lutils -L ../network -lnetwork -Wl,-Bstatic -lev -ljansson -lmpdec -lmysqlclient -lz -lhiredis -Wl,-Bdynamic -lm -lpthread -ldl +LIBS = -L ../utils -lutils -L ../network -lnetwork -Wl,-Bstatic -lev -ljansson -lmpdec -lz -lhiredis -Wl,-Bdynamic -lm -lpthread -ldl -lmysqlclient include ../makefile.inc diff --git a/readhistory/restart.sh b/readhistory/restart.sh index 11ead470..f4d6eaad 100755 --- a/readhistory/restart.sh +++ b/readhistory/restart.sh @@ -1,5 +1,5 @@ #!/bin/bash -killall -s SIGQUIT readhistory.exe +killall -s SIGQUIT readhistory sleep 1 -./readhistory.exe config.json +./readhistory config.json diff --git a/test/marketprice/makefile b/test/marketprice/makefile index ace60e8e..f8fb89c7 100755 --- a/test/marketprice/makefile +++ b/test/marketprice/makefile @@ -1,4 +1,4 @@ -TARGET := marketprice.exe +TARGET := marketprice INCS = -I ../../network -I ../../utils LIBS = -L ../../utils -lutils -L ../../network -lnetwork -Wl,-Bstatic -lev -ljansson -lmpdec -lrdkafka -lz -lssl -lcrypto -lhiredis -Wl,-Bdynamic -lm -lpthread -ldl include ../../makefile.inc diff --git a/test/matchengine/makefile b/test/matchengine/makefile index 2e227c04..e6814d54 100644 --- a/test/matchengine/makefile +++ b/test/matchengine/makefile @@ -1,5 +1,5 @@ all: - gcc -o cli.exe -g -std=gnu99 cli.c -I ../../network -I ../../utils -L ../../utils -lutils -L ../../network -lnetwork -lev -ljansson -lmpdec -lm + gcc -o cli -g -std=gnu99 cli.c -I ../../network -I ../../utils -L ../../utils -lutils -L ../../network -lnetwork -lev -ljansson -lmpdec -lm clearn: rm -f cli.exe From 04d598338ef3d9dd25097387dcb59a7a59981a0d Mon Sep 17 00:00:00 2001 From: Andrey Samokhvalov Date: Thu, 5 Oct 2017 12:31:03 +0000 Subject: [PATCH 03/38] test: update scripts --- test/matchengine/get_balance.sh | 4 ++-- test/matchengine/get_order.sh | 8 ++++---- test/matchengine/set_balance.sh | 16 ++++------------ test/matchengine/set_order.sh | 22 +++++++++------------- 4 files changed, 19 insertions(+), 31 deletions(-) diff --git a/test/matchengine/get_balance.sh b/test/matchengine/get_balance.sh index 6bf0308c..9cdac3b5 100755 --- a/test/matchengine/get_balance.sh +++ b/test/matchengine/get_balance.sh @@ -1,4 +1,4 @@ #!/bin/bash -./cli.exe 127.0.0.1 7316 101 '[1]' -./cli.exe 127.0.0.1 7316 101 '[2]' +./cli 127.0.0.1 7316 101 '[1]' +./cli 127.0.0.1 7316 101 '[2]' diff --git a/test/matchengine/get_order.sh b/test/matchengine/get_order.sh index 1a809ee2..214d922f 100755 --- a/test/matchengine/get_order.sh +++ b/test/matchengine/get_order.sh @@ -1,7 +1,7 @@ #!/bin/bash -./cli.exe 127.0.0.1 7316 205 '["BTCCNY", 1, 0, 100]' -./cli.exe 127.0.0.1 7316 205 '["BTCCNY", 2, 0, 100]' +./cli 127.0.0.1 7316 205 '["BTCCNY", 1, 0, 100]' +./cli 127.0.0.1 7316 205 '["BTCCNY", 2, 0, 100]' -./cli.exe 127.0.0.1 7316 206 '["BTCCNY", 10, "0"]' -./cli.exe 127.0.0.1 7316 207 '["BTCCNY", 10 ]' +./cli 127.0.0.1 7316 206 '["BTCCNY", 10, "0"]' +./cli 127.0.0.1 7316 207 '["BTCCNY", 10 ]' diff --git a/test/matchengine/set_balance.sh b/test/matchengine/set_balance.sh index 54c7b3d7..969d0062 100755 --- a/test/matchengine/set_balance.sh +++ b/test/matchengine/set_balance.sh @@ -1,14 +1,6 @@ #!/bin/bash -#./cli.exe 127.0.0.1 7316 102 '[1, 1, "BTC", "deposit", 1, "100"]' - -./cli.exe 127.0.0.1 7316 101 '[1, "CNY" ]' -./cli.exe 127.0.0.1 7316 102 '[1, "CNY", "deposit", 10001, "100000",null]' -./cli.exe 127.0.0.1 7316 101 '[1, "CNY" ]' -./cli.exe 127.0.0.1 7316 102 '[1, "CNY", "withdraw", 10002, "-10"]' -./cli.exe 127.0.0.1 7316 101 '[1, "CNY" ]' - -./cli.exe 127.0.0.1 7316 103 '[1, "CNY" ,"",0,0,0,10]' - -#./cli.exe 127.0.0.1 7316 102 '[2, 1, "BTC", "deposit", 1, "100"]' -#./cli.exe 127.0.0.1 7316 102 '[2, 1, "CNY", "deposit", 1, "10000"]' +./cli 127.0.0.1 7316 102 '[1, "CNY", "deposit", 10001, "100000",{}]' +./cli 127.0.0.1 7316 102 '[2, "CNY", "deposit", 10001, "100000",{}]' +./cli 127.0.0.1 7316 102 '[1, "BTC", "deposit", 10001, "100000",{}]' +./cli 127.0.0.1 7316 102 '[2, "BTC", "deposit", 10001, "100000",{}]' diff --git a/test/matchengine/set_order.sh b/test/matchengine/set_order.sh index 1bbf9f96..a94d161e 100755 --- a/test/matchengine/set_order.sh +++ b/test/matchengine/set_order.sh @@ -1,30 +1,26 @@ #!/bin/bash -#buy 2 bitcoins at 8000 CNY -./cli.exe 127.0.0.1 7316 201 '[1, "BTCCNY", 2, "3", "8000", "0.002", "0.001","api.v1"]' +#buy 3 bitcoins at 8000 CNY +#./cli 127.0.0.1 7316 201 '[1, "BTCCNY", 2, "3", "8000", "0.002", "0.001","api.v1"]' #query my pending order list -./cli.exe 127.0.0.1 7316 203 '[1, "BTCCNY", 0,10]' +#./cli 127.0.0.1 7316 203 '[1, "BTCCNY", 0,10]' -#sell 1 bitcoins at 8010 CNY -./cli.exe 127.0.0.1 7316 201 '[1, "BTCCNY", 1, "1", "8010", "0.002", "0.001","api.v1"]' - -#query my pending order list -./cli.exe 127.0.0.1 7316 203 '[1, "BTCCNY", 0,10]' +#sell 3 bitcoins at 8010 CNY +./cli 127.0.0.1 7316 201 '[2, "BTCCNY", 1, "3", "7999", "0.002", "0.001","api.v1"]' #query all pending list on sell direction -./cli.exe 127.0.0.1 7316 205 '[ "BTCCNY", 1, 0,10]' +#./cli 127.0.0.1 7316 205 '[ "BTCCNY", 1, 0,10]' #query all pending list on buy direction -./cli.exe 127.0.0.1 7316 205 '[ "BTCCNY", 2, 0,10]' +#./cli 127.0.0.1 7316 205 '[ "BTCCNY", 2, 0,10]' #query market depth -./cli.exe 127.0.0.1 7316 206 '[ "BTCCNY",10,0]' +#./cli 127.0.0.1 7316 206 '[ "BTCCNY",10,0]' #query pending order detail -./cli.exe 127.0.0.1 7316 207 '[ "BTCCNY",order_id]' +#./cli 127.0.0.1 7316 207 '[ "BTCCNY",order_id]' -#./cli.exe 127.0.0.1 7316 201 '[2, "BTCCNY", 2, "1", "7000", "0.002", "0.001"]' From e7a022c4a2c950deede580f9d26efd2689ad0944 Mon Sep 17 00:00:00 2001 From: Andrey Samokhvalov Date: Thu, 5 Oct 2017 15:43:07 +0300 Subject: [PATCH 04/38] matchengine: make return code consistent --- matchengine/me_server.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/matchengine/me_server.c b/matchengine/me_server.c index ca5d0e6e..f390a5de 100644 --- a/matchengine/me_server.c +++ b/matchengine/me_server.c @@ -278,9 +278,9 @@ static int on_cmd_balance_update(nw_ses *ses, rpc_pkg *pkg, json_t *params) int ret = update_user_balance(true, user_id, asset, business, business_id, change, detail); mpd_del(change); if (ret == -1) { - return reply_error(ses, pkg, 10, "repeat update"); + return reply_error(ses, pkg, 11, "repeat update"); } else if (ret == -2) { - return reply_error(ses, pkg, 11, "balance not enough"); + return reply_error(ses, pkg, 10, "balance not enough"); } else if (ret < 0) { return reply_error_internal_error(ses, pkg); } From 6e44b117d9eb22d2ecfc1abc179e8eeadf14aa36 Mon Sep 17 00:00:00 2001 From: Andrey Samokhvalov Date: Sat, 28 Oct 2017 16:12:53 +0300 Subject: [PATCH 05/38] matchengine: add new cryptocurrencies --- matchengine/config.json | 148 ++++++++++++++++++++++++---------------- 1 file changed, 91 insertions(+), 57 deletions(-) diff --git a/matchengine/config.json b/matchengine/config.json index c15218a3..9455a851 100644 --- a/matchengine/config.json +++ b/matchengine/config.json @@ -37,117 +37,151 @@ }, "assets": [ { - "name": "BCH", - "prec_save": 20, + "name": "ETH", + "prec_save": 8, "prec_show": 8 }, { - "name": "BTC", - "prec_save": 12, + "name": "ETC", + "prec_save": 8, "prec_show": 8 }, { - "name": "LTC", - "prec_save": 12, + "name": "BTC", + "prec_save": 16, "prec_show": 8 }, { - "name": "ETH", - "prec_save": 12, + "name": "BCH", + "prec_save": 16, "prec_show": 8 }, { - "name": "ETC", + "name": "LTC", "prec_save": 12, "prec_show": 8 }, { - "name": "ZEC", - "prec_save": 12, + "name": "XRP", + "prec_save": 8, "prec_show": 8 }, { - "name": "XMR", - "prec_save": 12, + "name": "DASH", + "prec_save": 8, "prec_show": 8 }, { - "name": "DSH", - "prec_save": 12, + "name": "ZEC", + "prec_save": 8, "prec_show": 8 } ], + "markets": [ { - "name": "BTCBCH", + "name": "BTCETH", "stock": { "name": "BTC", "prec": 8 }, "money": { - "name": "BCH", - "prec": 8 - }, - "min_amount": "0.001" + "name": "ETH", + "prec": 4 + } }, { - "name": "LTCBCH", + "name": "BTCETC", "stock": { - "name": "LTC", - "prec": 8 + "name": "BTC", + "prec": 4 }, "money": { - "name": "BCH", - "prec": 8 - }, - "min_amount": "0.001" + "name": "ETC", + "prec": 4 + } }, { - "name": "ETHBCH", + "name": "BTCBCH", "stock": { - "name": "ETH", - "prec": 8 + "name": "BTC", + "prec": 4 }, "money": { "name": "BCH", - "prec": 8 - }, - "min_amount": "0.001" + "prec": 4 + } }, { - "name": "ETCBCH", + "name": "BTCLTC", "stock": { - "name": "ETC", - "prec": 8 + "name": "BTC", + "prec": 4 }, "money": { - "name": "BCH", - "prec": 8 + "name": "LTC", + "prec": 4 + } + }, { + "name": "BTCXRP", + "stock": { + "name": "BTC", + "prec": 4 }, - "min_amount": "0.001" + "money": { + "name": "XRP", + "prec": 4 + } }, { - "name": "ZECBCH", + "name": "BTCDASH", "stock": { - "name": "ZEC", - "prec": 8 + "name": "BTC", + "prec": 4 }, "money": { - "name": "BCH", - "prec": 8 + "name": "DASH", + "prec": 4 + } + }, { + "name": "BTCZEC", + "stock": { + "name": "BTC", + "prec": 4 }, - "min_amount": "0.001" + "money": { + "name": "ZEC", + "prec": 4 + } }, { - "name": "XMRBCH", + "name": "ETHETC", "stock": { - "name": "XMR", - "prec": 8 + "name": "ETH", + "prec": 4 }, "money": { - "name": "BCH", - "prec": 8 + "name": "ETC", + "prec": 4 + } + }, { + "name": "ETHLTC", + "stock": { + "name": "ETH", + "prec": 4 }, - "min_amount": "0.001" + "money": { + "name": "LTC", + "prec": 4 + } }, { - "name": "DSHBCH", + "name": "ETHXRP", "stock": { - "name": "DSH", - "prec": 8 + "name": "ETH", + "prec": 4 }, "money": { - "name": "BCH", - "prec": 8 + "name": "XRP", + "prec": 4 + } + }, { + "name": "ETHZEC", + "stock": { + "name": "ETH", + "prec": 4 }, - "min_amount": "0.001" + "money": { + "name": "ZEC", + "prec": 4 + } } ], "brokers": "127.0.0.1:9092", From 983e52a5a891640c79df6ed26ab12112555a3aba Mon Sep 17 00:00:00 2001 From: Andrey Samokhvalov Date: Wed, 1 Nov 2017 03:07:53 +0300 Subject: [PATCH 06/38] matchengine: switch stock and money in config --- matchengine/config.json | 46 ++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/matchengine/config.json b/matchengine/config.json index 9455a851..6089ee51 100644 --- a/matchengine/config.json +++ b/matchengine/config.json @@ -75,111 +75,111 @@ { "name": "BTCETH", "stock": { - "name": "BTC", - "prec": 8 + "name": "ETH", + "prec": 4 }, "money": { - "name": "ETH", + "name": "BTC", "prec": 4 } }, { "name": "BTCETC", "stock": { - "name": "BTC", + "name": "ETC", "prec": 4 }, "money": { - "name": "ETC", + "name": "BTC", "prec": 4 } }, { "name": "BTCBCH", "stock": { - "name": "BTC", + "name": "BCH", "prec": 4 }, "money": { - "name": "BCH", + "name": "BTC", "prec": 4 } }, { "name": "BTCLTC", "stock": { - "name": "BTC", + "name": "LTC", "prec": 4 }, "money": { - "name": "LTC", + "name": "BTC", "prec": 4 } }, { "name": "BTCXRP", "stock": { - "name": "BTC", + "name": "XRP", "prec": 4 }, "money": { - "name": "XRP", + "name": "BTC", "prec": 4 } }, { "name": "BTCDASH", "stock": { - "name": "BTC", + "name": "DASH", "prec": 4 }, "money": { - "name": "DASH", + "name": "BTC", "prec": 4 } }, { "name": "BTCZEC", "stock": { - "name": "BTC", + "name": "ZEC", "prec": 4 }, "money": { - "name": "ZEC", + "name": "BTC", "prec": 4 } }, { "name": "ETHETC", "stock": { - "name": "ETH", + "name": "ETC", "prec": 4 }, "money": { - "name": "ETC", + "name": "ETH", "prec": 4 } }, { "name": "ETHLTC", "stock": { - "name": "ETH", + "name": "LTC", "prec": 4 }, "money": { - "name": "LTC", + "name": "ETH", "prec": 4 } }, { "name": "ETHXRP", "stock": { - "name": "ETH", + "name": "XRP", "prec": 4 }, "money": { - "name": "XRP", + "name": "ETH", "prec": 4 } }, { "name": "ETHZEC", "stock": { - "name": "ETH", + "name": "ZEC", "prec": 4 }, "money": { - "name": "ZEC", + "name": "ETH", "prec": 4 } } From c6bfedbaad6d61062e36170f47bd9e5d3cb53665 Mon Sep 17 00:00:00 2001 From: Andrey Samokhvalov Date: Thu, 2 Nov 2017 23:16:11 +0300 Subject: [PATCH 07/38] matchengine: change currency precious --- matchengine/config.json | 56 ++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/matchengine/config.json b/matchengine/config.json index 6089ee51..1c28947e 100644 --- a/matchengine/config.json +++ b/matchengine/config.json @@ -38,11 +38,11 @@ "assets": [ { "name": "ETH", - "prec_save": 8, + "prec_save": 16, "prec_show": 8 }, { "name": "ETC", - "prec_save": 8, + "prec_save": 16, "prec_show": 8 }, { "name": "BTC", @@ -54,19 +54,19 @@ "prec_show": 8 }, { "name": "LTC", - "prec_save": 12, + "prec_save": 16, "prec_show": 8 }, { "name": "XRP", - "prec_save": 8, + "prec_save": 16, "prec_show": 8 }, { "name": "DASH", - "prec_save": 8, + "prec_save": 16, "prec_show": 8 }, { "name": "ZEC", - "prec_save": 8, + "prec_save": 16, "prec_show": 8 } ], @@ -76,111 +76,111 @@ "name": "BTCETH", "stock": { "name": "ETH", - "prec": 4 + "prec": 8 }, "money": { "name": "BTC", - "prec": 4 + "prec": 8 } }, { "name": "BTCETC", "stock": { "name": "ETC", - "prec": 4 + "prec": 8 }, "money": { "name": "BTC", - "prec": 4 + "prec": 8 } }, { "name": "BTCBCH", "stock": { "name": "BCH", - "prec": 4 + "prec": 8 }, "money": { "name": "BTC", - "prec": 4 + "prec": 8 } }, { "name": "BTCLTC", "stock": { "name": "LTC", - "prec": 4 + "prec": 8 }, "money": { "name": "BTC", - "prec": 4 + "prec": 8 } }, { "name": "BTCXRP", "stock": { "name": "XRP", - "prec": 4 + "prec": 8 }, "money": { "name": "BTC", - "prec": 4 + "prec": 8 } }, { "name": "BTCDASH", "stock": { "name": "DASH", - "prec": 4 + "prec": 8 }, "money": { "name": "BTC", - "prec": 4 + "prec": 8 } }, { "name": "BTCZEC", "stock": { "name": "ZEC", - "prec": 4 + "prec": 8 }, "money": { "name": "BTC", - "prec": 4 + "prec": 8 } }, { "name": "ETHETC", "stock": { "name": "ETC", - "prec": 4 + "prec": 8 }, "money": { "name": "ETH", - "prec": 4 + "prec": 8 } }, { "name": "ETHLTC", "stock": { "name": "LTC", - "prec": 4 + "prec": 8 }, "money": { "name": "ETH", - "prec": 4 + "prec": 8 } }, { "name": "ETHXRP", "stock": { "name": "XRP", - "prec": 4 + "prec": 8 }, "money": { "name": "ETH", - "prec": 4 + "prec": 8 } }, { "name": "ETHZEC", "stock": { "name": "ZEC", - "prec": 4 + "prec": 8 }, "money": { "name": "ETH", - "prec": 4 + "prec": 8 } } ], From c96608af3b2afd1b3c7df641c45202ab7d27a0d6 Mon Sep 17 00:00:00 2001 From: Andrey Samokhvalov Date: Mon, 26 Feb 2018 16:00:51 +0300 Subject: [PATCH 08/38] matchengine: change availabel pairs --- matchengine/config.json | 88 +++++------------------------------------ 1 file changed, 10 insertions(+), 78 deletions(-) diff --git a/matchengine/config.json b/matchengine/config.json index 1c28947e..ef697506 100644 --- a/matchengine/config.json +++ b/matchengine/config.json @@ -40,11 +40,7 @@ "name": "ETH", "prec_save": 16, "prec_show": 8 - }, { - "name": "ETC", - "prec_save": 16, - "prec_show": 8 - }, { + },{ "name": "BTC", "prec_save": 16, "prec_show": 8 @@ -56,21 +52,12 @@ "name": "LTC", "prec_save": 16, "prec_show": 8 - }, { - "name": "XRP", - "prec_save": 16, - "prec_show": 8 - }, { + },{ "name": "DASH", "prec_save": 16, "prec_show": 8 - }, { - "name": "ZEC", - "prec_save": 16, - "prec_show": 8 } ], - "markets": [ { "name": "BTCETH", @@ -81,17 +68,8 @@ "money": { "name": "BTC", "prec": 8 - } - }, { - "name": "BTCETC", - "stock": { - "name": "ETC", - "prec": 8 }, - "money": { - "name": "BTC", - "prec": 8 - } + "min_amount":"0.00001" }, { "name": "BTCBCH", "stock": { @@ -101,7 +79,8 @@ "money": { "name": "BTC", "prec": 8 - } + }, + "min_amount":"0.00001" }, { "name": "BTCLTC", "stock": { @@ -111,17 +90,8 @@ "money": { "name": "BTC", "prec": 8 - } - }, { - "name": "BTCXRP", - "stock": { - "name": "XRP", - "prec": 8 }, - "money": { - "name": "BTC", - "prec": 8 - } + "min_amount":"0.00001" }, { "name": "BTCDASH", "stock": { @@ -131,27 +101,8 @@ "money": { "name": "BTC", "prec": 8 - } - }, { - "name": "BTCZEC", - "stock": { - "name": "ZEC", - "prec": 8 - }, - "money": { - "name": "BTC", - "prec": 8 - } - }, { - "name": "ETHETC", - "stock": { - "name": "ETC", - "prec": 8 }, - "money": { - "name": "ETH", - "prec": 8 - } + "min_amount":"0.00001" }, { "name": "ETHLTC", "stock": { @@ -161,30 +112,11 @@ "money": { "name": "ETH", "prec": 8 - } - }, { - "name": "ETHXRP", - "stock": { - "name": "XRP", - "prec": 8 - }, - "money": { - "name": "ETH", - "prec": 8 - } - }, { - "name": "ETHZEC", - "stock": { - "name": "ZEC", - "prec": 8 }, - "money": { - "name": "ETH", - "prec": 8 - } + "min_amount":"0.00001" } ], "brokers": "127.0.0.1:9092", "slice_interval": 3600, - "slice_keeptime": 259200 -} + "slice_keeptime": 259200 +} \ No newline at end of file From 2be8e939228b0f0b816a2766cf78c50b93776e3e Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Mon, 19 Feb 2018 19:40:43 +0500 Subject: [PATCH 09/38] docker: add mysql docker --- docker/docker-compose.yml | 9 ++ docker/mysql/Dockerfile | 25 +++++ docker/mysql/build.sh | 3 + docker/mysql/create_trade_history.sql | 88 +++++++++++++++ docker/mysql/create_trade_log.sql | 40 +++++++ docker/mysql/entrypoint.sh | 111 +++++++++++++++++++ docker/mysql/init_trade_history.sh | 36 +++++++ docker/mysql/init_trade_log.sh | 25 +++++ docker/mysql/my.cnf | 149 ++++++++++++++++++++++++++ docker/mysql/run.sh | 11 ++ 10 files changed, 497 insertions(+) create mode 100644 docker/docker-compose.yml create mode 100644 docker/mysql/Dockerfile create mode 100755 docker/mysql/build.sh create mode 100644 docker/mysql/create_trade_history.sql create mode 100644 docker/mysql/create_trade_log.sql create mode 100755 docker/mysql/entrypoint.sh create mode 100755 docker/mysql/init_trade_history.sh create mode 100755 docker/mysql/init_trade_log.sh create mode 100644 docker/mysql/my.cnf create mode 100755 docker/mysql/run.sh diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 00000000..c86950d3 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,9 @@ +version: '2.1' +services: + mysql: + build: mysql + image: bitlum/exchange-mysql + container_name: exchange_mysql + restart: always + volumes: + - /tmp/bitlum/exchange/mysql/lib:/var/lib/mysql diff --git a/docker/mysql/Dockerfile b/docker/mysql/Dockerfile new file mode 100644 index 00000000..a962f506 --- /dev/null +++ b/docker/mysql/Dockerfile @@ -0,0 +1,25 @@ +FROM alpine:3.7 +RUN apk update +RUN apk upgrade +RUN apk --no-cache add mariadb mariadb-client + +VOLUME /var/lib/mysql +EXPOSE 3306 + +COPY my.cnf /etc/mysql/my.cnf +RUN chown -R mysql.mysql /etc/mysql + +RUN mkdir /run/mysqld +RUN chown -R mysql.mysql /run/mysqld + +COPY entrypoint.sh docker-entrypoint.sh + +RUN mkdir /docker-entrypoint-initdb.d +COPY init_trade_log.sh /docker-entrypoint-initdb.d +COPY init_trade_history.sh /docker-entrypoint-initdb.d +COPY create_trade_history.sql /docker-entrypoint-initdb.d +COPY create_trade_log.sql /docker-entrypoint-initdb.d +RUN ls -la /docker-entrypoint-initdb.d + +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["mysqld"] diff --git a/docker/mysql/build.sh b/docker/mysql/build.sh new file mode 100755 index 00000000..5d95fe57 --- /dev/null +++ b/docker/mysql/build.sh @@ -0,0 +1,3 @@ +#!/bin/sh +WD=`dirname $0` +cd $WD && docker build -t bitlum/exchange-mysql . diff --git a/docker/mysql/create_trade_history.sql b/docker/mysql/create_trade_history.sql new file mode 100644 index 00000000..6d7afb1a --- /dev/null +++ b/docker/mysql/create_trade_history.sql @@ -0,0 +1,88 @@ +CREATE TABLE `balance_history_example` ( + `id` BIGINT UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT, + `time` DOUBLE NOT NULL, + `user_id` INT UNSIGNED NOT NULL, + `asset` VARCHAR(30) NOT NULL, + `business` VARCHAR(30) NOT NULL, + `change` DECIMAL(30,8) NOT NULL, + `balance` DECIMAL(30,16) NOT NULL, + `detail` TEXT NOT NULL, + INDEX `idx_user_asset` (`user_id`, `asset`), + INDEX `idx_user_asset_business` (`user_id`, `asset`, `business`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- split by user_id +CREATE TABLE `order_history_example` ( + `id` BIGINT UNSIGNED NOT NULL PRIMARY KEY, + `create_time` DOUBLE NOT NULL, + `finish_time` DOUBLE NOT NULL, + `user_id` INT UNSIGNED NOT NULL, + `market` VARCHAR(30) NOT NULL, + `source` VARCHAR(30) NOT NULL, + `t` TINYINT UNSIGNED NOT NULL, + `side` TINYINT UNSIGNED NOT NULL, + `price` DECIMAL(30,8) NOT NULL, + `amount` DECIMAL(30,8) NOT NULL, + `taker_fee` DECIMAL(30,4) NOT NULL, + `maker_fee` DECIMAL(30,4) NOT NULL, + `deal_stock` DECIMAL(30,8) NOT NULL, + `deal_money` DECIMAL(30,16) NOT NULL, + `deal_fee` DECIMAL(30,16) NOT NULL, + INDEX `idx_user_market` (`user_id`, `market`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- split by id, aka orer_id +CREATE TABLE `order_detail_example` ( + `id` BIGINT UNSIGNED NOT NULL PRIMARY KEY, + `create_time` DOUBLE NOT NULL, + `finish_time` DOUBLE NOT NULL, + `user_id` INT UNSIGNED NOT NULL, + `market` VARCHAR(30) NOT NULL, + `source` VARCHAR(30) NOT NULL, + `t` TINYINT UNSIGNED NOT NULL, + `side` TINYINT UNSIGNED NOT NULL, + `price` DECIMAL(30,8) NOT NULL, + `amount` DECIMAL(30,8) NOT NULL, + `taker_fee` DECIMAL(30,4) NOT NULL, + `maker_fee` DECIMAL(30,4) NOT NULL, + `deal_stock` DECIMAL(30,8) NOT NULL, + `deal_money` DECIMAL(30,16) NOT NULL, + `deal_fee` DECIMAL(30,16) NOT NULL, + INDEX `idx_user_market` (`user_id`, `market`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- split by order_id +CREATE TABLE `deal_history_example` ( + `id` BIGINT UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT, + `time` DOUBLE NOT NULL, + `user_id` INT UNSIGNED NOT NULL, + `deal_id` BIGINT UNSIGNED NOT NULL, + `order_id` BIGINT UNSIGNED NOT NULL, + `deal_order_id` BIGINT UNSIGNED NOT NULL, + `role` TINYINT UNSIGNED NOT NULL, + `price` DECIMAL(30,8) NOT NULL, + `amount` DECIMAL(30,8) NOT NULL, + `deal` DECIMAL(30,16) NOT NULL, + `fee` DECIMAL(30,16) NOT NULL, + `deal_fee` DECIMAL(30,16) NOT NULL, + INDEX `idx_order_id` (`order_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- split by user_id +CREATE TABLE `user_deal_history_example` ( + `id` BIGINT UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT, + `time` DOUBLE NOT NULL, + `user_id` INT UNSIGNED NOT NULL, + `market` VARCHAR(30) NOT NULL, + `deal_id` BIGINT UNSIGNED NOT NULL, + `order_id` BIGINT UNSIGNED NOT NULL, + `deal_order_id` BIGINT UNSIGNED NOT NULL, + `side` TINYINT UNSIGNED NOT NULL, + `role` TINYINT UNSIGNED NOT NULL, + `price` DECIMAL(30,8) NOT NULL, + `amount` DECIMAL(30,8) NOT NULL, + `deal` DECIMAL(30,16) NOT NULL, + `fee` DECIMAL(30,16) NOT NULL, + `deal_fee` DECIMAL(30,16) NOT NULL, + INDEX `idx_user_market` (`user_id`, `market`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/docker/mysql/create_trade_log.sql b/docker/mysql/create_trade_log.sql new file mode 100644 index 00000000..8a7acd8d --- /dev/null +++ b/docker/mysql/create_trade_log.sql @@ -0,0 +1,40 @@ +CREATE TABLE `slice_balance_example` ( + `id` INT UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT, + `user_id` INT UNSIGNED NOT NULL, + `asset` VARCHAR(30) NOT NULL, + `t` TINYINT UNSIGNED NOT NULL, + `balance` DECIMAL(30,16) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE `slice_order_example` ( + `id` BIGINT UNSIGNED NOT NULL PRIMARY KEY, + `t` TINYINT UNSIGNED NOT NULL, + `side` TINYINT UNSIGNED NOT NULL, + `create_time` DOUBLE NOT NULL, + `update_time` DOUBLE NOT NULL, + `user_id` INT UNSIGNED NOT NULL, + `market` VARCHAR(30) NOT NULL, + `price` DECIMAL(30,8) NOT NULL, + `amount` DECIMAL(30,8) NOT NULL, + `taker_fee` DECIMAL(30,4) NOT NULL, + `maker_fee` DECIMAL(30,4) NOT NULL, + `left` DECIMAL(30,8) NOT NULL, + `freeze` DECIMAL(30,8) NOT NULL, + `deal_stock` DECIMAL(30,8) NOT NULL, + `deal_money` DECIMAL(30,16) NOT NULL, + `deal_fee` DECIMAL(30,12) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE `slice_history` ( + `id` INT UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT, + `time` BIGINT NOT NULL, + `end_oper_id` BIGINT UNSIGNED NOT NULL, + `end_order_id` BIGINT UNSIGNED NOT NULL, + `end_deals_id` BIGINT UNSIGNED NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE `operlog_example` ( + `id` BIGINT UNSIGNED NOT NULL PRIMARY KEY, + `time` DOUBLE NOT NULL, + `detail` TEXT +) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/docker/mysql/entrypoint.sh b/docker/mysql/entrypoint.sh new file mode 100755 index 00000000..4bc21f00 --- /dev/null +++ b/docker/mysql/entrypoint.sh @@ -0,0 +1,111 @@ +#!/bin/sh +MYSQL_ROOT_PASSWORD=root +MYSQL_DATABASE_HISTORY=trade_history +MYSQL_DATABASE_LOG=trade_log +MYSQL_USER=exchange +MYSQL_PASSWORD=exchange +SOCKET=/var/run/mysqld/mysqld.sock + +_check_config() { + toRun=`"$@" --verbose --help` + if ! errors="`"${toRun[@]}" 2>&1 >/dev/null`"; then + cat >&2 <<-EOM + ERROR: mysqld failed while attempting to check config + command was: "${toRun[*]}" + $errors + EOM + exit 1 + fi +} + +_get_config() { + local conf="$1"; shift + "$@" --verbose --help --log-bin-index="`mktemp -u`" 2>/dev/null | awk '$1 == "'"$conf"'" { print $2; exit }' +} + +if [ "$1" = 'mysqld' -a "`id -u`" = '0' ]; then + # _check_config "$@" + + DATADIR="`_get_config 'datadir' "$@"`" + mkdir -p "$DATADIR" + chown -R mysql:mysql "$DATADIR" + mkdir -p `dirname $SOCKET` + chown -R mysql:mysql `dirname $SOCKET` + + if [ ! -d "$DATADIR/mysql" ]; then + echo "Initializing database in $DATADIR/mysql" + mysql_install_db --verbose --user=mysql + echo 'Database initialized' + + echo 'Start mysql to change root password' + su -c "$@ --skip-networking --socket=${SOCKET}&" mysql -s /bin/sh + + mysql="mysql --protocol=socket -uroot -hlocalhost --socket=${SOCKET}" + + i=0 + while [ $i -lt 30 ]; do + if echo 'SELECT 1' | /bin/sh -c "$mysql" &> /dev/null; then + break + fi + echo 'MySQL init process in progress...' + sleep 1 + i=`echo $((i + 1))` + done + + if [ $i -eq 30 ]; then + echo >&2 'MySQL init process failed.' + exit 1 + fi + + echo 'SET PASSWORD' + + /bin/sh -c "${mysql}" <<-EOSQL + -- What's done in this file shouldn't be replicated + -- or products like mysql-fabric won't work + SET @@SESSION.SQL_LOG_BIN=0; + DELETE FROM mysql.user WHERE user NOT IN ('mysql.session', 'mysql.sys', 'root') OR host NOT IN ('localhost') ; + SET PASSWORD FOR 'root'@'localhost'=PASSWORD('${MYSQL_ROOT_PASSWORD}') ; + GRANT ALL ON *.* TO 'root'@'localhost' WITH GRANT OPTION ; + DROP DATABASE IF EXISTS test ; + FLUSH PRIVILEGES ; + CREATE USER '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD'; + CREATE DATABASE $MYSQL_DATABASE_HISTORY; + CREATE DATABASE $MYSQL_DATABASE_LOG; + GRANT ALL ON $MYSQL_DATABASE_HISTORY.* TO '$MYSQL_USER'@'%' ; + GRANT ALL ON $MYSQL_DATABASE_LOG.* TO '$MYSQL_USER'@'%' ; + FLUSH PRIVILEGES ; + EOSQL + + ls -la /docker-entrypoint-initdb.d + mysql="mysql --protocol=socket -u${MYSQL_USER} -p${MYSQL_PASSWORD} -hlocalhost --socket=${SOCKET} $MYSQL_DATABASE_HISTORY" + /bin/sh -c "${mysql}" < "/docker-entrypoint-initdb.d/create_trade_history.sql"; + + sed -i.bak "s/MYSQL_USER=.*$/MYSQL_USER=\"${MYSQL_USER}\"/" /docker-entrypoint-initdb.d/init_trade_history.sh + sed -i.bak "s/MYSQL_PASS=.*$/MYSQL_PASS=\"${MYSQL_PASSWORD}\"/" /docker-entrypoint-initdb.d/init_trade_history.sh + sed -i.bak "s/MYSQL_DB=.*$/MYSQL_DB=\"${MYSQL_DATABASE_HISTORY}\"/" /docker-entrypoint-initdb.d/init_trade_history.sh + /docker-entrypoint-initdb.d/init_trade_history.sh + + mysql="mysql --protocol=socket -u${MYSQL_USER} -p${MYSQL_PASSWORD} -hlocalhost --socket=${SOCKET} $MYSQL_DATABASE_LOG" + /bin/sh -c "${mysql}" < "/docker-entrypoint-initdb.d/create_trade_log.sql"; + sed -i.bak "s/MYSQL_USER=.*$/MYSQL_USER=\"${MYSQL_USER}\"/" /docker-entrypoint-initdb.d/init_trade_log.sh + sed -i.bak "s/MYSQL_PASS=.*$/MYSQL_PASS=\"${MYSQL_PASSWORD}\"/" /docker-entrypoint-initdb.d/init_trade_log.sh + sed -i.bak "s/MYSQL_DB=.*$/MYSQL_DB=\"${MYSQL_DATABASE_LOG}\"/" /docker-entrypoint-initdb.d/init_trade_log.sh + /docker-entrypoint-initdb.d/init_trade_log.sh + + PIDFILE="`_get_config 'pid-file' "$@"`" + pid=`cat $PIDFILE` + + if ! kill -s TERM "$pid" ; then + echo >&2 'MySQL kill process failed.' + if ! wait "$pid"; then + echo >&2 'MySQL wait process failed.' + exit 1 + fi + fi + + echo + echo 'MySQL init process done. Ready for start up.' + echo + fi +fi +exec su -c "$*" mysql -s /bin/sh diff --git a/docker/mysql/init_trade_history.sh b/docker/mysql/init_trade_history.sh new file mode 100755 index 00000000..991a8935 --- /dev/null +++ b/docker/mysql/init_trade_history.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +MYSQL_HOST="localhost" +MYSQL_USER="root" +MYSQL_PASS="shit" +MYSQL_DB="trade_history" + +for i in `seq 0 99` +do + echo "create table balance_history_$i" + mysql -h$MYSQL_HOST -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_DB -e "CREATE TABLE balance_history_$i LIKE balance_history_example;" +done + +for i in `seq 0 99` +do + echo "create table order_history_$i" + mysql -h$MYSQL_HOST -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_DB -e "CREATE TABLE order_history_$i LIKE order_history_example;" +done + +for i in `seq 0 99` +do + echo "create table order_detail_$i" + mysql -h$MYSQL_HOST -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_DB -e "CREATE TABLE order_detail_$i LIKE order_detail_example;" +done + +for i in `seq 0 99` +do + echo "create table deal_history_$i" + mysql -h$MYSQL_HOST -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_DB -e "CREATE TABLE deal_history_$i LIKE deal_history_example;" +done + +for i in `seq 0 99` +do + echo "create table user_deal_history_$i" + mysql -h$MYSQL_HOST -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_DB -e "CREATE TABLE user_deal_history_$i LIKE user_deal_history_example;" +done diff --git a/docker/mysql/init_trade_log.sh b/docker/mysql/init_trade_log.sh new file mode 100755 index 00000000..1bdf92db --- /dev/null +++ b/docker/mysql/init_trade_log.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +MYSQL_HOST="localhost" +MYSQL_USER="user" +MYSQL_PASS="pass" +MYSQL_DB="trade_log" + +for i in `seq 0 99` +do + echo "create table slice_balance_example$i" + mysql -h$MYSQL_HOST -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_DB -e "CREATE TABLE slice_balance_$i LIKE slice_balance_example;" +done + +for i in `seq 0 99` +do + echo "create table slice_order_$i" + mysql -h$MYSQL_HOST -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_DB -e "CREATE TABLE slice_order_$i LIKE slice_order_example;" +done + +for i in `seq 0 99` +do + echo "create table operlog_$i" + mysql -h$MYSQL_HOST -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_DB -e "CREATE TABLE operlog_$i LIKE operlog_example;" +done + diff --git a/docker/mysql/my.cnf b/docker/mysql/my.cnf new file mode 100644 index 00000000..0ae350f0 --- /dev/null +++ b/docker/mysql/my.cnf @@ -0,0 +1,149 @@ +# Example MariaDB config file for medium systems. +# +# This is for a system with little memory (32M - 64M) where MariaDB plays +# an important part, or systems up to 128M where MariaDB is used together with +# other programs (such as a web server) +# +# MariaDB programs look for option files in a set of +# locations which depend on the deployment platform. +# You can copy this option file to one of those +# locations. For information about these locations, do: +# 'my_print_defaults --help' and see what is printed under +# Default options are read from the following files in the given order: +# More information at: http://dev.mysql.com/doc/mysql/en/option-files.html +# +# In this file, you can use all long options that a program supports. +# If you want to know which options a program supports, run the program +# with the "--help" option. + +# The following options will be passed to all MariaDB clients +[client] +#password = your_password +port = 3306 +socket = /run/mysqld/mysqld.sock + +# Here follows entries for some specific programs + +# The MariaDB server +[mysqld] +port = 3306 +socket = /run/mysqld/mysqld.sock +skip-external-locking +key_buffer_size = 16M +max_allowed_packet = 1M +table_open_cache = 64 +sort_buffer_size = 512K +net_buffer_length = 8K +read_buffer_size = 256K +read_rnd_buffer_size = 512K +myisam_sort_buffer_size = 8M + +# Point the following paths to different dedicated disks +#tmpdir = /tmp/ + +# Don't listen on a TCP/IP port at all. This can be a security enhancement, +# if all processes that need to connect to mysqld run on the same host. +# All interaction with mysqld must be made via Unix sockets or named pipes. +# Note that using this option without enabling named pipes on Windows +# (via the "enable-named-pipe" option) will render mysqld useless! +# +#skip-networking + +# Replication Master Server (default) +# binary logging is required for replication +log-bin=mysql-bin + +# binary logging format - mixed recommended +binlog_format=mixed + +# required unique id between 1 and 2^32 - 1 +# defaults to 1 if master-host is not set +# but will not function as a master if omitted +server-id = 1 + +# Replication Slave (comment out master section to use this) +# +# To configure this host as a replication slave, you can choose between +# two methods : +# +# 1) Use the CHANGE MASTER TO command (fully described in our manual) - +# the syntax is: +# +# CHANGE MASTER TO MASTER_HOST=, MASTER_PORT=, +# MASTER_USER=, MASTER_PASSWORD= ; +# +# where you replace , , by quoted strings and +# by the master's port number (3306 by default). +# +# Example: +# +# CHANGE MASTER TO MASTER_HOST='125.564.12.1', MASTER_PORT=3306, +# MASTER_USER='joe', MASTER_PASSWORD='secret'; +# +# OR +# +# 2) Set the variables below. However, in case you choose this method, then +# start replication for the first time (even unsuccessfully, for example +# if you mistyped the password in master-password and the slave fails to +# connect), the slave will create a master.info file, and any later +# change in this file to the variables' values below will be ignored and +# overridden by the content of the master.info file, unless you shutdown +# the slave server, delete master.info and restart the slaver server. +# For that reason, you may want to leave the lines below untouched +# (commented) and instead use CHANGE MASTER TO (see above) +# +# required unique id between 2 and 2^32 - 1 +# (and different from the master) +# defaults to 2 if master-host is set +# but will not function as a slave if omitted +#server-id = 2 +# +# The replication master for this slave - required +#master-host = +# +# The username the slave will use for authentication when connecting +# to the master - required +#master-user = +# +# The password the slave will authenticate with when connecting to +# the master - required +#master-password = +# +# The port the master is listening on. +# optional - defaults to 3306 +#master-port = +# +# binary logging - not required for slaves, but recommended +#log-bin=mysql-bin + +# Uncomment the following if you are using InnoDB tables +#innodb_data_home_dir = /var/lib/mysql +#innodb_data_file_path = ibdata1:10M:autoextend +#innodb_log_group_home_dir = /var/lib/mysql +# You can set .._buffer_pool_size up to 50 - 80 % +# of RAM but beware of setting memory usage too high +#innodb_buffer_pool_size = 16M +#innodb_additional_mem_pool_size = 2M +# Set .._log_file_size to 25 % of buffer pool size +#innodb_log_file_size = 5M +#innodb_log_buffer_size = 8M +#innodb_flush_log_at_trx_commit = 1 +#innodb_lock_wait_timeout = 50 + +[mysqldump] +quick +max_allowed_packet = 16M + +[mysql] +no-auto-rehash +# Remove the next comment character if you are not familiar with SQL +#safe-updates + +[myisamchk] +key_buffer_size = 20M +sort_buffer_size = 20M +read_buffer = 2M +write_buffer = 2M + +[mysqlhotcopy] +interactive-timeout diff --git a/docker/mysql/run.sh b/docker/mysql/run.sh new file mode 100755 index 00000000..07664c03 --- /dev/null +++ b/docker/mysql/run.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +V="-v/tmp/bitlum/exchange/mysql:/var/lib/mysql" + +docker stop exchange_mysql +docker rm exchange_mysql +if [ -z $SH ];then +docker run --name=exchange_mysql $V $@ bitlum/exchange-mysql +else +docker run --name=exchange_mysql -it --entrypoint=/bin/sh $V $@ bitlum/exchange-mysql +fi From ffb8b82f6bbae848b898f1c49af214d8fcfcab11 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Mon, 19 Feb 2018 19:43:25 +0500 Subject: [PATCH 10/38] docker: add kafka to docker-compose --- docker/docker-compose.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index c86950d3..e9c12ff8 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -7,3 +7,10 @@ services: restart: always volumes: - /tmp/bitlum/exchange/mysql/lib:/var/lib/mysql + kafka: + build: kafka + container_name: exchange_kafka + restart: always + image: bitlum/exchange-kafka + volumes: + - /tmp/bitlum/exchange/kafka/logs:/tmp/kafka-logs From ec061a49457775f770060e8dfc6ba3abb29422b6 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Mon, 19 Feb 2018 19:45:57 +0500 Subject: [PATCH 11/38] docker: rm update && upgrade from mysql Dockerfile for sake of stability --- docker/mysql/Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker/mysql/Dockerfile b/docker/mysql/Dockerfile index a962f506..920c8bbb 100644 --- a/docker/mysql/Dockerfile +++ b/docker/mysql/Dockerfile @@ -1,6 +1,4 @@ FROM alpine:3.7 -RUN apk update -RUN apk upgrade RUN apk --no-cache add mariadb mariadb-client VOLUME /var/lib/mysql From 8469ffb22f824d2a6938cffe455a9b0ab54cbc45 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Tue, 20 Feb 2018 01:48:34 +0500 Subject: [PATCH 12/38] docker: add kafka --- docker/docker-compose.yml | 2 +- docker/kafka/Dockerfile | 45 +++++++++++ docker/kafka/build.sh | 3 + docker/kafka/run.sh | 11 +++ docker/kafka/server.properties | 141 +++++++++++++++++++++++++++++++++ 5 files changed, 201 insertions(+), 1 deletion(-) create mode 100644 docker/kafka/Dockerfile create mode 100755 docker/kafka/build.sh create mode 100755 docker/kafka/run.sh create mode 100644 docker/kafka/server.properties diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index e9c12ff8..a90caa26 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -13,4 +13,4 @@ services: restart: always image: bitlum/exchange-kafka volumes: - - /tmp/bitlum/exchange/kafka/logs:/tmp/kafka-logs + - /tmp/bitlum/exchange/kafka/logs:/var/log/kafka diff --git a/docker/kafka/Dockerfile b/docker/kafka/Dockerfile new file mode 100644 index 00000000..a42e7747 --- /dev/null +++ b/docker/kafka/Dockerfile @@ -0,0 +1,45 @@ +FROM alpine:3.7 + +# Default to UTF-8 file.encoding +ENV LANG C.UTF-8 + +# add a simple script that can auto-detect the appropriate JAVA_HOME value +# based on whether the JDK or only the JRE is installed +RUN { \ + echo '#!/bin/sh'; \ + echo 'set -e'; \ + echo; \ + echo 'dirname "$(dirname "$(readlink -f "$(which javac || which java)")")"'; \ + } > /usr/local/bin/docker-java-home \ + && chmod +x /usr/local/bin/docker-java-home +ENV JAVA_HOME /usr/lib/jvm/java-1.8-openjdk/jre +ENV PATH $PATH:/usr/lib/jvm/java-1.8-openjdk/jre/bin:/usr/lib/jvm/java-1.8-openjdk/bin + +ENV JAVA_VERSION 8u151 +ENV JAVA_ALPINE_VERSION 8.151.12-r0 + +RUN set -x \ + && apk add --no-cache \ + openjdk8-jre="$JAVA_ALPINE_VERSION" \ + && [ "$JAVA_HOME" = "$(docker-java-home)" ] + +RUN apk --no-cache add wget + +RUN mkdir -p /opt/kafka +RUN mkdir -p /etc/kafka +RUN mkdir -p /var/log/kafka +RUN mkdir -p /var/lib/kafka + +RUN wget http://mirror.linux-ia64.org/apache/kafka/1.0.0/kafka_2.11-1.0.0.tgz && \ + tar -xvf kafka_2.11-1.0.0.tgz -C /opt/kafka/ && \ + rm kafka_2.11-1.0.0.tgz + +COPY server.properties /etc/kafka/server.properties + +RUN apk --no-cache add bash + +EXPOSE 9092 +CMD cd /opt/kafka/kafka_2.11-1.0.0 && \ + (nohup bin/zookeeper-server-start.sh config/zookeeper.properties &) && \ + sleep 3 && \ + bin/kafka-server-start.sh config/server.properties diff --git a/docker/kafka/build.sh b/docker/kafka/build.sh new file mode 100755 index 00000000..9cc3f6fc --- /dev/null +++ b/docker/kafka/build.sh @@ -0,0 +1,3 @@ +#!/bin/sh +WD=`dirname $0` +cd $WD && docker build -t bitlum/exchange-kafka . diff --git a/docker/kafka/run.sh b/docker/kafka/run.sh new file mode 100755 index 00000000..af98547b --- /dev/null +++ b/docker/kafka/run.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +V="-v/tmp/bitlum/exchange/kafka/logs:/var/log/kafka" + +docker stop exchange_kafka +docker rm exchange_kafka +if [ -z $SH ];then +docker run --name=exchange_kafka -it $V $@ bitlum/exchange-kafka +else +docker run --name=exchange_kafka -it --entrypoint=/bin/sh $V $@ bitlum/exchange-kafka +fi diff --git a/docker/kafka/server.properties b/docker/kafka/server.properties new file mode 100644 index 00000000..df8e8200 --- /dev/null +++ b/docker/kafka/server.properties @@ -0,0 +1,141 @@ +# Ansible managed. Do not edit by hands. + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# see kafka.server.KafkaConfig for additional details and defaults + +############################# Server Basics ############################# + +# The id of the broker. This must be set to a unique integer for each broker. +broker.id=0 + +# Switch to enable topic deletion or not, default value is false +#delete.topic.enable=true + +############################# Socket Server Settings ############################# + +# The address the socket server listens on. It will get the value returned from +# java.net.InetAddress.getCanonicalHostName() if not configured. +# FORMAT: +# listeners = listener_name://host_name:port +# EXAMPLE: +# listeners = PLAINTEXT://your.host.name:9092 +#listeners=PLAINTEXT://:9092 + +# Hostname and port the broker will advertise to producers and consumers. If not set, +# it uses the value for "listeners" if configured. Otherwise, it will use the value +# returned from java.net.InetAddress.getCanonicalHostName(). +#advertised.listeners=PLAINTEXT://your.host.name:9092 + +# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details +#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL + +# The number of threads that the server uses for receiving requests from the network and sending responses to the network +num.network.threads=3 + +# The number of threads that the server uses for processing requests, which may include disk I/O +num.io.threads=8 + +# The send buffer (SO_SNDBUF) used by the socket server +socket.send.buffer.bytes=102400 + +# The receive buffer (SO_RCVBUF) used by the socket server +socket.receive.buffer.bytes=102400 + +# The maximum size of a request that the socket server will accept (protection against OOM) +socket.request.max.bytes=104857600 + + +############################# Log Basics ############################# + +# A comma seperated list of directories under which to store log files +log.dirs=/var/log/kafka + +# The default number of log partitions per topic. More partitions allow greater +# parallelism for consumption, but this will also result in more files across +# the brokers. +num.partitions=1 + +# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. +# This value is recommended to be increased for installations with data dirs located in RAID array. +num.recovery.threads.per.data.dir=1 + +############################# Internal Topic Settings ############################# +# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" +# For anything other than development testing, a value greater than 1 is recommended for to ensure availability such as 3. +offsets.topic.replication.factor=1 +transaction.state.log.replication.factor=1 +transaction.state.log.min.isr=1 + +############################# Log Flush Policy ############################# + +# Messages are immediately written to the filesystem but by default we only fsync() to sync +# the OS cache lazily. The following configurations control the flush of data to disk. +# There are a few important trade-offs here: +# 1. Durability: Unflushed data may be lost if you are not using replication. +# 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. +# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. +# The settings below allow one to configure the flush policy to flush data after a period of time or +# every N messages (or both). This can be done globally and overridden on a per-topic basis. + +# The number of messages to accept before forcing a flush of data to disk +#log.flush.interval.messages=10000 + +# The maximum amount of time a message can sit in a log before we force a flush +#log.flush.interval.ms=1000 + +############################# Log Retention Policy ############################# + +# The following configurations control the disposal of log segments. The policy can +# be set to delete segments after a period of time, or after a given size has accumulated. +# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens +# from the end of the log. + +# The minimum age of a log file to be eligible for deletion due to age +log.retention.hours=168 + +# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining +# segments don't drop below log.retention.bytes. Functions independently of log.retention.hours. +#log.retention.bytes=1073741824 + +# The maximum size of a log segment file. When this size is reached a new log segment will be created. +log.segment.bytes=1073741824 + +# The interval at which log segments are checked to see if they can be deleted according +# to the retention policies +log.retention.check.interval.ms=300000 + +############################# Zookeeper ############################# + +# Zookeeper connection string (see zookeeper docs for details). +# This is a comma separated host:port pairs, each corresponding to a zk +# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". +# You can also append an optional chroot string to the urls to specify the +# root directory for all kafka znodes. +zookeeper.connect=localhost:2181 + +# Timeout in ms for connecting to zookeeper +zookeeper.connection.timeout.ms=6000 + + +############################# Group Coordinator Settings ############################# + +# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. +# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms. +# The default value for this is 3 seconds. +# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing. +# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup. +group.initial.rebalance.delay.ms=0 From 10b2f3f1530545dc399c160e7697a62dc1f4fa65 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Tue, 20 Feb 2018 02:15:05 +0500 Subject: [PATCH 13/38] docker: add redis --- docker/docker-compose.yml | 9 +++++++++ docker/redis/Dockerfile | 14 ++++++++++++++ docker/redis/build.sh | 3 +++ docker/redis/redis.conf | 23 +++++++++++++++++++++++ docker/redis/run.sh | 18 ++++++++++++++++++ 5 files changed, 67 insertions(+) create mode 100644 docker/redis/Dockerfile create mode 100755 docker/redis/build.sh create mode 100644 docker/redis/redis.conf create mode 100755 docker/redis/run.sh diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index a90caa26..a188068b 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -14,3 +14,12 @@ services: image: bitlum/exchange-kafka volumes: - /tmp/bitlum/exchange/kafka/logs:/var/log/kafka + redis1: + build: + context: redis + args: + PORT: 6381 + image: bitlum/exchange-redis-1 + container_name: exchange_redis_1 + volumes: + - /tmp/bitlum/exchange/redis/1/lib:/var/lib/redis diff --git a/docker/redis/Dockerfile b/docker/redis/Dockerfile new file mode 100644 index 00000000..56a939ff --- /dev/null +++ b/docker/redis/Dockerfile @@ -0,0 +1,14 @@ +FROM alpine:3.7 + +RUN addgroup -S redis && adduser -S -G redis redis +RUN apk add --no-cache redis + +ARG PORT=6379 +COPY redis.conf /etc/redis/redis.conf +RUN sed -i.bak "s/^port .*$/port ${PORT}/" /etc/redis/redis.conf && rm /etc/redis/redis.conf.bak +RUN chown redis:redis /etc/redis/redis.conf + +EXPOSE ${PORT} + +#CMD echo never > /sys/kernel/mm/transparent_hugepage/enabled; redis-server /etc/redis/redis.conf +CMD redis-server /etc/redis/redis.conf diff --git a/docker/redis/build.sh b/docker/redis/build.sh new file mode 100755 index 00000000..2a4d748a --- /dev/null +++ b/docker/redis/build.sh @@ -0,0 +1,3 @@ +#!/bin/sh +WD=`dirname $0` +cd $WD && docker build -t bitlum/exchange-redis . diff --git a/docker/redis/redis.conf b/docker/redis/redis.conf new file mode 100644 index 00000000..1eb4fea6 --- /dev/null +++ b/docker/redis/redis.conf @@ -0,0 +1,23 @@ +daemonize no +pidfile /var/run/redis/redis-sentinel.pid +port 26379 +timeout 0 +tcp-keepalive 60 +loglevel notice +logfile "" +syslog-enabled no +# syslog-ident redis +# syslog-facility local0 +databases 2 +save 900 1 +save 300 10 +save 60 10000 +stop-writes-on-bgsave-error yes +rdbcompression yes +rdbchecksum yes +dbfilename dump.rdb +dir /var/lib/redis +slowlog-log-slower-than 10000 +slowlog-max-len 128 +latency-monitor-threshold 0 +notify-keyspace-events "" diff --git a/docker/redis/run.sh b/docker/redis/run.sh new file mode 100755 index 00000000..9f71f9ce --- /dev/null +++ b/docker/redis/run.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +if [ $# -lt 1 ]; then + N=1 +else + N=$1 + shift +fi + +V="-v/tmp/bitlum/exchange/redis/$N/lib:/var/lib/redis" + +docker stop exchange_redis_$N +docker rm exchange_redis_$N +if [ -z $SH ];then +docker run --name=exchange_redis_$N -it $V $@ bitlum/exchange-redis +else +docker run --name=exchange_redis_$N -it --entrypoint=/bin/sh $V $@ bitlum/exchange-redis +fi From 64b251c686647e268bf66f8319531162a7b1f19f Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Tue, 20 Feb 2018 10:00:06 +0500 Subject: [PATCH 14/38] docker: redis run from reference docker-compose.yml --- docker/docker-compose.yml | 28 ++++++++++++++-------------- docker/redis/Dockerfile | 6 ++++-- docker/redis/build.sh | 8 +++++++- docker/redis/entrypoint.sh | 3 +++ docker/redis/run.sh | 4 ++-- 5 files changed, 30 insertions(+), 19 deletions(-) create mode 100755 docker/redis/entrypoint.sh diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index a188068b..c304ed3b 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,19 +1,19 @@ version: '2.1' services: - mysql: - build: mysql - image: bitlum/exchange-mysql - container_name: exchange_mysql - restart: always - volumes: - - /tmp/bitlum/exchange/mysql/lib:/var/lib/mysql - kafka: - build: kafka - container_name: exchange_kafka - restart: always - image: bitlum/exchange-kafka - volumes: - - /tmp/bitlum/exchange/kafka/logs:/var/log/kafka +# mysql: +# build: mysql +# image: bitlum/exchange-mysql +# container_name: exchange_mysql +# restart: always +# volumes: +# - /tmp/bitlum/exchange/mysql/lib:/var/lib/mysql +# kafka: +# build: kafka +# container_name: exchange_kafka +# restart: always +# image: bitlum/exchange-kafka +# volumes: +# - /tmp/bitlum/exchange/kafka/logs:/var/log/kafka redis1: build: context: redis diff --git a/docker/redis/Dockerfile b/docker/redis/Dockerfile index 56a939ff..ab241e42 100644 --- a/docker/redis/Dockerfile +++ b/docker/redis/Dockerfile @@ -5,10 +5,12 @@ RUN apk add --no-cache redis ARG PORT=6379 COPY redis.conf /etc/redis/redis.conf +COPY entrypoint.sh /docker-entrypoint.sh RUN sed -i.bak "s/^port .*$/port ${PORT}/" /etc/redis/redis.conf && rm /etc/redis/redis.conf.bak RUN chown redis:redis /etc/redis/redis.conf + EXPOSE ${PORT} -#CMD echo never > /sys/kernel/mm/transparent_hugepage/enabled; redis-server /etc/redis/redis.conf -CMD redis-server /etc/redis/redis.conf +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["redis-server", "/etc/redis/redis.conf"] diff --git a/docker/redis/build.sh b/docker/redis/build.sh index 2a4d748a..6e16f820 100755 --- a/docker/redis/build.sh +++ b/docker/redis/build.sh @@ -1,3 +1,9 @@ #!/bin/sh WD=`dirname $0` -cd $WD && docker build -t bitlum/exchange-redis . +if [ $# -lt 1 ]; then + N=1 +else + N=$1 + shift +fi +cd $WD && docker build -t bitlum/exchange-redis-$N . diff --git a/docker/redis/entrypoint.sh b/docker/redis/entrypoint.sh new file mode 100755 index 00000000..0daad140 --- /dev/null +++ b/docker/redis/entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +exec $@ diff --git a/docker/redis/run.sh b/docker/redis/run.sh index 9f71f9ce..e226f717 100755 --- a/docker/redis/run.sh +++ b/docker/redis/run.sh @@ -12,7 +12,7 @@ V="-v/tmp/bitlum/exchange/redis/$N/lib:/var/lib/redis" docker stop exchange_redis_$N docker rm exchange_redis_$N if [ -z $SH ];then -docker run --name=exchange_redis_$N -it $V $@ bitlum/exchange-redis +docker run --name=exchange_redis_$N -it $V $@ bitlum/exchange-redis-$N else -docker run --name=exchange_redis_$N -it --entrypoint=/bin/sh $V $@ bitlum/exchange-redis +docker run --name=exchange_redis_$N -it --entrypoint=/bin/sh $V $@ bitlum/exchange-redis-$N fi From 6405328c81fd06e9d7bf90b0fea09e6178853e53 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Tue, 20 Feb 2018 10:16:26 +0500 Subject: [PATCH 15/38] docker: add sentinel docker --- docker/docker-compose.yml | 14 ++++++++++++++ docker/sentinel/Dockerfile | 22 ++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 docker/sentinel/Dockerfile diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index c304ed3b..20a75adf 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -23,3 +23,17 @@ services: container_name: exchange_redis_1 volumes: - /tmp/bitlum/exchange/redis/1/lib:/var/lib/redis + sentinel1: + build: + context: sentinel + args: + PORT: 26381 + REDIS_HOST: redis1 + REDIS_PORT: 6381 + REDIS_QUORUM: 1 + image: bitlum/exchange-sentinel-1 + container_name: exchange_sentinel_1 + depends_on: + - redis1 + volumes: + - /tmp/bitlum/exchange/sentinel/1/log:/var/log/sentinel diff --git a/docker/sentinel/Dockerfile b/docker/sentinel/Dockerfile new file mode 100644 index 00000000..074028ac --- /dev/null +++ b/docker/sentinel/Dockerfile @@ -0,0 +1,22 @@ +FROM alpine:3.7 + +RUN addgroup -S redis && adduser -S -G redis redis +RUN apk add --no-cache redis + +ARG PORT=26379 +ARG REDIS_PORT=6379 +ARG REDIS_HOST=redis +ARG REDIS_QUORUM=1 + +COPY sentinel.conf /etc/redis/sentinel.conf + +RUN sed -i.bak "s/^port .*$/port ${PORT}/" /etc/redis/sentinel.conf && \ + cat /etc/redis/sentinel.conf && \ + sed -i.bak "s/^\(sentinel monitor [^ ]*\).*$/\1 ${REDIS_HOST} ${REDIS_PORT} ${REDIS_QUORUM}/" /etc/redis/sentinel.conf && \ + rm /etc/redis/sentinel.conf.bak && \ + cat /etc/redis/sentinel.conf +RUN chown redis:redis /etc/redis/sentinel.conf + +EXPOSE ${PORT} + +CMD ["redis-server", "/etc/redis/sentinel.conf", "--sentinel"] From 9449b9787fc7f83ade2f5cd766d6b9965e5cd293 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Tue, 20 Feb 2018 10:16:40 +0500 Subject: [PATCH 16/38] docker: add sentinel docker --- docker/sentinel/sentinel.conf | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docker/sentinel/sentinel.conf diff --git a/docker/sentinel/sentinel.conf b/docker/sentinel/sentinel.conf new file mode 100644 index 00000000..e7c86dec --- /dev/null +++ b/docker/sentinel/sentinel.conf @@ -0,0 +1,6 @@ +port 26379 +sentinel monitor exchange-sentinel redis 6379 2 +sentinel down-after-milliseconds exchange-sentinel 30000 +sentinel parallel-syncs exchange-sentinel 1 +sentinel failover-timeout exchange-sentinel 180000 +daemonize no From b0a52b157ec685a5d4efda510f56ad35340f45ba Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Tue, 20 Feb 2018 10:17:29 +0500 Subject: [PATCH 17/38] docker: rm entrypoint from redis --- docker/redis/Dockerfile | 3 --- docker/redis/entrypoint.sh | 3 --- 2 files changed, 6 deletions(-) delete mode 100755 docker/redis/entrypoint.sh diff --git a/docker/redis/Dockerfile b/docker/redis/Dockerfile index ab241e42..4817aaf7 100644 --- a/docker/redis/Dockerfile +++ b/docker/redis/Dockerfile @@ -5,12 +5,9 @@ RUN apk add --no-cache redis ARG PORT=6379 COPY redis.conf /etc/redis/redis.conf -COPY entrypoint.sh /docker-entrypoint.sh RUN sed -i.bak "s/^port .*$/port ${PORT}/" /etc/redis/redis.conf && rm /etc/redis/redis.conf.bak RUN chown redis:redis /etc/redis/redis.conf - EXPOSE ${PORT} -ENTRYPOINT ["/docker-entrypoint.sh"] CMD ["redis-server", "/etc/redis/redis.conf"] diff --git a/docker/redis/entrypoint.sh b/docker/redis/entrypoint.sh deleted file mode 100755 index 0daad140..00000000 --- a/docker/redis/entrypoint.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -exec $@ From 10c7e9eda4cf65df7ab40a404a9bf4cd99635c08 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Tue, 20 Feb 2018 22:31:16 +0500 Subject: [PATCH 18/38] docker: docker-compose for exchange draft --- docker/docker-compose.yml | 162 +++++++++++++++++++-- docker/exchange/Dockerfile | 91 ++++++++++++ docker/exchange/Dockerfile.alpine | 31 ++++ docker/exchange/accesshttp.json | 49 +++++++ docker/exchange/accessws.json | 71 +++++++++ docker/exchange/alertcenter.json | 26 ++++ docker/exchange/build.sh | 3 + docker/exchange/makefile.accessws.ubuntu | 4 + docker/exchange/makefile.inc.ubuntu | 21 +++ docker/exchange/marketprice.json | 38 +++++ docker/exchange/matchengine.json | 122 ++++++++++++++++ docker/exchange/readhistory.json | 32 ++++ docker/exchange/run.sh | 177 +++++++++++++++++++++++ docker/exchange/test.sh | 4 + 14 files changed, 817 insertions(+), 14 deletions(-) create mode 100644 docker/exchange/Dockerfile create mode 100644 docker/exchange/Dockerfile.alpine create mode 100644 docker/exchange/accesshttp.json create mode 100644 docker/exchange/accessws.json create mode 100644 docker/exchange/alertcenter.json create mode 100755 docker/exchange/build.sh create mode 100644 docker/exchange/makefile.accessws.ubuntu create mode 100644 docker/exchange/makefile.inc.ubuntu create mode 100644 docker/exchange/marketprice.json create mode 100644 docker/exchange/matchengine.json create mode 100644 docker/exchange/readhistory.json create mode 100755 docker/exchange/run.sh create mode 100755 docker/exchange/test.sh diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 20a75adf..d22c6e2e 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,19 +1,25 @@ version: '2.1' services: -# mysql: -# build: mysql -# image: bitlum/exchange-mysql -# container_name: exchange_mysql -# restart: always -# volumes: -# - /tmp/bitlum/exchange/mysql/lib:/var/lib/mysql -# kafka: -# build: kafka -# container_name: exchange_kafka -# restart: always -# image: bitlum/exchange-kafka -# volumes: -# - /tmp/bitlum/exchange/kafka/logs:/var/log/kafka + mysql: + build: mysql + image: bitlum/exchange-mysql + container_name: exchange_mysql + restart: always + volumes: + - /tmp/bitlum/exchange/mysql/lib:/var/lib/mysql + networks: + exchange_net: + ipv4_address: 192.168.18.10 + kafka: + build: kafka + container_name: exchange_kafka + restart: always + image: bitlum/exchange-kafka + volumes: + - /tmp/bitlum/exchange/kafka/logs:/var/log/kafka + networks: + exchange_net: + ipv4_address: 192.168.18.20 redis1: build: context: redis @@ -23,6 +29,9 @@ services: container_name: exchange_redis_1 volumes: - /tmp/bitlum/exchange/redis/1/lib:/var/lib/redis + networks: + exchange_net: + ipv4_address: 192.168.18.31 sentinel1: build: context: sentinel @@ -37,3 +46,128 @@ services: - redis1 volumes: - /tmp/bitlum/exchange/sentinel/1/log:/var/log/sentinel + networks: + exchange_net: + ipv4_address: 192.168.18.32 + alertcenter: + build: + context: .. + dockerfile: docker/exchange/Dockerfile + command: alertcenter + image: bitlum/exchange-alertcenter + container_name: exchange_trade_alertcenter + restart: always + depends_on: + - sentinel1 + volumes: + - /tmp/bitlum/exchange/trade/alertcenter/log:/var/log/trade + networks: + exchange_net: + ipv4_address: 192.168.18.41 + matchengine: + build: + context: .. + dockerfile: docker/exchange/Dockerfile + command: matchengine + image: bitlum/exchange-matchengine + container_name: exchange_trade_matchengine + restart: always + depends_on: + - mysql + - kafka + - sentinel1 + - alertcenter + volumes: + - /tmp/bitlum/exchange/trade/matchengine/log:/var/log/trade + networks: + exchange_net: + ipv4_address: 192.168.18.42 + marketprice: + build: + context: .. + dockerfile: docker/exchange/Dockerfile + args: + SERVICE: marketprice + image: bitlum/exchange-marketprice + container_name: exchange_trade_marketprice + restart: always + depends_on: + - kafka + - sentinel1 + - alertcenter + volumes: + - /tmp/bitlum/exchange/trade/marketprice/log:/var/log/trade + networks: + exchange_net: + ipv4_address: 192.168.18.43 + readhistory: + build: + context: .. + dockerfile: docker/exchange/Dockerfile + args: + SERVICE: readhistory + image: bitlum/exchange-readhistory + container_name: exchange_trade_readhistory + restart: always + depends_on: + - mysql + - alertcenter + volumes: + - /tmp/bitlum/exchange/trade/readhistory/log:/var/log/trade + networks: + exchange_net: + ipv4_address: 192.168.18.44 + accesshttp: + build: + context: .. + dockerfile: docker/exchange/Dockerfile + args: + SERVICE: accesshttp + image: bitlum/exchange-accesshttp + container_name: exchange_trade_accesshttp + restart: always + depends_on: + - alertcenter + - matchengine + - marketprice + - readhistory + volumes: + - /tmp/bitlum/exchange/trade/accesshttp/log:/var/log/trade + ports: + - "8080:8080" + - "8081:8081" + networks: + exchange_net: + ipv4_address: 192.168.18.45 + accessws: + build: + context: .. + dockerfile: docker/exchange/Dockerfile + args: + SERVICE: accessws + image: bitlum/exchange-accessws + container_name: exchange_trade_accessws + restart: always + depends_on: + - kafka + - alertcenter + - matchengine + - marketprice + - readhistory + volumes: + - /tmp/bitlum/exchange/trade/accessws/log:/var/log/trade + ports: + - "8090:8090" + - "8090:8091" + networks: + exchange_net: + ipv4_address: 192.168.18.46 +networks: + exchange_net: + driver: bridge + enable_ipv6: false + ipam: + driver: default + config: + - subnet: 192.168.18.0/24 + gateway: 192.168.18.254 diff --git a/docker/exchange/Dockerfile b/docker/exchange/Dockerfile new file mode 100644 index 00000000..a11c29a4 --- /dev/null +++ b/docker/exchange/Dockerfile @@ -0,0 +1,91 @@ +FROM ubuntu:16.04 AS BUILDSTEP +LABEL BUILDSTEP NNNN +MAINTAINER Sergey Dolin + +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y \ + jq \ + netcat \ + inetutils-ping \ + vim \ + git \ + wget \ + make \ + cmake \ + gcc \ + g++ \ + python \ + libev-dev \ + libmpdec-dev \ + libjansson-dev \ + libssl-dev \ + libgnutls-dev \ + libmysqlclient-dev \ + libhttp-parser-dev \ + libcurl4-openssl-dev \ + libldap2-dev \ + libkrb5-dev \ + libalberta-dev \ + libgss-dev \ + libidn11-dev \ + librtmp-dev &&\ + rm -rf /var/lib/apt/lists/* + +RUN git clone https://github.com/edenhill/librdkafka.git && cd librdkafka && ./configure && make && make install && cd / && rm -rf librdkafka + +RUN mkdir viabtc_exchange_server +COPY depends viabtc_exchange_server/depends +COPY network viabtc_exchange_server/network +COPY utils viabtc_exchange_server/utils +COPY alertcenter viabtc_exchange_server/alertcenter +COPY matchengine viabtc_exchange_server/matchengine +COPY marketprice viabtc_exchange_server/marketprice +COPY readhistory viabtc_exchange_server/readhistory +COPY accesshttp viabtc_exchange_server/accesshttp +COPY accessws viabtc_exchange_server/accessws +COPY docker/exchange/makefile.inc.ubuntu viabtc_exchange_server/makefile.inc + +RUN cd viabtc_exchange_server/depends/hiredis && make && mv libhiredis.* /usr/lib && cd .. && mv hiredis /usr/include +RUN cd viabtc_exchange_server/network && make +RUN cd viabtc_exchange_server/utils && make +RUN cd viabtc_exchange_server/alertcenter && make +RUN cd viabtc_exchange_server/matchengine && make +RUN cd viabtc_exchange_server/marketprice && make +RUN cd viabtc_exchange_server/readhistory && make +RUN cd viabtc_exchange_server/accesshttp && make +COPY docker/exchange/makefile.accessws.ubuntu viabtc_exchange_server/accessws/makefile +RUN cd viabtc_exchange_server/accessws && make + +FROM ubuntu:16.04 + +RUN apt-get update \ + && apt-get upgrade -y \ + && apt-get install -y \ + jq \ + netcat \ + && rm -rf /var/lib/apt/lists/* + +# keep .exe for better grep of ps output +COPY --from=BUILDSTEP /viabtc_exchange_server/alertcenter/alertcenter.exe /viabtc_exchange_server/alertcenter.exe +COPY docker/exchange/alertcenter.json /viabtc_exchange_server + +COPY --from=BUILDSTEP /viabtc_exchange_server/matchengine/matchengine.exe /viabtc_exchange_server/matchengine.exe +COPY docker/exchange/matchengine.json /viabtc_exchange_server + +COPY --from=BUILDSTEP /viabtc_exchange_server/marketprice/marketprice.exe /viabtc_exchange_server/marketprice.exe +COPY docker/exchange/marketprice.json /viabtc_exchange_server + +COPY --from=BUILDSTEP /viabtc_exchange_server/readhistory/readhistory.exe /viabtc_exchange_server/readhistory.exe +COPY docker/exchange/readhistory.json /viabtc_exchange_server + +COPY --from=BUILDSTEP /viabtc_exchange_server/accesshttp/accesshttp.exe /viabtc_exchange_server/accesshttp.exe +COPY docker/exchange/accesshttp.json /viabtc_exchange_server + +COPY --from=BUILDSTEP /viabtc_exchange_server/accessws/accessws.exe /viabtc_exchange_server/accessws.exe +COPY docker/exchange/accessws.json /viabtc_exchange_server + +COPY docker/exchange/run.sh run.sh + +ENTRYPOINT [ "/run.sh" ] +# CMD /run.sh diff --git a/docker/exchange/Dockerfile.alpine b/docker/exchange/Dockerfile.alpine new file mode 100644 index 00000000..48725886 --- /dev/null +++ b/docker/exchange/Dockerfile.alpine @@ -0,0 +1,31 @@ +FROM alpine:3.7 + +RUN apk add --no-cache git make gcc g++ bash python wget libev-dev jansson-dev mariadb-dev libexecinfo-dev curl-dev + +RUN git clone https://github.com/edenhill/librdkafka.git && cd librdkafka && ./configure && make && make install && cd / && rm -rf librdkafka + + +RUN wget http://www.bytereef.org/software/mpdecimal/releases/mpdecimal-2.4.2.tar.gz +RUN tar -xzf mpdecimal-2.4.2.tar.gz +RUN cd mpdecimal-2.4.2 && ./configure && make && make install + + +RUN mkdir viabtc_exchange_server + +COPY depends viabtc_exchange_server/depends +COPY network viabtc_exchange_server/network +COPY utils viabtc_exchange_server/utils + +COPY alertcenter viabtc_exchange_server/alertcenter +COPY matchengine viabtc_exchange_server/matchengine +COPY marketprice viabtc_exchange_server/marketprice +COPY readhistory viabtc_exchange_server/readhistory +COPY accesshttp viabtc_exchange_server/accesshttp +COPY accessws viabtc_exchange_server/accessws + +COPY makefile.inc viabtc_exchange_server/makefile.inc + +RUN cd viabtc_exchange_server/depends/hiredis && make && mv libhiredis.* /usr/lib && cd .. && mv hiredis /usr/include +RUN cd viabtc_exchange_server/network && make +RUN cd viabtc_exchange_server/utils && make +RUN cd viabtc_exchange_server/alertcenter && make diff --git a/docker/exchange/accesshttp.json b/docker/exchange/accesshttp.json new file mode 100644 index 00000000..ed74028c --- /dev/null +++ b/docker/exchange/accesshttp.json @@ -0,0 +1,49 @@ +{ + "debug": true, + "process": { + "file_limit": 1000000, + "core_limit": 1000000000 + }, + "log": { + "path": "/var/log/trade/accesshttp", + "flag": "fatal,error,warn,info,debug,trace", + "num": 10 + }, + "alert": { + "host": "accesshttp", + "addr": "192.168.18.41:4444" + }, + "svr": { + "bind": [ + "tcp@0.0.0.0:8080" + ], + "max_pkg_size": 102400 + }, + "monitor": { + "bind": "tcp@0.0.0.0:8081", + "max_pkg_size": 1024 + }, + "worker_num": 4, + "timeout": 1.0, + "matchengine": { + "name": "matchengine", + "addr": [ + "tcp@192.168.18.42:7316" + ], + "max_pkg_size": 2000000 + }, + "marketprice": { + "name": "marketprice", + "addr": [ + "tcp@192.168.18.43:7416" + ], + "max_pkg_size": 2000000 + }, + "readhistory": { + "name": "readhistory", + "addr": [ + "tcp@192.168.18.44:7424" + ], + "max_pkg_size": 2000000 + } +} diff --git a/docker/exchange/accessws.json b/docker/exchange/accessws.json new file mode 100644 index 00000000..962ea428 --- /dev/null +++ b/docker/exchange/accessws.json @@ -0,0 +1,71 @@ +{ + "debug": true, + "process": { + "file_limit": 1000000, + "core_limit": 1000000000 + }, + "log": { + "path": "/var/log/trade/accessws", + "flag": "fatal,error,warn,info,debug,trace", + "num": 10 + }, + "alert": { + "host": "accessws", + "addr": "192.168.18.41:4444" + }, + "svr": { + "bind": [ + "tcp@0.0.0.0:8090" + ], + "max_pkg_size": 102400, + "protocol": "chat" + }, + "monitor": { + "bind": "tcp@0.0.0.0:8091", + "max_pkg_size": 1024 + }, + "worker_num": 1, + "timeout": 1.0, + "matchengine": { + "name": "matchengine", + "addr": [ + "tcp@192.168.18.42:7316" + ], + "max_pkg_size": 2000000 + }, + "marketprice": { + "name": "marketprice", + "addr": [ + "tcp@192.168.18.43:7416" + ], + "max_pkg_size": 2000000 + }, + "readhistory": { + "name": "readhistory", + "addr": [ + "tcp@192.168.18.44:7424" + ], + "max_pkg_size": 2000000 + }, + "deals": { + "brokers": "kafka:9092", + "topic": "deals", + "partition": 0 + }, + "orders": { + "brokers": "kafka:9092", + "topic": "orders", + "partition": 0 + }, + "balances": { + "brokers": "127.0.0.1:9092", + "topic": "balances", + "partition": 0 + }, + "backend_timeout": 1.0, + "cache_timeout": 10.0, + "auth_url": "http://192.168.1.6:8000/internal/exchange/user/auth", + "sign_url": "http://192.168.1.6:8000/internal/exchange/user/api/auth", + "depth_limit": [1, 5, 10, 20, 30, 50, 100], + "depth_merge": ["0", "0.00000001", "0.0000001", "0.000001", "0.00001", "0.0001", "0.001", "0.01", "0.1"] +} diff --git a/docker/exchange/alertcenter.json b/docker/exchange/alertcenter.json new file mode 100644 index 00000000..490e23bb --- /dev/null +++ b/docker/exchange/alertcenter.json @@ -0,0 +1,26 @@ +{ + "process": { + "file_limit": 1000000, + "core_limit": 1000000000 + }, + "log": { + "path": "/var/log/trade/alertcenter", + "flag": "fatal,error,warn,info,debug" + }, + "svr": { + "bind": [ + "tcp@0.0.0.0:4444", + "udp@0.0.0.0:4444" + ], + "max_pkg_size": 10240 + }, + "redis": { + "name": "exchange-sentinel", + "addr": [ + "192.168.18.32:26381", + "192.168.18.32:26382", + "192.168.18.32:26383" + ] + } +} + diff --git a/docker/exchange/build.sh b/docker/exchange/build.sh new file mode 100755 index 00000000..cd45c5e0 --- /dev/null +++ b/docker/exchange/build.sh @@ -0,0 +1,3 @@ +#!/bin/sh +WD=`dirname $0`/../.. +cd $WD && docker build -t bitlum/exchange . -f docker/exchange/Dockerfile diff --git a/docker/exchange/makefile.accessws.ubuntu b/docker/exchange/makefile.accessws.ubuntu new file mode 100644 index 00000000..ac0b7639 --- /dev/null +++ b/docker/exchange/makefile.accessws.ubuntu @@ -0,0 +1,4 @@ +TARGET := accessws.exe +INCS = -I ../network -I ../utils +LIBS = -L ../utils -lutils -L ../network -lnetwork -Wl,-Bstatic -lev -ljansson -lmpdec -lrdkafka -lcurl -lz -lssl -lcrypt -lcrypto -lhiredis -Wl,-Bdynamic -lm -lpthread -ldl -lldap -lkrb5 -llber -lgss -lidn -lrtmp +include ../makefile.inc diff --git a/docker/exchange/makefile.inc.ubuntu b/docker/exchange/makefile.inc.ubuntu new file mode 100644 index 00000000..518dc183 --- /dev/null +++ b/docker/exchange/makefile.inc.ubuntu @@ -0,0 +1,21 @@ +SOURCE := $(wildcard *.c) +OBJS := $(patsubst %.c, %.o, $(SOURCE)) +CC := gcc +LD := g++ #we will need to link c++ mysql client library +CFLAGS := -Wall -Wno-strict-aliasing -Wno-uninitialized -g -rdynamic -std=gnu99 +LFLAGS := -g -rdynamic + +.PHONY : all clean install + +all : $(TARGET) + +clean : + rm -rf *.d *.o $(TARGET) + +$(TARGET) : $(OBJS) + $(LD) $(LFLAGS) -o $@ $(OBJS) $(LIBS) +.c.o : + $(CC) $(CFLAGS) -c -o $@ $< $(INCS) + +install : + diff --git a/docker/exchange/marketprice.json b/docker/exchange/marketprice.json new file mode 100644 index 00000000..493e4a11 --- /dev/null +++ b/docker/exchange/marketprice.json @@ -0,0 +1,38 @@ +{ + "debug": true, + "process": { + "file_limit": 1000000, + "core_limit": 1000000000 + }, + "log": { + "path": "/var/log/trade/marketprice", + "flag": "fatal,error,warn,info,debug,trace", + "num": 10 + }, + "alert": { + "host": "marketprice", + "addr": "192.168.18.41:4444" + }, + "svr": { + "bind": [ + "tcp@0.0.0.0:7416", + "udp@0.0.0.0:7416" + ], + "buf_limit": 100, + "max_pkg_size": 1000000, + "heartbeat_check": false + }, + "deals": { + "brokers": "192.168.18.20:9092", + "topic": "deals", + "partition": 0 + }, + "redis": { + "name": "exchange-sentinel", + "addr": [ + "192.168.18.32:26381" + ] + }, + "accesshttp": "http://accesshttp:8080" +} + diff --git a/docker/exchange/matchengine.json b/docker/exchange/matchengine.json new file mode 100644 index 00000000..33ca6eaa --- /dev/null +++ b/docker/exchange/matchengine.json @@ -0,0 +1,122 @@ +{ + "debug": true, + "process": { + "file_limit": 1000000, + "core_limit": 1000000000 + }, + "log": { + "path": "/var/log/trade/matchengine", + "flag": "fatal,error,warn,info,debug,trace", + "num": 10 + }, + "alert": { + "host": "matchengine", + "addr": "192.168.18.41:4444" + }, + "svr": { + "bind": [ + "tcp@0.0.0.0:7316", + "udp@0.0.0.0:7316" + ], + "buf_limit": 100, + "max_pkg_size": 10240, + "heartbeat_check": false + }, + "cli": "tcp@127.0.0.1:7317", + "db_log": { + "host": "mysql", + "user": "exchange", + "pass": "exchange", + "name": "trade_log" + }, + "db_history": { + "host": "mysql", + "user": "exchange", + "pass": "exchange", + "name": "trade_history" + }, + "assets": [ + { + "name": "ETH", + "prec_save": 16, + "prec_show": 8 + },{ + "name": "BTC", + "prec_save": 16, + "prec_show": 8 + }, { + "name": "BCH", + "prec_save": 16, + "prec_show": 8 + }, { + "name": "LTC", + "prec_save": 16, + "prec_show": 8 + },{ + "name": "DASH", + "prec_save": 16, + "prec_show": 8 + } + ], + "markets": [ + { + "name": "BTCETH", + "stock": { + "name": "ETH", + "prec": 8 + }, + "money": { + "name": "BTC", + "prec": 8 + }, + "min_amount":"0.00001" + }, { + "name": "BTCBCH", + "stock": { + "name": "BCH", + "prec": 8 + }, + "money": { + "name": "BTC", + "prec": 8 + }, + "min_amount":"0.00001" + }, { + "name": "BTCLTC", + "stock": { + "name": "LTC", + "prec": 8 + }, + "money": { + "name": "BTC", + "prec": 8 + }, + "min_amount":"0.00001" + }, { + "name": "BTCDASH", + "stock": { + "name": "DASH", + "prec": 8 + }, + "money": { + "name": "BTC", + "prec": 8 + }, + "min_amount":"0.00001" + }, { + "name": "ETHLTC", + "stock": { + "name": "LTC", + "prec": 8 + }, + "money": { + "name": "ETH", + "prec": 8 + }, + "min_amount":"0.00001" + } + ], + "brokers": "kafka:9092", + "slice_interval": 3600, + "slice_keeptime": 259200 +} diff --git a/docker/exchange/readhistory.json b/docker/exchange/readhistory.json new file mode 100644 index 00000000..2da30ccb --- /dev/null +++ b/docker/exchange/readhistory.json @@ -0,0 +1,32 @@ +{ + "debug": true, + "process": { + "file_limit": 1000000, + "core_limit": 1000000000 + }, + "log": { + "path": "/var/log/trade/readhistory", + "flag": "fatal,error,warn,info,debug,trace", + "num": 10 + }, + "alert": { + "host": "readhistory", + "addr": "192.168.18.41:4444" + }, + "svr": { + "bind": [ + "tcp@0.0.0.0:7424", + "udp@0.0.0.0:7424" + ], + "buf_limit": 100, + "max_pkg_size": 1000000, + "heartbeat_check": false + }, + "db_history": { + "host": "mysql", + "user": "exchange", + "pass": "exchange", + "name": "trade_history" + }, + "worker_num": 10 +} diff --git a/docker/exchange/run.sh b/docker/exchange/run.sh new file mode 100755 index 00000000..bb8972f7 --- /dev/null +++ b/docker/exchange/run.sh @@ -0,0 +1,177 @@ +#!/bin/bash +help () { + echo "Available daemons are: " + echo " accesshttp" + echo " accessws" + echo " alertcenter" + echo " marketprice" + echo " matchengine" + echo " readhistory" +} + +echo "@=$@" + +if [ $# -lt 1 ];then + help + exit 0 +fi + +SERVICE=$1 + + +case $SERVICE in + alertcenter) + DAEMON=alertcenter + ;; + matchengine) + DAEMON=matchengine + ;; + marketprice) + DAEMON=marketprice + ;; + readhistory) + DAEMON=readhistory + ;; + accesshttp) + DAEMON=accesshttp + ;; + accessws) + DAEMON=accessws + ;; + --help) + help + ;; + -h) + help + ;; + *) + exec $@ + ;; +esac + +if [ $DAEMON ]; then + cd /viabtc_exchange_server + echo "=== $DAEMON is starting" + redis=`jq .redis.addr[0] < ${DAEMON}.json 2>/dev/null` + test -z $redis || test $redis = 'null' && redis='' + if [ -z $redis ];then + redis=`jq .redis.addr < ${DAEMON}.json 2>/dev/null` + fi + test -z $redis || test $redis = 'null' && redis='' + if [ $redis ];then + echo "=== $DAEMON contains redis config: $redis" + redis=`echo $redis|sed -e 's/"//g'` + echo "=== $DAEMON: sentinel addr found: $redis" + host=`echo $redis | cut -f1 -d':'` + port=`echo $redis | cut -f2 -d':'` + nc -z $host $port + result=$? + while [ $result -ne 0 ]; do + echo "=== $DAEMON: is waiting for sentinel on host=$host, port=$port ..." + sleep 10 + nc -z $host $port + result=$? + done + echo "=== $DAEMON: sentinel is up" + fi + kafka=`jq .brokers[0] < ${DAEMON}.json 2>/dev/null` + test -z $kafka || test $kafka = 'null' && kafka='' + if [ -z $kafka ];then + kafka=`jq .brokers < ${DAEMON}.json 2>/dev/null` + fi + test -z $kafka || test $kafka = 'null' && kafka='' + if [ -z $kafka ];then + kafka=`jq .deals.brokers[0] < ${DAEMON}.json 2>/dev/null` + fi + test -z $kafka || test $kafka = 'null' && kafka='' + if [ -z $kafka ];then + kafka=`jq .deals.brokers < ${DAEMON}.json 2>/dev/null` + fi + test -z $kafka || test $kafka = 'null' && kafka='' + if [ $kafka ];then + echo "=== $DAEMON contains kafka config: $kafka" + kafka=`echo $kafka|sed -e 's/"//g'` + echo "=== $DAEMON: kafka addr found: $kafka" + host=`echo $kafka | cut -f1 -d':'` + port=`echo $kafka | cut -f2 -d':'` + nc -z $host $port + result=$? + while [ $result -ne 0 ]; do + echo "=== $DAEMON: is waiting for kafka on host=$host, port=$port ..." + sleep 10 + nc -z $host $port + result=$? + done + echo "=== $DAEMON: kafka is up" + fi + host=`jq .db_history.host < ${DAEMON}.json 2>/dev/null` + test -z $host || test "$host" = 'null' && host='' + if [ "$host" ];then + port=`jq .db_history.port < ${DAEMON}.json 2>/dev/null` + test -z $port || test $port = 'null' && port='' + test $port || port=3306 + echo "=== $DAEMON contains db_history mysql host: $host and port: $port" + host=`echo $host|sed -e 's/"//g'` + port=`echo $port|sed -e 's/"//g'` + nc -z $host $port + result=$? + while [ $result -ne 0 ]; do + echo "=== $DAEMON: is waiting for db_history on host=$host, port=$port ..." + sleep 10 + nc -z $host $port + result=$? + done + echo "=== $DAEMON: db_history is up" + fi + host=`jq .db_log.host < ${DAEMON}.json 2>/dev/null` + test -z $host || test "$host" = 'null' && host='' + if [ "$host" ];then + port=`jq .db_log.port < ${DAEMON}.json 2>/dev/null` + test -z $port || test $port = 'null' && port='' + test $port || port=3306 + echo "=== $DAEMON contains db_log mysql host: $host and port: $port" + host=`echo $host|sed -e 's/"//g'` + port=`echo $port|sed -e 's/"//g'` + nc -z $host $port + result=$? + while [ $result -ne 0 ]; do + echo "=== $DAEMON: is waiting for db_log on host=$host, port=$port ..." + sleep 10 + nc -z $host $port + result=$? + done + echo "=== $DAEMON: db_log is up" + fi + echo "=== $DAEMON: ready to exec" + ./$DAEMON.exe ${DAEMON}.json + NEW=1 + while true; do + sleep 5 + ps aux|grep ${DAEMON}.exe|grep -v grep > /dev/null + RESULT=$? + if [ $RESULT -ne 0 ];then + # some of services forks with _worker suffix + ps aux|grep ${DAEMON}_worker |grep -v grep > /dev/null + RESULT=$? + if [ $RESULT -ne 0 ];then + echo "=== $DAEMON has died, exitting docker..."; + break; + else + if [ $NEW -eq 1 ]; then + echo "Microservice $DAEMON(worker) has started"; + NEW=0 + fi + fi + else + if [ $NEW -eq 1 ]; then + echo "=== $DAEMON(exe) has started"; + NEW=0 + fi + fi + done + exit 100 +elif [ $BASH ]; then + while true;do sleep 3600;done +else + exit 101 +fi diff --git a/docker/exchange/test.sh b/docker/exchange/test.sh new file mode 100755 index 00000000..bb6cea1e --- /dev/null +++ b/docker/exchange/test.sh @@ -0,0 +1,4 @@ +#!/bin/bash +docker stop exchange_test +docker rm exchange_test +docker run --name=exchange_test -it bitlum/exchange $@ From c7e6c29b3bf838e17408b996b0288da37c24d412 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Tue, 20 Feb 2018 22:39:04 +0500 Subject: [PATCH 19/38] docker: add mysql docker diagnostic --- docker/mysql/Dockerfile | 1 - docker/mysql/entrypoint.sh | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/mysql/Dockerfile b/docker/mysql/Dockerfile index 920c8bbb..280b2a07 100644 --- a/docker/mysql/Dockerfile +++ b/docker/mysql/Dockerfile @@ -17,7 +17,6 @@ COPY init_trade_log.sh /docker-entrypoint-initdb.d COPY init_trade_history.sh /docker-entrypoint-initdb.d COPY create_trade_history.sql /docker-entrypoint-initdb.d COPY create_trade_log.sql /docker-entrypoint-initdb.d -RUN ls -la /docker-entrypoint-initdb.d ENTRYPOINT ["/docker-entrypoint.sh"] CMD ["mysqld"] diff --git a/docker/mysql/entrypoint.sh b/docker/mysql/entrypoint.sh index 4bc21f00..76e9c238 100755 --- a/docker/mysql/entrypoint.sh +++ b/docker/mysql/entrypoint.sh @@ -76,8 +76,8 @@ if [ "$1" = 'mysqld' -a "`id -u`" = '0' ]; then FLUSH PRIVILEGES ; EOSQL - ls -la /docker-entrypoint-initdb.d mysql="mysql --protocol=socket -u${MYSQL_USER} -p${MYSQL_PASSWORD} -hlocalhost --socket=${SOCKET} $MYSQL_DATABASE_HISTORY" + echo "${mysql} < /docker-entrypoint-initdb.d/create_trade_history.sql"; /bin/sh -c "${mysql}" < "/docker-entrypoint-initdb.d/create_trade_history.sql"; sed -i.bak "s/MYSQL_USER=.*$/MYSQL_USER=\"${MYSQL_USER}\"/" /docker-entrypoint-initdb.d/init_trade_history.sh @@ -86,6 +86,7 @@ if [ "$1" = 'mysqld' -a "`id -u`" = '0' ]; then /docker-entrypoint-initdb.d/init_trade_history.sh mysql="mysql --protocol=socket -u${MYSQL_USER} -p${MYSQL_PASSWORD} -hlocalhost --socket=${SOCKET} $MYSQL_DATABASE_LOG" + echo "${mysql} < /docker-entrypoint-initdb.d/create_trade_log.sql"; /bin/sh -c "${mysql}" < "/docker-entrypoint-initdb.d/create_trade_log.sql"; sed -i.bak "s/MYSQL_USER=.*$/MYSQL_USER=\"${MYSQL_USER}\"/" /docker-entrypoint-initdb.d/init_trade_log.sh sed -i.bak "s/MYSQL_PASS=.*$/MYSQL_PASS=\"${MYSQL_PASSWORD}\"/" /docker-entrypoint-initdb.d/init_trade_log.sh From e9c7ca2c27bb037adcaf30cfba7303311c3d33bd Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Wed, 21 Feb 2018 19:06:59 +0500 Subject: [PATCH 20/38] docker: ensure sentinel cluster is up; ensure all services are up; add cli container --- docker/docker-compose.yml | 148 +++++++++++++++++++------- docker/exchange/Dockerfile | 8 +- docker/exchange/alertcenter.json | 6 +- docker/exchange/marketprice.json | 6 +- docker/exchange/run.sh | 177 ------------------------------- docker/kafka/Dockerfile | 2 +- docker/redis/Dockerfile | 8 +- docker/redis/redis.conf | 8 +- docker/sentinel/Dockerfile | 17 +-- docker/sentinel/sentinel.conf | 16 ++- 10 files changed, 146 insertions(+), 250 deletions(-) delete mode 100755 docker/exchange/run.sh diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index d22c6e2e..b66b02c9 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -20,42 +20,100 @@ services: networks: exchange_net: ipv4_address: 192.168.18.20 - redis1: + redis-master: build: context: redis - args: - PORT: 6381 - image: bitlum/exchange-redis-1 - container_name: exchange_redis_1 + environment: + PORT: 6381 + image: bitlum/exchange-redis + container_name: exchange_redis_master volumes: - - /tmp/bitlum/exchange/redis/1/lib:/var/lib/redis + - /tmp/bitlum/exchange/redis/master/lib:/var/lib/redis networks: exchange_net: - ipv4_address: 192.168.18.31 + ipv4_address: 192.168.18.36 + redis-replica-1: + build: + context: redis + environment: + PORT: 6382 + SLAVEOF: "redis-master 6381" + image: bitlum/exchange-redis + container_name: exchange_redis_replica_1 + depends_on: + - redis-master + volumes: + - /tmp/bitlum/exchange/redis/replica1/lib:/var/lib/redis + networks: + exchange_net: + ipv4_address: 192.168.18.37 + redis-replica-2: + build: + context: redis + environment: + PORT: 6383 + SLAVEOF: "redis-master 6381" + image: bitlum/exchange-redis + container_name: exchange_redis_replica_2 + depends_on: + - redis-master + volumes: + - /tmp/bitlum/exchange/redis/replica2/lib:/var/lib/redis + networks: + exchange_net: + ipv4_address: 192.168.18.38 sentinel1: build: context: sentinel - args: - PORT: 26381 - REDIS_HOST: redis1 - REDIS_PORT: 6381 - REDIS_QUORUM: 1 - image: bitlum/exchange-sentinel-1 + environment: + PORT: 26381 + REDIS_HOST: redis-master + REDIS_PORT: 6381 + image: bitlum/exchange-sentinel container_name: exchange_sentinel_1 depends_on: - - redis1 - volumes: - - /tmp/bitlum/exchange/sentinel/1/log:/var/log/sentinel + - redis-master + networks: + exchange_net: + ipv4_address: 192.168.18.31 + sentinel2: + build: + context: sentinel + environment: + PORT: 26382 + REDIS_HOST: redis-replica-2 + REDIS_PORT: 6382 + image: bitlum/exchange-sentinel + container_name: exchange_sentinel_2 + depends_on: + - redis-replica-1 + - sentinel1 networks: exchange_net: ipv4_address: 192.168.18.32 + sentinel3: + build: + context: sentinel + environment: + PORT: 26383 + REDIS_HOST: redis-replica-2 + REDIS_PORT: 6383 + image: bitlum/exchange-sentinel + container_name: exchange_sentinel_3 + depends_on: + - redis-replica-2 + - sentinel1 + - sentinel2 + networks: + exchange_net: + ipv4_address: 192.168.18.33 alertcenter: build: context: .. dockerfile: docker/exchange/Dockerfile command: alertcenter - image: bitlum/exchange-alertcenter - container_name: exchange_trade_alertcenter + image: bitlum/exchange + container_name: exchange_alertcenter restart: always depends_on: - sentinel1 @@ -69,13 +127,12 @@ services: context: .. dockerfile: docker/exchange/Dockerfile command: matchengine - image: bitlum/exchange-matchengine - container_name: exchange_trade_matchengine + image: bitlum/exchange + container_name: exchange_matchengine restart: always depends_on: - mysql - kafka - - sentinel1 - alertcenter volumes: - /tmp/bitlum/exchange/trade/matchengine/log:/var/log/trade @@ -86,10 +143,9 @@ services: build: context: .. dockerfile: docker/exchange/Dockerfile - args: - SERVICE: marketprice - image: bitlum/exchange-marketprice - container_name: exchange_trade_marketprice + image: bitlum/exchange + command: marketprice + container_name: exchange_marketprice restart: always depends_on: - kafka @@ -104,10 +160,9 @@ services: build: context: .. dockerfile: docker/exchange/Dockerfile - args: - SERVICE: readhistory - image: bitlum/exchange-readhistory - container_name: exchange_trade_readhistory + command: readhistory + image: bitlum/exchange + container_name: exchange_readhistory restart: always depends_on: - mysql @@ -121,10 +176,9 @@ services: build: context: .. dockerfile: docker/exchange/Dockerfile - args: - SERVICE: accesshttp - image: bitlum/exchange-accesshttp - container_name: exchange_trade_accesshttp + command: accesshttp + image: bitlum/exchange + container_name: exchange_accesshttp restart: always depends_on: - alertcenter @@ -143,10 +197,9 @@ services: build: context: .. dockerfile: docker/exchange/Dockerfile - args: - SERVICE: accessws - image: bitlum/exchange-accessws - container_name: exchange_trade_accessws + command: accessws + image: bitlum/exchange + container_name: exchange_accessws restart: always depends_on: - kafka @@ -158,10 +211,29 @@ services: - /tmp/bitlum/exchange/trade/accessws/log:/var/log/trade ports: - "8090:8090" - - "8090:8091" + - "8091:8091" networks: exchange_net: ipv4_address: 192.168.18.46 + cli: + build: cli + image: bitlum/exchange-cli + container_name: exchange_cli + restart: always + volumes: + - /tmp/bitlum/exchange/mysql/lib + - /tmp/bitlum/exchange/kafka/logs + - /tmp/bitlum/exchange/redis/master/lib + - /tmp/bitlum/exchange/redis/replica1/lib + - /tmp/bitlum/exchange/redis/replica2/lib + - /tmp/bitlum/exchange/trade/alertcenter + - /tmp/bitlum/exchange/trade/matchengine + - /tmp/bitlum/exchange/trade/marketprice + - /tmp/bitlum/exchange/trade/readhistory + - /tmp/bitlum/exchange/trade/accesshttp + networks: + exchange_net: + ipv4_address: 192.168.18.50 networks: exchange_net: driver: bridge diff --git a/docker/exchange/Dockerfile b/docker/exchange/Dockerfile index a11c29a4..8a07b9b2 100644 --- a/docker/exchange/Dockerfile +++ b/docker/exchange/Dockerfile @@ -62,6 +62,9 @@ FROM ubuntu:16.04 RUN apt-get update \ && apt-get upgrade -y \ && apt-get install -y \ + libldap-2.4 \ + libgss3 \ + librtmp1 \ jq \ netcat \ && rm -rf /var/lib/apt/lists/* @@ -85,7 +88,6 @@ COPY docker/exchange/accesshttp.json /viabtc_exchange_server COPY --from=BUILDSTEP /viabtc_exchange_server/accessws/accessws.exe /viabtc_exchange_server/accessws.exe COPY docker/exchange/accessws.json /viabtc_exchange_server -COPY docker/exchange/run.sh run.sh +COPY docker/exchange/entrypoint.sh /docker-entrypoint.sh -ENTRYPOINT [ "/run.sh" ] -# CMD /run.sh +ENTRYPOINT [ "/docker-entrypoint.sh" ] diff --git a/docker/exchange/alertcenter.json b/docker/exchange/alertcenter.json index 490e23bb..7a4bff8e 100644 --- a/docker/exchange/alertcenter.json +++ b/docker/exchange/alertcenter.json @@ -15,11 +15,11 @@ "max_pkg_size": 10240 }, "redis": { - "name": "exchange-sentinel", + "name": "exchange_sentinel", "addr": [ - "192.168.18.32:26381", + "192.168.18.31:26381", "192.168.18.32:26382", - "192.168.18.32:26383" + "192.168.18.33:26383" ] } } diff --git a/docker/exchange/marketprice.json b/docker/exchange/marketprice.json index 493e4a11..19b02592 100644 --- a/docker/exchange/marketprice.json +++ b/docker/exchange/marketprice.json @@ -28,9 +28,11 @@ "partition": 0 }, "redis": { - "name": "exchange-sentinel", + "name": "exchange_sentinel", "addr": [ - "192.168.18.32:26381" + "192.168.18.31:26381", + "192.168.18.32:26382", + "192.168.18.33:26383" ] }, "accesshttp": "http://accesshttp:8080" diff --git a/docker/exchange/run.sh b/docker/exchange/run.sh deleted file mode 100755 index bb8972f7..00000000 --- a/docker/exchange/run.sh +++ /dev/null @@ -1,177 +0,0 @@ -#!/bin/bash -help () { - echo "Available daemons are: " - echo " accesshttp" - echo " accessws" - echo " alertcenter" - echo " marketprice" - echo " matchengine" - echo " readhistory" -} - -echo "@=$@" - -if [ $# -lt 1 ];then - help - exit 0 -fi - -SERVICE=$1 - - -case $SERVICE in - alertcenter) - DAEMON=alertcenter - ;; - matchengine) - DAEMON=matchengine - ;; - marketprice) - DAEMON=marketprice - ;; - readhistory) - DAEMON=readhistory - ;; - accesshttp) - DAEMON=accesshttp - ;; - accessws) - DAEMON=accessws - ;; - --help) - help - ;; - -h) - help - ;; - *) - exec $@ - ;; -esac - -if [ $DAEMON ]; then - cd /viabtc_exchange_server - echo "=== $DAEMON is starting" - redis=`jq .redis.addr[0] < ${DAEMON}.json 2>/dev/null` - test -z $redis || test $redis = 'null' && redis='' - if [ -z $redis ];then - redis=`jq .redis.addr < ${DAEMON}.json 2>/dev/null` - fi - test -z $redis || test $redis = 'null' && redis='' - if [ $redis ];then - echo "=== $DAEMON contains redis config: $redis" - redis=`echo $redis|sed -e 's/"//g'` - echo "=== $DAEMON: sentinel addr found: $redis" - host=`echo $redis | cut -f1 -d':'` - port=`echo $redis | cut -f2 -d':'` - nc -z $host $port - result=$? - while [ $result -ne 0 ]; do - echo "=== $DAEMON: is waiting for sentinel on host=$host, port=$port ..." - sleep 10 - nc -z $host $port - result=$? - done - echo "=== $DAEMON: sentinel is up" - fi - kafka=`jq .brokers[0] < ${DAEMON}.json 2>/dev/null` - test -z $kafka || test $kafka = 'null' && kafka='' - if [ -z $kafka ];then - kafka=`jq .brokers < ${DAEMON}.json 2>/dev/null` - fi - test -z $kafka || test $kafka = 'null' && kafka='' - if [ -z $kafka ];then - kafka=`jq .deals.brokers[0] < ${DAEMON}.json 2>/dev/null` - fi - test -z $kafka || test $kafka = 'null' && kafka='' - if [ -z $kafka ];then - kafka=`jq .deals.brokers < ${DAEMON}.json 2>/dev/null` - fi - test -z $kafka || test $kafka = 'null' && kafka='' - if [ $kafka ];then - echo "=== $DAEMON contains kafka config: $kafka" - kafka=`echo $kafka|sed -e 's/"//g'` - echo "=== $DAEMON: kafka addr found: $kafka" - host=`echo $kafka | cut -f1 -d':'` - port=`echo $kafka | cut -f2 -d':'` - nc -z $host $port - result=$? - while [ $result -ne 0 ]; do - echo "=== $DAEMON: is waiting for kafka on host=$host, port=$port ..." - sleep 10 - nc -z $host $port - result=$? - done - echo "=== $DAEMON: kafka is up" - fi - host=`jq .db_history.host < ${DAEMON}.json 2>/dev/null` - test -z $host || test "$host" = 'null' && host='' - if [ "$host" ];then - port=`jq .db_history.port < ${DAEMON}.json 2>/dev/null` - test -z $port || test $port = 'null' && port='' - test $port || port=3306 - echo "=== $DAEMON contains db_history mysql host: $host and port: $port" - host=`echo $host|sed -e 's/"//g'` - port=`echo $port|sed -e 's/"//g'` - nc -z $host $port - result=$? - while [ $result -ne 0 ]; do - echo "=== $DAEMON: is waiting for db_history on host=$host, port=$port ..." - sleep 10 - nc -z $host $port - result=$? - done - echo "=== $DAEMON: db_history is up" - fi - host=`jq .db_log.host < ${DAEMON}.json 2>/dev/null` - test -z $host || test "$host" = 'null' && host='' - if [ "$host" ];then - port=`jq .db_log.port < ${DAEMON}.json 2>/dev/null` - test -z $port || test $port = 'null' && port='' - test $port || port=3306 - echo "=== $DAEMON contains db_log mysql host: $host and port: $port" - host=`echo $host|sed -e 's/"//g'` - port=`echo $port|sed -e 's/"//g'` - nc -z $host $port - result=$? - while [ $result -ne 0 ]; do - echo "=== $DAEMON: is waiting for db_log on host=$host, port=$port ..." - sleep 10 - nc -z $host $port - result=$? - done - echo "=== $DAEMON: db_log is up" - fi - echo "=== $DAEMON: ready to exec" - ./$DAEMON.exe ${DAEMON}.json - NEW=1 - while true; do - sleep 5 - ps aux|grep ${DAEMON}.exe|grep -v grep > /dev/null - RESULT=$? - if [ $RESULT -ne 0 ];then - # some of services forks with _worker suffix - ps aux|grep ${DAEMON}_worker |grep -v grep > /dev/null - RESULT=$? - if [ $RESULT -ne 0 ];then - echo "=== $DAEMON has died, exitting docker..."; - break; - else - if [ $NEW -eq 1 ]; then - echo "Microservice $DAEMON(worker) has started"; - NEW=0 - fi - fi - else - if [ $NEW -eq 1 ]; then - echo "=== $DAEMON(exe) has started"; - NEW=0 - fi - fi - done - exit 100 -elif [ $BASH ]; then - while true;do sleep 3600;done -else - exit 101 -fi diff --git a/docker/kafka/Dockerfile b/docker/kafka/Dockerfile index a42e7747..eb752907 100644 --- a/docker/kafka/Dockerfile +++ b/docker/kafka/Dockerfile @@ -42,4 +42,4 @@ EXPOSE 9092 CMD cd /opt/kafka/kafka_2.11-1.0.0 && \ (nohup bin/zookeeper-server-start.sh config/zookeeper.properties &) && \ sleep 3 && \ - bin/kafka-server-start.sh config/server.properties + bin/kafka-server-start.sh /etc/kafka/server.properties diff --git a/docker/redis/Dockerfile b/docker/redis/Dockerfile index 4817aaf7..6a767ef5 100644 --- a/docker/redis/Dockerfile +++ b/docker/redis/Dockerfile @@ -3,11 +3,11 @@ FROM alpine:3.7 RUN addgroup -S redis && adduser -S -G redis redis RUN apk add --no-cache redis -ARG PORT=6379 COPY redis.conf /etc/redis/redis.conf -RUN sed -i.bak "s/^port .*$/port ${PORT}/" /etc/redis/redis.conf && rm /etc/redis/redis.conf.bak -RUN chown redis:redis /etc/redis/redis.conf +COPY entrypoint.sh /docker-entrypoint.sh -EXPOSE ${PORT} +ENV PORT=6379 +ENV SLAVEOF= +ENTRYPOINT ["/docker-entrypoint.sh"] CMD ["redis-server", "/etc/redis/redis.conf"] diff --git a/docker/redis/redis.conf b/docker/redis/redis.conf index 1eb4fea6..975df310 100644 --- a/docker/redis/redis.conf +++ b/docker/redis/redis.conf @@ -1,13 +1,11 @@ daemonize no -pidfile /var/run/redis/redis-sentinel.pid +protected-mode no port 26379 -timeout 0 -tcp-keepalive 60 loglevel notice logfile "" +timeout 0 +tcp-keepalive 60 syslog-enabled no -# syslog-ident redis -# syslog-facility local0 databases 2 save 900 1 save 300 10 diff --git a/docker/sentinel/Dockerfile b/docker/sentinel/Dockerfile index 074028ac..ec176921 100644 --- a/docker/sentinel/Dockerfile +++ b/docker/sentinel/Dockerfile @@ -3,20 +3,13 @@ FROM alpine:3.7 RUN addgroup -S redis && adduser -S -G redis redis RUN apk add --no-cache redis -ARG PORT=26379 -ARG REDIS_PORT=6379 -ARG REDIS_HOST=redis -ARG REDIS_QUORUM=1 COPY sentinel.conf /etc/redis/sentinel.conf +COPY entrypoint.sh /docker-entrypoint.sh -RUN sed -i.bak "s/^port .*$/port ${PORT}/" /etc/redis/sentinel.conf && \ - cat /etc/redis/sentinel.conf && \ - sed -i.bak "s/^\(sentinel monitor [^ ]*\).*$/\1 ${REDIS_HOST} ${REDIS_PORT} ${REDIS_QUORUM}/" /etc/redis/sentinel.conf && \ - rm /etc/redis/sentinel.conf.bak && \ - cat /etc/redis/sentinel.conf -RUN chown redis:redis /etc/redis/sentinel.conf - -EXPOSE ${PORT} +ENV PORT=26379 +ENV REDIS_PORT=6379 +ENV REDIS_HOST=redis +ENTRYPOINT ["/docker-entrypoint.sh"] CMD ["redis-server", "/etc/redis/sentinel.conf", "--sentinel"] diff --git a/docker/sentinel/sentinel.conf b/docker/sentinel/sentinel.conf index e7c86dec..aae5d1fe 100644 --- a/docker/sentinel/sentinel.conf +++ b/docker/sentinel/sentinel.conf @@ -1,6 +1,12 @@ -port 26379 -sentinel monitor exchange-sentinel redis 6379 2 -sentinel down-after-milliseconds exchange-sentinel 30000 -sentinel parallel-syncs exchange-sentinel 1 -sentinel failover-timeout exchange-sentinel 180000 daemonize no +protected-mode no +port 26379 +loglevel notice +logfile "" +timeout 0 +tcp-keepalive 60 +syslog-enabled no +sentinel monitor exchange_sentinel redis 6379 2 +sentinel down-after-milliseconds exchange_sentinel 20000 +sentinel failover-timeout exchange_sentinel 60000 +sentinel parallel-syncs exchange_sentinel 1 From cf1589d2dbeb603163891e13cb381b6bc6b7d8d4 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Fri, 23 Feb 2018 12:59:28 +0500 Subject: [PATCH 21/38] docker: add cli dockerfile --- docker/cli/Dockerfile | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 docker/cli/Dockerfile diff --git a/docker/cli/Dockerfile b/docker/cli/Dockerfile new file mode 100644 index 00000000..67957fe4 --- /dev/null +++ b/docker/cli/Dockerfile @@ -0,0 +1,16 @@ +FROM ubuntu:16.04 +MAINTAINER Sergey Dolin + +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y \ + mysql-client \ + inetutils-ping \ + net-tools \ + lsof \ + telnet && \ + rm -rf /var/lib/apt/lists/* + +RUN echo 'while true;do sleep 3600;done' > /wait +RUN chmod +x /wait +CMD /wait From 703d5f123e477fd37e213030774a1ab4c0919d0f Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Fri, 23 Feb 2018 23:29:59 +0500 Subject: [PATCH 22/38] docker: mount /var/zookeeper to host filesystem --- docker/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index b66b02c9..09afd379 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -17,6 +17,7 @@ services: image: bitlum/exchange-kafka volumes: - /tmp/bitlum/exchange/kafka/logs:/var/log/kafka + - /tmp/bitlum/exchange/zookeeper/logs:/var/log/zookeeper networks: exchange_net: ipv4_address: 192.168.18.20 From 7563869f99aa3fad2f3fa753c2b54074a2d8f77b Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Sat, 24 Feb 2018 00:04:26 +0500 Subject: [PATCH 23/38] docker: add logging option to limit console log size to 10mb --- docker/docker-compose.yml | 53 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 09afd379..fca5eedc 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -5,6 +5,9 @@ services: image: bitlum/exchange-mysql container_name: exchange_mysql restart: always + logging: + options: + max-size: 10m volumes: - /tmp/bitlum/exchange/mysql/lib:/var/lib/mysql networks: @@ -12,9 +15,12 @@ services: ipv4_address: 192.168.18.10 kafka: build: kafka + image: bitlum/exchange-kafka container_name: exchange_kafka restart: always - image: bitlum/exchange-kafka + logging: + options: + max-size: 10m volumes: - /tmp/bitlum/exchange/kafka/logs:/var/log/kafka - /tmp/bitlum/exchange/zookeeper/logs:/var/log/zookeeper @@ -28,6 +34,10 @@ services: PORT: 6381 image: bitlum/exchange-redis container_name: exchange_redis_master + restart: always + logging: + options: + max-size: 10m volumes: - /tmp/bitlum/exchange/redis/master/lib:/var/lib/redis networks: @@ -41,6 +51,10 @@ services: SLAVEOF: "redis-master 6381" image: bitlum/exchange-redis container_name: exchange_redis_replica_1 + restart: always + logging: + options: + max-size: 10m depends_on: - redis-master volumes: @@ -56,6 +70,10 @@ services: SLAVEOF: "redis-master 6381" image: bitlum/exchange-redis container_name: exchange_redis_replica_2 + restart: always + logging: + options: + max-size: 10m depends_on: - redis-master volumes: @@ -72,6 +90,10 @@ services: REDIS_PORT: 6381 image: bitlum/exchange-sentinel container_name: exchange_sentinel_1 + restart: always + logging: + options: + max-size: 10m depends_on: - redis-master networks: @@ -86,6 +108,10 @@ services: REDIS_PORT: 6382 image: bitlum/exchange-sentinel container_name: exchange_sentinel_2 + restart: always + logging: + options: + max-size: 10m depends_on: - redis-replica-1 - sentinel1 @@ -101,6 +127,10 @@ services: REDIS_PORT: 6383 image: bitlum/exchange-sentinel container_name: exchange_sentinel_3 + restart: always + logging: + options: + max-size: 10m depends_on: - redis-replica-2 - sentinel1 @@ -116,6 +146,9 @@ services: image: bitlum/exchange container_name: exchange_alertcenter restart: always + logging: + options: + max-size: 10m depends_on: - sentinel1 volumes: @@ -131,6 +164,9 @@ services: image: bitlum/exchange container_name: exchange_matchengine restart: always + logging: + options: + max-size: 10m depends_on: - mysql - kafka @@ -148,6 +184,9 @@ services: command: marketprice container_name: exchange_marketprice restart: always + logging: + options: + max-size: 10m depends_on: - kafka - sentinel1 @@ -165,6 +204,9 @@ services: image: bitlum/exchange container_name: exchange_readhistory restart: always + logging: + options: + max-size: 10m depends_on: - mysql - alertcenter @@ -181,6 +223,9 @@ services: image: bitlum/exchange container_name: exchange_accesshttp restart: always + logging: + options: + max-size: 10m depends_on: - alertcenter - matchengine @@ -202,6 +247,9 @@ services: image: bitlum/exchange container_name: exchange_accessws restart: always + logging: + options: + max-size: 10m depends_on: - kafka - alertcenter @@ -221,6 +269,9 @@ services: image: bitlum/exchange-cli container_name: exchange_cli restart: always + logging: + options: + max-size: 10m volumes: - /tmp/bitlum/exchange/mysql/lib - /tmp/bitlum/exchange/kafka/logs From a34a16d52a7b8783616f0eec0494582f2e025d7b Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Wed, 21 Feb 2018 19:19:17 +0500 Subject: [PATCH 24/38] utils: general: add `/bin` directory with set of shell scripts mirroring httpAccess rpc --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 1410591e..1eeee165 100644 --- a/.gitignore +++ b/.gitignore @@ -267,7 +267,6 @@ $RECYCLE.BIN/ x64/ x86/ bld/ -[Bb]in/ [Oo]bj/ [Ll]og/ From 10094cd81bb298341463dc18cd8337cb0a3830c3 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Wed, 21 Feb 2018 19:19:28 +0500 Subject: [PATCH 25/38] utils: general: add `/bin` directory with set of shell scripts mirroring httpAccess rpc --- bin/accesshttp | 90 ++++++++++++++++++++++++++++++++++++++++++++ bin/balance.query | 5 +++ bin/balance.update | 10 +++++ bin/config.sh | 42 +++++++++++++++++++++ bin/market.status | 6 +++ bin/order.put_market | 6 +++ 6 files changed, 159 insertions(+) create mode 100755 bin/accesshttp create mode 100755 bin/balance.query create mode 100755 bin/balance.update create mode 100755 bin/config.sh create mode 100755 bin/market.status create mode 100755 bin/order.put_market diff --git a/bin/accesshttp b/bin/accesshttp new file mode 100755 index 00000000..601071ac --- /dev/null +++ b/bin/accesshttp @@ -0,0 +1,90 @@ +#!/bin/bash + +echo @=$@ + +if [ x$1 = x--help -o x$1 = x-h ];then + echo "With params calls http access service" + echo " $0 METHOD [comma separated params]" + echo "Without params adds the highlevel functions to the environment" + echo " $0" + echo "Exported functions are:" + echo " rpc METHOD [comma separated params]" + echo " balance_update user_id asset deposit | withdraw op_id amount" + echo " order_put_market user_id 1(sell)|2(buy) amount taker_fee_rate [source]" + exit 0 +fi + +shdir=`dirname ${0}` + +if [ -z $HTTP_URL ];then + . ${shdir}/config.sh + HTTP_URL="http://$( accesshttp_bind )" +fi + +function rpc () { + method=$1 + shift + if [ ! -z $DEBUG ];then + echo "{\"method\":\"${method}\",\"params\":[$1],\"id\":${RANDOM}}" + fi + curl -i -H 'Accept: application/json' \ + -X POST \ + -d "{\"method\":\"${method}\",\"params\":[$1],\"id\":${RANDOM}}" \ + $HTTP_URL + RESULT=$? + echo + return $RESULT +} + +function balance.query () { + if [ $# -lt 1 ]; then + echo "bad arguments: $@" + echo 'usage: user_id' + return 1 + fi + rpc balance.query $1 +} + +# user_id +# asset +# deposit | withdraw +# op_id +# amount +function balance.update () { + rpc balance.update "$1,\"$2\",\"$3\",$4,\"$5\",{}" +} + +function order.put_market () { + if [ $# -lt 5 ]; then + echo "bad arguments: $@" + echo 'usage: user_id market 1(sell)|2(buy) amount taker_fee_rate [source]' + return 1 + fi + source=$6 + test $source || source="shell command" + rpc order.put_market "$1,\"$2\",$3,\"$4\",\"$5\",\"$source\"" +} + +function order.put_market () { + if [ $# -lt 5 ]; then + echo "bad arguments: $@" + echo 'usage: user_id market 1(sell)|2(buy) amount taker_fee_rate [source]' + return 1 + fi + source=$6 + test $source || source="shell command" + rpc order.put_market "$1,\"$2\",$3,\"$4\",\"$5\",\"$source\"" +} + +function market.status () { + if [ $# -lt 2 ]; then + echo "bad arguments: $@" + echo 'usage: market period' + return 1 + fi + rpc "market.status" "\"$1\",$2" +} + +# if [ $# -gt 0 ];then +# rpc $@ +# fi diff --git a/bin/balance.query b/bin/balance.query new file mode 100755 index 00000000..c444b392 --- /dev/null +++ b/bin/balance.query @@ -0,0 +1,5 @@ +#!/bin/bash +shdir=`dirname ${0}` +. ${shdir}/accesshttp + +balance.query $@ diff --git a/bin/balance.update b/bin/balance.update new file mode 100755 index 00000000..9f8fee74 --- /dev/null +++ b/bin/balance.update @@ -0,0 +1,10 @@ +#!/bin/bash +if [ $# -eq 0 -o x$1 = x--help -o x$1 = x-h ];then + echo "$0 user_id asset deposit | withdraw op_id amount" + exit 0 +fi + +shdir=`dirname ${0}` +. ${shdir}/accesshttp + +balance.update $@ diff --git a/bin/config.sh b/bin/config.sh new file mode 100755 index 00000000..58d65703 --- /dev/null +++ b/bin/config.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Exports functions parsing the services configs + +# Internal utils +servicesdir="`dirname ${0}`/.." + +function get () { + filter=$1 + service=$2 + config="${servicesdir}/$service/config.json" + ret=`jq "$filter[0]" < $config` + test -z $ret || test $ret = null && $ret='' + if [ -z $ret ]; then + ret=`jq $filter[0] < $config` + test -z ret && return 1 + fi + echo $ret +} + +function trim_quotes () { + echo $1|sed -e's/^"//'|sed -e's/"$//' +} + +function trim_port() { + echo $1|sed -e's/^[^@]*@//' +} + +function trim_quotes_port() { + echo $1|sed -e's/^"//'|sed -e's/"$//'|sed -e's/^[^@]*@//' +} + +# Exported functions +function accesshttp_bind () { + if [ -z $REMOTE ]; then + # get .svr.bind accesshttp + ip=$( get .svr.bind accesshttp ) + # echo "ip=$ip" + trim_quotes_port $ip + else + echo $REMOTE + fi +} diff --git a/bin/market.status b/bin/market.status new file mode 100755 index 00000000..3cd3d7d7 --- /dev/null +++ b/bin/market.status @@ -0,0 +1,6 @@ +#!/bin/bash + +shdir=`dirname ${0}` +. ${shdir}/accesshttp + +market.status $@ diff --git a/bin/order.put_market b/bin/order.put_market new file mode 100755 index 00000000..c31d0908 --- /dev/null +++ b/bin/order.put_market @@ -0,0 +1,6 @@ +#!/bin/bash + +shdir=`dirname ${0}` +. ${shdir}/accesshttp + +order.put_market $@ From 54c43a40c18ba93f6d66a9653d92d221ca15a81d Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Mon, 26 Feb 2018 14:41:07 +0500 Subject: [PATCH 26/38] docker: configuration: kafka: change log rotation params --- docker/kafka/server.properties | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/kafka/server.properties b/docker/kafka/server.properties index df8e8200..eee49144 100644 --- a/docker/kafka/server.properties +++ b/docker/kafka/server.properties @@ -105,14 +105,15 @@ transaction.state.log.min.isr=1 # from the end of the log. # The minimum age of a log file to be eligible for deletion due to age -log.retention.hours=168 +log.retention.hours=4 # A size-based retention policy for logs. Segments are pruned from the log as long as the remaining # segments don't drop below log.retention.bytes. Functions independently of log.retention.hours. #log.retention.bytes=1073741824 # The maximum size of a log segment file. When this size is reached a new log segment will be created. -log.segment.bytes=1073741824 +# 64M +log.segment.bytes=67108864 # The interval at which log segments are checked to see if they can be deleted according # to the retention policies From 66d3506a90928ea0db544d6afeba2188b336c725 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Mon, 26 Feb 2018 15:01:47 +0500 Subject: [PATCH 27/38] dockerize: rm COPY entrypoint.sh from redis --- docker/redis/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/redis/Dockerfile b/docker/redis/Dockerfile index 6a767ef5..b98a4f77 100644 --- a/docker/redis/Dockerfile +++ b/docker/redis/Dockerfile @@ -9,5 +9,4 @@ COPY entrypoint.sh /docker-entrypoint.sh ENV PORT=6379 ENV SLAVEOF= -ENTRYPOINT ["/docker-entrypoint.sh"] CMD ["redis-server", "/etc/redis/redis.conf"] From a0aa6250482f6e2f68cd165a323abe0f3d37b7ba Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Mon, 26 Feb 2018 15:03:24 +0500 Subject: [PATCH 28/38] dockerize: rm COPY entrypoint.sh from redis --- docker/redis/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/redis/Dockerfile b/docker/redis/Dockerfile index b98a4f77..62b46771 100644 --- a/docker/redis/Dockerfile +++ b/docker/redis/Dockerfile @@ -4,7 +4,6 @@ RUN addgroup -S redis && adduser -S -G redis redis RUN apk add --no-cache redis COPY redis.conf /etc/redis/redis.conf -COPY entrypoint.sh /docker-entrypoint.sh ENV PORT=6379 ENV SLAVEOF= From 6bde2130c026cd6f3d4af8e409c889c59ec2fdc4 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Mon, 26 Feb 2018 15:06:39 +0500 Subject: [PATCH 29/38] dockerize: rm COPY entrypoint.sh from sentinel --- docker/sentinel/Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker/sentinel/Dockerfile b/docker/sentinel/Dockerfile index ec176921..fe32b2ad 100644 --- a/docker/sentinel/Dockerfile +++ b/docker/sentinel/Dockerfile @@ -5,11 +5,9 @@ RUN apk add --no-cache redis COPY sentinel.conf /etc/redis/sentinel.conf -COPY entrypoint.sh /docker-entrypoint.sh ENV PORT=26379 ENV REDIS_PORT=6379 ENV REDIS_HOST=redis -ENTRYPOINT ["/docker-entrypoint.sh"] CMD ["redis-server", "/etc/redis/sentinel.conf", "--sentinel"] From 80d1f7c3ff8105d59ffb59645f2e8ab7463186d4 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Mon, 26 Feb 2018 15:45:33 +0500 Subject: [PATCH 30/38] dockerize: add docker/exchange/entrypoint.sh - it was lost somehow --- docker/exchange/entrypoint.sh | 177 ++++++++++++++++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 docker/exchange/entrypoint.sh diff --git a/docker/exchange/entrypoint.sh b/docker/exchange/entrypoint.sh new file mode 100644 index 00000000..bb8972f7 --- /dev/null +++ b/docker/exchange/entrypoint.sh @@ -0,0 +1,177 @@ +#!/bin/bash +help () { + echo "Available daemons are: " + echo " accesshttp" + echo " accessws" + echo " alertcenter" + echo " marketprice" + echo " matchengine" + echo " readhistory" +} + +echo "@=$@" + +if [ $# -lt 1 ];then + help + exit 0 +fi + +SERVICE=$1 + + +case $SERVICE in + alertcenter) + DAEMON=alertcenter + ;; + matchengine) + DAEMON=matchengine + ;; + marketprice) + DAEMON=marketprice + ;; + readhistory) + DAEMON=readhistory + ;; + accesshttp) + DAEMON=accesshttp + ;; + accessws) + DAEMON=accessws + ;; + --help) + help + ;; + -h) + help + ;; + *) + exec $@ + ;; +esac + +if [ $DAEMON ]; then + cd /viabtc_exchange_server + echo "=== $DAEMON is starting" + redis=`jq .redis.addr[0] < ${DAEMON}.json 2>/dev/null` + test -z $redis || test $redis = 'null' && redis='' + if [ -z $redis ];then + redis=`jq .redis.addr < ${DAEMON}.json 2>/dev/null` + fi + test -z $redis || test $redis = 'null' && redis='' + if [ $redis ];then + echo "=== $DAEMON contains redis config: $redis" + redis=`echo $redis|sed -e 's/"//g'` + echo "=== $DAEMON: sentinel addr found: $redis" + host=`echo $redis | cut -f1 -d':'` + port=`echo $redis | cut -f2 -d':'` + nc -z $host $port + result=$? + while [ $result -ne 0 ]; do + echo "=== $DAEMON: is waiting for sentinel on host=$host, port=$port ..." + sleep 10 + nc -z $host $port + result=$? + done + echo "=== $DAEMON: sentinel is up" + fi + kafka=`jq .brokers[0] < ${DAEMON}.json 2>/dev/null` + test -z $kafka || test $kafka = 'null' && kafka='' + if [ -z $kafka ];then + kafka=`jq .brokers < ${DAEMON}.json 2>/dev/null` + fi + test -z $kafka || test $kafka = 'null' && kafka='' + if [ -z $kafka ];then + kafka=`jq .deals.brokers[0] < ${DAEMON}.json 2>/dev/null` + fi + test -z $kafka || test $kafka = 'null' && kafka='' + if [ -z $kafka ];then + kafka=`jq .deals.brokers < ${DAEMON}.json 2>/dev/null` + fi + test -z $kafka || test $kafka = 'null' && kafka='' + if [ $kafka ];then + echo "=== $DAEMON contains kafka config: $kafka" + kafka=`echo $kafka|sed -e 's/"//g'` + echo "=== $DAEMON: kafka addr found: $kafka" + host=`echo $kafka | cut -f1 -d':'` + port=`echo $kafka | cut -f2 -d':'` + nc -z $host $port + result=$? + while [ $result -ne 0 ]; do + echo "=== $DAEMON: is waiting for kafka on host=$host, port=$port ..." + sleep 10 + nc -z $host $port + result=$? + done + echo "=== $DAEMON: kafka is up" + fi + host=`jq .db_history.host < ${DAEMON}.json 2>/dev/null` + test -z $host || test "$host" = 'null' && host='' + if [ "$host" ];then + port=`jq .db_history.port < ${DAEMON}.json 2>/dev/null` + test -z $port || test $port = 'null' && port='' + test $port || port=3306 + echo "=== $DAEMON contains db_history mysql host: $host and port: $port" + host=`echo $host|sed -e 's/"//g'` + port=`echo $port|sed -e 's/"//g'` + nc -z $host $port + result=$? + while [ $result -ne 0 ]; do + echo "=== $DAEMON: is waiting for db_history on host=$host, port=$port ..." + sleep 10 + nc -z $host $port + result=$? + done + echo "=== $DAEMON: db_history is up" + fi + host=`jq .db_log.host < ${DAEMON}.json 2>/dev/null` + test -z $host || test "$host" = 'null' && host='' + if [ "$host" ];then + port=`jq .db_log.port < ${DAEMON}.json 2>/dev/null` + test -z $port || test $port = 'null' && port='' + test $port || port=3306 + echo "=== $DAEMON contains db_log mysql host: $host and port: $port" + host=`echo $host|sed -e 's/"//g'` + port=`echo $port|sed -e 's/"//g'` + nc -z $host $port + result=$? + while [ $result -ne 0 ]; do + echo "=== $DAEMON: is waiting for db_log on host=$host, port=$port ..." + sleep 10 + nc -z $host $port + result=$? + done + echo "=== $DAEMON: db_log is up" + fi + echo "=== $DAEMON: ready to exec" + ./$DAEMON.exe ${DAEMON}.json + NEW=1 + while true; do + sleep 5 + ps aux|grep ${DAEMON}.exe|grep -v grep > /dev/null + RESULT=$? + if [ $RESULT -ne 0 ];then + # some of services forks with _worker suffix + ps aux|grep ${DAEMON}_worker |grep -v grep > /dev/null + RESULT=$? + if [ $RESULT -ne 0 ];then + echo "=== $DAEMON has died, exitting docker..."; + break; + else + if [ $NEW -eq 1 ]; then + echo "Microservice $DAEMON(worker) has started"; + NEW=0 + fi + fi + else + if [ $NEW -eq 1 ]; then + echo "=== $DAEMON(exe) has started"; + NEW=0 + fi + fi + done + exit 100 +elif [ $BASH ]; then + while true;do sleep 3600;done +else + exit 101 +fi From 639cb19e7a17cc113af106ba07cdfc59b0d522d7 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Mon, 26 Feb 2018 15:48:50 +0500 Subject: [PATCH 31/38] dockerize: chmod +x docker/exchange/entrypoint.sh --- docker/exchange/entrypoint.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 docker/exchange/entrypoint.sh diff --git a/docker/exchange/entrypoint.sh b/docker/exchange/entrypoint.sh old mode 100644 new mode 100755 From fc547a8c1336abfb8bcdf829a3ddc7ce43de8de6 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Tue, 27 Feb 2018 10:41:52 +0500 Subject: [PATCH 32/38] docker: connect sentinel2 to redis-replica-2 & add dependicies of exchange service on all 3 sentinels to make sure the whole cluster is up --- docker/docker-compose.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index fca5eedc..ccdc271e 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -88,6 +88,7 @@ services: PORT: 26381 REDIS_HOST: redis-master REDIS_PORT: 6381 + QUORUM: 2 image: bitlum/exchange-sentinel container_name: exchange_sentinel_1 restart: always @@ -104,8 +105,9 @@ services: context: sentinel environment: PORT: 26382 - REDIS_HOST: redis-replica-2 + REDIS_HOST: redis-replica-1 REDIS_PORT: 6382 + QUORUM: 2 image: bitlum/exchange-sentinel container_name: exchange_sentinel_2 restart: always @@ -125,6 +127,7 @@ services: PORT: 26383 REDIS_HOST: redis-replica-2 REDIS_PORT: 6383 + QUORUM: 2 image: bitlum/exchange-sentinel container_name: exchange_sentinel_3 restart: always @@ -151,6 +154,8 @@ services: max-size: 10m depends_on: - sentinel1 + - sentinel2 + - sentinel3 volumes: - /tmp/bitlum/exchange/trade/alertcenter/log:/var/log/trade networks: @@ -190,6 +195,8 @@ services: depends_on: - kafka - sentinel1 + - sentinel2 + - sentinel3 - alertcenter volumes: - /tmp/bitlum/exchange/trade/marketprice/log:/var/log/trade From 6accd514540c4a4f32f8119eec2a2cb2e9f3951b Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Tue, 27 Feb 2018 10:49:18 +0500 Subject: [PATCH 33/38] docker: add entrypoint to redis --- docker/redis/Dockerfile | 2 ++ docker/redis/entrypoint.sh | 10 ++++++++++ 2 files changed, 12 insertions(+) create mode 100755 docker/redis/entrypoint.sh diff --git a/docker/redis/Dockerfile b/docker/redis/Dockerfile index 62b46771..6a767ef5 100644 --- a/docker/redis/Dockerfile +++ b/docker/redis/Dockerfile @@ -4,8 +4,10 @@ RUN addgroup -S redis && adduser -S -G redis redis RUN apk add --no-cache redis COPY redis.conf /etc/redis/redis.conf +COPY entrypoint.sh /docker-entrypoint.sh ENV PORT=6379 ENV SLAVEOF= +ENTRYPOINT ["/docker-entrypoint.sh"] CMD ["redis-server", "/etc/redis/redis.conf"] diff --git a/docker/redis/entrypoint.sh b/docker/redis/entrypoint.sh new file mode 100755 index 00000000..59dfefab --- /dev/null +++ b/docker/redis/entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/sh +sed -i.bak "s/^port .*$/port ${PORT}/" /etc/redis/redis.conf && rm /etc/redis/redis.conf.bak +if [ "x$SLAVEOF" != x ];then + echo "slaveof $SLAVEOF" >> /etc/redis/redis.conf +fi + +chown redis:redis /etc/redis/redis.conf +cat /etc/redis/redis.conf +echo $@ +exec $@ From d4cee4d1c777a70bf52b23b837585d14a60daae8 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Tue, 27 Feb 2018 10:51:28 +0500 Subject: [PATCH 34/38] docker: add entrypoint to sentinel --- docker/sentinel/Dockerfile | 4 +++- docker/sentinel/entrypoint.sh | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100755 docker/sentinel/entrypoint.sh diff --git a/docker/sentinel/Dockerfile b/docker/sentinel/Dockerfile index fe32b2ad..e44798b5 100644 --- a/docker/sentinel/Dockerfile +++ b/docker/sentinel/Dockerfile @@ -3,11 +3,13 @@ FROM alpine:3.7 RUN addgroup -S redis && adduser -S -G redis redis RUN apk add --no-cache redis - COPY sentinel.conf /etc/redis/sentinel.conf +COPY entrypoint.sh /docker-entrypoint.sh ENV PORT=26379 ENV REDIS_PORT=6379 ENV REDIS_HOST=redis +ENV QUORUM=2 +ENTRYPOINT ["/docker-entrypoint.sh"] CMD ["redis-server", "/etc/redis/sentinel.conf", "--sentinel"] diff --git a/docker/sentinel/entrypoint.sh b/docker/sentinel/entrypoint.sh new file mode 100755 index 00000000..7fb8485f --- /dev/null +++ b/docker/sentinel/entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +sed -i.bak "s/^port .*$/port ${PORT}/" /etc/redis/sentinel.conf +sed -i.bak "s/^\(sentinel monitor [^ ]*\).*$/\1 ${REDIS_HOST} ${REDIS_PORT} ${QUORUM}/" /etc/redis/sentinel.conf +rm /etc/redis/sentinel.conf.bak +chown redis:redis /etc/redis/sentinel.conf +cat /etc/redis/sentinel.conf +echo $@ +exec $@ From f50c90186864a42bd383e4613206f3097b4e239a Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Tue, 27 Feb 2018 11:01:39 +0500 Subject: [PATCH 35/38] docker: add libmysqlclient20 to exchange run time dependencies and fix binary names --- docker/exchange/Dockerfile | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docker/exchange/Dockerfile b/docker/exchange/Dockerfile index 8a07b9b2..bcb1f38d 100644 --- a/docker/exchange/Dockerfile +++ b/docker/exchange/Dockerfile @@ -51,6 +51,7 @@ RUN cd viabtc_exchange_server/network && make RUN cd viabtc_exchange_server/utils && make RUN cd viabtc_exchange_server/alertcenter && make RUN cd viabtc_exchange_server/matchengine && make +COPY docker/exchange/makefile.marketprice.ubuntu viabtc_exchange_server/marketprice/makefile RUN cd viabtc_exchange_server/marketprice && make RUN cd viabtc_exchange_server/readhistory && make RUN cd viabtc_exchange_server/accesshttp && make @@ -65,27 +66,29 @@ RUN apt-get update \ libldap-2.4 \ libgss3 \ librtmp1 \ + libmysqlclient20 \ jq \ netcat \ && rm -rf /var/lib/apt/lists/* +RUN mkdir /viabtc_exchange_server # keep .exe for better grep of ps output -COPY --from=BUILDSTEP /viabtc_exchange_server/alertcenter/alertcenter.exe /viabtc_exchange_server/alertcenter.exe +COPY --from=BUILDSTEP /viabtc_exchange_server/alertcenter/alertcenter /viabtc_exchange_server/alertcenter.exe COPY docker/exchange/alertcenter.json /viabtc_exchange_server -COPY --from=BUILDSTEP /viabtc_exchange_server/matchengine/matchengine.exe /viabtc_exchange_server/matchengine.exe +COPY --from=BUILDSTEP /viabtc_exchange_server/matchengine/matchengine /viabtc_exchange_server/matchengine.exe COPY docker/exchange/matchengine.json /viabtc_exchange_server -COPY --from=BUILDSTEP /viabtc_exchange_server/marketprice/marketprice.exe /viabtc_exchange_server/marketprice.exe +COPY --from=BUILDSTEP /viabtc_exchange_server/marketprice/marketprice /viabtc_exchange_server/marketprice.exe COPY docker/exchange/marketprice.json /viabtc_exchange_server -COPY --from=BUILDSTEP /viabtc_exchange_server/readhistory/readhistory.exe /viabtc_exchange_server/readhistory.exe +COPY --from=BUILDSTEP /viabtc_exchange_server/readhistory/readhistory /viabtc_exchange_server/readhistory.exe COPY docker/exchange/readhistory.json /viabtc_exchange_server -COPY --from=BUILDSTEP /viabtc_exchange_server/accesshttp/accesshttp.exe /viabtc_exchange_server/accesshttp.exe +COPY --from=BUILDSTEP /viabtc_exchange_server/accesshttp/accesshttp /viabtc_exchange_server/accesshttp.exe COPY docker/exchange/accesshttp.json /viabtc_exchange_server -COPY --from=BUILDSTEP /viabtc_exchange_server/accessws/accessws.exe /viabtc_exchange_server/accessws.exe +COPY --from=BUILDSTEP /viabtc_exchange_server/accessws/accessws /viabtc_exchange_server/accessws.exe COPY docker/exchange/accessws.json /viabtc_exchange_server COPY docker/exchange/entrypoint.sh /docker-entrypoint.sh From 804605a63eacbbeabaaa930a16d7fe6d46bc2dda Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Tue, 27 Feb 2018 11:06:40 +0500 Subject: [PATCH 36/38] docker: fix makefiles in order to compile services with ubuntu binaries --- docker/exchange/makefile.accessws.ubuntu | 2 +- docker/exchange/makefile.marketprice.ubuntu | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 docker/exchange/makefile.marketprice.ubuntu diff --git a/docker/exchange/makefile.accessws.ubuntu b/docker/exchange/makefile.accessws.ubuntu index ac0b7639..51ba5269 100644 --- a/docker/exchange/makefile.accessws.ubuntu +++ b/docker/exchange/makefile.accessws.ubuntu @@ -1,4 +1,4 @@ -TARGET := accessws.exe +TARGET := accessws INCS = -I ../network -I ../utils LIBS = -L ../utils -lutils -L ../network -lnetwork -Wl,-Bstatic -lev -ljansson -lmpdec -lrdkafka -lcurl -lz -lssl -lcrypt -lcrypto -lhiredis -Wl,-Bdynamic -lm -lpthread -ldl -lldap -lkrb5 -llber -lgss -lidn -lrtmp include ../makefile.inc diff --git a/docker/exchange/makefile.marketprice.ubuntu b/docker/exchange/makefile.marketprice.ubuntu new file mode 100644 index 00000000..41645b22 --- /dev/null +++ b/docker/exchange/makefile.marketprice.ubuntu @@ -0,0 +1,4 @@ +TARGET := marketprice +INCS = -I ../network -I ../utils +LIBS = -L ../utils -lutils -L ../network -lnetwork -Wl,-Bstatic -lev -ljansson -lmpdec -lrdkafka -lcurl -lz -lssl -lcrypt -lcrypto -lhiredis -Wl,-Bdynamic -lm -lpthread -ldl -lldap -lkrb5 -llber -lgss -lidn -lrtmp +include ../makefile.inc From a77f08ff46af80ade7b459c19869b72f39ab46b3 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Tue, 27 Feb 2018 12:56:47 +0500 Subject: [PATCH 37/38] utilities: add scripts asset.list, market.list, init --- bin/asset.list | 6 ++++++ bin/init | 27 +++++++++++++++++++++++++++ bin/market.list | 6 ++++++ 3 files changed, 39 insertions(+) create mode 100755 bin/asset.list create mode 100755 bin/init create mode 100755 bin/market.list diff --git a/bin/asset.list b/bin/asset.list new file mode 100755 index 00000000..19b7820c --- /dev/null +++ b/bin/asset.list @@ -0,0 +1,6 @@ +#!/bin/bash + +shdir=`dirname ${0}` +. ${shdir}/accesshttp + +asset.list diff --git a/bin/init b/bin/init new file mode 100755 index 00000000..2d824462 --- /dev/null +++ b/bin/init @@ -0,0 +1,27 @@ +#!/bin/bash + +shdir=`dirname ${0}` +. ${shdir}/accesshttp + +OP=3 + +balance.update 1 BTC deposit $OP 1000 +balance.update 1 ETH deposit $OP 1000 +balance.update 1 BCH deposit $OP 1000 +balance.update 1 LTC deposit $OP 1000 +balance.update 1 DASH deposit $OP 1000 + +balance.update 2 BTC deposit $OP 1000 +balance.update 2 ETH deposit $OP 1000 +balance.update 2 BCH deposit $OP 1000 +balance.update 2 LTC deposit $OP 1000 +balance.update 2 DASH deposit $OP 1000 + +balance.update 1121516550 BTC deposit $OP 1000 +balance.update 1121516550 ETH deposit $OP 1000 +balance.update 1121516550 BCH deposit $OP 1000 +balance.update 1121516550 LTC deposit $OP 1000 +balance.update 1121516550 DASH deposit $OP 1000 + +order.put_market 1121516550 BTCETH 1 0.00001 0.0000001 +order.put_market 1121516550 BTCBCH 1 0.00001 0.0000001 diff --git a/bin/market.list b/bin/market.list new file mode 100755 index 00000000..939f14f3 --- /dev/null +++ b/bin/market.list @@ -0,0 +1,6 @@ +#!/bin/bash + +shdir=`dirname ${0}` +. ${shdir}/accesshttp + +market.list From 5b535f0a0a09de894049989dc55098dfb7b0d6a5 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Tue, 27 Feb 2018 13:25:28 +0500 Subject: [PATCH 38/38] utilities: add asset.list function --- bin/accesshttp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bin/accesshttp b/bin/accesshttp index 601071ac..6eb022a5 100755 --- a/bin/accesshttp +++ b/bin/accesshttp @@ -85,6 +85,14 @@ function market.status () { rpc "market.status" "\"$1\",$2" } +function market.list () { + rpc "market.list" +} + +function asset.list () { + rpc "asset.list" +} + # if [ $# -gt 0 ];then # rpc $@ # fi