From 25b58d1cc11cc664ec2ac977103c6efacdb9f182 Mon Sep 17 00:00:00 2001 From: denglong Date: Tue, 25 May 2021 18:03:23 +0800 Subject: [PATCH 1/9] feat: add Fedb put --- .../_4paradigm/benchmark/memory/FedbTest.java | 116 ++++++++++++++++++ .../benchmark/memory/SingleStoreTest.java | 60 +++++++++ java/src/main/resources/benchmark.properties | 10 ++ java/src/main/resources/log4j.properties | 8 ++ 4 files changed, 194 insertions(+) create mode 100644 java/src/main/java/com/_4paradigm/benchmark/memory/FedbTest.java create mode 100644 java/src/main/java/com/_4paradigm/benchmark/memory/SingleStoreTest.java create mode 100644 java/src/main/resources/benchmark.properties create mode 100644 java/src/main/resources/log4j.properties diff --git a/java/src/main/java/com/_4paradigm/benchmark/memory/FedbTest.java b/java/src/main/java/com/_4paradigm/benchmark/memory/FedbTest.java new file mode 100644 index 0000000..22df13e --- /dev/null +++ b/java/src/main/java/com/_4paradigm/benchmark/memory/FedbTest.java @@ -0,0 +1,116 @@ +package com._4paradigm.benchmark.memory; + +import com._4paradigm.sql.sdk.SqlExecutor; +import com._4paradigm.sql.sdk.SdkOption; +import com._4paradigm.sql.sdk.impl.SqlClusterExecutor; +import com._4paradigm.sql.SQLInsertRow; +import com._4paradigm.sql.SQLInsertRows; + +import java.util.Properties; +import java.util.Random; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +public class FedbTest { + private static String zkCluster; + private static String zkRootPath; + private static String tableName; + private static String dbName; + private static int threadNum; + private static int pkCnt; + private static int tsCnt; + private static String baseKey; + private static boolean needCreate; + + private SqlExecutor executor; + private String createDDL = "create table " + tableName + " (col1 string, col2 bigint, " + + "col3 float," + + "col4 double," + + "col5 string," + + "index(key=(col1),ts=col2)) partitionnum=4;"; + private String dropDDL = "drop table " + tableName + " ;"; + private String format = "insert into " + tableName + " values(?, ?, 100.0, 200.0, 'hello world');"; + private ExecutorService executorService; + private Random random = new Random(System.currentTimeMillis()); + + static { + try { + Properties prop = new Properties(); + prop.load(SingleStoreTest.class.getClassLoader().getResourceAsStream("benchmark.properties")); + zkCluster = prop.getProperty("zk_cluster"); + zkRootPath = prop.getProperty("zk_root_path"); + tableName = prop.getProperty("table_name"); + dbName = prop.getProperty("db_name"); + baseKey = prop.getProperty("base_key"); + threadNum = Integer.parseInt(prop.getProperty("thread_num", "1")); + pkCnt = Integer.parseInt(prop.getProperty("pk_cnt", "1")); + tsCnt = Integer.parseInt(prop.getProperty("ts_cnt", "1")); + needCreate = Boolean.parseBoolean(prop.getProperty("need_create", "true")); + } catch (Exception e) { + e.printStackTrace(); + } + } + + public boolean init() { + executorService = Executors.newFixedThreadPool(threadNum); + SdkOption sdkOption = new SdkOption(); + sdkOption.setSessionTimeout(30000); + sdkOption.setZkCluster(zkCluster); + sdkOption.setZkPath(zkRootPath); + sdkOption.setEnableDebug(false); + try { + executor = new SqlClusterExecutor(sdkOption); + } catch (Exception e) { + e.printStackTrace(); + return false; + } + /* if (!executor.createDB(dbName)) { + return false; + }*/ + if (needCreate) { + if (!executor.executeDDL(dbName, createDDL)) { + return false; + } + } + return true; + } + + public void put() { + while(true) { + SQLInsertRows rows = executor.getInsertRows(dbName, format); + int num = random.nextInt(pkCnt) + pkCnt; + String key = baseKey + String.valueOf(num); + for (int i = 0; i < tsCnt; i++) { + SQLInsertRow row = rows.NewRow(); + row.Init(key.length()); + row.AppendString(key); + row.AppendInt64(System.currentTimeMillis()); + } + try { + executor.executeInsert(dbName, format, rows); + //counter ++; + } catch (Exception e) { + e.printStackTrace(); + } + } + + } + + public void run() { + for (int i = 0; i < threadNum; i++) { + executorService.submit(new Runnable() { + @Override + public void run() { + put(); + } + }); + } + } + + public static void main(String[] args) { + FedbTest test = new FedbTest(); + test.init(); + test.run(); + } + +} diff --git a/java/src/main/java/com/_4paradigm/benchmark/memory/SingleStoreTest.java b/java/src/main/java/com/_4paradigm/benchmark/memory/SingleStoreTest.java new file mode 100644 index 0000000..ecae0c3 --- /dev/null +++ b/java/src/main/java/com/_4paradigm/benchmark/memory/SingleStoreTest.java @@ -0,0 +1,60 @@ +package com._4paradigm.benchmark.memory; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import java.util.Properties; + +class SingleStoreTest { + private static String singlestoreUrl; + private static String tableName; + private static String dbName; + private static int threadNum; + private Connection cnn; + private String createDDL = "create table " + tableName + " (col1 varchar(20), col2 bigint, " + + "col3 float," + + "col4 double," + + "col5 varchar(2)," + + "KEY (col1, col2));"; + private String dropDDL = "drop table " + tableName + " ;"; + private String format = "insert into %s values('%s', %d," + + "100.0, 200.0, 'hello world');"; + + static { + try { + Properties prop = new Properties(); + prop.load(SingleStoreTest.class.getClassLoader().getResourceAsStream("benchmark.properties")); + singlestoreUrl = prop.getProperty("singlestore_url"); + tableName = prop.getProperty("table_name"); + dbName = prop.getProperty("db_name"); + threadNum = Integer.parseInt(prop.getProperty("thread_num", "1")); + + } catch (Exception e) { + e.printStackTrace(); + } + } + + public boolean init() { + try { + cnn = DriverManager.getConnection(singlestoreUrl); + Statement st = cnn.createStatement(); + try { + st.execute(dropDDL); + } catch (Exception e) { + e.printStackTrace(); + } + st = cnn.createStatement(); + st.execute(createDDL); + //st.execute(ddl1); + return true; + } catch (Exception e) { + e.printStackTrace(); + return false; + } + } + + public static void main(String[] args) { + SingleStoreTest test = new SingleStoreTest(); + //test.init(); + System.out.println(threadNum); + } +} \ No newline at end of file diff --git a/java/src/main/resources/benchmark.properties b/java/src/main/resources/benchmark.properties new file mode 100644 index 0000000..9e16db6 --- /dev/null +++ b/java/src/main/resources/benchmark.properties @@ -0,0 +1,10 @@ +db_name=memory +table_name=perf +singlestore_url=jdbc:mysql://172.24.4.40:3306/memory?user=root&password=root +thread_num=2 +zk_cluster=172.24.4.55:12200 +zk_root_path=/rtidb_scan +base_key=10000000000 +pk_cnt=1000000 +ts_cnt=10 +need_create=true \ No newline at end of file diff --git a/java/src/main/resources/log4j.properties b/java/src/main/resources/log4j.properties new file mode 100644 index 0000000..d9bb827 --- /dev/null +++ b/java/src/main/resources/log4j.properties @@ -0,0 +1,8 @@ +# Root logger option +log4j.rootLogger=INFO, stdout, file + +# Redirect log messages to console +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.ayout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n \ No newline at end of file From 774118c0a30d60fda628154e9edf9834ec6e4d41 Mon Sep 17 00:00:00 2001 From: denglong Date: Wed, 26 May 2021 11:48:44 +0800 Subject: [PATCH 2/9] feat: add singlestore put --- .../_4paradigm/benchmark/memory/FedbTest.java | 3 + .../benchmark/memory/SingleStoreTest.java | 67 +++++++++++++++---- 2 files changed, 57 insertions(+), 13 deletions(-) diff --git a/java/src/main/java/com/_4paradigm/benchmark/memory/FedbTest.java b/java/src/main/java/com/_4paradigm/benchmark/memory/FedbTest.java index 22df13e..6ef4fd7 100644 --- a/java/src/main/java/com/_4paradigm/benchmark/memory/FedbTest.java +++ b/java/src/main/java/com/_4paradigm/benchmark/memory/FedbTest.java @@ -85,12 +85,15 @@ public void put() { row.Init(key.length()); row.AppendString(key); row.AppendInt64(System.currentTimeMillis()); + row.delete(); } try { executor.executeInsert(dbName, format, rows); //counter ++; } catch (Exception e) { e.printStackTrace(); + } finally { + rows.delete(); } } diff --git a/java/src/main/java/com/_4paradigm/benchmark/memory/SingleStoreTest.java b/java/src/main/java/com/_4paradigm/benchmark/memory/SingleStoreTest.java index ecae0c3..783bb0d 100644 --- a/java/src/main/java/com/_4paradigm/benchmark/memory/SingleStoreTest.java +++ b/java/src/main/java/com/_4paradigm/benchmark/memory/SingleStoreTest.java @@ -1,23 +1,32 @@ package com._4paradigm.benchmark.memory; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.PreparedStatement; import java.sql.Statement; import java.util.Properties; +import java.util.Random; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; class SingleStoreTest { private static String singlestoreUrl; private static String tableName; private static String dbName; private static int threadNum; + private static int pkCnt; + private static int tsCnt; + private static String baseKey; + private static boolean needCreate; private Connection cnn; + private ExecutorService executorService; private String createDDL = "create table " + tableName + " (col1 varchar(20), col2 bigint, " + "col3 float," + "col4 double," + - "col5 varchar(2)," + + "col5 varchar(12)," + "KEY (col1, col2));"; - private String dropDDL = "drop table " + tableName + " ;"; - private String format = "insert into %s values('%s', %d," + - "100.0, 200.0, 'hello world');"; + private String sql = "insert into " + tableName + " values(?, ?, 100.0, 200.0, 'hello world');"; + private Random random = new Random(System.currentTimeMillis()); static { try { @@ -27,23 +36,25 @@ class SingleStoreTest { tableName = prop.getProperty("table_name"); dbName = prop.getProperty("db_name"); threadNum = Integer.parseInt(prop.getProperty("thread_num", "1")); - + baseKey = prop.getProperty("base_key"); + threadNum = Integer.parseInt(prop.getProperty("thread_num", "1")); + pkCnt = Integer.parseInt(prop.getProperty("pk_cnt", "1")); + tsCnt = Integer.parseInt(prop.getProperty("ts_cnt", "1")); + needCreate = Boolean.parseBoolean(prop.getProperty("need_create", "true")); } catch (Exception e) { e.printStackTrace(); } } public boolean init() { + executorService = Executors.newFixedThreadPool(threadNum); try { cnn = DriverManager.getConnection(singlestoreUrl); Statement st = cnn.createStatement(); - try { - st.execute(dropDDL); - } catch (Exception e) { - e.printStackTrace(); - } st = cnn.createStatement(); - st.execute(createDDL); + if (needCreate) { + st.execute(createDDL); + } //st.execute(ddl1); return true; } catch (Exception e) { @@ -52,9 +63,39 @@ public boolean init() { } } + public void put() { + while(true) { + int num = random.nextInt(pkCnt) + pkCnt; + String key = baseKey + String.valueOf(num); + long ts = System.currentTimeMillis(); + for (int i = 0; i < tsCnt; i++) { + try { + PreparedStatement st = cnn.prepareStatement(sql); + st.setString(1, key); + st.setLong(2, ts - i); + st.executeUpdate(); + st.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + } + public void run() { + for (int i = 0; i < threadNum; i++) { + executorService.submit(new Runnable() { + @Override + public void run() { + put(); + } + }); + } + } + public static void main(String[] args) { SingleStoreTest test = new SingleStoreTest(); - //test.init(); - System.out.println(threadNum); + test.init(); + //test.put(); + test.run(); } } \ No newline at end of file From 44e7bd7c178b9950a7dab64631a82135fe8f9e0b Mon Sep 17 00:00:00 2001 From: denglong Date: Thu, 27 May 2021 11:39:49 +0800 Subject: [PATCH 3/9] feat: add jdbc put --- .../_4paradigm/benchmark/memory/FedbTest.java | 31 ++---------- ...{SingleStoreTest.java => JDBCSqlTest.java} | 49 ++++++------------- .../benchmark/memory/MemoryBenchmark.java | 47 ++++++++++++++++++ .../com/_4paradigm/benchmark/memory/Test.java | 6 +++ java/src/main/resources/benchmark.properties | 6 ++- 5 files changed, 74 insertions(+), 65 deletions(-) rename java/src/main/java/com/_4paradigm/benchmark/memory/{SingleStoreTest.java => JDBCSqlTest.java} (64%) create mode 100644 java/src/main/java/com/_4paradigm/benchmark/memory/MemoryBenchmark.java create mode 100644 java/src/main/java/com/_4paradigm/benchmark/memory/Test.java diff --git a/java/src/main/java/com/_4paradigm/benchmark/memory/FedbTest.java b/java/src/main/java/com/_4paradigm/benchmark/memory/FedbTest.java index 6ef4fd7..e8b4d7c 100644 --- a/java/src/main/java/com/_4paradigm/benchmark/memory/FedbTest.java +++ b/java/src/main/java/com/_4paradigm/benchmark/memory/FedbTest.java @@ -8,15 +8,12 @@ import java.util.Properties; import java.util.Random; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -public class FedbTest { +public class FedbTest implements Test { private static String zkCluster; private static String zkRootPath; private static String tableName; private static String dbName; - private static int threadNum; private static int pkCnt; private static int tsCnt; private static String baseKey; @@ -30,7 +27,6 @@ public class FedbTest { "index(key=(col1),ts=col2)) partitionnum=4;"; private String dropDDL = "drop table " + tableName + " ;"; private String format = "insert into " + tableName + " values(?, ?, 100.0, 200.0, 'hello world');"; - private ExecutorService executorService; private Random random = new Random(System.currentTimeMillis()); static { @@ -42,7 +38,6 @@ public class FedbTest { tableName = prop.getProperty("table_name"); dbName = prop.getProperty("db_name"); baseKey = prop.getProperty("base_key"); - threadNum = Integer.parseInt(prop.getProperty("thread_num", "1")); pkCnt = Integer.parseInt(prop.getProperty("pk_cnt", "1")); tsCnt = Integer.parseInt(prop.getProperty("ts_cnt", "1")); needCreate = Boolean.parseBoolean(prop.getProperty("need_create", "true")); @@ -50,9 +45,8 @@ public class FedbTest { e.printStackTrace(); } } - + @Override public boolean init() { - executorService = Executors.newFixedThreadPool(threadNum); SdkOption sdkOption = new SdkOption(); sdkOption.setSessionTimeout(30000); sdkOption.setZkCluster(zkCluster); @@ -75,6 +69,7 @@ public boolean init() { return true; } + @Override public void put() { while(true) { SQLInsertRows rows = executor.getInsertRows(dbName, format); @@ -89,31 +84,11 @@ public void put() { } try { executor.executeInsert(dbName, format, rows); - //counter ++; } catch (Exception e) { e.printStackTrace(); } finally { rows.delete(); } } - - } - - public void run() { - for (int i = 0; i < threadNum; i++) { - executorService.submit(new Runnable() { - @Override - public void run() { - put(); - } - }); - } - } - - public static void main(String[] args) { - FedbTest test = new FedbTest(); - test.init(); - test.run(); } - } diff --git a/java/src/main/java/com/_4paradigm/benchmark/memory/SingleStoreTest.java b/java/src/main/java/com/_4paradigm/benchmark/memory/JDBCSqlTest.java similarity index 64% rename from java/src/main/java/com/_4paradigm/benchmark/memory/SingleStoreTest.java rename to java/src/main/java/com/_4paradigm/benchmark/memory/JDBCSqlTest.java index 783bb0d..a579ed0 100644 --- a/java/src/main/java/com/_4paradigm/benchmark/memory/SingleStoreTest.java +++ b/java/src/main/java/com/_4paradigm/benchmark/memory/JDBCSqlTest.java @@ -1,61 +1,57 @@ package com._4paradigm.benchmark.memory; + import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.Statement; import java.util.Properties; import java.util.Random; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -class SingleStoreTest { - private static String singlestoreUrl; +public class JDBCSqlTest implements Test { + private static String connectURL; private static String tableName; private static String dbName; - private static int threadNum; private static int pkCnt; private static int tsCnt; private static String baseKey; private static boolean needCreate; + private static String method; private Connection cnn; - private ExecutorService executorService; private String createDDL = "create table " + tableName + " (col1 varchar(20), col2 bigint, " + "col3 float," + - "col4 double," + + "col4 float," + "col5 varchar(12)," + - "KEY (col1, col2));"; + "PRIMARY KEY (col1, col2));"; private String sql = "insert into " + tableName + " values(?, ?, 100.0, 200.0, 'hello world');"; private Random random = new Random(System.currentTimeMillis()); - static { try { Properties prop = new Properties(); - prop.load(SingleStoreTest.class.getClassLoader().getResourceAsStream("benchmark.properties")); - singlestoreUrl = prop.getProperty("singlestore_url"); + prop.load(JDBCSqlTest.class.getClassLoader().getResourceAsStream("benchmark.properties")); + connectURL = prop.getProperty("connect_url"); tableName = prop.getProperty("table_name"); dbName = prop.getProperty("db_name"); - threadNum = Integer.parseInt(prop.getProperty("thread_num", "1")); baseKey = prop.getProperty("base_key"); - threadNum = Integer.parseInt(prop.getProperty("thread_num", "1")); pkCnt = Integer.parseInt(prop.getProperty("pk_cnt", "1")); tsCnt = Integer.parseInt(prop.getProperty("ts_cnt", "1")); needCreate = Boolean.parseBoolean(prop.getProperty("need_create", "true")); + method = prop.getProperty("method"); } catch (Exception e) { e.printStackTrace(); } } public boolean init() { - executorService = Executors.newFixedThreadPool(threadNum); try { - cnn = DriverManager.getConnection(singlestoreUrl); + if (method.equals("voltdb")) { + Class.forName("org.voltdb.jdbc.Driver"); + } + cnn = DriverManager.getConnection(connectURL); Statement st = cnn.createStatement(); st = cnn.createStatement(); if (needCreate) { st.execute(createDDL); } - //st.execute(ddl1); return true; } catch (Exception e) { e.printStackTrace(); @@ -81,21 +77,4 @@ public void put() { } } } - public void run() { - for (int i = 0; i < threadNum; i++) { - executorService.submit(new Runnable() { - @Override - public void run() { - put(); - } - }); - } - } - - public static void main(String[] args) { - SingleStoreTest test = new SingleStoreTest(); - test.init(); - //test.put(); - test.run(); - } -} \ No newline at end of file +} diff --git a/java/src/main/java/com/_4paradigm/benchmark/memory/MemoryBenchmark.java b/java/src/main/java/com/_4paradigm/benchmark/memory/MemoryBenchmark.java new file mode 100644 index 0000000..d1f9aaa --- /dev/null +++ b/java/src/main/java/com/_4paradigm/benchmark/memory/MemoryBenchmark.java @@ -0,0 +1,47 @@ +package com._4paradigm.benchmark.memory; +import java.util.Properties; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +public class MemoryBenchmark { + private ExecutorService executorService; + private int threadNum; + private String method; + public MemoryBenchmark() { + try { + Properties prop = new Properties(); + prop.load(MemoryBenchmark.class.getClassLoader().getResourceAsStream("benchmark.properties")); + threadNum = Integer.parseInt(prop.getProperty("thread_num", "1")); + method = prop.getProperty("method"); + } catch (Exception e) { + e.printStackTrace(); + } + executorService = Executors.newFixedThreadPool(threadNum); + } + public void run() { + Test tmp = null; + if (method.equals("rtidb")) { + tmp = new FedbTest(); + } else { + tmp = new JDBCSqlTest(); + } + if (!tmp.init()) { + System.out.println("init failed"); + return; + } + final Test test = tmp; + for (int i = 0; i < threadNum; i++) { + System.out.println("create thread " + String.valueOf(i)); + executorService.submit(new Runnable() { + @Override + public void run() { + test.put(); + } + }); + } + } + + public static void main(String[] args) { + MemoryBenchmark benchmark = new MemoryBenchmark(); + benchmark.run(); + } +} diff --git a/java/src/main/java/com/_4paradigm/benchmark/memory/Test.java b/java/src/main/java/com/_4paradigm/benchmark/memory/Test.java new file mode 100644 index 0000000..9b76a5d --- /dev/null +++ b/java/src/main/java/com/_4paradigm/benchmark/memory/Test.java @@ -0,0 +1,6 @@ +package com._4paradigm.benchmark.memory; + +public interface Test { + public abstract boolean init(); + public abstract void put(); +} diff --git a/java/src/main/resources/benchmark.properties b/java/src/main/resources/benchmark.properties index 9e16db6..051e879 100644 --- a/java/src/main/resources/benchmark.properties +++ b/java/src/main/resources/benchmark.properties @@ -1,10 +1,12 @@ db_name=memory table_name=perf -singlestore_url=jdbc:mysql://172.24.4.40:3306/memory?user=root&password=root +#connect_url=jdbc:mysql://172.24.4.40:3306/memory?user=root&password=root +connect_url=jdbc:voltdb://172.24.4.55:21212 thread_num=2 zk_cluster=172.24.4.55:12200 zk_root_path=/rtidb_scan base_key=10000000000 pk_cnt=1000000 ts_cnt=10 -need_create=true \ No newline at end of file +need_create=false +method=voltdb \ No newline at end of file From 2fee4d34296f998017f49131cb008d679a0883f3 Mon Sep 17 00:00:00 2001 From: denglong Date: Thu, 27 May 2021 12:18:41 +0800 Subject: [PATCH 4/9] refact: use create if not exist --- .../java/com/_4paradigm/benchmark/memory/FedbTest.java | 2 +- .../java/com/_4paradigm/benchmark/memory/JDBCSqlTest.java | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/java/src/main/java/com/_4paradigm/benchmark/memory/FedbTest.java b/java/src/main/java/com/_4paradigm/benchmark/memory/FedbTest.java index e8b4d7c..9b18ebf 100644 --- a/java/src/main/java/com/_4paradigm/benchmark/memory/FedbTest.java +++ b/java/src/main/java/com/_4paradigm/benchmark/memory/FedbTest.java @@ -32,7 +32,7 @@ public class FedbTest implements Test { static { try { Properties prop = new Properties(); - prop.load(SingleStoreTest.class.getClassLoader().getResourceAsStream("benchmark.properties")); + prop.load(FedbTest.class.getClassLoader().getResourceAsStream("benchmark.properties")); zkCluster = prop.getProperty("zk_cluster"); zkRootPath = prop.getProperty("zk_root_path"); tableName = prop.getProperty("table_name"); diff --git a/java/src/main/java/com/_4paradigm/benchmark/memory/JDBCSqlTest.java b/java/src/main/java/com/_4paradigm/benchmark/memory/JDBCSqlTest.java index a579ed0..4eb6140 100644 --- a/java/src/main/java/com/_4paradigm/benchmark/memory/JDBCSqlTest.java +++ b/java/src/main/java/com/_4paradigm/benchmark/memory/JDBCSqlTest.java @@ -14,10 +14,9 @@ public class JDBCSqlTest implements Test { private static int pkCnt; private static int tsCnt; private static String baseKey; - private static boolean needCreate; private static String method; private Connection cnn; - private String createDDL = "create table " + tableName + " (col1 varchar(20), col2 bigint, " + + private String createDDL = "create table if not exists" + tableName + " (col1 varchar(20), col2 bigint, " + "col3 float," + "col4 float," + "col5 varchar(12)," + @@ -34,7 +33,6 @@ public class JDBCSqlTest implements Test { baseKey = prop.getProperty("base_key"); pkCnt = Integer.parseInt(prop.getProperty("pk_cnt", "1")); tsCnt = Integer.parseInt(prop.getProperty("ts_cnt", "1")); - needCreate = Boolean.parseBoolean(prop.getProperty("need_create", "true")); method = prop.getProperty("method"); } catch (Exception e) { e.printStackTrace(); @@ -49,9 +47,7 @@ public boolean init() { cnn = DriverManager.getConnection(connectURL); Statement st = cnn.createStatement(); st = cnn.createStatement(); - if (needCreate) { - st.execute(createDDL); - } + st.execute(createDDL); return true; } catch (Exception e) { e.printStackTrace(); From 24eb7a1a784f7d825345030d30ead07284f66ff9 Mon Sep 17 00:00:00 2001 From: denglong Date: Thu, 27 May 2021 12:28:28 +0800 Subject: [PATCH 5/9] fix: use different connection in jdbc for per thread --- .../_4paradigm/benchmark/memory/JDBCSqlTest.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/java/src/main/java/com/_4paradigm/benchmark/memory/JDBCSqlTest.java b/java/src/main/java/com/_4paradigm/benchmark/memory/JDBCSqlTest.java index 4eb6140..aaa210d 100644 --- a/java/src/main/java/com/_4paradigm/benchmark/memory/JDBCSqlTest.java +++ b/java/src/main/java/com/_4paradigm/benchmark/memory/JDBCSqlTest.java @@ -15,7 +15,6 @@ public class JDBCSqlTest implements Test { private static int tsCnt; private static String baseKey; private static String method; - private Connection cnn; private String createDDL = "create table if not exists" + tableName + " (col1 varchar(20), col2 bigint, " + "col3 float," + "col4 float," + @@ -44,10 +43,11 @@ public boolean init() { if (method.equals("voltdb")) { Class.forName("org.voltdb.jdbc.Driver"); } - cnn = DriverManager.getConnection(connectURL); + Connection cnn = DriverManager.getConnection(connectURL); Statement st = cnn.createStatement(); - st = cnn.createStatement(); st.execute(createDDL); + st.close(); + cnn.close(); return true; } catch (Exception e) { e.printStackTrace(); @@ -56,6 +56,13 @@ public boolean init() { } public void put() { + Connection cnn = null; + try { + cnn = DriverManager.getConnection(connectURL); + } catch (Exception e) { + e.printStackTrace(); + return; + } while(true) { int num = random.nextInt(pkCnt) + pkCnt; String key = baseKey + String.valueOf(num); From cd4f73afa7cd69b1b57a1b424a9aef6cf64ce9b6 Mon Sep 17 00:00:00 2001 From: denglong Date: Mon, 31 May 2021 10:49:26 +0800 Subject: [PATCH 6/9] feat: add pom --- java/pom.xml | 170 ++++++++++++++++++ .../benchmark/memory/JDBCSqlTest.java | 16 +- 2 files changed, 180 insertions(+), 6 deletions(-) create mode 100644 java/pom.xml diff --git a/java/pom.xml b/java/pom.xml new file mode 100644 index 0000000..7f23df7 --- /dev/null +++ b/java/pom.xml @@ -0,0 +1,170 @@ + + 4.0.0 + com._4paradigm + benchmark + 0.0.1 + jar + + + + org.testng + testng + 6.9.4 + test + + + mysql + mysql-connector-java + 5.1.49 + + + mysql + mysql-connector-java + 5.1.49 + + + org.voltdb + voltdbclient + 10.1.1 + + + com._4paradigm.sql + sql-jdbc + 2.1.0.0-SNAPSHOT + + + junit + junit + 4.10 + test + + + com.google.guava + guava + 19.0 + + + + org.slf4j + slf4j-api + 1.7.21 + + + + + joda-time + joda-time + 2.10 + + + + ch.qos.logback + logback-core + 1.1.2 + test + + + ch.qos.logback + logback-classic + 1.1.2 + test + + + slf4j-api + org.slf4j + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.7 + 1.7 + UTF-8 + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + 0.5.0 + +/rtidb/thirdparty/bin/protoc + + + + + compile + test-compile + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.22.0 + + once + -Dfile.encoding=UTF-8 + + + junit + false + + + 1 + + + + org.apache.maven.surefire + surefire-junit47 + 2.22.0 + + + org.apache.maven.surefire + surefire-testng + 2.22.0 + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.4.1 + + + + jar-with-dependencies + + + + + com._4paradigm.benchmark.memory.FedbTest + + + + + + + make-assembly + + package + + single + + + + + + + + + + diff --git a/java/src/main/java/com/_4paradigm/benchmark/memory/JDBCSqlTest.java b/java/src/main/java/com/_4paradigm/benchmark/memory/JDBCSqlTest.java index aaa210d..1ec4414 100644 --- a/java/src/main/java/com/_4paradigm/benchmark/memory/JDBCSqlTest.java +++ b/java/src/main/java/com/_4paradigm/benchmark/memory/JDBCSqlTest.java @@ -15,7 +15,8 @@ public class JDBCSqlTest implements Test { private static int tsCnt; private static String baseKey; private static String method; - private String createDDL = "create table if not exists" + tableName + " (col1 varchar(20), col2 bigint, " + + private static boolean needCreate; + private String createDDL = "create table " + tableName + " (col1 varchar(20), col2 bigint, " + "col3 float," + "col4 float," + "col5 varchar(12)," + @@ -33,6 +34,7 @@ public class JDBCSqlTest implements Test { pkCnt = Integer.parseInt(prop.getProperty("pk_cnt", "1")); tsCnt = Integer.parseInt(prop.getProperty("ts_cnt", "1")); method = prop.getProperty("method"); + needCreate = Boolean.parseBoolean(prop.getProperty("need_create")); } catch (Exception e) { e.printStackTrace(); } @@ -43,11 +45,13 @@ public boolean init() { if (method.equals("voltdb")) { Class.forName("org.voltdb.jdbc.Driver"); } - Connection cnn = DriverManager.getConnection(connectURL); - Statement st = cnn.createStatement(); - st.execute(createDDL); - st.close(); - cnn.close(); + if (needCreate) { + Connection cnn = DriverManager.getConnection(connectURL); + Statement st = cnn.createStatement(); + st.execute(createDDL); + st.close(); + cnn.close(); + } return true; } catch (Exception e) { e.printStackTrace(); From 5251a540f3096c8c8bc52d895573cfa98eec0a80 Mon Sep 17 00:00:00 2001 From: denglong Date: Mon, 31 May 2021 11:44:55 +0800 Subject: [PATCH 7/9] refact: rm duplicated dependency --- java/pom.xml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 7f23df7..68e710e 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -18,11 +18,6 @@ mysql-connector-java 5.1.49 - - mysql - mysql-connector-java - 5.1.49 - org.voltdb voltdbclient From 95990a7e01186b14cc820bab0bcc2b9c869fc419 Mon Sep 17 00:00:00 2001 From: dl239 Date: Wed, 2 Jun 2021 16:11:58 +0800 Subject: [PATCH 8/9] feat: add redis --- java/pom.xml | 11 ++++++++--- .../com/_4paradigm/benchmark/memory/FedbTest.java | 10 +++++----- .../_4paradigm/benchmark/memory/MemoryBenchmark.java | 2 ++ java/src/main/resources/benchmark.properties | 3 ++- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 68e710e..0e34eee 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -24,9 +24,14 @@ 10.1.1 - com._4paradigm.sql - sql-jdbc - 2.1.0.0-SNAPSHOT + com.4paradigm.hybridsql + fedb-jdbc + 2.2.2 + + + redis.clients + jedis + 3.6.0 junit diff --git a/java/src/main/java/com/_4paradigm/benchmark/memory/FedbTest.java b/java/src/main/java/com/_4paradigm/benchmark/memory/FedbTest.java index 9b18ebf..2924110 100644 --- a/java/src/main/java/com/_4paradigm/benchmark/memory/FedbTest.java +++ b/java/src/main/java/com/_4paradigm/benchmark/memory/FedbTest.java @@ -1,10 +1,10 @@ package com._4paradigm.benchmark.memory; -import com._4paradigm.sql.sdk.SqlExecutor; -import com._4paradigm.sql.sdk.SdkOption; -import com._4paradigm.sql.sdk.impl.SqlClusterExecutor; -import com._4paradigm.sql.SQLInsertRow; -import com._4paradigm.sql.SQLInsertRows; +import com._4paradigm.hybridsql.fedb.sdk.SqlExecutor; +import com._4paradigm.hybridsql.fedb.sdk.SdkOption; +import com._4paradigm.hybridsql.fedb.sdk.impl.SqlClusterExecutor; +import com._4paradigm.hybridsql.fedb.SQLInsertRow; +import com._4paradigm.hybridsql.fedb.SQLInsertRows; import java.util.Properties; import java.util.Random; diff --git a/java/src/main/java/com/_4paradigm/benchmark/memory/MemoryBenchmark.java b/java/src/main/java/com/_4paradigm/benchmark/memory/MemoryBenchmark.java index d1f9aaa..018f925 100644 --- a/java/src/main/java/com/_4paradigm/benchmark/memory/MemoryBenchmark.java +++ b/java/src/main/java/com/_4paradigm/benchmark/memory/MemoryBenchmark.java @@ -21,6 +21,8 @@ public void run() { Test tmp = null; if (method.equals("rtidb")) { tmp = new FedbTest(); + } else if (method.equals("redis")) { + tmp = new RedisTest(); } else { tmp = new JDBCSqlTest(); } diff --git a/java/src/main/resources/benchmark.properties b/java/src/main/resources/benchmark.properties index 051e879..5438953 100644 --- a/java/src/main/resources/benchmark.properties +++ b/java/src/main/resources/benchmark.properties @@ -5,8 +5,9 @@ connect_url=jdbc:voltdb://172.24.4.55:21212 thread_num=2 zk_cluster=172.24.4.55:12200 zk_root_path=/rtidb_scan +redis_addr=172.24.4.40:6001,172.24.4.40:6002,172.24.4.40:6003 base_key=10000000000 pk_cnt=1000000 ts_cnt=10 need_create=false -method=voltdb \ No newline at end of file +method=redis \ No newline at end of file From d74c5889223abf059ce677371862ac4d20259780 Mon Sep 17 00:00:00 2001 From: dl239 Date: Mon, 19 Jul 2021 11:16:42 +0800 Subject: [PATCH 9/9] feat: update benchmark --- java/pom.xml | 11 +++- .../_4paradigm/benchmark/memory/FedbTest.java | 10 ++-- .../benchmark/memory/MemoryBenchmark.java | 2 + .../benchmark/memory/RedisTest.java | 56 +++++++++++++++++++ java/src/main/resources/benchmark.properties | 3 +- 5 files changed, 73 insertions(+), 9 deletions(-) create mode 100644 java/src/main/java/com/_4paradigm/benchmark/memory/RedisTest.java diff --git a/java/pom.xml b/java/pom.xml index 68e710e..0e34eee 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -24,9 +24,14 @@ 10.1.1 - com._4paradigm.sql - sql-jdbc - 2.1.0.0-SNAPSHOT + com.4paradigm.hybridsql + fedb-jdbc + 2.2.2 + + + redis.clients + jedis + 3.6.0 junit diff --git a/java/src/main/java/com/_4paradigm/benchmark/memory/FedbTest.java b/java/src/main/java/com/_4paradigm/benchmark/memory/FedbTest.java index 9b18ebf..2924110 100644 --- a/java/src/main/java/com/_4paradigm/benchmark/memory/FedbTest.java +++ b/java/src/main/java/com/_4paradigm/benchmark/memory/FedbTest.java @@ -1,10 +1,10 @@ package com._4paradigm.benchmark.memory; -import com._4paradigm.sql.sdk.SqlExecutor; -import com._4paradigm.sql.sdk.SdkOption; -import com._4paradigm.sql.sdk.impl.SqlClusterExecutor; -import com._4paradigm.sql.SQLInsertRow; -import com._4paradigm.sql.SQLInsertRows; +import com._4paradigm.hybridsql.fedb.sdk.SqlExecutor; +import com._4paradigm.hybridsql.fedb.sdk.SdkOption; +import com._4paradigm.hybridsql.fedb.sdk.impl.SqlClusterExecutor; +import com._4paradigm.hybridsql.fedb.SQLInsertRow; +import com._4paradigm.hybridsql.fedb.SQLInsertRows; import java.util.Properties; import java.util.Random; diff --git a/java/src/main/java/com/_4paradigm/benchmark/memory/MemoryBenchmark.java b/java/src/main/java/com/_4paradigm/benchmark/memory/MemoryBenchmark.java index d1f9aaa..018f925 100644 --- a/java/src/main/java/com/_4paradigm/benchmark/memory/MemoryBenchmark.java +++ b/java/src/main/java/com/_4paradigm/benchmark/memory/MemoryBenchmark.java @@ -21,6 +21,8 @@ public void run() { Test tmp = null; if (method.equals("rtidb")) { tmp = new FedbTest(); + } else if (method.equals("redis")) { + tmp = new RedisTest(); } else { tmp = new JDBCSqlTest(); } diff --git a/java/src/main/java/com/_4paradigm/benchmark/memory/RedisTest.java b/java/src/main/java/com/_4paradigm/benchmark/memory/RedisTest.java new file mode 100644 index 0000000..8a60858 --- /dev/null +++ b/java/src/main/java/com/_4paradigm/benchmark/memory/RedisTest.java @@ -0,0 +1,56 @@ +package com._4paradigm.benchmark.memory; +import redis.clients.jedis.*; + +import java.util.Properties; +import java.util.Random; +import java.util.Set; +import java.util.LinkedHashSet; +import java.util.concurrent.atomic.AtomicLong; + +public class RedisTest implements Test{ + private JedisPoolConfig poolConfig; + private JedisCluster cluster; + static private String endpoints; + + private static String baseKey; + private long base; + private AtomicLong deta = new AtomicLong(0); + private Random random = new Random(System.currentTimeMillis()); + static { + try { + Properties prop = new Properties(); + prop.load(FedbTest.class.getClassLoader().getResourceAsStream("benchmark.properties")); + endpoints = prop.getProperty("redis_addr"); + baseKey = prop.getProperty("base_key"); + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Override + public boolean init() { + poolConfig = new JedisPoolConfig(); + poolConfig.setMaxTotal(128); + Set nodes = new LinkedHashSet(); + + for (String endpoint : endpoints.trim().split(",")) { + String[] arr = endpoint.split(":"); + if (arr.length != 2) { + continue; + } + System.out.println("add endpoint " + endpoint); + nodes.add(new HostAndPort(arr[0], Integer.valueOf(arr[1]))); + } + base = Long.valueOf(baseKey); + cluster = new JedisCluster(nodes, poolConfig); + return true; + } + + @Override + public void put() { + while (true) { + String key = baseKey + (base + deta.getAndIncrement()); + cluster.set(key, key); + } + } +} diff --git a/java/src/main/resources/benchmark.properties b/java/src/main/resources/benchmark.properties index 051e879..5438953 100644 --- a/java/src/main/resources/benchmark.properties +++ b/java/src/main/resources/benchmark.properties @@ -5,8 +5,9 @@ connect_url=jdbc:voltdb://172.24.4.55:21212 thread_num=2 zk_cluster=172.24.4.55:12200 zk_root_path=/rtidb_scan +redis_addr=172.24.4.40:6001,172.24.4.40:6002,172.24.4.40:6003 base_key=10000000000 pk_cnt=1000000 ts_cnt=10 need_create=false -method=voltdb \ No newline at end of file +method=redis \ No newline at end of file