Skip to content

Commit

Permalink
Fallback for old sqlites, reqs for plugins
Browse files Browse the repository at this point in the history
  • Loading branch information
@EggPool
EggPool committed Nov 12, 2019
1 parent d1a8f30 commit 9e8bfa2
Show file tree
Hide file tree
Showing 6 changed files with 277 additions and 57 deletions.
35 changes: 26 additions & 9 deletions apihandler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -700,9 +700,14 @@ def api_gettransaction(self, socket_handler, db_handler, peers):
# and format
format = connections.receive(socket_handler)
# raw tx details
db_handler.execute_param(db_handler.h,
"SELECT * FROM transactions WHERE substr(signature,1,4)=substr(?1,1,4) and signature like ?1",
(transaction_id+'%',))
if self.config.old_sqlite:
db_handler.execute_param(db_handler.h,
"SELECT * FROM transactions WHERE signature like ?1",
(transaction_id + '%',))
else:
db_handler.execute_param(db_handler.h,
"SELECT * FROM transactions WHERE substr(signature,1,4)=substr(?1,1,4) and signature like ?1",
(transaction_id+'%',))
raw = db_handler.h.fetchone()
if not format:
connections.send(socket_handler, raw)
Expand Down Expand Up @@ -757,9 +762,14 @@ def api_gettransactionbysignature(self, socket_handler, db_handler, peers):
# and format
format = connections.receive(socket_handler)
# raw tx details
db_handler.execute_param(db_handler.h,
"SELECT * FROM transactions WHERE substr(signature,1,4)=substr(?1,1,4) and signature = ?1",
(signature,))
if self.config.old_sqlite:
db_handler.execute_param(db_handler.h,
"SELECT * FROM transactions WHERE signature = ?1",
(signature,))
else:
db_handler.execute_param(db_handler.h,
"SELECT * FROM transactions WHERE substr(signature,1,4)=substr(?1,1,4) and signature = ?1",
(signature,))
raw = db_handler.h.fetchone()
if not format:
connections.send(socket_handler, raw)
Expand Down Expand Up @@ -836,9 +846,16 @@ def api_gettransaction_for_recipients(self, socket_handler, db_handler, peers):
format = connections.receive(socket_handler)
recipients = json.dumps(addresses).replace("[", "(").replace(']', ')') # format as sql
# raw tx details
db_handler.execute_param(db_handler.h,
"SELECT * FROM transactions WHERE recipient IN {} AND substr(signature,1,4)=substr(?1,1,4) and signature LIKE ?1".format(recipients),
(transaction_id + '%', ))
if self.config.old_sqlite:
db_handler.execute_param(db_handler.h,
"SELECT * FROM transactions WHERE recipient IN {} AND signature LIKE ?1".format(recipients),
(transaction_id + '%', ))
else:
db_handler.execute_param(db_handler.h,
"SELECT * FROM transactions WHERE recipient IN {} AND substr(signature,1,4)=substr(?1,1,4) and signature LIKE ?1".format(
recipients),
(transaction_id + '%',))

raw = db_handler.h.fetchone()
if not format:
connections.send(socket_handler, raw)
Expand Down
19 changes: 14 additions & 5 deletions digest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,17 +108,26 @@ def check_signature(block):
if entry_signature: # prevent empty signature database retry hack
signature_list.append(entry_signature)
# reject block with transactions which are already in the ledger ram
if node.old_sqlite:
db_handler.execute_param(db_handler.h, "SELECT block_height FROM transactions WHERE signature = ?1;",
(entry_signature,))
else:
db_handler.execute_param(db_handler.h,
"SELECT block_height FROM transactions WHERE substr(signature,1,4) = substr(?1,1,4) and signature = ?1;",
(entry_signature,))

db_handler.execute_param(db_handler.h, "SELECT block_height FROM transactions WHERE substr(signature,1,4) = substr(?1,1,4) and signature = ?1;",
(entry_signature,))
tx_presence_check = db_handler.h.fetchone()
if tx_presence_check:
# print(node.last_block)
raise ValueError(f"That transaction {entry_signature[:10]} is already in our ledger, "
f"block_height {tx_presence_check[0]}")

db_handler.execute_param(db_handler.c, "SELECT block_height FROM transactions WHERE substr(signature,1,4) = substr(?1,1,4) and signature = ?1;",
(entry_signature,))
if node.old_sqlite:
db_handler.execute_param(db_handler.c, "SELECT block_height FROM transactions WHERE signature = ?1;",
(entry_signature,))
else:
db_handler.execute_param(db_handler.c,
"SELECT block_height FROM transactions WHERE substr(signature,1,4) = substr(?1,1,4) and signature = ?1;",
(entry_signature,))
tx_presence_check = db_handler.c.fetchone()
if tx_presence_check:
# print(node.last_block)
Expand Down
26 changes: 21 additions & 5 deletions mempool.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,11 @@

# Check for presence of a given tx signature
SQL_SIG_CHECK = 'SELECT timestamp FROM transactions WHERE substr(signature,1,4) = substr(?1,1,4) and signature = ?1'
SQL_SIG_CHECK_OLD = 'SELECT timestamp FROM transactions WHERE signature = ?1'

# delete a single tx
SQL_DELETE_TX = 'DELETE FROM transactions WHERE substr(signature,1,4) = substr(?1,1,4) and signature = ?1'
SQL_DELETE_TX_OLD = 'DELETE FROM transactions WHERE signature = ?1'

# Selects all tx from mempool - list fields so we don't send mergedts and keep compatibility
SQL_SELECT_ALL_TXS = 'SELECT timestamp, address, recipient, amount, signature, public_key, operation, openfield FROM transactions'
Expand Down Expand Up @@ -289,7 +291,10 @@ def delete_transaction(self, signature):
:return:
"""
with self.lock:
self.execute(SQL_DELETE_TX, (signature,))
if self.config.old_sqlite:
self.execute(SQL_DELETE_TX_OLD, (signature,))
else:
self.execute(SQL_DELETE_TX, (signature,))
self.commit()

def sig_check(self, signature):
Expand All @@ -298,7 +303,10 @@ def sig_check(self, signature):
:param signature:
:return: boolean
"""
return bool(self.fetchone(SQL_SIG_CHECK, (signature,)))
if self.config.old_sqlite:
return bool(self.fetchone(SQL_SIG_CHECK_OLD, (signature,)))
else:
return bool(self.fetchone(SQL_SIG_CHECK, (signature,)))

def status(self):
"""
Expand Down Expand Up @@ -578,14 +586,22 @@ def merge(self, data: list, peer_ip: str, c, size_bypass: bool=False, wait: bool
# reject transactions which are already in the ledger
# TODO: not clean, will need to have ledger as a module too.
# TODO: need better txid index, this is very sloooooooow
essentials.execute_param_c(c, "SELECT timestamp FROM transactions WHERE substr(signature,1,4) = substr(?1,1,4) AND signature = ?1",
(mempool_signature_enc,), self.app_log)
if self.config.old_sqlite:
essentials.execute_param_c(c, "SELECT timestamp FROM transactions WHERE signature = ?1",
(mempool_signature_enc,), self.app_log)
else:
essentials.execute_param_c(c,
"SELECT timestamp FROM transactions WHERE substr(signature,1,4) = substr(?1,1,4) AND signature = ?1",
(mempool_signature_enc,), self.app_log)
ledger_in = bool(c.fetchone())
# remove from mempool if it's in both ledger and mempool already
if mempool_in and ledger_in:
try:
# Do not lock, we already have the lock for the whole merge.
self.execute(SQL_DELETE_TX, (mempool_signature_enc,))
if self.config.old_sqlite:
self.execute(SQL_DELETE_TX_OLD, (mempool_signature_enc,))
else:
self.execute(SQL_DELETE_TX, (mempool_signature_enc,))
self.commit()
mempool_result.append("Mempool: Transaction deleted from our mempool")
except: # experimental try and except
Expand Down
13 changes: 10 additions & 3 deletions node.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1974,15 +1974,20 @@ def add_indices(db_handler: dbhandler.DbHandler):
node.logger.app_log.warning("Creating indices")

# ledger.db
db_handler.execute(db_handler.h, CREATE_TXID4_INDEX_IF_NOT_EXISTS)
if not node.old_sqlite:
db_handler.execute(db_handler.h, CREATE_TXID4_INDEX_IF_NOT_EXISTS)
else:
node.logger.app_log.warning("Setting old_sqlite is True, lookups will be slower.")
db_handler.execute(db_handler.h, CREATE_MISC_BLOCK_HEIGHT_INDEX_IF_NOT_EXISTS)

# hyper.db
db_handler.execute(db_handler.h2, CREATE_TXID4_INDEX_IF_NOT_EXISTS)
if not node.old_sqlite:
db_handler.execute(db_handler.h2, CREATE_TXID4_INDEX_IF_NOT_EXISTS)
db_handler.execute(db_handler.h2, CREATE_MISC_BLOCK_HEIGHT_INDEX_IF_NOT_EXISTS)

# RAM or hyper.db
db_handler.execute(db_handler.c, CREATE_TXID4_INDEX_IF_NOT_EXISTS)
if not node.old_sqlite:
db_handler.execute(db_handler.c, CREATE_TXID4_INDEX_IF_NOT_EXISTS)
db_handler.execute(db_handler.c, CREATE_MISC_BLOCK_HEIGHT_INDEX_IF_NOT_EXISTS)

node.logger.app_log.warning("Finished creating indices")
Expand All @@ -2009,6 +2014,7 @@ def add_indices(db_handler: dbhandler.DbHandler):
# or just do node.config = config
# and use node.config.port... aso

# TODO: Simplify. Just do node.config = config, then use node.config.required_option
node.version = config.version
node.debug_level = config.debug_level
node.port = config.port
Expand All @@ -2032,6 +2038,7 @@ def add_indices(db_handler: dbhandler.DbHandler):
node.full_ledger = config.full_ledger
node.trace_db_calls = config.trace_db_calls
node.heavy3_path = config.heavy3_path
node.old_sqlite = config.old_sqlite

node.logger.app_log = log.log("node.log", node.debug_level, node.terminal_output)
node.logger.app_log.warning("Configuration settings loaded")
Expand Down
75 changes: 40 additions & 35 deletions options.py
View file
Original file line number Diff line number Diff line change
@@ -1,37 +1,38 @@
import json
import os.path as path
from sys import exit
import json


class Get:
# "param_name":["type"] or "param_name"=["type","property_name"]
vars={
"port":["str"],
"verify":["bool","verify"],
"testnet":["bool"],
"regnet":["bool"],
"version":["str","version"],
"version_allow":["list"],
"thread_limit":["int","thread_limit"],
"rebuild_db":["bool","rebuild_db"],
"debug":["bool","debug"],
"purge":["bool","purge"],
"pause":["int","pause"],
"ledger_path":["str","ledger_path"],
"hyper_path":["str","hyper_path"],
"hyper_recompress":["bool","hyper_recompress"],
"full_ledger":["bool","full_ledger"],
"ban_threshold":["int"],
"tor":["bool","tor"],
"debug_level":["str","debug_level"],
"allowed":["str","allowed"],
"ram":["bool","ram"],
"node_ip":["str","node_ip"],
"light_ip":["dict"],
"reveal_address":["bool"],
"accept_peers":["bool"],
"banlist":["list"],
"whitelist":["list"],
"nodes_ban_reset":["int"],
vars = {
"port": ["str"],
"verify": ["bool", "verify"],
"testnet": ["bool"],
"regnet": ["bool"],
"version": ["str", "version"],
"version_allow": ["list"],
"thread_limit": ["int", "thread_limit"],
"rebuild_db": ["bool", "rebuild_db"],
"debug": ["bool", "debug"],
"purge": ["bool", "purge"],
"pause": ["int", "pause"],
"ledger_path": ["str", "ledger_path"],
"hyper_path": ["str", "hyper_path"],
"hyper_recompress": ["bool", "hyper_recompress"],
"full_ledger": ["bool", "full_ledger"],
"ban_threshold": ["int"],
"tor": ["bool", "tor"],
"debug_level": ["str", "debug_level"],
"allowed": ["str", "allowed"],
"ram": ["bool", "ram"],
"node_ip": ["str", "node_ip"],
"light_ip": ["dict"],
"reveal_address": ["bool"],
"accept_peers": ["bool"],
"banlist": ["list"],
"whitelist": ["list"],
"nodes_ban_reset": ["int"],
"mempool_allowed": ["list"],
"terminal_output": ["bool"],
"gui_scaling": ["str"],
Expand All @@ -40,6 +41,7 @@ class Get:
"trace_db_calls": ["bool"],
"heavy3_path": ["str"],
"mempool_path": ["str"],
"old_sqlite": ["bool"],
}

# Optional default values so we don't bug if they are not in the config.
Expand All @@ -51,16 +53,19 @@ class Get:
"mempool_ram": True,
"heavy3_path": "./heavy3a.bin",
"mempool_path": "./mempool.db",
"old_sqlite": False,
}

def load_file(self, filename):
#print("Loading",filename)
# print("Loading",filename)
with open(filename) as fp:
for line in fp:
if '=' in line:
left,right = map(str.strip,line.rstrip("\n").split("="))
if "=" in line:
left, right = map(str.strip, line.rstrip("\n").split("="))
if "mempool_ram_conf" == left:
print("Inconsistent config, param is now mempool_ram in config.txt")
print(
"Inconsistent config, param is now mempool_ram in config.txt"
)
exit()
if not left in self.vars:
# Warn for unknown param?
Expand All @@ -71,7 +76,7 @@ def load_file(self, filename):
elif params[0] == "dict":
try:
right = json.loads(right)
except: #compatibility
except: # compatibility
right = [item.strip() for item in right.split(",")]
elif params[0] == "list":
right = [item.strip() for item in right.split(",")]
Expand All @@ -94,7 +99,7 @@ def load_file(self, filename):
if key not in self.__dict__:
setattr(self, key, default)

#print(self.__dict__)
# print(self.__dict__)

def read(self):
# first of all, load from default config so we have all needed params
Expand Down
Loading

0 comments on commit 9e8bfa2

Please sign in to comment.