Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

decrease limit of get_account_history #235

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
decrease limit of get_account_history(not pass the unittest)
ety001 committed Jul 6, 2022
commit 94e3ef24bc0d21d35cad352a763937234853daff
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -129,3 +129,4 @@ Untitled.ipynb
build
prof/
local/
.vscode
3 changes: 2 additions & 1 deletion DEV_config.json
Original file line number Diff line number Diff line change
@@ -2,7 +2,8 @@
"limits": {
"blacklist_accounts": [
"non-steemit"
]
],
"account_history_limit": 100
},
"upstreams": [
{
4 changes: 3 additions & 1 deletion jussi/errors.py
Original file line number Diff line number Diff line change
@@ -295,7 +295,6 @@ def to_dict(self):
logger.info('error adding timing data to RequestTimeoutError', e=e)
return data


class UpstreamResponseError(JsonRpcError):
code = 1100
message = 'Upstream response error'
@@ -330,6 +329,9 @@ class JussiLimitsError(JsonRpcError):
code = 1700
message = 'Request exceeded limit'

class JussiAccountHistoryLimitsError(JsonRpcError):
code = 1701
message = 'Account History request exceeded limit. The max limit is 100. Your limit is {your_limit}'

class JussiCustomJsonOpLengthError(JsonRpcError):
code = 1800
2 changes: 1 addition & 1 deletion jussi/listeners.py
Original file line number Diff line number Diff line change
@@ -121,7 +121,7 @@ async def setup_limits(app: WebApp, loop) -> None:
config_file = args.upstream_config_file
with open(config_file) as f:
config = json.load(f)
app.config.limits = config.get('limits', {'accounts_blacklist': set()})
app.config.limits = config.get('limits', {'accounts_blacklist': set(), 'account_history_limit': 100})

app.config.jsonrpc_batch_size_limit = args.jsonrpc_batch_size_limit

2 changes: 2 additions & 0 deletions jussi/middlewares/__init__.py
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@
from .jussi import initialize_jussi_request
from .jussi import finalize_jussi_response
from .limits import check_limits
from .limits import account_history_limit
from .caching import get_response
from .caching import cache_response
from .update_block_num import update_last_irreversible_block_num
@@ -20,6 +21,7 @@ def setup_middlewares(app):
app.request_middleware.append(init_stats)
app.request_middleware.append(check_limits)
app.request_middleware.append(get_response)
app.request_middleware.append(account_history_limit)

# response middlware
app.response_middleware.append(finalize_jussi_response)
26 changes: 26 additions & 0 deletions jussi/middlewares/limits.py
Original file line number Diff line number Diff line change
@@ -3,9 +3,11 @@

from ..errors import JsonRpcBatchSizeError
from ..errors import JsonRpcError
from ..errors import JussiAccountHistoryLimitsError
from ..typedefs import HTTPRequest
from ..typedefs import HTTPResponse
from ..validators import limit_broadcast_transaction_request
from ..validators import limit_account_history_count_request


async def check_limits(request: HTTPRequest) -> Optional[HTTPResponse]:
@@ -28,3 +30,27 @@ async def check_limits(request: HTTPRequest) -> Optional[HTTPResponse]:
except Exception as e:
return JsonRpcError(http_request=request,
exception=e).to_sanic_response()

# This is a temporary way to improve the ahnode backend perform
async def account_history_limit(request: HTTPRequest) -> Optional[HTTPResponse]:
# pylint: disable=no-member
if request.app.config.limits['account_history_limit']:
limits = request.app.config.limits['account_history_limit']
else:
limits = 100
try:
print(request.is_single_jrpc)
if request.is_single_jrpc:
limit_account_history_count_request(request.jsonrpc,
limits=limits)
elif request.is_batch_jrpc:
_ = [limit_account_history_count_request(r, limits=limits)
for r in request.jsonrpc
]
except JussiAccountHistoryLimitsError as e:
e.add_http_request(http_request=request)
return e.to_sanic_response()
except Exception as e:
return JsonRpcError(http_request=request,
exception=e).to_sanic_response()

2 changes: 1 addition & 1 deletion jussi/upstream.py
Original file line number Diff line number Diff line change
@@ -114,7 +114,7 @@ def ttl(self, request_urn) -> int:
@functools.lru_cache(8192)
def timeout(self, request_urn) -> int:
_, timeout = self.__TIMEOUTS.longest_prefix(str(request_urn))
if timeout is 0:
if timeout == 0:
timeout = None
return timeout

9 changes: 9 additions & 0 deletions jussi/validators.py
Original file line number Diff line number Diff line change
@@ -9,6 +9,7 @@
#from .errors import InvalidRequest
from .errors import JussiCustomJsonOpLengthError
from .errors import JussiLimitsError
from .errors import JussiAccountHistoryLimitsError
from .typedefs import JrpcRequest
from .typedefs import JrpcResponse
from .typedefs import RawRequest
@@ -197,6 +198,14 @@ def limit_broadcast_transaction_request(request: JSONRPCRequest, limits=None) ->
limit_custom_json_op_length(ops, size_limit=CUSTOM_JSON_SIZE_LIMIT)
limit_custom_json_account(ops, blacklist_accounts=blacklist_accounts)

def limit_account_history_count_request(request: JSONRPCRequest, limits=100) -> NoReturn:
if request.urn.method == 'get_account_history':
if isinstance(request.urn.params, list):
limit_count = request.urn.params[2]
elif isinstance(request.urn.params, dict):
limit_count = request.urn.params['limit']
if limit_count > limits:
raise JussiAccountHistoryLimitsError(your_limit=limit_count)

def limit_custom_json_op_length(ops: list, size_limit=None):
if any(len(op[1]['json'].encode('utf-8')) > size_limit for op in ops):