diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..691a2a3 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,37 @@ +name: build-and-push-docker-image + +on: + push: + branches: + - 'master' + workflow_dispatch: + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - + name: Build and push + uses: docker/build-push-action@v2 + with: + context: . + push: true + tags: steemit/jussi:latest + - + name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.gitignore b/.gitignore index b2b9fb2..1b55e84 100644 --- a/.gitignore +++ b/.gitignore @@ -129,3 +129,4 @@ Untitled.ipynb build prof/ local/ +.vscode diff --git a/DEV_config.json b/DEV_config.json index 7a2fac4..8e0c77f 100644 --- a/DEV_config.json +++ b/DEV_config.json @@ -2,7 +2,8 @@ "limits": { "blacklist_accounts": [ "non-steemit" - ] + ], + "account_history_limit": 100 }, "upstreams": [ { diff --git a/jussi/errors.py b/jussi/errors.py index 2bd714b..50c40c3 100644 --- a/jussi/errors.py +++ b/jussi/errors.py @@ -295,7 +295,6 @@ def to_dict(self): logger.info('error adding timing data to RequestTimeoutError', e=e) return data - class UpstreamResponseError(JsonRpcError): code = 1100 message = 'Upstream response error' @@ -330,6 +329,9 @@ class JussiLimitsError(JsonRpcError): code = 1700 message = 'Request exceeded limit' +class JussiAccountHistoryLimitsError(JsonRpcError): + code = 1701 + message = 'Account History request exceeded limit. The max limit is 100. Your limit is {your_limit}' class JussiCustomJsonOpLengthError(JsonRpcError): code = 1800 diff --git a/jussi/listeners.py b/jussi/listeners.py index 856fdaa..ebcda5d 100644 --- a/jussi/listeners.py +++ b/jussi/listeners.py @@ -121,7 +121,7 @@ async def setup_limits(app: WebApp, loop) -> None: config_file = args.upstream_config_file with open(config_file) as f: config = json.load(f) - app.config.limits = config.get('limits', {'accounts_blacklist': set()}) + app.config.limits = config.get('limits', {'accounts_blacklist': set(), 'account_history_limit': 100}) app.config.jsonrpc_batch_size_limit = args.jsonrpc_batch_size_limit diff --git a/jussi/middlewares/__init__.py b/jussi/middlewares/__init__.py index 2e18533..4a36458 100644 --- a/jussi/middlewares/__init__.py +++ b/jussi/middlewares/__init__.py @@ -3,6 +3,7 @@ from .jussi import initialize_jussi_request from .jussi import finalize_jussi_response from .limits import check_limits +from .limits import account_history_limit from .caching import get_response from .caching import cache_response from .update_block_num import update_last_irreversible_block_num @@ -20,6 +21,7 @@ def setup_middlewares(app): app.request_middleware.append(init_stats) app.request_middleware.append(check_limits) app.request_middleware.append(get_response) + app.request_middleware.append(account_history_limit) # response middlware app.response_middleware.append(finalize_jussi_response) diff --git a/jussi/middlewares/limits.py b/jussi/middlewares/limits.py index b8903e2..3e2567b 100644 --- a/jussi/middlewares/limits.py +++ b/jussi/middlewares/limits.py @@ -3,9 +3,11 @@ from ..errors import JsonRpcBatchSizeError from ..errors import JsonRpcError +from ..errors import JussiAccountHistoryLimitsError from ..typedefs import HTTPRequest from ..typedefs import HTTPResponse from ..validators import limit_broadcast_transaction_request +from ..validators import limit_account_history_count_request async def check_limits(request: HTTPRequest) -> Optional[HTTPResponse]: @@ -28,3 +30,26 @@ async def check_limits(request: HTTPRequest) -> Optional[HTTPResponse]: except Exception as e: return JsonRpcError(http_request=request, exception=e).to_sanic_response() + +# This is a temporary way to improve the ahnode backend perform +async def account_history_limit(request: HTTPRequest) -> Optional[HTTPResponse]: + # pylint: disable=no-member + if 'account_history_limit' in request.app.config.limits: + limits = request.app.config.limits['account_history_limit'] + else: + limits = 100 + try: + if request.is_single_jrpc: + limit_account_history_count_request(request.jsonrpc, + limits=limits) + elif request.is_batch_jrpc: + _ = [limit_account_history_count_request(r, limits=limits) + for r in request.jsonrpc + ] + except JussiAccountHistoryLimitsError as e: + e.add_http_request(http_request=request) + return e.to_sanic_response() + except Exception as e: + return JsonRpcError(http_request=request, + exception=e).to_sanic_response() + diff --git a/jussi/upstream.py b/jussi/upstream.py index e25ff58..859ebc6 100644 --- a/jussi/upstream.py +++ b/jussi/upstream.py @@ -114,7 +114,7 @@ def ttl(self, request_urn) -> int: @functools.lru_cache(8192) def timeout(self, request_urn) -> int: _, timeout = self.__TIMEOUTS.longest_prefix(str(request_urn)) - if timeout is 0: + if timeout == 0: timeout = None return timeout diff --git a/jussi/validators.py b/jussi/validators.py index 9e5c086..03a30ff 100644 --- a/jussi/validators.py +++ b/jussi/validators.py @@ -9,6 +9,7 @@ #from .errors import InvalidRequest from .errors import JussiCustomJsonOpLengthError from .errors import JussiLimitsError +from .errors import JussiAccountHistoryLimitsError from .typedefs import JrpcRequest from .typedefs import JrpcResponse from .typedefs import RawRequest @@ -197,6 +198,14 @@ def limit_broadcast_transaction_request(request: JSONRPCRequest, limits=None) -> limit_custom_json_op_length(ops, size_limit=CUSTOM_JSON_SIZE_LIMIT) limit_custom_json_account(ops, blacklist_accounts=blacklist_accounts) +def limit_account_history_count_request(request: JSONRPCRequest, limits=100) -> NoReturn: + if request.urn.method == 'get_account_history': + if isinstance(request.urn.params, list): + limit_count = request.urn.params[2] + elif isinstance(request.urn.params, dict): + limit_count = request.urn.params['limit'] + if limit_count > limits: + raise JussiAccountHistoryLimitsError(your_limit=limit_count) def limit_custom_json_op_length(ops: list, size_limit=None): if any(len(op[1]['json'].encode('utf-8')) > size_limit for op in ops):