Add async support (serum-community#82)

* add async utils and connection

* refactor market.py before adding async

* add async open orders account

* add async_market

* add type hint

* replace pytest-tornasync with pytest-asyncio

* add async tests

* add async tests

* linting

* add async exmplae to README

* fix unit test selection

* bump minor version number

* chmod

* fix keygen error when key already exists

* use --cov-append

* fix coverage for multi test

* fix typo

Co-authored-by: kevinheavey <[email protected]>

.github/workflows/integration.yml

@@ -35,3 +35,6 @@ jobs:
 
       - name: Run integration tests
         run: scripts/run_int_tests.sh
+
+      - name: Run async integration tests
+        run: scripts/run_async_int_tests.sh

.github/workflows/main.yml

@@ -42,7 +42,7 @@ jobs:
 
       - name: Run unit tests
         run: |
-          pipenv run pytest -v -m "not integration"
+          pipenv run pytest -v -m "not integration and not async_integration"
 
   coverage:
     # The type of runner that the job will run on

.pylintrc

@@ -141,7 +141,8 @@ disable=missing-class-docstring,
         xreadlines-attribute,
         deprecated-sys-function,
         exception-escape,
-        comprehension-escape
+        comprehension-escape,
+        duplicate-code
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

Makefile

@@ -25,11 +25,14 @@ test-publish:
 	pipenv run twine upload -r testpypi -u serum-community dist/*
 
 unit-tests:
-	pipenv run pytest -v -m "not integration"
+	pipenv run pytest -v -m "not integration and not async_integration"
 
 int-tests:
 	bash scripts/run_int_tests.sh
 
+async-int-tests:
+	bash scripts/run_async_int_tests.sh
+
 # Minimal makefile for Sphinx documentation
 #
 

Pipfile

@@ -14,7 +14,6 @@ jupyterlab = "*"
 black = "*"
 pytest = "*"
 pylint = "*"
-pytest-tornasync = "*"
 mypy = "*"
 pydocstyle = "*"
 flake8 = "*"
@@ -25,6 +24,8 @@ twine = "*"
 setuptools = "*"
 sphinx = "*"
 sphinxemoji = "*"
+pytest-asyncio = "*"
+types-requests = "*"
 
 [packages]
 solana = {version = ">=0.11.3"}

README.md

@@ -50,8 +50,37 @@ print("\n")
 print("Bid Orders:")
 bids = market.load_bids()
 for bid in bids:
-    print("Order id: %d, price: %f, size: %f." % (
-          bid.order_id, bid.info.price, bid.info.size))
+    print(f"Order id: {bid.order_id}, price: {bid.info.price}, size: {bid.info.size}.")
+```
+
+### Get Orderbook (Async)
+
+```python
+import asyncio
+from pyserum.async_connection import async_conn
+from pyserum.market import AsyncMarket
+
+
+async def main():
+    market_address = "5LgJphS6D5zXwUVPU7eCryDBkyta3AidrJ5vjNU6BcGW"  # Address for BTC/USDC
+    async with async_conn("https://api.mainnet-beta.solana.com/") as cc:
+        # Load the given market
+        market = await AsyncMarket.load(cc, market_address)
+        asks = await market.load_asks()
+        # Show all current ask order
+        print("Ask Orders:")
+        for ask in asks:
+            print(f"Order id: {ask.order_id}, price: {ask.info.price}, size: {ask.info.size}.")
+        print("\n")
+        # Show all current bid order
+        print("Bid Orders:")
+        bids = await market.load_bids()
+        for bid in bids:
+            print(f"Order id: {bid.order_id}, price: {bid.info.price}, size: {bid.info.size}.")
+
+
+asyncio.run(main())
+
 ```
 
 ### Support

pyserum/async_connection.py

@@ -0,0 +1,16 @@
+from typing import List
+import httpx
+from solana.rpc.async_api import AsyncClient as async_conn  # pylint: disable=unused-import # noqa:F401
+
+from .market.types import MarketInfo, TokenInfo
+from .connection import LIVE_MARKETS_URL, TOKEN_MINTS_URL, parse_live_markets, parse_token_mints
+
+
+async def get_live_markets(httpx_client: httpx.AsyncClient) -> List[MarketInfo]:
+    resp = await httpx_client.get(LIVE_MARKETS_URL)
+    return parse_live_markets(resp.json())
+
+
+async def get_token_mints(httpx_client: httpx.AsyncClient) -> List[TokenInfo]:
+    resp = await httpx_client.get(TOKEN_MINTS_URL)
+    return parse_token_mints(resp.json())

pyserum/async_open_orders_account.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from typing import List
+from solana.rpc.async_api import AsyncClient
+from solana.publickey import PublicKey
+from solana.rpc.types import Commitment
+from solana.rpc.commitment import Recent
+
+from .async_utils import load_bytes_data
+from .open_orders_account import _OpenOrdersAccountCore
+
+
+class AsyncOpenOrdersAccount(_OpenOrdersAccountCore):
+    @classmethod
+    async def find_for_market_and_owner(  # pylint: disable=too-many-arguments
+        cls,
+        conn: AsyncClient,
+        market: PublicKey,
+        owner: PublicKey,
+        program_id: PublicKey,
+        commitment: Commitment = Recent,
+    ) -> List[AsyncOpenOrdersAccount]:
+        args = cls._build_get_program_accounts_args(
+            market=market, program_id=program_id, owner=owner, commitment=commitment
+        )
+        resp = await conn.get_program_accounts(*args)
+        return cls._process_get_program_accounts_resp(resp)
+
+    @classmethod
+    async def load(cls, conn: AsyncClient, address: str) -> AsyncOpenOrdersAccount:
+        addr_pub_key = PublicKey(address)
+        bytes_data = await load_bytes_data(addr_pub_key, conn)
+        return cls.from_bytes(addr_pub_key, bytes_data)

pyserum/async_utils.py

@@ -0,0 +1,19 @@
+from solana.publickey import PublicKey
+from solana.rpc.async_api import AsyncClient
+from spl.token.constants import WRAPPED_SOL_MINT
+
+from pyserum.utils import parse_bytes_data, parse_mint_decimals
+
+
+async def load_bytes_data(addr: PublicKey, conn: AsyncClient) -> bytes:
+    res = await conn.get_account_info(addr)
+    return parse_bytes_data(res)
+
+
+async def get_mint_decimals(conn: AsyncClient, mint_pub_key: PublicKey) -> int:
+    """Get the mint decimals for a token mint"""
+    if mint_pub_key == WRAPPED_SOL_MINT:
+        return 9
+
+    bytes_data = await load_bytes_data(mint_pub_key, conn)
+    return parse_mint_decimals(bytes_data)

pyserum/connection.py

@@ -1,20 +1,28 @@
-from typing import List
+from typing import List, Dict, Any
 
-from solana.rpc.api import Client as conn  # pylint: disable=unused-import # noqa:F401
-from solana.rpc.providers.http import requests
+import requests
 
+from solana.rpc.api import Client as conn  # pylint: disable=unused-import # noqa:F401
+from solana.publickey import PublicKey
 from .market.types import MarketInfo, TokenInfo
 
+LIVE_MARKETS_URL = "https://raw.githubusercontent.com/project-serum/serum-ts/master/packages/serum/src/markets.json"
+TOKEN_MINTS_URL = "https://raw.githubusercontent.com/project-serum/serum-ts/master/packages/serum/src/token-mints.json"
 
-def get_live_markets() -> List[MarketInfo]:
-    url = "https://raw.githubusercontent.com/project-serum/serum-ts/master/packages/serum/src/markets.json"
+
+def parse_live_markets(data: List[Dict[str, Any]]) -> List[MarketInfo]:
     return [
-        MarketInfo(name=m["name"], address=m["address"], program_id=m["programId"])
-        for m in requests.get(url).json()
-        if not m["deprecated"]
+        MarketInfo(name=m["name"], address=m["address"], program_id=m["programId"]) for m in data if not m["deprecated"]
     ]
 
 
+def parse_token_mints(data: List[Dict[str, str]]) -> List[TokenInfo]:
+    return [TokenInfo(name=t["name"], address=PublicKey(t["address"])) for t in data]
+
+
+def get_live_markets() -> List[MarketInfo]:
+    return parse_live_markets(requests.get(LIVE_MARKETS_URL).json())
+
+
 def get_token_mints() -> List[TokenInfo]:
-    url = "https://raw.githubusercontent.com/project-serum/serum-ts/master/packages/serum/src/token-mints.json"
-    return [TokenInfo(**t) for t in requests.get(url).json()]
+    return parse_token_mints(requests.get(TOKEN_MINTS_URL).json())

pyserum/market/__init__.py

@@ -1,3 +1,4 @@
 from .market import Market  # noqa: F401
+from .async_market import AsyncMarket  # noqa: F401
 from .orderbook import OrderBook  # noqa: F401
 from .state import MarketState as State  # noqa: F401