From da156fa1394498b0fc56d0e8fd673b972481be1d Mon Sep 17 00:00:00 2001 From: Alexey Shamrin Date: Mon, 28 Dec 2020 14:04:40 +0200 Subject: [PATCH 1/2] strict typecheck db module, basic typecheck for the rest --- .gitignore | 1 + README.md | 2 + lint | 2 + package-lock.json | 12 +++ package.json | 10 ++ poetry.lock | 52 ++++++++++- pyproject.toml | 1 + pyrightconfig.json | 8 ++ sitewatch/db.py | 19 ++-- sitewatch/model.py | 12 +-- typings/triopg/__init__.pyi | 10 ++ typings/triopg/_triopg.pyi | 166 ++++++++++++++++++++++++++++++++++ typings/triopg/exceptions.pyi | 9 ++ 13 files changed, 289 insertions(+), 15 deletions(-) create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 pyrightconfig.json create mode 100644 typings/triopg/__init__.pyi create mode 100644 typings/triopg/_triopg.pyi create mode 100644 typings/triopg/exceptions.pyi diff --git a/.gitignore b/.gitignore index cc25207..0c9b6e9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ __pycache__ /.env +/node_modules diff --git a/README.md b/README.md index d6b499f..53273e0 100644 --- a/README.md +++ b/README.md @@ -26,11 +26,13 @@ Prerequisites: * [Google Cloud SDK](https://cloud.google.com/sdk/docs/install) * [jq](https://stedolan.github.io/jq/) * Aiven project with Kafka and Postgres running +* npm Setup: ``` poetry env use python3.9 poetry install +npm install ``` Load Aiven credentials and service URIs to `.env/` (replace `pg-123456` and `kafka-123456` with correct service names in Aiven): diff --git a/lint b/lint index 8b578ce..e042cd6 100755 --- a/lint +++ b/lint @@ -3,6 +3,8 @@ set -eux SOURCES=(sitewatch tests) +npm run typecheck + # stop the build if there are Python syntax errors or undefined names poetry run flake8 sitewatch --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..644b07b --- /dev/null +++ b/package-lock.json @@ -0,0 +1,12 @@ +{ + "name": "sitewatch", + "requires": true, + "lockfileVersion": 1, + "dependencies": { + "pyright": { + "version": "1.1.97", + "resolved": "https://registry.npmjs.org/pyright/-/pyright-1.1.97.tgz", + "integrity": "sha512-6k+Soj2KlsBpnWMhP14PEJU2ozxUuJaiNG2hAySeOdAc5y4CuPVJ5F2InySFXdNIHGzgPGkA8/v33yh8OvVWbw==" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..b6f0023 --- /dev/null +++ b/package.json @@ -0,0 +1,10 @@ +{ + "name": "sitewatch", + "scripts": { + "typecheck": "pyright", + "typecheck-watch": "pyright -w" + }, + "dependencies": { + "pyright": "^1.1.97" + } +} diff --git a/poetry.lock b/poetry.lock index b75ca1f..f771aaf 100644 --- a/poetry.lock +++ b/poetry.lock @@ -239,6 +239,22 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "mypy" +version = "0.790" +description = "Optional static typing for Python" +category = "dev" +optional = false +python-versions = ">=3.5" + +[package.dependencies] +mypy-extensions = ">=0.4.3,<0.5.0" +typed-ast = ">=1.4.0,<1.5.0" +typing-extensions = ">=3.7.4" + +[package.extras] +dmypy = ["psutil (>=4.0)"] + [[package]] name = "mypy-extensions" version = "0.4.3" @@ -491,6 +507,20 @@ trio = ">=0.12.0" outcome = "*" async-generator = ">=1.6" +[[package]] +name = "trio-typing" +version = "0.5.0" +description = "Static type checking support for Trio and related projects" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +mypy-extensions = ">=0.4.2" +trio = ">=0.16.0" +mypy = {version = ">=0.780", markers = "implementation_name == \"cpython\""} +typing-extensions = ">=3.7.4" + [[package]] name = "triopg" version = "0.5.0" @@ -552,7 +582,7 @@ python-versions = "*" [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "b5f258a254062e1faf4a6e2e3a4e04cf2b04cc70a7263803de23710a8cb3d93d" +content-hash = "2afdaf069bfdb70e602d5daf243cd1292e80b786fed0ec32a9b714ce78ebd530" [metadata.files] aiokafka = [ @@ -724,6 +754,22 @@ mccabe = [ {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, ] +mypy = [ + {file = "mypy-0.790-cp35-cp35m-macosx_10_6_x86_64.whl", hash = "sha256:bd03b3cf666bff8d710d633d1c56ab7facbdc204d567715cb3b9f85c6e94f669"}, + {file = "mypy-0.790-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:2170492030f6faa537647d29945786d297e4862765f0b4ac5930ff62e300d802"}, + {file = "mypy-0.790-cp35-cp35m-win_amd64.whl", hash = "sha256:e86bdace26c5fe9cf8cb735e7cedfe7850ad92b327ac5d797c656717d2ca66de"}, + {file = "mypy-0.790-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e97e9c13d67fbe524be17e4d8025d51a7dca38f90de2e462243ab8ed8a9178d1"}, + {file = "mypy-0.790-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:0d34d6b122597d48a36d6c59e35341f410d4abfa771d96d04ae2c468dd201abc"}, + {file = "mypy-0.790-cp36-cp36m-win_amd64.whl", hash = "sha256:72060bf64f290fb629bd4a67c707a66fd88ca26e413a91384b18db3876e57ed7"}, + {file = "mypy-0.790-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:eea260feb1830a627fb526d22fbb426b750d9f5a47b624e8d5e7e004359b219c"}, + {file = "mypy-0.790-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:c614194e01c85bb2e551c421397e49afb2872c88b5830e3554f0519f9fb1c178"}, + {file = "mypy-0.790-cp37-cp37m-win_amd64.whl", hash = "sha256:0a0d102247c16ce93c97066443d11e2d36e6cc2a32d8ccc1f705268970479324"}, + {file = "mypy-0.790-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cf4e7bf7f1214826cf7333627cb2547c0db7e3078723227820d0a2490f117a01"}, + {file = "mypy-0.790-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:af4e9ff1834e565f1baa74ccf7ae2564ae38c8df2a85b057af1dbbc958eb6666"}, + {file = "mypy-0.790-cp38-cp38-win_amd64.whl", hash = "sha256:da56dedcd7cd502ccd3c5dddc656cb36113dd793ad466e894574125945653cea"}, + {file = "mypy-0.790-py3-none-any.whl", hash = "sha256:2842d4fbd1b12ab422346376aad03ff5d0805b706102e475e962370f874a5122"}, + {file = "mypy-0.790.tar.gz", hash = "sha256:2b21ba45ad9ef2e2eb88ce4aeadd0112d0f5026418324176fd494a6824b74975"}, +] mypy-extensions = [ {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, @@ -853,6 +899,10 @@ trio-asyncio = [ {file = "trio_asyncio-0.11.0-py3-none-any.whl", hash = "sha256:74249ab3d38aac50ada3de51425781bc93d2c27ce89179f110847d33c3e5f91b"}, {file = "trio_asyncio-0.11.0.tar.gz", hash = "sha256:33113d6c9ba13c643de9d688f1f604b0a40e986754677373cb023ee2867e8c27"}, ] +trio-typing = [ + {file = "trio-typing-0.5.0.tar.gz", hash = "sha256:f2007df617a6c26a2294db0dd63645b5451149757e1bde4cb8dbf3e1369174fb"}, + {file = "trio_typing-0.5.0-py3-none-any.whl", hash = "sha256:35f1bec8df2150feab6c8b073b54135321722c9d9289bbffa78a9a091ea83b72"}, +] triopg = [ {file = "triopg-0.5.0-py3-none-any.whl", hash = "sha256:71096987b443281a16aff3f6750daafbfc64758598d0016116b65a363b41b78f"}, {file = "triopg-0.5.0.tar.gz", hash = "sha256:fe485c6f3aba703577397843b14cd91e389a5b13e9736d7449ee2bd41d0a960d"}, diff --git a/pyproject.toml b/pyproject.toml index ec5eaff..be5fe50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ pytest = "^6.2.1" snapshottest = "^0.6.0" black = "^20.8b1" pytest-trio = "^0.7.0" +trio-typing = "^0.5.0" [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 0000000..8e60170 --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,8 @@ +{ + "include": [ + "sitewatch" + ], + "pythonVersion": "3.9", + "venvPath": ".", + "venv": ".venv" +} diff --git a/sitewatch/db.py b/sitewatch/db.py index 9e56643..bfdd2ab 100644 --- a/sitewatch/db.py +++ b/sitewatch/db.py @@ -1,13 +1,16 @@ """Datatabase connection, tables init and main operations""" +# pyright: strict + import os from datetime import timedelta import re -from typing import List +from typing import List, Any from contextlib import asynccontextmanager import trio import triopg +from triopg import Connection from .model import Report, Page @@ -25,7 +28,7 @@ def connect(): @asynccontextmanager -async def listen(conn, channel): +async def listen(conn: Connection, channel: str): """LISTEN on `channel` notifications and return memory channel to iterate over For example: @@ -37,9 +40,9 @@ async def listen(conn, channel): # based on https://gitter.im/python-trio/general?at=5fe10d762084ee4b78650fc8 - send_channel, receive_channel = trio.open_memory_channel(1) + send_channel, receive_channel = trio.open_memory_channel[str](1) - def _listen_callback(c, pid, chan, payload): + def _listen_callback(c: Any, pid: Any, chan: str, payload: str): send_channel.send_nowait(payload) await conn.add_listener(channel, _listen_callback) @@ -48,7 +51,7 @@ def _listen_callback(c, pid, chan, payload): await conn.remove_listener(channel, _listen_callback) -async def init_page_table(conn): +async def init_page_table(conn: Connection): """Initialize `page` table and add fixtures (idempotent)""" await conn.execute( @@ -104,7 +107,7 @@ async def init_page_table(conn): ) -async def init_report_table(conn): +async def init_report_table(conn: Connection): """Initialize `report` table (idempotent)""" await conn.execute( @@ -121,7 +124,7 @@ async def init_report_table(conn): ) -async def fetch_pages(conn) -> List[Page]: +async def fetch_pages(conn: Connection) -> List[Page]: pages = [ Page( row['pageid'], @@ -135,7 +138,7 @@ async def fetch_pages(conn) -> List[Page]: return pages -async def save_report(conn, r: Report): +async def save_report(conn: Connection, r: Report): await conn.execute( ''' INSERT INTO report(pageid, elapsed, statuscode, sent, found) diff --git a/sitewatch/model.py b/sitewatch/model.py index 524b44a..133f419 100644 --- a/sitewatch/model.py +++ b/sitewatch/model.py @@ -1,6 +1,6 @@ from typing import Optional from dataclasses import dataclass -from datetime import timedelta, datetime +from datetime import timedelta import json import re @@ -21,11 +21,11 @@ class Page: class Report(typesystem.Schema): """Web page check result""" - pageid: int = typesystem.Integer(minimum=0) - sent: datetime = typesystem.DateTime() - elapsed: float = typesystem.Float(minimum=0) - status_code: int = typesystem.Integer() - found: Optional[bool] = typesystem.Boolean(default=None, allow_null=True) + pageid = typesystem.Integer(minimum=0) + sent = typesystem.DateTime() + elapsed = typesystem.Float(minimum=0) + status_code = typesystem.Integer() + found = typesystem.Boolean(default=None, allow_null=True) def tobytes(self) -> bytes: """Serialize to JSON""" diff --git a/typings/triopg/__init__.pyi b/typings/triopg/__init__.pyi new file mode 100644 index 0000000..a255736 --- /dev/null +++ b/typings/triopg/__init__.pyi @@ -0,0 +1,10 @@ +""" +This type stub file was generated by pyright. +""" + +from ._triopg import connect, create_pool, TrioConnectionProxy as Connection +from .exceptions import * + +""" +This type stub file was generated by pyright. +""" diff --git a/typings/triopg/_triopg.pyi b/typings/triopg/_triopg.pyi new file mode 100644 index 0000000..00e7596 --- /dev/null +++ b/typings/triopg/_triopg.pyi @@ -0,0 +1,166 @@ +""" +This type stub file was generated by pyright. +""" + +import trio_asyncio + +from typing import Callable, Any, List + +""" +This type stub file was generated by pyright. +""" +def connect(connection: str) -> TrioConnectionProxy: + ... + +def create_pool(*args, **kwargs): + ... + +class TrioTransactionProxy: + def __init__(self, asyncpg_transaction) -> None: + ... + + @trio_asyncio.aio_as_trio + async def __aenter__(self, *args): + ... + + @_shielded + @trio_asyncio.aio_as_trio + async def __aexit__(self, *args): + ... + + +class TrioCursorProxy: + def __init__(self, asyncpg_cursor) -> None: + ... + + @trio_asyncio.aio_as_trio + async def fetch(self, *args, **kwargs): + ... + + @trio_asyncio.aio_as_trio + async def fetchrow(self, *args, **kwargs): + ... + + @trio_asyncio.aio_as_trio + async def forward(self, *args, **kwargs): + ... + + + +class TrioCursorFactoryProxy: + def __init__(self, asyncpg_transaction_factory) -> None: + ... + + def __await__(self): + ... + + def __aiter__(self): + ... + + @trio_asyncio.aio_as_trio + async def __anext__(self): + ... + + + +class TrioStatementProxy: + def __init__(self, asyncpg_statement) -> None: + ... + + def cursor(self, *args, **kwargs): + ... + + def __getattr__(self, attr): + ... + + + +class TrioConnectionProxy: + def __init__(self, *args, **kwargs) -> None: + ... + + def transaction(self, *args, **kwargs): + ... + + async def prepare(self, *args, **kwargs): + ... + + def __getattr__(self, attr): + ... + + def cursor(self, *args, **kwargs): + ... + + @_shielded + @trio_asyncio.aio_as_trio + async def close(self): + ... + + async def __aenter__(self): + ... + + async def __aexit__(self, *exc): + ... + + async def add_listener(self, channel: str, callback: Callable[..., Any])-> None: ... + + async def remove_listener(self, channel: str, callback: Callable[..., Any])-> None: ... + + async def execute(self, query: str, *params: ...) -> None: ... + async def executemany(self, query: str, params: List[Any]) -> None: ... + async def fetch(self, query: str) -> List[Any]: ... + + +class TrioPoolAcquireContextProxy: + def __init__(self, asyncpg_acquire_context) -> None: + ... + + @trio_asyncio.aio_as_trio + async def __aenter__(self, *args): + ... + + @_shielded + @trio_asyncio.aio_as_trio + async def __aexit__(self, *args): + ... + + + +class TrioPoolProxy: + def __init__(self, *args, **kwargs) -> None: + ... + + def acquire(self): + ... + + async def execute(self, statement: str, *args, timeout: float = ...): + ... + + async def executemany(self, statement: str, args, *, timeout: float = ...): + ... + + async def fetch(self, query, *args, timeout: float = ...): + ... + + async def fetchval(self, query, *args, timeout: float = ...): + ... + + async def fetchrow(self, query, *args, timeout: float = ...): + ... + + @_shielded + @trio_asyncio.aio_as_trio + async def close(self): + ... + + def terminate(self): + ... + + async def __aenter__(self): + ... + + async def __aexit__(self, *exc): + ... + + + diff --git a/typings/triopg/exceptions.pyi b/typings/triopg/exceptions.pyi new file mode 100644 index 0000000..9a9b633 --- /dev/null +++ b/typings/triopg/exceptions.pyi @@ -0,0 +1,9 @@ +""" +This type stub file was generated by pyright. +""" + +from asyncpg.exceptions import * + +""" +This type stub file was generated by pyright. +""" From cc93787831ba118810d2461f755387a392689a29 Mon Sep 17 00:00:00 2001 From: Alexey Shamrin Date: Mon, 28 Dec 2020 14:15:12 +0200 Subject: [PATCH 2/2] ci --- .github/workflows/test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6f68b2a..a176934 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,10 +20,14 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.9 + - uses: actions/setup-node@v2 + with: + node-version: '12' - name: Install dependencies run: | pip install poetry==1.1.4 poetry install + npm install - name: Lint run: | ./lint