Merge #1484: Migrate to modern packaging and src layout

c8eef50 Migrate to modern packaging and src layout (roshii)

Pull request description:

  - Migrate to src-layout following https://setuptools.pypa.io/en/latest/userguide/package_discovery.html#src-layout to facilitate automatic package discovery and editable installs.
  - Create `joinmarket` namespace distribution package in line with https://packaging.python.org/en/latest/guides/packaging-namespace-packages/
  - Migrate to `pyproject.toml` file for project configuration instead of legacy `setup.py`

  In practice:
  - Joinmarket packages are moved to `src` folder, with their respective test moved to `test` folder at project root
  - Optional dependencies definitions groups allows a granular installation depending on user needs
  - Integrated test cannot be run prior to subpackages' respective test, `reactor` is somehow left unclean by the latter., the former test definitions are therefore moved to a dedicated `unified` folder.
  - A private module is created in `jmqtui` to be run upon UI update (not tested)

  Fixes #1491

Top commit has no ACKs.

Tree-SHA512: f88c4f58c114700a68cbc451fa37ad53e9e2e07b837c20ac4d83a73e6803a28a4a38ec68320932ba309a72107d6747f37465f9898a9cdf0e2f210aa067146e1b

.coveragerc

@@ -1,15 +1,4 @@
 # .coveragerc to control coverage.py
 [run]
 omit =
- jmclient/jmclient/jsonrpc.py
- jmclient/jmclient/electruminterface.py
- jmclient/jmclient/btc.py
- jmclient/test/*
- jmclient/setup.py
- jmbitcoin/test/*
- jmbitcoin/setup.py
- jmbase/test/*
- jmbase/setup.py
- jmdaemon/test/*
- jmdaemon/setup.py
- jmdaemon/jmdaemon/socks.py
+ src/jmclient/jsonrpc.py

.github/workflows/unittests.yml

@@ -18,23 +18,21 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-#      - name: Cache venv
-#        id: cache-venv
-#        uses: actions/cache@v3
-#        env:
-#          cache-name: venv
-#        with:
-#          path: jmvenv
-#          key: ${{ runner.os }}-${{ matrix.python-version }}-${{ env.cache-name }}-${{ hashFiles('requirements/*.txt', 'install.sh', '*/setup.py') }}
+      - name: Cache venv
+        id: cache-venv
+        uses: actions/cache@v3
+        env:
+          cache-name: venv
+        with:
+          path: jmvenv
+          key: ${{ runner.os }}-${{ matrix.python-version }}-${{ env.cache-name }}-${{ hashFiles('requirements/*.txt', 'install.sh', '*/setup.py') }}
       - name: Setup joinmarket + virtualenv
-#        if: steps.cache-venv.outputs.cache-hit != 'true'
+        if: steps.cache-venv.outputs.cache-hit != 'true'
         run: |
-          sed -i.bak 's/-e //g' requirements/base.txt
-          sed -i.bak 's/-e //g' requirements/gui.txt
           ./install.sh --develop --with-qt
-          ./jmvenv/bin/pip install -r requirements/testing.txt
+          ./jmvenv/bin/python -m pip install --upgrade pip
       - name: Lint with flake8
-        run: ./jmvenv/bin/flake8 -v jmclient jmbase jmbitcoin jmdaemon scripts
+        run: ./jmvenv/bin/flake8 -v src
       - name: Cache bitcoind
         uses: actions/cache@v3
         env:

README.md

@@ -93,7 +93,7 @@ If binaries are built, they will be gpg signed and announced on the Releases pag
 If you haven't chosen the Qt option during installation with `install.sh`, then to run the script `joinmarket-qt.py` from the command line you will need to install two more packages.  Use these 2 commands while the `jmvenv` virtual environment is activated:
 
 ```
-pip install -r requirements/gui.txt
+pip install .[gui]
 ```
 After this, the command `python joinmarket-qt.py` from within the `scripts` subdirectory should work.
 There is a [walkthrough](docs/JOINMARKET-QT-GUIDE.md) for what to do next.

docs/INSTALL.md

@@ -75,7 +75,7 @@ Then build and install a local copy of libsecp256k1 for python-bitcointx:
 
 > *NOTE*: It is very important to have activated the virtual environment before running this step. Otherwise, `pip install` will fail, you may be tempted to re-run it with `sudo pip install` which will cause problems in the future.
 
-    pip install -r requirements/base.txt
+    pip install .[services]
 
 If you have installed this "full" version of the client, you can use it with the command line scripts as explained in the [usage guide](USAGE.md).
 
@@ -122,7 +122,7 @@ If you have installed this "full" version of the client, you can use it with the
 
 7) Setup joinmarket-qt
     ```
-    pip install -r requirements/gui.txt
+    pip install .[gui]
     ```
 8) Start joinmarket-qt
     ```
@@ -165,10 +165,9 @@ Unzip the `joinmarket-clientserver-x.x.x` (where `x.x.x` is the release number)
 
 Using the command prompt in Administrator mode, go to that directory and run the commands:
 
-`pip install -r requirements\base.txt`
-`pip install -r requirements\gui.txt`
+`pip install joinmarket[services]`
 
-(the latter is needed for Joinmarket-Qt).
+(replace `services` with `gui` for Joinmarket-Qt).
 
 The final step is to manually add the libsodium dependency, as mentioned. Do the following:
 

docs/JSON-RPC-API-using-jmwalletd.md

@@ -28,7 +28,7 @@ On initially creating, unlocking or recovering a wallet, a new access and refres
 
 Tokens are signed with HS256 (HMAC with SHA-256), a symmetric keyed hashing algorithm that uses one secret key. Signature keys (differentiated between access and refresh tokens) are generated from random bytes upon the following events, implying that any previously issued token is invalidated.
 
-- starting Joinmarket wallet deamon
+- starting Joinmarket wallet daemon
 - creating, unlocking or recovering a wallet if RPC API is already serving another wallet
 - locking wallet
 

docs/TESTING.md

@@ -7,7 +7,7 @@ Work in your `jmvenv` virtual environment as for all Joinmarket work. Make sure
 
 Install the test requirements:
 
-    (jmvenv)$ pip install -r requirements/testing.txt
+    (jmvenv)$ pip install .[test]
 
 #### Running the test suite.
 

install.sh

@@ -434,16 +434,17 @@ CookieAuthentication 1
 
 joinmarket_install ()
 {
-    reqs=( 'base.txt' )
+    reqs='services'
 
     if [[ ${with_qt} == "1" ]]; then
-        reqs+=( 'gui.txt' )
+        reqs='gui'
+    fi
+    if [[ ${develop} == "1" ]]; then
+        reqs+=',test'
     fi
 
-    for req in "${reqs[@]}"; do
-        if [ "$with_jmvenv" == 1 ]; then pip_command=pip; else pip_command=pip3; fi
-        $pip_command install -r "requirements/${req}" || return 1
-    done
+    if [ "$with_jmvenv" == 1 ]; then pip_command=pip; else pip_command=pip3; fi
+    $pip_command install -e ".[${reqs}]" || return 1
 
     if [[ ${with_qt} == "1" ]]; then
         if [[ -d ~/.local/share/icons ]] && [[ -d ~/.local/share/applications ]]; then
@@ -462,7 +463,9 @@ parse_flags ()
     while :; do
         case $1 in
             --develop)
-                # no-op for backwards compatibility
+                # editable install is currently always on
+                # option solely triggers test dependencies installation for now
+                develop='1'
                 ;;
             --disable-os-deps-check)
                 use_os_deps_check='0'

pyproject.toml

@@ -0,0 +1,81 @@
+[build-system]
+requires = ["setuptools>=64", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "joinmarket"
+version = "0.9.11dev"
+description = "Joinmarket client library for Bitcoin coinjoins"
+readme = "README.md"
+requires-python = ">=3.7"
+license = {file = "LICENSE"}
+dependencies = [
+    "chromalog==1.0.5",
+    "pyaes==1.6.1",
+    "service-identity==21.1.0",
+    "txtorcon==22.0.0",
+    "twisted==22.4.0",
+]
+
+[project.optional-dependencies]
+jmbitcoin = [
+    "python-bitcointx==1.1.3",
+]
+jmclient = [
+    "argon2_cffi==21.3.0",
+    "autobahn==20.12.3",
+    "bencoder.pyx==3.0.1",
+    "klein==20.6.0",
+    "mnemonic==0.20",
+    "pyjwt==2.4.0",
+    "werkzeug==2.2.3",
+]
+jmdaemon = [
+    "libnacl==1.8.0",
+    "pyopenssl==23.2.0",
+    'cryptography==3.3.2; platform_machine != "aarch64" and platform_machine != "amd64" and platform_machine != "x86_64"',
+    'cryptography==41.0.2; platform_machine == "aarch64" or platform_machine == "amd64" or platform_machine == "x86_64"',
+]
+jmqtui = [
+    "PyQt5!=5.15.0,!=5.15.1,!=5.15.2,!=6.0",
+    "PySide2!=5.15.0,!=5.15.1,!=5.15.2,!=6.0", # https://bugreports.qt.io/browse/QTBUG-88688
+    "qrcode[pil]==7.3.1",
+    'pywin32; platform_system == "Windows"',
+    "qt5reactor==0.6.3",
+]
+client = [
+    "joinmarket[jmclient]",
+    "joinmarket[jmbitcoin]",
+]
+daemon = [
+    "joinmarket[jmdaemon]",
+]
+services = [
+    "joinmarket[client]",
+    "joinmarket[daemon]",
+]
+test = [
+    "joinmarket[services]",
+    "coverage==5.2.1",
+    "flake8",
+    "freezegun",
+    "mock",
+    "pexpect",
+    "pytest-cov>=2.4.0,<2.6",
+    "pytest==6.2.5",
+    "python-coveralls",
+]
+gui = [
+    "joinmarket[services]",
+    "joinmarket[jmqtui]",
+]
+
+[project.urls]
+repository = "https://github.com/JoinMarket-Org/joinmarket-clientserver.git"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+exclude = ["*.test"]
+
+[tool.pytest.ini_options]
+testpaths = ["test"]

setup.cfg

@@ -1,13 +1,10 @@
-# content of pytest.ini
-[tool:pytest]
-testpaths = jmbitcoin jmclient jmbase jmdaemon test
-
 [flake8]
 exclude =
     # defaults
     .svn,CVS,.bzr,.hg,.git,__pycache__,.tox,
     # lots of "unused imports"
     __init__.py
+    jmqtui
 
 max-line-length = 79