Traditional non docker environment for backend development

.env-sample

@@ -11,14 +11,24 @@ MACAROON_PATH='data/chain/bitcoin/testnet/admin.macaroon'
 LND_CERT_BASE64='LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNLVENDQWRDZ0F3SUJBZ0lRQ0VoeGpPZXY1bGQyVFNPTXhKalFvekFLQmdncWhrak9QUVFEQWpBNE1SOHcKSFFZRFZRUUtFeFpzYm1RZ1lYVjBiMmRsYm1WeVlYUmxaQ0JqWlhKME1SVXdFd1lEVlFRREV3d3dNakJtTVRnMQpZelkwTnpVd0hoY05Nakl3TWpBNE1UWXhOalV3V2hjTk1qTXdOREExTVRZeE5qVXdXakE0TVI4d0hRWURWUVFLCkV4WnNibVFnWVhWMGIyZGxibVZ5WVhSbFpDQmpaWEowTVJVd0V3WURWUVFERXd3d01qQm1NVGcxWXpZME56VXcKV1RBVEJnY3Foa2pPUFFJQkJnZ3Foa2pPUFFNQkJ3TkNBQVNJVWdkcVMrWFZKL3EzY0JZeWd6ZDc2endaanlmdQpLK3BzcWNYVkFyeGZjU2NXQ25jbXliNGRaMy9Lc3lLWlRaamlySDE3aEY0OGtIMlp5clRZSW9hZG80RzdNSUc0Ck1BNEdBMVVkRHdFQi93UUVBd0lDcERBVEJnTlZIU1VFRERBS0JnZ3JCZ0VGQlFjREFUQVBCZ05WSFJNQkFmOEUKQlRBREFRSC9NQjBHQTFVZERnUVdCQlEwWUJjZXdsd1BqYTJPRXFyTGxzZnJscEswUFRCaEJnTlZIUkVFV2pCWQpnZ3d3TWpCbU1UZzFZelkwTnpXQ0NXeHZZMkZzYUc5emRJSUVkVzVwZUlJS2RXNXBlSEJoWTJ0bGRJSUhZblZtClkyOXVib2NFZndBQUFZY1FBQUFBQUFBQUFBQUFBQUFBQUFBQUFZY0V3S2dRQW9jRUFBQUFBREFLQmdncWhrak8KUFFRREFnTkhBREJFQWlBd0dMY05qNXVZSkVwanhYR05OUnNFSzAwWmlSUUh2Qm50NHp6M0htWHBiZ0lnSWtvUQo3cHFvNGdWNGhiczdrSmt1bnk2bkxlNVg0ZzgxYjJQOW52ZnZ2bkk9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K'
 # base64 ~/.lnd/data/chain/bitcoin/testnet/admin.macaroon | tr -d '\n'
 LND_MACAROON_BASE64='AgEDbG5kAvgBAwoQsyI+PK+fyb7F2UyTeZ4seRIBMBoWCgdhZGRyZXNzEgRyZWFkEgV3cml0ZRoTCgRpbmZvEgRyZWFkEgV3cml0ZRoXCghpbnZvaWNlcxIEcmVhZBIFd3JpdGUaIQoIbWFjYXJvb24SCGdlbmVyYXRlEgRyZWFkEgV3cml0ZRoWCgdtZXNzYWdlEgRyZWFkEgV3cml0ZRoXCghvZmZjaGFpbhIEcmVhZBIFd3JpdGUaFgoHb25jaGFpbhIEcmVhZBIFd3JpdGUaFAoFcGVlcnMSBHJlYWQSBXdyaXRlGhgKBnNpZ25lchIIZ2VuZXJhdGUSBHJlYWQAAAYgMt90uD6v4truTadWCjlppoeJ4hZrL1SBb09Y+4WOiI0='
+LND_GRPC_HOST='localhost'
+LND_GRPC_PORT='10009'
+# Auto unlock LND password. Only used in development docker-compose environment.
+# It will fail starting up the node without it.
+# To disable auto unlock, comment out 'wallet-unlock-password-file=/tmp/pwd' from 'docker/lnd/lnd.conf'
+AUTO_UNLOCK_PWD='1234'
 
 # CLN directory
 CLN_DIR='/cln/testnet/'
-CLN_GRPC_HOST='localhost:9999'
-CLN_GRPC_HOLD_HOST='localhost:9998'
+CLN_GRPC_HOST='localhost'
+CLN_GRPC_PORT='9999'
+CLN_GRPC_HOLD_HOST='localhost'
+CLN_GRPC_HOLD_PORT='9998'
 
 # Bitcoin Core Daemon RPC, used to validate addresses
-BITCOIND_RPCURL = 'http://127.0.0.1:18332'
+# mainnet: 8332, testnet: 18332, signet: 38332, regtest: 18443
+BITCOIND_RPCHOST = '127.0.0.1'
+BITCOIND_RPCPORT = '18332'
 BITCOIND_RPCUSER = 'robodev'
 BITCOIND_RPCPASSWORD = 'robodev'
 
@@ -33,14 +43,12 @@ POSTGRES_PORT='5432'
 USE_TOR=True
 TOR_PROXY='127.0.0.1:9050'
 
-# Auto unlock LND password. Only used in development docker-compose environment.
-# It will fail starting up the node without it.
-# To disable auto unlock, comment out 'wallet-unlock-password-file=/tmp/pwd' from 'docker/lnd/lnd.conf'
-AUTO_UNLOCK_PWD='1234'
-
-LND_GRPC_HOST='localhost:10009'
+REDIS_HOST="localhost"
+REDIS_PORT="6379"
+REDIS_DB_NUMBER="1"
 
-REDIS_URL='redis://localhost:6379/1'
+LOG_TO_CONSOLE=False
+LOGGER_LEVEL="WARNING"
 
 # List of market price public APIs. If the currency is available in more than 1 API, will use median price.
 MARKET_PRICE_APIS = https://blockchain.info/ticker, https://api.yadio.io/exrates/BTC, https://bitpay.com/rates/BTC, https://criptoya.com/api/btc

.gitignore

@@ -653,3 +653,6 @@ mobile/html/Web.bundle/assets*
 
 # Protocol Buffers
 api/lightning/*.proto
+
+# Traditional environment
+regtest/

api/lightning/cln.py

@@ -29,8 +29,12 @@
     server_cert = f.read()
 
 
-CLN_GRPC_HOST = config("CLN_GRPC_HOST", cast=str, default="localhost:9999")
-CLN_GRPC_HOLD_HOST = config("CLN_GRPC_HOLD_HOST", cast=str, default="localhost:9998")
+CLN_GRPC_HOST = \
+    str(config("CLN_GRPC_HOST", cast=str, default="localhost")) + \
+    ":" + str(config("CLN_GRPC_PORT", cast=str, default="9999"))
+CLN_GRPC_HOLD_HOST = \
+    str(config("CLN_GRPC_HOLD_HOST", cast=str, default="localhost")) + \
+    ":" + str(config("CLN_GRPC_HOLD_PORT", cast=str, default="9998"))
 DISABLE_ONCHAIN = config("DISABLE_ONCHAIN", cast=bool, default=True)
 MAX_SWAP_AMOUNT = config("MAX_SWAP_AMOUNT", cast=int, default=500000)
 

api/lightning/lnd.py

@@ -42,7 +42,9 @@
 except Exception:
     MACAROON = b64decode(config("LND_MACAROON_BASE64"))
 
-LND_GRPC_HOST = config("LND_GRPC_HOST")
+LND_GRPC_HOST = \
+    str(config("LND_GRPC_HOST", cast=str, default="localhost")) + \
+    ":" + str(config("LND_GRPC_PORT", cast=str, default="10009"))
 DISABLE_ONCHAIN = config("DISABLE_ONCHAIN", cast=bool, default=True)
 MAX_SWAP_AMOUNT = config("MAX_SWAP_AMOUNT", cast=int, default=500_000)
 

api/tests/test_utils.py

@@ -35,7 +35,7 @@ def test_get_session(self, mock_session, mock_config):
     @patch("api.utils.config")
     @patch("api.utils.requests.post")
     def test_bitcoind_rpc(self, mock_post, mock_config):
-        mock_config.side_effect = ["url", "user", "password"]
+        mock_config.side_effect = ["host", "port", "user", "password"]
         mock_post.return_value.json.return_value = {"result": "response"}
         response = bitcoind_rpc("method", ["params"])
         self.assertEqual(response, "response")

api/utils.py

@@ -37,7 +37,9 @@ def bitcoind_rpc(method, params=None):
     :return:
     """
 
-    BITCOIND_RPCURL = config("BITCOIND_RPCURL")
+    BITCOIND_RPCHOST = str(config("BITCOIND_RPCHOST", cast=str, default="127.0.0.1"))
+    BITCOIND_RPCPORT = str(config("BITCOIND_RPCPORT", cast=str, default="18332"))
+    BITCOIND_RPCURL = "http://" + BITCOIND_RPCHOST + ":" + BITCOIND_RPCPORT
     BITCOIND_RPCUSER = config("BITCOIND_RPCUSER")
     BITCOIND_RPCPASSWORD = config("BITCOIND_RPCPASSWORD")
 
@@ -385,7 +387,7 @@ def compute_avg_premium(queryset):
 
 def validate_pgp_keys(pub_key, enc_priv_key):
     """Validates PGP valid keys. Formats them in a way understandable by the frontend"""
-    gpg = gnupg.GPG()
+    gpg = gnupg.GPG(gnupghome=config("GNUPG_DIR", default=None))
 
     # Standardize format with linux linebreaks '\n'. Windows users submitting their own keys have '\r\n' breaking communication.
     enc_priv_key = enc_priv_key.replace("\r\n", "\n").replace("\\", "\n")
@@ -439,7 +441,7 @@ def verify_signed_message(pub_key, signed_message):
     Verifies a signed cleartext PGP message. Returns whether the signature
     is valid (was made by the given pub_key) and the content of the message.
     """
-    gpg = gnupg.GPG()
+    gpg = gnupg.GPG(gnupghome=config("GNUPG_DIR", default=None))
 
     # import the public key
     import_result = gpg.import_keys(pub_key)

api/views.py

@@ -16,6 +16,7 @@
 from rest_framework.views import APIView
 
 from api.logics import Logics
+from api.tasks import cache_market
 from api.models import (
     Currency,
     LNPayment,
@@ -146,6 +147,9 @@ def post(self, request):
                 status.HTTP_400_BAD_REQUEST,
             )
 
+        if len(Currency.objects.all()) == 0:
+            cache_market()
+
         # Creates a new order
         order = Order(
             type=type,

docker-compose.yml

@@ -115,7 +115,9 @@ services:
     container_name: cele-worker-dev
     restart: always
     environment:
-      REDIS_URL: redis://localhost:6379
+      REDIS_HOST: localhost
+      REDIS_PORT: 6379
+      REDIS_DB_NUMBER: 1
       SKIP_COLLECT_STATIC: "true"
     volumes:
       - .:/usr/src/robosats
@@ -132,7 +134,9 @@ services:
     container_name: cele-beat-dev
     restart: always
     environment:
-      REDIS_URL: redis://localhost:6379
+      REDIS_HOST: localhost
+      REDIS_PORT: 6379
+      REDIS_DB_NUMBER: 1
       SKIP_COLLECT_STATIC: "true"
     command: celery -A robosats beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler
     volumes:

docker-tests.yml

@@ -149,7 +149,8 @@ services:
       USE_TOR: False
       MACAROON_PATH: 'data/chain/bitcoin/regtest/admin.macaroon'
       CLN_DIR: '/cln/regtest/'
-      BITCOIND_RPCURL: 'http://127.0.0.1:18443'
+      BITCOIND_RPCHOST: '127.0.0.1'
+      BITCOIND_RPCPORT: '18443'
       BITCOIND_RPCUSER: 'test'
       BITCOIND_RPCPASSWORD: 'test'
     env_file:
@@ -190,7 +191,8 @@ services:
   #     USE_TOR: False
   #     MACAROON_PATH: 'data/chain/bitcoin/regtest/admin.macaroon'
   #     CLN_DIR: '/cln/regtest/'
-  #     BITCOIND_RPCURL: 'http://127.0.0.1:18443'
+  #     BITCOIND_RPCHOST: '127.0.0.1'
+  #     BITCOIND_RPCPORT: '18443'
   #     BITCOIND_RPCUSER: 'test'
   #     BITCOIND_RPCPASSWORD: 'test'
   #     SKIP_COLLECT_STATIC: "true"
@@ -210,4 +212,4 @@ volumes:
   bitcoin:
   lnd:
   cln:
-  lndrobot:
+  lndrobot:

requirements_dev.txt

@@ -1,4 +1,5 @@
 coverage==7.5.0
 ruff==0.5.1
 git+https://github.com/Reckless-Satoshi/drf-openapi-tester.git@soften-django-requirements
-pre-commit==3.7.0
+pre-commit==3.7.0
+python-dotenv[cli]==1.0.1

robosats/celery/__init__.py

@@ -3,11 +3,15 @@
 import os
 from datetime import timedelta
 
+from decouple import config
 from celery import Celery
 from celery.schedules import crontab
 
 # You can use rabbitmq instead here.
-BASE_REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379")
+BASE_REDIS_URL = \
+    "redis://" + str(config("REDIS_HOST", cast=str, default="localhost")) + \
+    ":" + str(config("REDIS_PORT", cast=str, default="6379")) + \
+    "/" + str(config("REDIS_DB_NUMBER", cast=str, default="1"))
 
 # set the default Django settings module for the 'celery' program.
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "robosats.settings")

robosats/settings.py

@@ -59,7 +59,7 @@
 SESSION_COOKIE_HTTPONLY = False
 
 # Logging settings
-if os.environ.get("LOG_TO_CONSOLE"):
+if config("LOG_TO_CONSOLE", cast=bool, default=False):
     LOGGING = {
         "version": 1,
         "disable_existing_loggers": False,
@@ -70,12 +70,12 @@
         },
         "root": {
             "handlers": ["console"],
-            "level": "WARNING",
+            "level": str(config("LOGGER_LEVEL", cast=str, default="WARNING")),
         },
         "loggers": {
             "api.utils": {
                 "handlers": ["console"],
-                "level": "WARNING",
+                "level": str(config("LOGGER_LEVEL", cast=str, default="WARNING")),
             },
         },
     }
@@ -231,15 +231,22 @@
     "default": {
         "BACKEND": "channels_redis.core.RedisChannelLayer",
         "CONFIG": {
-            "hosts": [config("REDIS_URL")],
+            "hosts": [
+                "redis://" + str(config("REDIS_HOST", cast=str, default="localhost")) + \
+                ":" + str(config("REDIS_PORT", cast=str, default="6379")) + \
+                "/" + str(config("REDIS_DB_NUMBER", cast=str, default="1"))
+            ],
         },
     },
 }
 
 CACHES = {
     "default": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": config("REDIS_URL"),
+        "LOCATION": \
+            "redis://" + str(config("REDIS_HOST", cast=str, default="localhost")) + \
+            ":" + str(config("REDIS_PORT", cast=str, default="6379")) + \
+            "/" + str(config("REDIS_DB_NUMBER", cast=str, default="1")),
         "OPTIONS": {"CLIENT_CLASS": "django_redis.client.DefaultClient"},
     }
 }