v1.7.0 (#293)

.gitignore

@@ -4,4 +4,8 @@ lndg/settings.py
 frontend
 node_modules
 lndg-admin.txt
-.vscode
+.vscode
+.vs
+gui/static/admin
+gui/static/rest_framework
+data/

gui/af.py

@@ -0,0 +1,110 @@
+import django
+from django.db.models import Sum
+from datetime import datetime, timedelta
+from os import environ
+from pandas import DataFrame, concat
+environ['DJANGO_SETTINGS_MODULE'] = 'lndg.settings'
+django.setup()
+from gui.models import Forwards, Channels, LocalSettings, FailedHTLCs
+
+def main(channels):
+    channels_df = DataFrame.from_records(channels.values())
+    filter_1day = datetime.now() - timedelta(days=1)
+    filter_7day = datetime.now() - timedelta(days=7)
+    if channels_df.shape[0] > 0:
+        if LocalSettings.objects.filter(key='AF-MaxRate').exists():
+            max_rate = int(LocalSettings.objects.filter(key='AF-MaxRate')[0].value)
+        else:
+            LocalSettings(key='AF-MaxRate', value='2500').save()
+            max_rate = 2500
+        if LocalSettings.objects.filter(key='AF-MinRate').exists():
+            min_rate = int(LocalSettings.objects.filter(key='AF-MinRate')[0].value)
+        else:
+            LocalSettings(key='AF-MinRate', value='0').save()
+            min_rate = 0
+        if LocalSettings.objects.filter(key='AF-Increment').exists():
+            increment = int(LocalSettings.objects.filter(key='AF-Increment')[0].value)
+        else:
+            LocalSettings(key='AF-Increment', value='5').save()
+            increment = 5
+        if LocalSettings.objects.filter(key='AF-Multiplier').exists():
+            multiplier = int(LocalSettings.objects.filter(key='AF-Multiplier')[0].value)
+        else:
+            LocalSettings(key='AF-Multiplier', value='5').save()
+            multiplier = 5
+        if LocalSettings.objects.filter(key='AF-FailedHTLCs').exists():
+            failed_htlc_limit = int(LocalSettings.objects.filter(key='AF-FailedHTLCs')[0].value)
+        else:
+            LocalSettings(key='AF-FailedHTLCs', value='25').save()
+            failed_htlc_limit = 25
+        if LocalSettings.objects.filter(key='AF-UpdateHours').exists():
+            update_hours = int(LocalSettings.objects.filter(key='AF-UpdateHours').get().value)
+        else:
+            LocalSettings(key='AF-UpdateHours', value='24').save()
+            update_hours = 24
+        if LocalSettings.objects.filter(key='AF-LowLiqLimit').exists():
+            lowliq_limit = int(LocalSettings.objects.filter(key='AF-LowLiqLimit').get().value)
+        else:
+            LocalSettings(key='AF-LowLiqLimit', value='5').save()
+            lowliq_limit = 15
+        if LocalSettings.objects.filter(key='AF-ExcessLimit').exists():
+            excess_limit = int(LocalSettings.objects.filter(key='AF-ExcessLimit').get().value)
+        else:
+            LocalSettings(key='AF-ExcessLimit', value='95').save()
+            excess_limit = 90
+        if lowliq_limit >= excess_limit:
+            print('Invalid thresholds detected, using defaults...')
+            lowliq_limit = 5
+            excess_limit = 95
+        forwards = Forwards.objects.filter(forward_date__gte=filter_7day, amt_out_msat__gte=1000000)
+        if forwards.exists():
+            forwards_df_in_7d_sum = DataFrame.from_records(forwards.values('chan_id_in').annotate(amt_out_msat=Sum('amt_out_msat'), fee=Sum('fee')), 'chan_id_in')
+            forwards_df_out_7d_sum = DataFrame.from_records(forwards.values('chan_id_out').annotate(amt_out_msat=Sum('amt_out_msat'), fee=Sum('fee')), 'chan_id_out')
+        else:
+            forwards_df_in_7d_sum = DataFrame()
+            forwards_df_out_7d_sum = DataFrame()
+        channels_df['amt_routed_in_7day'] = channels_df.apply(lambda row: int(forwards_df_in_7d_sum.loc[row.chan_id].amt_out_msat/1000) if (forwards_df_in_7d_sum.index == row.chan_id).any() else 0, axis=1)
+        channels_df['amt_routed_out_7day'] = channels_df.apply(lambda row: int(forwards_df_out_7d_sum.loc[row.chan_id].amt_out_msat/1000) if (forwards_df_out_7d_sum.index == row.chan_id).any() else 0, axis=1)
+        channels_df['net_routed_7day'] = channels_df.apply(lambda row: round((row['amt_routed_out_7day']-row['amt_routed_in_7day'])/row['capacity'], 1), axis=1)
+        channels_df['local_balance'] = channels_df.apply(lambda row: row.local_balance + row.pending_outbound, axis=1)
+        channels_df['remote_balance'] = channels_df.apply(lambda row: row.remote_balance + row.pending_inbound, axis=1)
+        channels_df['in_percent'] = channels_df.apply(lambda row: int(round((row['remote_balance']/row['capacity'])*100, 0)), axis=1)
+        channels_df['out_percent'] = channels_df.apply(lambda row: int(round((row['local_balance']/row['capacity'])*100, 0)), axis=1)
+        channels_df['eligible'] = channels_df.apply(lambda row: (datetime.now()-row['fees_updated']).total_seconds() > (update_hours*3600), axis=1)
+
+        # Low Liquidity
+        lowliq_df = channels_df[channels_df['out_percent'] <= lowliq_limit].copy()
+        failed_htlc_df = DataFrame.from_records(FailedHTLCs.objects.exclude(wire_failure=99).filter(timestamp__gte=filter_1day).order_by('-id').values())
+        if failed_htlc_df.shape[0] > 0:
+            failed_htlc_df = failed_htlc_df[(failed_htlc_df['wire_failure']==15) & (failed_htlc_df['failure_detail']==6) & (failed_htlc_df['amount']>failed_htlc_df['chan_out_liq']+failed_htlc_df['chan_out_pending'])]
+        lowliq_df['failed_out_1day'] = 0 if failed_htlc_df.empty else lowliq_df.apply(lambda row: len(failed_htlc_df[failed_htlc_df['chan_id_out']==row.chan_id]), axis=1)
+        # INCREASE IF (failed htlc > threshhold) && (flow in == 0)
+        lowliq_df['new_rate'] = lowliq_df.apply(lambda row: row['local_fee_rate']+(5*multiplier) if row['failed_out_1day']>failed_htlc_limit and row['amt_routed_in_7day'] == 0 else row['local_fee_rate'], axis=1)
+
+        # Balanced Liquidity
+        balanced_df = channels_df[(channels_df['out_percent'] > lowliq_limit) & (channels_df['out_percent'] < excess_limit)].copy()
+        # IF NO FLOW THEN DECREASE FEE AND IF HIGH FLOW THEN SLOWLY INCREASE FEE
+        balanced_df['new_rate'] = balanced_df.apply(lambda row: row['local_fee_rate']+((2*multiplier)*(1+(row['net_routed_7day']/row['capacity']))) if row['net_routed_7day'] > row['capacity'] else row['local_fee_rate'], axis=1)
+        balanced_df['new_rate'] = balanced_df.apply(lambda row: row['local_fee_rate']-(3*multiplier) if (row['amt_routed_in_7day']+row['amt_routed_out_7day']) == 0 else row['local_fee_rate'], axis=1)
+
+        # Excess Liquidity
+        excess_df = channels_df[channels_df['out_percent'] >= excess_limit].copy()
+        excess_df['revenue_7day'] = excess_df.apply(lambda row: int(forwards_df_out_7d_sum.loc[row.chan_id].fee) if forwards_df_out_7d_sum.empty == False and (forwards_df_out_7d_sum.index == row.chan_id).any() else 0, axis=1)
+        excess_df['revenue_assist_7day'] = excess_df.apply(lambda row: int(forwards_df_in_7d_sum.loc[row.chan_id].fee) if forwards_df_in_7d_sum.empty == False and (forwards_df_in_7d_sum.index == row.chan_id).any() else 0, axis=1)
+        # DECREASE IF (assisting channel or stagnant liq)
+        excess_df['new_rate'] = excess_df.apply(lambda row: row['local_fee_rate']-(5*multiplier) if row['net_routed_7day'] < 0 and row['revenue_assist_7day'] > (row['revenue_7day']*10) else row['local_fee_rate'], axis=1)
+        excess_df['new_rate'] = excess_df.apply(lambda row: row['local_fee_rate']-(5*multiplier) if (row['amt_routed_in_7day']+row['amt_routed_out_7day']) == 0 else row['local_fee_rate'], axis=1)
+
+        #Merge back results
+        result_df = concat([lowliq_df, balanced_df, excess_df])
+        result_df['new_rate'] = result_df.apply(lambda row: int(round(row['new_rate']/increment, 0)*increment), axis=1)
+        result_df['new_rate'] = result_df.apply(lambda row: max_rate if max_rate < row['new_rate'] else row['new_rate'], axis=1)
+        result_df['new_rate'] = result_df.apply(lambda row: min_rate if min_rate > row['new_rate'] else row['new_rate'], axis=1)
+        result_df['adjustment'] = result_df.apply(lambda row: int(row['new_rate']-row['local_fee_rate']), axis=1)
+        return result_df
+    else:
+        return DataFrame()
+
+
+if __name__ == '__main__':
+    print(main(Channels.objects.filter(is_open=True)))

gui/forms.py

@@ -86,6 +86,8 @@ class AutoFeesForm(AutoRebalanceForm):
     af_multiplier = forms.IntegerField(label='af_multiplier', required=False)	
     af_failedHTLCs = forms.IntegerField(label='af_failedHTLCs', required=False)	
     af_updateHours = forms.IntegerField(label='af_updateHours', required=False)
+    af_lowliq = forms.IntegerField(label='af_lowliq', required=False)
+    af_excess = forms.IntegerField(label='af_excess', required=False)
 
 class GUIForm(AutoFeesForm):	
     gui_graphLinks = forms.CharField(label='gui_graphLinks', required=False)	

gui/migrations/0036_peers.py

@@ -0,0 +1,53 @@
+from django.db import migrations, models
+from gui.lnd_deps import lightning_pb2 as ln
+from gui.lnd_deps import lightning_pb2_grpc as lnrpc
+from gui.lnd_deps.lnd_connect import lnd_connect
+
+def migrate_update_channels(apps, schedma_editor):
+    try:
+        stub = lnrpc.LightningStub(lnd_connect())
+        lnd_channels = stub.ListChannels(ln.ListChannelsRequest()).channels
+        db_channels = apps.get_model('gui', 'channels')
+        for channel in lnd_channels:
+            if db_channels.objects.filter(chan_id=channel.chan_id).exists():
+                db_channel = db_channels.objects.filter(chan_id=channel.chan_id)[0]
+                db_channel.push_amt = channel.push_amount_sat
+                db_channel.close_address = channel.close_address
+                db_channel.save()
+    except Exception as e:
+        print('Unable to get current channel data:', str(e))
+
+def revert_update_channels(apps, schedma_editor):
+    pass
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gui', '0035_histfailedhtlc'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='peers',
+            name='ping_time',
+            field=models.BigIntegerField(default=0),
+        ),
+        migrations.AddField(
+            model_name='channels',
+            name='notes',
+            field=models.TextField(blank=True, default=''),
+        ),
+        migrations.AddField(
+            model_name='channels',
+            name='close_address',
+            field=models.CharField(default='', max_length=100),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='channels',
+            name='push_amt',
+            field=models.BigIntegerField(default=0),
+            preserve_default=False,
+        ),
+        migrations.RunPython(migrate_update_channels, revert_update_channels),
+    ]

gui/models.py

@@ -96,6 +96,8 @@ class Channels(models.Model):
     remote_cltv = models.IntegerField()
     remote_min_htlc_msat = models.BigIntegerField()
     remote_max_htlc_msat = models.BigIntegerField()
+    push_amt = models.BigIntegerField()
+    close_address = models.CharField(max_length=100)
     is_active = models.BooleanField()
     is_open = models.BooleanField()
     last_update = models.DateTimeField()
@@ -106,6 +108,7 @@ class Channels(models.Model):
     ar_max_cost = models.IntegerField()
     fees_updated = models.DateTimeField(default=timezone.now)
     auto_fees = models.BooleanField()
+    notes = models.TextField(default='', blank=True)
 
     def save(self, *args, **kwargs):
         if self.auto_fees is None:
@@ -157,6 +160,7 @@ class Peers(models.Model):
     inbound = models.BooleanField()
     connected = models.BooleanField()
     last_reconnected = models.DateTimeField(null=True, default=None)
+    ping_time = models.BigIntegerField(default=0)
     class Meta:
         app_label = 'gui'
 

gui/serializers.py

@@ -1,6 +1,6 @@
 from rest_framework import serializers
 from rest_framework.relations import PrimaryKeyRelatedField
-from .models import LocalSettings, Payments, PaymentHops, Invoices, Forwards, Channels, Rebalancer, Peers, Onchain, PendingHTLCs, FailedHTLCs, Closures, Resolutions
+from .models import LocalSettings, Payments, PaymentHops, Invoices, Forwards, Channels, Rebalancer, Peers, Onchain, PendingHTLCs, FailedHTLCs, Closures, Resolutions, PeerEvents
 
 ##FUTURE UPDATE 'exclude' TO 'fields'
 
@@ -71,6 +71,9 @@ class ChannelSerializer(serializers.HyperlinkedModelSerializer):
     remote_max_htlc_msat = serializers.ReadOnlyField()
     alias = serializers.ReadOnlyField()
     fees_updated = serializers.ReadOnlyField()
+    push_amt = serializers.ReadOnlyField()
+    close_address = serializers.ReadOnlyField()
+    opened_in = serializers.SerializerMethodField()
     ar_max_cost = serializers.IntegerField(required=False)
     ar_amt_target = serializers.IntegerField(required=False)
     ar_out_target = serializers.IntegerField(required=False)
@@ -80,6 +83,9 @@ class Meta:
         model = Channels
         exclude = []
 
+    def get_opened_in(self, obj):
+        return int(obj.short_chan_id.split('x')[0])
+
 class RebalancerSerializer(serializers.HyperlinkedModelSerializer):
     id = serializers.ReadOnlyField()
     requested = serializers.ReadOnlyField()
@@ -110,12 +116,27 @@ class BumpFeeSerializer(serializers.Serializer):
     target_fee = serializers.IntegerField(label='target_fee')
     force = serializers.BooleanField(default=False)
 
+class BroadcastTXSerializer(serializers.Serializer):
+    raw_tx = serializers.CharField(label='raw_tx')
+
 class AddInvoiceSerializer(serializers.Serializer):
     value = serializers.IntegerField(label='value')
 
 class UpdateAliasSerializer(serializers.Serializer):
     peer_pubkey = serializers.CharField(label='peer_pubkey', max_length=66)
 
+class UpdateChanPolicy(serializers.Serializer):
+    chan_id = serializers.CharField(max_length=20)
+    base_fee = serializers.IntegerField(required=False, default=None)
+    fee_rate = serializers.IntegerField(required=False, default=None)
+    disabled = serializers.IntegerField(required=False, default=None)
+    cltv = serializers.IntegerField(required=False, default=None)
+    min_htlc = serializers.FloatField(required=False, default=None)
+    max_htlc = serializers.FloatField(required=False, default=None)
+
+class NewAddressSerializer(serializers.Serializer):
+    legacy = serializers.BooleanField(required=False, default=False)
+
 class PeerSerializer(serializers.HyperlinkedModelSerializer):
     pubkey = serializers.ReadOnlyField()
     class Meta:
@@ -158,6 +179,17 @@ class Meta:
         model = PendingHTLCs
         exclude = []
 
+class PeerEventsSerializer(serializers.HyperlinkedModelSerializer):
+    id = serializers.ReadOnlyField()
+    out_liq_percent = serializers.SerializerMethodField()
+    class Meta:
+        model = PeerEvents
+        exclude = []
+
+    def get_out_liq_percent(self, obj):
+        capacity = Channels.objects.filter(chan_id=obj.chan_id).get().capacity
+        return int(round((obj.out_liq/capacity)*100, 1))
+
 class FailedHTLCSerializer(serializers.HyperlinkedModelSerializer):
     id = serializers.ReadOnlyField()
     class Meta:

gui/static/api.js

@@ -1,4 +1,4 @@
-async function GET(url, {method = 'GET', data = null}){
+async function GET(url, {method = 'GET', data} = {}){
     if(!data.limit) data.limit = 100000
     if(!data.format) data.format = 'json'
     return call({url, method, data})
@@ -16,7 +16,7 @@ async function PATCH(url, {method = 'PATCH', body}){
     return call({url, method, body})
 }
 
-async function DELETE(url, {method = 'DELETE'}){
+async function DELETE(url, {method = 'DELETE'} = {}){
     return call({url, method})
 }
 
@@ -31,89 +31,7 @@ class Sync{
     static PUT(url, {method = 'PUT', body}, callback){
         call({url, method, body}).then(res => callback(res))
     }
-}
-
-function showBannerMsg(h1Msg, result){
-    document.getElementById('content').insertAdjacentHTML("beforebegin", `<div style="top:5px" class="w3-panel w3-orange w3-display-container"><span onclick="this.parentElement.style.display='none'" class="w3-button w3-hover-red w3-display-topright">X</span><h1 style="word-wrap: break-word">${h1Msg} updated to: ${result}</h1></div>`);
-    window.scrollTo(0, 0);
-}
-
-function flash(element, response){
-    if (response != element.value) {
-        element.value = response
-        return
-    }
-    var rgb = window.getComputedStyle(element).backgroundColor;
-    rgb = rgb.substring(4, rgb.length-1).replace(/ /g, '').split(',');
-    var r = rgb[0], g = rgb[1], bOrigin = rgb[2], b = bOrigin;
-    var add = false;
-    var complete = false;
-    const increment = 15;
-    var flashOn = setInterval(() => {
-        if(add){
-            if(b < 255 && (b+increment) <= 255){
-                b += increment;
-            }else{
-                add = false;
-                complete = true;
-            }
-        }else{
-            if(complete == true && b < bOrigin){
-                b = bOrigin;
-                clearInterval(flashOn);
-            }
-            else if(b > 0 && (b-increment) >= 0){
-                b -= increment;
-            }else{
-                add = true;
-            }
-        }
-        element.style.backgroundColor = 'RGB('+r+','+g+','+b+')';
-        if(b == bOrigin) element.style.removeProperty("background-color");
-    }, 50);
-}
-
-function formatDate(start, end = new Date().getTime() + new Date().getTimezoneOffset()*60000){
-    if (end == null) return '---'
-    end = new Date(end)
-    if (start == null) return '---'
-    difference = (end - new Date(start))/1000
-    if (difference < 0) return 'Just now'
-    if (difference < 60) {
-        if (Math.floor(difference) == 1){
-            return `a second ago`;
-        }else{
-            return `${Math.floor(difference)} seconds ago`;
-        }
-    } else if (difference < 3600) {
-        if (Math.floor(difference / 60) == 1){
-            return `a minute ago`;
-        }else{
-            return `${Math.floor(difference / 60)} minutes ago`;
-        }
-    } else if (difference < 86400) {
-        if (Math.floor(difference / 3600) == 1){
-            return `an hour ago`;
-        }else{
-            return `${Math.floor(difference / 3600)} hours ago`;
-        }
-    } else if (difference < 2620800) {
-        if (Math.floor(difference / 86400) == 1){
-            return `a day ago`;
-        }else{
-            return `${Math.floor(difference / 86400)} days ago`;
-        }
-    } else if (difference < 31449600) {
-        if (Math.floor(difference / 2620800) == 1){
-            return `a month ago`;
-        }else{
-            return `${Math.floor(difference / 2620800)} months ago`;
-        }
-    } else {
-        if (Math.floor(difference / 31449600) == 1){
-            return `a year ago`;
-        }else{
-            return `${Math.floor(difference / 31449600)} years ago`;
-        }
+    static POST(url, {method = 'POST', body}, callback){
+        call({url, method, body}).then(res => callback(res))
     }
 }