Merge branch 'dev' of github.com:pmhalvor/website into dev

phsite/urls.py

@@ -17,12 +17,16 @@
 from django.urls import include, path
 from django.conf import settings
 from django.conf.urls.static import static
+from django.views.static import serve
 
 urlpatterns = [
     path('admin/', admin.site.urls),
     path('', include('home.urls')),
     path('radio/', include('radio.urls')),
+    path(r'^media/(?P<path>.*)$', serve,{'document_root': settings.MEDIA_ROOT}), 
+    path(r'^static/(?P<path>.*)$', serve,{'document_root': settings.STATIC_ROOT}),
 ]
 
-urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)
-urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
+
+# urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)
+# urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)

radio/templates/includes/plots.html

@@ -0,0 +1,10 @@
+<h1>favorites</h1>
+<div>
+    top artists past 6 months
+    {{ plot_artists.plot | safe}}
+</div>
+<br><br>
+<div>
+    top songs past 6 months
+    {{ plot_songs.plot | safe}}
+</div>

radio/templates/radio/index.html

@@ -75,7 +75,10 @@ <h1 style="text-align: right; padding-right:10px;">recently played</h1>
 					{% include "includes/recents.html" %}
 				</ul>
 			</div>
-
+			<br>
+			<div class=plots>
+				{% include "includes/plots.html" %}
+			</div>
 		</div>
 
 

radio/urls.py

@@ -1,12 +1,12 @@
 from django.contrib import admin
 from django.urls import include, path
 from . import views
-from .worker import search, submit
+from .worker import search, submit, plot
 
 urlpatterns = [
     path('', views.radio, name='radio'),
     path('current/', views.Http_current, name='current'),
     path('recents/', views.Http_recents, name='recents'),
     path('search/', search.Http_search, name='search'),
     path('submit/', submit.submit_suggestions , name='submit'),
-]
+]

radio/views.py

@@ -5,8 +5,10 @@
 from .worker.authorize import get_token
 from datetime import datetime, timedelta
 import json
-
-
+from .worker.plot import artist_duration, song_plays
+from django.views.generic.base import TemplateView
+import plotly.offline as opy
+import plotly.graph_objs as go
 
 def index(request):
 	return HttpResponse("Welcome to Per's Radio!")
@@ -23,6 +25,8 @@ def radio(request):
 						# """
 	context['recents'] = recents()
 	context['current'] = current()
+	context['plot_artists'] = artist_duration()
+	context['plot_songs'] = song_plays()
 	return render(request, 'radio/index.html', context)
 
 
@@ -101,3 +105,4 @@ def Http_current(request):
 
 def Http_recents(request):
 	return render(None, 'includes/recents.html', {'recents': recents()})
+

radio/worker/plot.py

@@ -0,0 +1,96 @@
+from datetime import datetime, timedelta
+try:
+    from .song_history import download_to_df, get_durations
+except:
+    from song_history import download_to_df, get_durations
+import pandas as pd 
+import plotly.offline as opy
+pd.options.plotting.backend = "plotly"
+
+####### being used
+def to_html(figure):
+    context = {}
+    div = opy.plot(figure, auto_open=False, output_type='div')
+    context['plot'] = div
+    return context
+
+def artist_duration(n=37):
+    df, mdf = download_to_df()
+
+    durations = get_durations(df.id.unique())
+    df = df.merge(durations, on='id', how='left')
+
+    df_artist_track = df.groupby(['artist', 'id'])
+    duration = df_artist_track['duration'].sum()
+    count = df_artist_track['count'].sum()
+
+    total_time_artist = (count*duration).groupby('artist').sum()
+    total_time_artist.sort_values(ascending=True, inplace=True)
+
+    top_artists_ms = total_time_artist.tail(n)
+
+    top_artists = top_artists_ms//(1000*60*60)
+
+    top_artists = pd.DataFrame(top_artists).reset_index()
+    top_artists.columns=['artist', 'time']
+    top_artists.index = [f'Rank: {n-i}' for i in range(n)]
+    print(top_artists)
+
+    figure = top_artists.plot.barh(
+        x = 'time',
+        y = 'artist',
+        color='time',
+        hover_name = top_artists.index,
+        template='plotly_dark')
+    figure.update(layout_showlegend=False)
+
+    return to_html(figure)
+
+def song_plays(n=37):
+    print('Downloading to df()...')
+    df, mdf = download_to_df()
+
+    df.rename(columns={'name':'track'}, inplace=True)
+    name_artist = df.groupby(['track', 'artist'])
+    _counts = name_artist.size().reset_index(name='count')
+    sorted_counts = _counts.sort_values('count', ascending=True)
+    top_songs = sorted_counts.tail(n)
+
+    top_songs.index = ['Rank: '+str(n-i) for i in range(n)]
+    print(top_songs)
+
+    figure = top_songs.plot.barh(
+        y = 'track',
+        x = 'count',
+        custom_data = ['artist'],
+        color='count',
+        hover_name = top_songs.index,
+        hover_data=['artist', 'track'],
+        template='plotly_dark')
+    figure.update_layout(
+        showlegend=False
+    )
+
+    return to_html(figure)
+
+######################
+
+'''
+Checklist of things to do:
+    X hours per artist
+    X songs plays with color 
+    - genre plots?
+'''
+
+if __name__=='__main__':
+    # print('Downloading to df()...')
+    # df, mdf = download_to_df()
+
+    # song_plays()
+    artist_duration()
+
+    print('end')
+
+
+
+

radio/worker/song_history.py

@@ -6,16 +6,12 @@
 import io, json, logging
 import pandas as pd 
 import requests as r
+import os 
 
-# Request recently played
-def get_recents(token=None) -> dict:
-    if not token:
-        token = get_token()
-    URL = "https://api.spotify.com/v1/me/player/recently-played"    # api-endpoint for recently played
-    HEAD = {'Authorization': 'Bearer '+token}                       # provide auth. crendtials
-    PARAMS = {'limit':50}	                                        # default here is 20
-    return r.get(url=URL, headers=HEAD, params=PARAMS).json()
+real_path = os.path.realpath(__file__)
+dir_path = os.path.dirname(real_path)
 
+####### CLOUD HANDLING #############
 # Download csv from FileShare to dataframe
 def download_to_df() -> pd.DataFrame:
     file_client = get_fileshare_client('history.csv')                # create FileShareClient
@@ -27,6 +23,9 @@ def download_to_df() -> pd.DataFrame:
 
 # Convert json data to dataframe
 def json_to_df(data=None, latest=None) -> pd.DataFrame:
+    '''
+    typical josn data format:
+    '''
     if not isinstance(data, dict):
         try:
             data = data.json()              # This needs to be cleaned up 
@@ -76,7 +75,10 @@ def df_to_csv(df=None) -> str:
         return csv_str
     except:
         return df
+#####################################
 
+
+#####    API CALLS     ##############
 # Get currently playing
 def get_current(token=None) -> dict:
     if not token:
@@ -89,6 +91,99 @@ def get_current(token=None) -> dict:
     else:
         return {}
 
+# Request recently played
+def get_recents(token=None) -> dict:
+    if not token:
+        token = get_token()
+    URL = "https://api.spotify.com/v1/me/player/recently-played"    # api-endpoint for recently played
+    HEAD = {'Authorization': 'Bearer '+token}                       # provide auth. crendtials
+    PARAMS = {'limit':50}	                                        # default here is 20
+    return r.get(url=URL, headers=HEAD, params=PARAMS).json()
+
+def get_durations(ids = '', token=None, store=True):
+    pth = os.path.join(dir_path, 'store', 'duration_df.pkl')
+
+    # ids.pop(ids.index[3]) # TODO: delete when left on checked and working
+    durations = pd.DataFrame({
+        'id':ids
+    })
+
+    print('Looking for cached durations')
+    try:
+        # local_durations = pd.read_csv(os.path.join(dir_path, 'store/durations.csv'))
+        local_durations = pd.read_pickle(pth)
+        id_ = local_durations.index
+        local_durations.reset_index(inplace = True)
+        local_durations['id'] = id_
+        del id_
+        print('Found local durations')
+    except Exception as e:
+        print('Nothing stored locally. Calling API...')
+        local_durations = pd.DataFrame(durations)
+
+    # durations = durations.merge(local_durations,how='outer', on='id')
+    durations = pd.concat([local_durations, durations])
+    durations = durations.drop_duplicates('id', keep='first', ignore_index=True)
+    durations.fillna(0, inplace=True)
+    try:
+        durations.drop(columns='count')
+    except:
+        print('No count colmun to drop.\n\
+        Remove this call on line 132 in song_history.py')
+
+    ids = durations.index[durations.duration<1]
+
+    if len(ids) > 0:
+        print(f'{len(ids)} new ids to check')
+
+        if not token:
+            token = get_token()
+        batches = (len(ids)//50) + 1
+        print(f'Will be executing {batches} API call(s)')
+
+        URL = "https://api.spotify.com/v1/tracks"    # api-endpoint for recently played  
+        HEAD = {'Authorization': 'Bearer '+token}                       # provide auth. crendtials
+
+        # batching the unstored indexes incase exceeds max
+        for i in range(batches):
+            if i==(batches-1):
+                batch_ids = ids[50*i:]  # last set of indices
+            else:
+                batch_ids = ids[50*i:50*(i+1)]  # forward indexing
+
+            b_ids = ','.join(batch_ids)
+
+            PARAMS = {'ids':b_ids}	
+            data = r.get(url=URL, headers=HEAD, params=PARAMS).json()
+
+            batch_dur = []
+
+            for track in data['tracks']:
+                batch_dur.append(track['duration_ms'])
+
+            durations.duration[batch_ids] = batch_dur
+
+            print(durations[durations.index.isin(batch_ids)])
+
+        # this only gets stored again when new ids are added
+        print(f'Storing at {pth}')
+        durations.to_pickle(pth)
+        print('Success!')
+
+    return durations
+
+#####################################
+
+####   OTHER TOOLS    #####
+# bacthes, taken from https://code.activestate.com/recipes/303279-getting-items-in-batches/
+def batch(iterable, size):
+    '''
+    Try replacing my bacthing with something similar
+    '''
+    sourceiter = iter(iterable)
+    while True:
+        batchiter = islice(sourceiter, size)
+        yield chain([batchiter.next()], batchiter)
 
 
 # Run
@@ -99,7 +194,7 @@ def run() -> bool:
     csv_df, latest = download_to_df()
     print(csv_df.tail())
     spot_df = json_to_df(data=data, latest=latest)
-    
+###########################
 
 
 if __name__=='__main__':