Ahmed develop

.gitignore

@@ -135,4 +135,18 @@ dmypy.json
 .pytype/
 
 # Cython debug symbols
-cython_debug/
+cython_debug/
+
+# VSCode settings
+.vscode
+
+# vscode workspace files
+*.code-workspace
+
+# misc
+.DS_Store
+.env
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local

Dockerfile

@@ -1,7 +1,5 @@
 FROM python:3.8
 
-MAINTAINER Part Time Larry "[email protected]"
-
 ADD . /app
 
 WORKDIR /app

data/alpaca.py

@@ -0,0 +1,15 @@
+import sys
+sys.path.append('/app')
+from data.api_data import ALPACA_API_KEY, ALPACA_END_POINT, ALPACA_SECRET_KEY
+import alpaca_trade_api as trade_api
+from datetime import date, datetime
+
+def get_all_assets():
+    api = trade_api.REST(ALPACA_API_KEY, ALPACA_SECRET_KEY, base_url=ALPACA_END_POINT)
+    return api.list_assets()
+
+def get_api_pointer():
+    return trade_api.REST(ALPACA_API_KEY, ALPACA_SECRET_KEY, base_url=ALPACA_END_POINT)
+
+def get_barset(api_pointer, symbols, period, from_date):
+    return api_pointer.get_barset(symbols, period, after=from_date)

data/alpaca_polygon_rest.py

@@ -0,0 +1,10 @@
+import sys
+sys.path.append('/app')
+from data.api_data import ALPACA_API_KEY, ALPACA_END_POINT, ALPACA_SECRET_KEY
+import alpaca_trade_api as trade_api
+
+def get_agg_bars(symbol, period, multiplier, start, end):
+
+    api = trade_api.REST(ALPACA_API_KEY, ALPACA_SECRET_KEY, base_url=ALPACA_END_POINT)
+
+    return api.polygon.historic_agg_v2(symbol, 1, period, _from=start, to=end)

data/alpaca_polygon_websocket.py

@@ -0,0 +1,29 @@
+import sys
+sys.path.append('/app')
+from data.api_data import ALPACA_API_KEY
+import websocket, json
+
+POLYGON_ALPACA_WS = 'wss://alpaca.socket.polygon.io/stocks'
+TICKERS_TO_STREAM = "Q.MSFT"
+
+def on_open(ws):
+    auth_data = {
+        "action":"auth",
+        "params": ALPACA_API_KEY
+    }
+    ws.send(json.dumps(auth_data))
+
+    sub_data = {
+        "action":"subscribe",
+        "params":TICKERS_TO_STREAM
+    }
+    ws.send(json.dumps(sub_data))
+
+def on_message(ws,message):
+    print(message)
+
+def on_close(ws):
+    print("Conn Closed")
+
+ws = websocket.WebSocketApp(POLYGON_ALPACA_WS, on_open=on_open, on_message=on_message, on_close=on_close)
+ws.run_forever()

data/api_data.py

@@ -0,0 +1,8 @@
+ALPACA_END_POINT = 'https://paper-api.alpaca.markets'
+ALPACA_API_KEY = 'PK45PMO9M1JFIAWKW1X1'
+ALPACA_SECRET_KEY = 'l6nOaQQvgPqE98MhKjXkrZFvoKPl6ikJ7inBrEed'
+
+POLYGON_ALPACA_WS = 'wss://alpaca.socket.polygon.io/stocks'
+
+# POLYGON_API_KEY = 'jrQpup0rXAdbYrIRXUTOI8bZ1BkQlJiR'
+POLYGON_API_KEY = 'PK45PMO9M1JFIAWKW1X1'

data/polygon.py

@@ -0,0 +1,60 @@
+import sys
+sys.path.append('/app')
+from data.api_data import POLYGON_API_KEY
+from datetime import datetime, date, timedelta
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+import pandas as pd
+import math
+
+# URL for all the tickers
+POLYGON_TICKERS_URL = 'https://api.polygon.io/v2/reference/tickers?page={}&apiKey={}'
+
+# URL for aggregate data - default limit 5000. Adjusted for splits and dividents
+POLYGON_AGGS_URL = 'https://api.polygon.io/v2/aggs/ticker/{}/range/{}/{}/{}/{}?unadjusted=false&sort=asc&apiKey={}'
+
+# URL For specific, but it looks like they are all served by the same endpoint, regardless of the type
+POLYGON_STOCKS_AGGS_URL = 'https://api.polygon.io/v2/aggs/ticker/{}/range/{}/{}/{}/{}?unadjusted=false&sort=asc&apiKey={}'
+POLYGON_FOREX_AGGS_URL  = 'https://api.polygon.io/v2/aggs/ticker/{}/range/{}/{}/{}/{}?unadjusted=false&sort=asc&apiKey={}'
+POLYGON_CRYPTO_AGGS_URL = 'https://api.polygon.io/v2/aggs/ticker/{}/range/{}/{}/{}/{}?unadjusted=false&sort=asc&apiKey={}'
+
+def get_agg_bars(symbol, period, multiplier, start, end):
+    session = requests.Session()
+    try:
+        r = session.get(POLYGON_AGGS_URL.format(symbol, multiplier, period, start.strftime("%Y-%m-%d"), end.strftime("%Y-%m-%d"), POLYGON_API_KEY))
+        if r:
+            data = r.json()
+            return data['results']
+        return None
+    except Exception as e:
+        print('****** getting bars errored for symbol: ' + str(symbol))
+        print(e, data)
+        return None
+
+def get_all_tickers():
+    page = 1
+
+    session = requests.Session()
+    # Initial request to get the ticker count
+    r = session.get(POLYGON_TICKERS_URL.format(page, POLYGON_API_KEY))
+    data = r.json()
+
+    count = data['count']
+    print('total tickers ' + str(count))
+    pages = math.ceil(count / data['perPage'])
+
+    #tickers = pd.DataFrame()
+    tickers = []
+    #for pages in range (2, pages+1):
+    for pages in range (2, 4): #for testing otherwise too long
+        r = session.get(POLYGON_TICKERS_URL.format(page, POLYGON_API_KEY))
+        data = r.json()
+        #tickers = df.append(pd.DataFrame(data['tickers']))
+        tickers = tickers + data['tickers']
+        page += 1 
+
+    return tickers
+
+def ts_to_datetime(ts) -> str:
+    return datetime.fromtimestamp(ts / 1000.0).strftime('%Y-%m-%d %H:%M')

data/yfinance.py

@@ -0,0 +1,4 @@
+import yfinance
+
+def get_agg_daily_bars(symbol, start, end)
+    return yfinance.download(symbol, start=start, end=end)

db/db_create.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python 
+
+import os
+import json
+import dbwrapper as dbw
+
+def initSchema():
+    basepath = '/app/db/tables'
+    entries = os.listdir(basepath)
+
+    with os.scandir(basepath) as entries:
+        for entry in entries:
+            if entry.is_file():
+                print('Running '+ entry.name)
+                # TODO: Can put this in a function and just yield entries intsead
+                with open(entry, 'r') as sqlFile:
+                    query = sqlFile.read()
+                    #with dbw.dbEngine.connect().execution_options(autocommit=True) as conn:
+                    with dbw.dbEngine.connect() as conn:
+                        result = conn.execute(query)
+
+dbw.initDb()
+initSchema()
+
+# #seed future strategies here
+# strategies = ['opening_range_breakout', 'opening_range_breakdown']
+
+# for strategy in strategies:
+#     cursor.execute("""
+#         INSERT INTO strategy (name) VALUES (?)
+#     """, (strategy,))
+
+# connection.commit()

db/db_drop.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python 
+
+import sys
+sys.path.append('/app')
+import dbwrapper as dbw
+
+
+dbw.initDb()
+
+with dbw.dbEngine.connect() as conn:
+
+    result = conn.execute("""
+            DO $$ DECLARE
+                r RECORD;
+            BEGIN
+                FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = 'public') LOOP
+                    EXECUTE 'DROP TABLE IF EXISTS ' || quote_ident(r.tablename) || ' CASCADE';
+                END LOOP;
+            END $$;
+        """)

db/db_populate_assets.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python 
+
+import sys
+sys.path.append('/app')
+import dbwrapper as dbw
+import data.polygon as d_poly
+import data.alpaca as d_alpaca
+
+USE_POLYGON = False
+
+def alpaca_populate_assets():
+    with dbw.dbEngine.connect() as conn:
+
+        #all the existing symbols and flags so that we don't add ones that already exist
+        result = conn.execute("""SELECT symbol, id FROM asset""")
+        symbols = [row['symbol'] for row in result]
+
+        assets = d_alpaca.get_all_assets()
+
+        for asset in assets:
+            try:
+                if asset.status == 'active' and asset.tradable:
+                    if asset.symbol not in symbols:
+                        print(f"Alpaca New asset: {asset.symbol} {asset.name}")
+                        conn.execute("""INSERT INTO asset (exchange, symbol, name, source, easy_to_borrow, marginable, shortable) VALUES (%s, %s, %s, %s, %s, %s, %s)""", 
+                        (asset.exchange, asset.symbol, asset.name, 'alpaca', asset.easy_to_borrow, asset.marginable, asset.shortable))
+                    else:
+                        conn.execute("""UPDATE asset SET easy_to_borrow = %s, marginable = %s, shortable = %s where symbol = %s""", 
+                        (asset.easy_to_borrow, asset.marginable, asset.shortable, asset.symbol))
+            except Exception as e:
+                print(asset.symbol)
+                print(e)
+
+def polygon_populate_assets():
+    with dbw.dbEngine.connect() as conn:
+        #all the existing symbols and flags so that we don't add ones that already exist
+        result = conn.execute("""SELECT symbol, id FROM asset""")
+        symbols = [row['symbol'] for row in result]
+
+        assets = d_poly.get_all_tickers()
+
+        for asset in assets:
+            try:
+                if asset['active'] == True and asset['ticker'] not in symbols:
+                    print(f"Polygon New asset {asset['primaryExch']} {asset['ticker']} {asset['name']} {asset['market']} {asset['locale']} {asset['currency']}")
+                    #type is included at times
+                    if 'type' not in asset:
+                        conn.execute("""
+                            INSERT INTO asset (exchange, symbol, name, market, locale, currency, source) VALUES (%s, %s, %s, %s, %s, %s, %s)
+                            """, (asset['primaryExch'], asset['ticker'], asset['name'], asset['market'], asset['locale'], asset['currency'],'polygon'))
+                    else:
+                        conn.execute("""
+                            INSERT INTO asset (exchange, symbol, name, market, locale, currency, source, type) VALUES (%s, %s, %s, %s, %s, %s, %s)
+                            """, (asset['primaryExch'], asset['ticker'], asset['name'], asset['market'], asset['locale'], asset['currency'],'polygon', asset['type']))
+            except Exception as e:
+                print(asset['ticker'])
+                print(e)
+
+dbw.initDb()
+
+if USE_POLYGON:
+    polygon_populate_assets()
+alpaca_populate_assets()