From 4d38d049c98808408d0a72c8e528d725c2402747 Mon Sep 17 00:00:00 2001 From: awb99 Date: Sat, 2 Sep 2023 21:40:34 -0500 Subject: [PATCH] tv search working --- app/resources/symbollist/futures-kibot.edn | 2 +- lib/data/src/ta/data/api/kibot.clj | 2 +- lib/data/src/ta/data/api_ds/kibot.clj | 2 +- lib/data/src/ta/data/import.clj | 23 ++++-- lib/data/src/ta/data/settings.clj | 13 +++- .../src/ta/tradingview/db_instrument.clj | 18 +++-- .../src/ta/tradingview/handler_datasource.clj | 71 ++++++++++--------- lib/warehouse/src/ta/warehouse.clj | 4 +- lib/warehouse/src/ta/warehouse/symbol_db.clj | 48 +++++++++++-- lib/warehouse/src/ta/warehouse/symbollist.clj | 2 + 10 files changed, 122 insertions(+), 63 deletions(-) diff --git a/app/resources/symbollist/futures-kibot.edn b/app/resources/symbollist/futures-kibot.edn index f121e02c..46cd1744 100644 --- a/app/resources/symbollist/futures-kibot.edn +++ b/app/resources/symbollist/futures-kibot.edn @@ -64,7 +64,7 @@ {:symbol "BR0" :kibot "BR" :name "CONTINUOUS BRAZILIAN REAL CONTRACT" :category :future} {:symbol "RY0" :kibot "RY" :name "CONTINUOUS EURO FX/JAPANESE YEN CONTRACT" :category :future} {:symbol "SEK0" :kibot "SEK" :name "CONTINUOUS SWEDISH KRONA CONTRACT" :category :future} -{:symbol "AJY0" :kibot "AKY" :name "CONTINUOUS AUSTRALIAN $/JAPANESE YEN CONTRACT" :category :future} +{:symbol "AJY0" :kibot "AJY" :name "CONTINUOUS AUSTRALIAN $/JAPANESE YEN CONTRACT" :category :future} {:symbol "NOK0" :kibot "NOK" :name "CONTINUOUS NORWEGIAN KRONE CONTRACT" :category :future} {:symbol "PJY0" :kibot "PJY" :name "CONTINUOUS BRITISH PD/JAPANSE YEN CONTRACT" :category :future} {:symbol "ECD0" :kibot "ECD" :name "CONTINUOUS EURO FX/CANADIAN DOLLAR CONTRACT" :category :future} diff --git a/lib/data/src/ta/data/api/kibot.clj b/lib/data/src/ta/data/api/kibot.clj index 6940ee46..3b2524a5 100644 --- a/lib/data/src/ta/data/api/kibot.clj +++ b/lib/data/src/ta/data/api/kibot.clj @@ -61,7 +61,7 @@ (defn history [opts] (let [{:keys [user password]} @api-key] - (info "login user: " user "pwd: " password) + ;(info "login user: " user "pwd: " password) (make-request base-url (merge {:action "history" diff --git a/lib/data/src/ta/data/api_ds/kibot.clj b/lib/data/src/ta/data/api_ds/kibot.clj index 41d590af..e00106a5 100644 --- a/lib/data/src/ta/data/api_ds/kibot.clj +++ b/lib/data/src/ta/data/api_ds/kibot.clj @@ -71,7 +71,7 @@ ] (-> (merge symbol-map {:interval period - :period 1 + :period 1000 :timezone "UTC" :splitadjusted 1}) (kibot/history) diff --git a/lib/data/src/ta/data/import.clj b/lib/data/src/ta/data/import.clj index e4913484..98b624f9 100644 --- a/lib/data/src/ta/data/import.clj +++ b/lib/data/src/ta/data/import.clj @@ -5,8 +5,9 @@ [ta.data.api-ds.kibot :as kibot] [ta.data.api-ds.alphavantage :as av] [ta.warehouse :as wh] - [ta.data.settings :refer [determine-wh]])) - + [ta.data.settings :refer [determine-wh]] + [ta.warehouse.symbollist :refer [load-list]] + )) (def dict-provider {:kibot kibot/get-series @@ -25,17 +26,25 @@ (wh/save-symbol w series-ds interval symbol)))) - -#_(defn init-symbols [w fn-get-history-since-as-ds frequency since symbols] +(defn import-symbols [provider symbols interval range opts] + (let [symbols (if (string? symbols) + (load-list symbols) + symbols)] (doall (map - (partial init-symbol w fn-get-history-since-as-ds frequency since) - symbols))) + #(import-series provider % interval range opts) + symbols)))) (comment (import-series :alphavantage "MSFT" "D" :full {}) (import-series :alphavantage "EURUSD" "D" :full {}) (import-series :alphavantage "BTCUSD" "D" :full {}) - + (import-series :kibot "SIL0" "D" :full {}) + + (import-symbols :kibot ["SIL0" "NG0" "IBM"] "D" :full {}) + (import-symbols :kibot "joseph" "D" :full {}) + + (import-symbols :kibot "futures-kibot" "D" :full {}) + ; ) diff --git a/lib/data/src/ta/data/settings.clj b/lib/data/src/ta/data/settings.clj index b2301cdd..7d7c1dbf 100644 --- a/lib/data/src/ta/data/settings.clj +++ b/lib/data/src/ta/data/settings.clj @@ -13,7 +13,18 @@ :future :futures :stocks))) - +;{:value "Corp" :name "Bonds"} +;{:value "Index" :name "Indices"} +;{:value "Curncy" :name "Currencies"} + +(def category-names + {:crypto "Crypto" + :etf "ETF" + :equity "Equity" + :fx "FX" + :future "Future" + :mutualfund "MutualFund" + }) (comment diff --git a/lib/tradingview/src/ta/tradingview/db_instrument.clj b/lib/tradingview/src/ta/tradingview/db_instrument.clj index 171281cd..1820f273 100644 --- a/lib/tradingview/src/ta/tradingview/db_instrument.clj +++ b/lib/tradingview/src/ta/tradingview/db_instrument.clj @@ -1,14 +1,10 @@ -(ns ta.tradingview.db-instrument) - -(def categories - {:crypto "Crypto" - :etf "ETF" - :mutualfund "MutualFund" - :equity "Equity"}) - +(ns ta.tradingview.db-instrument + (:require + [ta.data.settings :as settings] + )) (defn reverse-lookup [] - (->> categories + (->> settings/category-names (map (fn [[k v]] [v k])) (into {}))) @@ -19,7 +15,7 @@ (defn inst-type [i] ; this dict has to match above the server-config list of supported categories (let [c (:category i)] - (or (get categories c) "Equity"))) + (or (get settings/category-names c) "Equity"))) (defn category-name->category [c] @@ -32,6 +28,8 @@ (category-name->category "Equity") (category-name->category nil) (category-name->category "Crypto") + (category-name->category "Future") + ; ) diff --git a/lib/tradingview/src/ta/tradingview/handler_datasource.clj b/lib/tradingview/src/ta/tradingview/handler_datasource.clj index 48500f55..1395ad6e 100644 --- a/lib/tradingview/src/ta/tradingview/handler_datasource.clj +++ b/lib/tradingview/src/ta/tradingview/handler_datasource.clj @@ -16,9 +16,10 @@ ;[ta.tradingview.db-ts :refer [save-chart-boxed delete-chart load-chart-boxed chart-list now-epoch]] [ta.warehouse.symbol-db :refer [search instrument-details]] [ta.tradingview.db-instrument :refer [inst-type category-name->category]] - [ta.data.settings :refer [determine-wh]] + [ta.data.settings :refer [determine-wh category-names]] ;[ta.tradingview.db-marks :refer [load-marks convert-marks]] )) + (defn server-time [] (-> (now-datetime) datetime->epoch-second)) @@ -29,6 +30,17 @@ ;; CONFIG - Tell TradingView which featurs are supported by server. +(def supported-types + (->> (concat [{:value "" :name "All"}] + (map (fn [[k v]] + {:value k :name v}) category-names)) + (into []) + )) + + + + + (def server-config {:supports_time true ; we send our server-time :supports_search true ;search and individual symbol resolve logic. @@ -36,14 +48,12 @@ :supports_timescale_marks false :supports_group_request false :supported_resolutions ["15" "D"] ; ["1" "5" "15" "30" "60" "1D" "1W" "1M"] - :symbols_types [{:value "" :name "All"} + :symbols_types supported-types + #_[{:value "" :name "All"} {:value "Crypto" :name "Crypto"} {:value "Equity" :name "Equities"} {:value "Mutualfund" :name "Mutualfund"} {:value "ETF" :name "ETF"} - ;{:value "Corp" :name "Bonds"} - ;{:value "Index" :name "Indices"} - ;{:value "Curncy" :name "Currencies"} ] :exchanges [{:value "" :name "All Exchanges" :desc ""} {:value "BB" :name "Bybit" :desc ""} @@ -114,42 +124,37 @@ ;[{"symbol":"BLK","full_name":"BLK","description":"BlackRock, Inc.","exchange":"NYSE","type":"stock"}, ; {"symbol":"BA","full_name":"BA","description":"The Boeing Company","exchange":"NYSE","type":"stock"}] -(defn filter-exchange [exchange list] - (if (str/blank? exchange) - list - (filter #(= exchange (:exchange %)) list))) - -(defn filter-category [type list] - (if (str/blank? type) - list - (let [c (category-name->category type)] - (filter #(= c (:category %)) list)))) - -(defn symbol-search [query type exchange limit] - (let [sr (->> (search query) - (filter-exchange exchange) - (filter-category type)) - sr-limit (take limit sr) - sr-tv (map (fn [{:keys [symbol name] :as i}] - {:ticker symbol - :symbol symbol ; OUR SYMBOL FORMAT. TV uses exchange:symbol - :full_name symbol - :description (:name i) - :exchange (:exchange i) - :type (inst-type i)}) sr-limit)] - sr-tv)) + +(defn instrument->tradingview [{:keys [symbol name] :as i}] + {:ticker symbol + :symbol symbol ; OUR SYMBOL FORMAT. TV uses exchange:symbol + :full_name name + :description (:name i) + :exchange (:exchange i) + :type (inst-type i)}) + + +(defn symbol-search [query category exchange limit] + (info "symbol-search q:" query " category: " category "exchange: " exchange "limit: " limit) + (let [category (if (and type (string? category) (not (str/blank? category))) + (keyword category) nil) + result (search query category exchange) + result (take limit result) + result-tv (map instrument->tradingview result)] + result-tv)) (defn search-handler [{:keys [query-params] :as req}] (info "tv/search: " query-params) (let [{:keys [query type exchange limit]} (clojure.walk/keywordize-keys query-params) limit (Integer/parseInt limit) - sr-tv (symbol-search query type exchange limit)] - (res/response sr-tv))) + result-tv (symbol-search query type exchange limit) + ] + (res/response result-tv))) (comment - (-> (search-handler {:query-params {:query "E" + (-> (search-handler {:query-params {:query "B" :type "" - :exchange "BB" + :exchange "" :limit "10"}}) :body ;count diff --git a/lib/warehouse/src/ta/warehouse.clj b/lib/warehouse/src/ta/warehouse.clj index ce1a3c0b..13ad31f7 100644 --- a/lib/warehouse/src/ta/warehouse.clj +++ b/lib/warehouse/src/ta/warehouse.clj @@ -17,10 +17,10 @@ (defn save-ts [wkw ds name] (let [dir (get-in-config [:ta :warehouse :series wkw]) - _ (do ;(info "creating dir: " dir) + _ (do (debug "creating dir: " dir) (create-dirs dir)) s (io/gzip-output-stream! (filename-ts wkw name))] - (info "saving series " name " count: " (tc/row-count ds)) + (debug "saving series " name " count: " (tc/row-count ds)) (io/put-nippy! s ds))) (defn load-ts [wkw name] diff --git a/lib/warehouse/src/ta/warehouse/symbol_db.clj b/lib/warehouse/src/ta/warehouse/symbol_db.clj index d3b44e77..d92a7632 100644 --- a/lib/warehouse/src/ta/warehouse/symbol_db.clj +++ b/lib/warehouse/src/ta/warehouse/symbol_db.clj @@ -93,19 +93,53 @@ ; ) -(defn search [q] - (let [l (get-instruments) - q (lower-case q)] - (filter (fn [{:keys [name symbol]}] - (or (includes? (lower-case name) q) - (includes? (lower-case symbol) q))) - l))) +(defn q? [q] + (fn [{:keys [name symbol]}] + (or (includes? (lower-case name) q) + (includes? (lower-case symbol) q)))) + +(defn =exchange? [e] + (fn [{:keys [exchange]}] + (= exchange e))) + +(defn =category? [c] + (fn [{:keys [category]}] + (= category c))) + +(defn filter-eventually [make-pred target list] + (if target + (filter (make-pred target) list) + list)) + +(defn search + ([q] + (search q nil nil)) + ([q category] + (search q category nil)) + ([q category exchange] + + (let [list-full (get-instruments) + q (if (or (nil? q) (blank? q)) nil (lower-case q)) + e (if (or (nil? exchange) (blank? exchange)) nil exchange) + c (if (nil? category) nil category)] + (info "search q: " q "category: " c " exchange: " e) + (->> list-full + (filter-eventually =exchange? e) + (filter-eventually =category? c) + (filter-eventually q? q))))) + (comment ;(search "P") (search "Bitc") (search "BT") + (search "BT" :crypto) + (search "B" :equity) + (search "B" :equity "SG") + (search "B" nil nil) + (search "B" "" "") + (instrument-details "BTCUSD") (instrument-details "EURUSD") ; diff --git a/lib/warehouse/src/ta/warehouse/symbollist.clj b/lib/warehouse/src/ta/warehouse/symbollist.clj index be72d7b1..60f367ae 100644 --- a/lib/warehouse/src/ta/warehouse/symbollist.clj +++ b/lib/warehouse/src/ta/warehouse/symbollist.clj @@ -63,6 +63,8 @@ (def files (file-seq directory)) (take 10 files) + (load-list "bonds") + (load-list-full "fidelity-select") (load-lists-full ["crypto" "fidelity-select"