Update daily update script

daily_update.sh

@@ -0,0 +1,27 @@
+echo "Updating index weight"
+startdate=$(dolt sql -q "select DATE_FORMAT(DATE_ADD(max(trade_date), INTERVAL 1 DAY), '%Y%m%d') from ts_index_weight" -r csv | tail -1)
+python3 tushare/dump_index_weight.py --start_date=$startdate
+for file in $(ls tushare/index_weight/); 
+do  
+  dolt table import -u ts_index_weight tushare/index/$file; 
+done
+
+echo "Updating index price"
+python3 tushare/dump_index_eod_price.py 
+for file in $(ls tushare/index/); 
+do   
+  dolt table import -u ts_a_stock_eod_price tushare/index/$file; 
+done
+
+echo "Updating stock price"
+dolt sql-server &
+python3 tushare/update_a_stock_eod_price_to_latest.py
+killall dolt
+
+dolt sql --file ./tushare/regular_update.sql
+
+dolt add -A
+
+dolt commit -m "Daily update"
+
+dolt push

tushare/dump_index_eod_price.py

@@ -18,9 +18,11 @@ def get_trade_cal(start_date, end_date):
 
 index_list = ['399300.SZ', '000905.SH', '000300.SH']
 
-def dump_index_data(start_date, end_date, skip_exists=True):
+def dump_index_data(start_date="19900101", end_date="20500101", skip_exists=True):
     trade_date_df = get_trade_cal(start_date, end_date)
-    print(trade_date_df)
+
+    if not os.path.exists(f"{file_path}/index/"):
+        os.makedirs(f"{file_path}/index/")
 
     for index_name in index_list:
         filename = f'{file_path}/index/{index_name}.csv'
@@ -30,7 +32,11 @@ def dump_index_data(start_date, end_date, skip_exists=True):
             end_index = min((time_slice+1) * 4000 - 1, len(trade_date_df) - 1)
             end_date = trade_date_df["cal_date"][end_index]
             df = pro.index_daily(ts_code=index_name, start_date = start_date, end_date=end_date)
+            if df.empty:
+                continue
             result_df_list.append(df)
+        if len(result_df_list) == 0:
+            continue
         result_df = pandas.concat(result_df_list)
         result_df["tradedate"] = result_df["trade_date"]
         result_df["volume"] = result_df["vol"]

tushare/dump_index_weight.py

@@ -16,7 +16,9 @@
     ]
 
 def dump_index_data(start_date=None, end_date=None, skip_exists=True):
-
+    if not os.path.exists(f"{file_path}/index_weight/"):
+        os.makedirs(f"{file_path}/index_weight/")
+
     for index_name in index_list:
         time_step = datetime.timedelta(days=15)
         if start_date is None:
@@ -28,7 +30,7 @@ def dump_index_data(start_date=None, end_date=None, skip_exists=True):
             index_start_date = datetime.datetime.strptime(str(start_date), '%Y%m%d')
 
         if end_date is None:
-            index_end_date = list_date_obj + time_step
+            index_end_date = index_start_date + time_step
         else:
             index_end_date = datetime.datetime.strptime(str(end_date), '%Y%m%d')
 
@@ -37,10 +39,14 @@ def dump_index_data(start_date=None, end_date=None, skip_exists=True):
         result_df_list = []
         while index_end_date < datetime.datetime.now():
             df = pro.index_weight(index_code=index_name, start_date = index_start_date.strftime('%Y%m%d'), end_date=index_end_date.strftime('%Y%m%d'))
-            result_df_list.append(df)
             index_start_date += time_step
             index_end_date += time_step
+            if df.empty:
+                continue
+            result_df_list.append(df)
             time.sleep(0.5)
+        if len(result_df_list) == 0:
+            continue
         result_df = pandas.concat(result_df_list)
         result_df["stock_code"] = result_df["con_code"]
         result_df.to_csv(filename, index=False)

tushare/regular_update.sql

@@ -24,3 +24,48 @@ WHERE ts_a_stock_eod_price.symbol = missing_table.w_missing_symbol;
 
 /* Set new stock adj ratio to 1 */
 UPDATE ts_link_table  SET adj_ratio=1 WHERE adj_ratio is NULL;
+
+/* Fill in index price from ts */
+INSERT IGNORE INTO final_a_stock_eod_price (tradedate, symbol, high, low, open, close, volume, adjclose) 
+select ts_raw_table.tradedate, 
+			ts_link_table.w_symbol as symbol,
+			ts_raw_table.high,
+			ts_raw_table.low,
+			ts_raw_table.open,
+			ts_raw_table.close,
+			ts_raw_table.volume,
+			ROUND(ts_raw_table.adjclose / ts_link_table.adj_ratio, 2) as adjclose 
+FROM (
+SELECT * FROM ts_a_stock_eod_price
+WHERE tradedate > 
+		(
+			select max(tradedate) as tradedate
+			FROM final_a_stock_eod_price 
+			where symbol = "SZ399300"
+		) 
+) ts_raw_table
+LEFT JOIN ts_link_table ON ts_raw_table.symbol = ts_link_table.link_symbol;
+
+/* Fill in stock price from ts */
+INSERT IGNORE INTO final_a_stock_eod_price (tradedate, symbol, high, low, open, close, volume, adjclose) 
+select ts_raw_table.tradedate, 
+			ts_link_table.w_symbol as symbol,
+			ts_raw_table.high,
+			ts_raw_table.low,
+			ts_raw_table.open,
+			ts_raw_table.close,
+			ts_raw_table.volume,
+			ROUND(ts_raw_table.adjclose / ts_link_table.adj_ratio, 2) as adjclose 
+FROM (
+SELECT * FROM ts_a_stock_eod_price
+WHERE tradedate > (
+	select max(tradedate) as tradedate
+	FROM
+		(select tradedate, count(tradedate) as symbol_count 
+		FROM final_a_stock_eod_price 
+		where tradedate > "2022-07-01" 
+		group by tradedate) tradedate_record
+	WHERE symbol_count > 1000
+  )
+) ts_raw_table
+LEFT JOIN ts_link_table ON ts_raw_table.symbol = ts_link_table.link_symbol;