diff --git a/daily_update.sh b/daily_update.sh new file mode 100644 index 0000000..ed51ac9 --- /dev/null +++ b/daily_update.sh @@ -0,0 +1,27 @@ +echo "Updating index weight" +startdate=$(dolt sql -q "select DATE_FORMAT(DATE_ADD(max(trade_date), INTERVAL 1 DAY), '%Y%m%d') from ts_index_weight" -r csv | tail -1) +python3 tushare/dump_index_weight.py --start_date=$startdate +for file in $(ls tushare/index_weight/); +do + dolt table import -u ts_index_weight tushare/index/$file; +done + +echo "Updating index price" +python3 tushare/dump_index_eod_price.py +for file in $(ls tushare/index/); +do + dolt table import -u ts_a_stock_eod_price tushare/index/$file; +done + +echo "Updating stock price" +dolt sql-server & +python3 tushare/update_a_stock_eod_price_to_latest.py +killall dolt + +dolt sql --file ./tushare/regular_update.sql + +dolt add -A + +dolt commit -m "Daily update" + +dolt push diff --git a/tushare/dump_index_eod_price.py b/tushare/dump_index_eod_price.py index 072ce14..b296b8f 100644 --- a/tushare/dump_index_eod_price.py +++ b/tushare/dump_index_eod_price.py @@ -18,9 +18,11 @@ def get_trade_cal(start_date, end_date): index_list = ['399300.SZ', '000905.SH', '000300.SH'] -def dump_index_data(start_date, end_date, skip_exists=True): +def dump_index_data(start_date="19900101", end_date="20500101", skip_exists=True): trade_date_df = get_trade_cal(start_date, end_date) - print(trade_date_df) + + if not os.path.exists(f"{file_path}/index/"): + os.makedirs(f"{file_path}/index/") for index_name in index_list: filename = f'{file_path}/index/{index_name}.csv' @@ -30,7 +32,11 @@ def dump_index_data(start_date, end_date, skip_exists=True): end_index = min((time_slice+1) * 4000 - 1, len(trade_date_df) - 1) end_date = trade_date_df["cal_date"][end_index] df = pro.index_daily(ts_code=index_name, start_date = start_date, end_date=end_date) + if df.empty: + continue result_df_list.append(df) + if len(result_df_list) == 0: + continue result_df = pandas.concat(result_df_list) result_df["tradedate"] = result_df["trade_date"] result_df["volume"] = result_df["vol"] diff --git a/tushare/dump_index_weight.py b/tushare/dump_index_weight.py index db94780..123b43d 100644 --- a/tushare/dump_index_weight.py +++ b/tushare/dump_index_weight.py @@ -16,7 +16,9 @@ ] def dump_index_data(start_date=None, end_date=None, skip_exists=True): - + if not os.path.exists(f"{file_path}/index_weight/"): + os.makedirs(f"{file_path}/index_weight/") + for index_name in index_list: time_step = datetime.timedelta(days=15) if start_date is None: @@ -28,7 +30,7 @@ def dump_index_data(start_date=None, end_date=None, skip_exists=True): index_start_date = datetime.datetime.strptime(str(start_date), '%Y%m%d') if end_date is None: - index_end_date = list_date_obj + time_step + index_end_date = index_start_date + time_step else: index_end_date = datetime.datetime.strptime(str(end_date), '%Y%m%d') @@ -37,10 +39,14 @@ def dump_index_data(start_date=None, end_date=None, skip_exists=True): result_df_list = [] while index_end_date < datetime.datetime.now(): df = pro.index_weight(index_code=index_name, start_date = index_start_date.strftime('%Y%m%d'), end_date=index_end_date.strftime('%Y%m%d')) - result_df_list.append(df) index_start_date += time_step index_end_date += time_step + if df.empty: + continue + result_df_list.append(df) time.sleep(0.5) + if len(result_df_list) == 0: + continue result_df = pandas.concat(result_df_list) result_df["stock_code"] = result_df["con_code"] result_df.to_csv(filename, index=False) diff --git a/tushare/regular_update.sql b/tushare/regular_update.sql index 6fb13fa..f1a7abf 100644 --- a/tushare/regular_update.sql +++ b/tushare/regular_update.sql @@ -24,3 +24,48 @@ WHERE ts_a_stock_eod_price.symbol = missing_table.w_missing_symbol; /* Set new stock adj ratio to 1 */ UPDATE ts_link_table SET adj_ratio=1 WHERE adj_ratio is NULL; + +/* Fill in index price from ts */ +INSERT IGNORE INTO final_a_stock_eod_price (tradedate, symbol, high, low, open, close, volume, adjclose) +select ts_raw_table.tradedate, + ts_link_table.w_symbol as symbol, + ts_raw_table.high, + ts_raw_table.low, + ts_raw_table.open, + ts_raw_table.close, + ts_raw_table.volume, + ROUND(ts_raw_table.adjclose / ts_link_table.adj_ratio, 2) as adjclose +FROM ( +SELECT * FROM ts_a_stock_eod_price +WHERE tradedate > + ( + select max(tradedate) as tradedate + FROM final_a_stock_eod_price + where symbol = "SZ399300" + ) +) ts_raw_table +LEFT JOIN ts_link_table ON ts_raw_table.symbol = ts_link_table.link_symbol; + +/* Fill in stock price from ts */ +INSERT IGNORE INTO final_a_stock_eod_price (tradedate, symbol, high, low, open, close, volume, adjclose) +select ts_raw_table.tradedate, + ts_link_table.w_symbol as symbol, + ts_raw_table.high, + ts_raw_table.low, + ts_raw_table.open, + ts_raw_table.close, + ts_raw_table.volume, + ROUND(ts_raw_table.adjclose / ts_link_table.adj_ratio, 2) as adjclose +FROM ( +SELECT * FROM ts_a_stock_eod_price +WHERE tradedate > ( + select max(tradedate) as tradedate + FROM + (select tradedate, count(tradedate) as symbol_count + FROM final_a_stock_eod_price + where tradedate > "2022-07-01" + group by tradedate) tradedate_record + WHERE symbol_count > 1000 + ) +) ts_raw_table +LEFT JOIN ts_link_table ON ts_raw_table.symbol = ts_link_table.link_symbol; \ No newline at end of file