cnn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import datetime as dt\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from freqtrade.exchange import timeframe_to_minutes\n",
    "from keras import Sequential\n",
    "from keras.losses import categorical_crossentropy\n",
    "from lazyft.data_loader import load_pair_data\n",
    "\n",
    "from constants import IMAGES_PATH, REPO\n",
    "import keras\n",
    "from keras.initializers import initializers_v2 as initializer\n",
    "\n",
    "from preprocess import preprocess\n",
    "from constants import IMAGES_PATH\n",
    "from cnn_model import create_cnn\n",
    "from keras_preprocessing.image import ImageDataGenerator\n",
    "from keras.optimizer_v2.adam import Adam\n",
    "import tensorflow as tf\n",
    "\n",
    "tf.config.run_functions_eagerly(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "pair = \"BTC/USDT\"\n",
    "SPLIT = 0.30\n",
    "LR = 0.001\n",
    "TIMESTAMP = dt.datetime.now().strftime(\"%Y%m%d%H%M%S\")\n",
    "# cnn_networks = 1\n",
    "models: list[keras.Sequential] = []\n",
    "image_train_path = TRAIN_IMAGES_PATH / pair.replace(\"/\", \"_\")\n",
    "image_test_path = TEST_IMAGES_PATH / pair.replace(\"/\", \"_\")\n",
    "image_train_path.mkdir(parents=True, exist_ok=True)\n",
    "image_test_path.mkdir(parents=True, exist_ok=True)\n",
    "preprocess(\n",
    "    timerange=\"20170101-20211231\",\n",
    "    image_save_path=image_train_path,\n",
    "    pair=pair,\n",
    "    interval=\"1h\",\n",
    ")\n",
    "preprocess(\n",
    "    timerange=\"20220101-\", pair=pair, image_save_path=image_test_path\n",
    ")\n",
    "\n",
    "# global history, string_list, summary\n",
    "target_size = 40\n",
    "batch_size = 20\n",
    "EPOCHS = 10"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-04-03 09:45:23.250 | INFO     | lazyft.downloader:download_missing_historical_data:301 - Checking if download is needed for BTC/USDT @ 1h interval(s)\n",
      "2022-04-03 09:45:23.253 | INFO     | lazyft.downloader:download_missing_historical_data:338 - Data is up to date\n",
      "2022-04-03 09:45:23.315 | INFO     | lazyft.data_loader:load_pair_data:39 - Loaded 35041 rows for BTC/USDT, data starts at 2018-01-01 00:00:00+00:00\n",
      "Generating images...\n",
      "========PREPROCESS REPORT========:\n",
      "Total Data Points: 1461\n",
      "Total Images Created: 1441\n",
      "Total LONG positions: 700\n",
      "Total SHORT positions: 741\n",
      "2022-04-03 09:45:32.889 | INFO     | lazyft.downloader:download_missing_historical_data:301 - Checking if download is needed for BTC/USDT @ 1h interval(s)\n",
      "2022-04-03 09:45:32.891 | INFO     | lazyft.downloader:download_missing_historical_data:338 - Data is up to date\n",
      "2022-04-03 09:45:32.940 | INFO     | lazyft.data_loader:load_pair_data:39 - Loaded 2209 rows for BTC/USDT, data starts at 2022-01-01 00:00:00+00:00\n",
      "Generating images...\n",
      "========PREPROCESS REPORT========:\n",
      "Total Data Points: 93\n",
      "Total Images Created: 73\n",
      "Total LONG positions: 36\n",
      "Total SHORT positions: 37\n",
      "Found 73 images belonging to 2 classes.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "initializers_ = [\n",
    "    initializer.Orthogonal(),\n",
    "    initializer.LecunUniform(),\n",
    "    initializer.VarianceScaling(),\n",
    "    initializer.RandomNormal(),\n",
    "    initializer.RandomUniform(),\n",
    "    initializer.TruncatedNormal(),\n",
    "    initializer.GlorotNormal(),\n",
    "    initializer.GlorotUniform(),\n",
    "    initializer.HeNormal(),\n",
    "    initializer.HeUniform(),\n",
    "    initializer.Orthogonal(seed=42),\n",
    "    initializer.LecunUniform(seed=42),\n",
    "    initializer.VarianceScaling(seed=42),\n",
    "    initializer.RandomNormal(seed=42),\n",
    "    initializer.RandomUniform(seed=42),\n",
    "    initializer.TruncatedNormal(seed=42),\n",
    "    initializer.GlorotNormal(seed=42),\n",
    "    initializer.GlorotUniform(seed=42),\n",
    "    initializer.HeNormal(seed=42),\n",
    "    initializer.HeUniform(seed=42),\n",
    "]\n",
    "# for i, initializer_ in enumerate(initializers_):\n",
    "#     cnn = create_cnn(target_size, kernel_initializer=initializer_)\n",
    "#     # Compile each model\n",
    "#     cnn.compile(optimizer=Adam(learning_rate=LR), loss='binary_crossentropy', metrics=['acc'])\n",
    "#     models.append(cnn)\n",
    "\n",
    "# All images will be rescaled by 1./255\n",
    "train_validate_datagen = ImageDataGenerator(\n",
    "    rescale=1 / 255, validation_split=SPLIT\n",
    ")  # set validation split\n",
    "test_datagen = ImageDataGenerator(rescale=1 / 255)\n",
    "# data_chunks = ensemble_data(len(models), str(image_path))\n",
    "test_generator = test_datagen.flow_from_directory(\n",
    "    directory=image_test_path,\n",
    "    target_size=(target_size, target_size),\n",
    "    batch_size=batch_size,\n",
    ")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "def load_model(pair: str, time: str, model_name: str) -> Sequential:\n",
    "    model = create_cnn(40)\n",
    "    model_to_load = REPO / pair.replace(\"/\", \"_\") / time / \"models\" / model_name\n",
    "    model.load_weights(model_to_load)\n",
    "\n",
    "    return model\n",
    "\n",
    "\n",
    "model = load_model(pair, \"20220403092229\", \"HeUniform.h5\")\n",
    "model.compile(\n",
    "    optimizer=Adam(learning_rate=LR), loss=categorical_crossentropy, metrics=[\"acc\"]\n",
    ")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/4 [======>.......................] - ETA: 0s - loss: 0.6396 - acc: 0.7500"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/raphael/PycharmProjects/freqgym/venv/lib/python3.9/site-packages/tensorflow/python/data/ops/structured_function.py:264: UserWarning: Even though the `tf.config.experimental_run_functions_eagerly` option is set, this option does not apply to tf.data functions. To force eager execution of tf.data functions, please use `tf.data.experimental.enable_debug_mode()`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4/4 [==============================] - 0s 109ms/step - loss: 0.6582 - acc: 0.6712\n",
      "Test accs: 67.12%\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(test_generator)\n",
    "print(\"Test {0}s: {1:.2f}%\".format(model.metrics_names[1], scores[1] * 100))\n",
    "string_list = []\n",
    "model.summary(print_fn=lambda x: string_list.append(x))\n",
    "string_list.append(f\"test acc: {scores[1] * 100}\")\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [
    {
     "data": {
      "text/plain": "[0.6582338213920593, 0.6712328791618347]"
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [],
   "source": [
    "sample = test_generator.next()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [
    {
     "data": {
      "text/plain": "{'LONG': 0, 'SHORT': 1}"
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_generator.class_indices\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-04-08 16:13:02.069 | INFO     | lazyft.downloader:download_missing_historical_data:302 - Checking if download is needed for BTC/USDT @ 1h interval(s)\n",
      "2022-04-08 16:13:02.072 | INFO     | lazyft.downloader:download_missing_historical_data:339 - Data is up to date\n",
      "2022-04-08 16:13:02.136 | INFO     | lazyft.data_loader:load_pair_data:40 - Loaded 186 rows for BTC/USDT, data starts at 2022-04-01 00:00:00+00:00\n"
     ]
    }
   ],
   "source": [
    "\n",
    "btc = load_pair_data(\"BTC/USDT\", timeframe=\"1h\", timerange=\"20220401-\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "btc.groupby()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "data": {
      "text/plain": "<pandas.core.groupby.generic.DataFrameGroupBy object at 0x7f6513d40850>"
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from util import group_by\n",
    "from datetime import timedelta\n",
    "import time\n",
    "\n",
    "\n",
    "w = btc.head(480)\n",
    "\n",
    "# t1 = time.perf_counter()\n",
    "# sampled = btc.resample('4H').mean()\n",
    "# print('Elapsed time:', timedelta(seconds=time.perf_counter() - t1))\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "data": {
      "text/plain": "{Timestamp('2022-04-01 00:00:00+0000', tz='UTC', freq='D'): 24,\n Timestamp('2022-04-02 00:00:00+0000', tz='UTC', freq='D'): 48,\n Timestamp('2022-04-03 00:00:00+0000', tz='UTC', freq='D'): 72,\n Timestamp('2022-04-04 00:00:00+0000', tz='UTC', freq='D'): 96,\n Timestamp('2022-04-05 00:00:00+0000', tz='UTC', freq='D'): 120,\n Timestamp('2022-04-06 00:00:00+0000', tz='UTC', freq='D'): 144,\n Timestamp('2022-04-07 00:00:00+0000', tz='UTC', freq='D'): 168,\n Timestamp('2022-04-08 00:00:00+0000', tz='UTC', freq='D'): 186}"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w.groupby(pd.Grouper(key='date', freq='d')).get_group()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [
    {
     "data": {
      "text/plain": "                           open  high   low  close  volume\ndate                                                      \n2022-01-01 00:00:00+00:00 47.1K 47.3K 46.9K  47.1K 816.853\n2022-01-02 00:00:00+00:00 47.2K 47.4K 47.1K  47.2K 764.186\n2022-01-03 00:00:00+00:00 46.8K   47K 46.6K  46.8K   1.15K\n2022-01-04 00:00:00+00:00 46.4K 46.6K 46.2K  46.4K   1.48K\n2022-01-05 00:00:00+00:00   46K 46.2K 45.6K  45.9K   2.16K\n2022-01-06 00:00:00+00:00 43.1K 43.3K 42.9K  43.1K   1.62K\n2022-01-07 00:00:00+00:00 42.1K 42.3K 41.7K    42K   2.28K\n2022-01-08 00:00:00+00:00 41.7K 41.9K 41.5K  41.7K   1.37K\n2022-01-09 00:00:00+00:00 41.9K 42.1K 41.7K  41.9K 946.850\n2022-01-10 00:00:00+00:00 41.6K 41.9K 41.3K  41.6K   2.11K\n2022-01-11 00:00:00+00:00 42.2K 42.4K   42K  42.2K   1.55K\n2022-01-12 00:00:00+00:00 43.2K 43.4K   43K  43.2K   1.41K\n2022-01-13 00:00:00+00:00 43.5K 43.6K 43.2K  43.4K   1.45K\n2022-01-14 00:00:00+00:00 42.7K 42.9K 42.6K  42.8K   1.36K\n2022-01-15 00:00:00+00:00 43.1K 43.3K   43K  43.1K 914.002\n2022-01-16 00:00:00+00:00 43.1K 43.2K 42.9K  43.1K 858.431\n2022-01-17 00:00:00+00:00 42.5K 42.7K 42.4K  42.5K   1.15K\n2022-01-18 00:00:00+00:00 41.9K 42.1K 41.8K  41.9K   1.22K\n2022-01-19 00:00:00+00:00 41.9K 42.1K 41.7K  41.9K   1.32K\n2022-01-20 00:00:00+00:00 42.2K 42.4K   42K  42.2K   1.76K",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>open</th>\n      <th>high</th>\n      <th>low</th>\n      <th>close</th>\n      <th>volume</th>\n    </tr>\n    <tr>\n      <th>date</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>2022-01-01 00:00:00+00:00</th>\n      <td>47.1K</td>\n      <td>47.3K</td>\n      <td>46.9K</td>\n      <td>47.1K</td>\n      <td>816.853</td>\n    </tr>\n    <tr>\n      <th>2022-01-02 00:00:00+00:00</th>\n      <td>47.2K</td>\n      <td>47.4K</td>\n      <td>47.1K</td>\n      <td>47.2K</td>\n      <td>764.186</td>\n    </tr>\n    <tr>\n      <th>2022-01-03 00:00:00+00:00</th>\n      <td>46.8K</td>\n      <td>47K</td>\n      <td>46.6K</td>\n      <td>46.8K</td>\n      <td>1.15K</td>\n    </tr>\n    <tr>\n      <th>2022-01-04 00:00:00+00:00</th>\n      <td>46.4K</td>\n      <td>46.6K</td>\n      <td>46.2K</td>\n      <td>46.4K</td>\n      <td>1.48K</td>\n    </tr>\n    <tr>\n      <th>2022-01-05 00:00:00+00:00</th>\n      <td>46K</td>\n      <td>46.2K</td>\n      <td>45.6K</td>\n      <td>45.9K</td>\n      <td>2.16K</td>\n    </tr>\n    <tr>\n      <th>2022-01-06 00:00:00+00:00</th>\n      <td>43.1K</td>\n      <td>43.3K</td>\n      <td>42.9K</td>\n      <td>43.1K</td>\n      <td>1.62K</td>\n    </tr>\n    <tr>\n      <th>2022-01-07 00:00:00+00:00</th>\n      <td>42.1K</td>\n      <td>42.3K</td>\n      <td>41.7K</td>\n      <td>42K</td>\n      <td>2.28K</td>\n    </tr>\n    <tr>\n      <th>2022-01-08 00:00:00+00:00</th>\n      <td>41.7K</td>\n      <td>41.9K</td>\n      <td>41.5K</td>\n      <td>41.7K</td>\n      <td>1.37K</td>\n    </tr>\n    <tr>\n      <th>2022-01-09 00:00:00+00:00</th>\n      <td>41.9K</td>\n      <td>42.1K</td>\n      <td>41.7K</td>\n      <td>41.9K</td>\n      <td>946.850</td>\n    </tr>\n    <tr>\n      <th>2022-01-10 00:00:00+00:00</th>\n      <td>41.6K</td>\n      <td>41.9K</td>\n      <td>41.3K</td>\n      <td>41.6K</td>\n      <td>2.11K</td>\n    </tr>\n    <tr>\n      <th>2022-01-11 00:00:00+00:00</th>\n      <td>42.2K</td>\n      <td>42.4K</td>\n      <td>42K</td>\n      <td>42.2K</td>\n      <td>1.55K</td>\n    </tr>\n    <tr>\n      <th>2022-01-12 00:00:00+00:00</th>\n      <td>43.2K</td>\n      <td>43.4K</td>\n      <td>43K</td>\n      <td>43.2K</td>\n      <td>1.41K</td>\n    </tr>\n    <tr>\n      <th>2022-01-13 00:00:00+00:00</th>\n      <td>43.5K</td>\n      <td>43.6K</td>\n      <td>43.2K</td>\n      <td>43.4K</td>\n      <td>1.45K</td>\n    </tr>\n    <tr>\n      <th>2022-01-14 00:00:00+00:00</th>\n      <td>42.7K</td>\n      <td>42.9K</td>\n      <td>42.6K</td>\n      <td>42.8K</td>\n      <td>1.36K</td>\n    </tr>\n    <tr>\n      <th>2022-01-15 00:00:00+00:00</th>\n      <td>43.1K</td>\n      <td>43.3K</td>\n      <td>43K</td>\n      <td>43.1K</td>\n      <td>914.002</td>\n    </tr>\n    <tr>\n      <th>2022-01-16 00:00:00+00:00</th>\n      <td>43.1K</td>\n      <td>43.2K</td>\n      <td>42.9K</td>\n      <td>43.1K</td>\n      <td>858.431</td>\n    </tr>\n    <tr>\n      <th>2022-01-17 00:00:00+00:00</th>\n      <td>42.5K</td>\n      <td>42.7K</td>\n      <td>42.4K</td>\n      <td>42.5K</td>\n      <td>1.15K</td>\n    </tr>\n    <tr>\n      <th>2022-01-18 00:00:00+00:00</th>\n      <td>41.9K</td>\n      <td>42.1K</td>\n      <td>41.8K</td>\n      <td>41.9K</td>\n      <td>1.22K</td>\n    </tr>\n    <tr>\n      <th>2022-01-19 00:00:00+00:00</th>\n      <td>41.9K</td>\n      <td>42.1K</td>\n      <td>41.7K</td>\n      <td>41.9K</td>\n      <td>1.32K</td>\n    </tr>\n    <tr>\n      <th>2022-01-20 00:00:00+00:00</th>\n      <td>42.2K</td>\n      <td>42.4K</td>\n      <td>42K</td>\n      <td>42.2K</td>\n      <td>1.76K</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w.set_index(\"date\").resample(\"1d\").mean()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "data": {
      "text/plain": "5760"
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from freqtrade.exchange import timeframe_to_minutes\n",
    "\n",
    "MINUTES_IN_DAY = 1440\n",
    "tf_minutes = timeframe_to_minutes(\"5m\")\n",
    "20 * MINUTES_IN_DAY // tf_minutes\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "outputs": [
    {
     "data": {
      "text/plain": "0"
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "btc.head(1).index[0]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "import util\n",
    "from preprocess import alive_bar\n",
    "\n",
    "\n",
    "def split_timeframes(\n",
    "    df: pd.DataFrame,\n",
    "    window: int,\n",
    "    timeframes: list[str],\n",
    "    list_dates: list[str],\n",
    "    download_interval: str,\n",
    ") -> dict[str, list[pd.Series]]:\n",
    "    \"\"\"\n",
    "    Given a dataframe, a window size, a list of timeframes, and a list of dates,\n",
    "    split the dataframe into a list of lists of series, where each series is a list of prices\n",
    "    for a specific timeframe\n",
    "\n",
    "    :param df: The dataframe that contains the data\n",
    "    :param window: The size of the moving window. This is the number of observations used for\n",
    "    calculating the statistic. Each window will be a fixed size\n",
    "    :param timeframes: a list of timeframes to use for grouping the data\n",
    "    :param list_dates: list of dates\n",
    "    :return: A list of lists of series. Each list of series is a list of the closing prices of the\n",
    "    stocks for a given timeframe.\n",
    "    \"\"\"\n",
    "    series_dict: dict[str, list[pd.Series]] = {}\n",
    "    index = window\n",
    "    with alive_bar(\n",
    "        len(list_dates) - window - 1, title=\"Creating decision map...\", bar=\"filling\"\n",
    "    ) as bar:\n",
    "        while True:\n",
    "            idx = index\n",
    "            save_idx = index - 1\n",
    "\n",
    "            if idx == len(list_dates):\n",
    "                break\n",
    "            if index >= len(list_dates) - 1:\n",
    "                save_idx = -2\n",
    "                after = df[\"date\"] > list_dates[-window - 1]\n",
    "                data_slice = df.loc[after]\n",
    "            else:\n",
    "                after = df[\"date\"] > list_dates[idx - window]\n",
    "                before = df[\"date\"] <= list_dates[idx]\n",
    "                data_slice = df.loc[after & before]\n",
    "\n",
    "            # select appropriate timeframe\n",
    "            # print('len of data slice: ', len(data_slice), 'head: ', data_slice.head())\n",
    "            tf_series = []\n",
    "\n",
    "            # group dataslices by timeframe\n",
    "            for freq in timeframes:\n",
    "                if freq == download_interval:\n",
    "                    tf_series.append(data_slice[\"close\"].tail(20))\n",
    "                    continue\n",
    "                group_dt = util.group_by(data_slice, freq)\n",
    "                tf_series.append(group_dt[\"close\"].tail(20))\n",
    "            series_dict[list_dates[save_idx]] = tf_series\n",
    "            index += 1\n",
    "            bar()\n",
    "    return series_dict\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-04-07 06:52:45.380 | INFO     | lazyft.downloader:download_missing_historical_data:301 - Checking if download is needed for BTC/USDT @ 1h interval(s)\n",
      "2022-04-07 06:52:45.382 | INFO     | lazyft.downloader:download_missing_historical_data:338 - Data is up to date\n",
      "2022-04-07 06:52:45.438 | INFO     | lazyft.data_loader:load_pair_data:40 - Loaded 581 rows for BTC/USDT, data starts at 2022-03-13 00:00:00+00:00\n"
     ]
    }
   ],
   "source": [
    "btc = load_pair_data(\"BTC/USDT\", timeframe=\"1h\", timerange=\"20220313-\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "split_timeframes(\n",
    "    btc, 20, [\"1h\", \"4h\", \"8h\", \"1d\"], util.get_dates_from_df(btc, \"1d\"), \"1h\"\n",
    ")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [],
   "source": [
    "series_dict: dict[str, list[pd.Series]] = {}\n",
    "window = 2\n",
    "df = btc\n",
    "list_dates = util.get_dates_from_df(btc, \"1d\")\n",
    "timeframes = [\"1h\", \"4h\", \"8h\", \"1d\"]\n",
    "download_interval = \"1h\"\n",
    "index = window"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [],
   "source": [
    "idx = index\n",
    "save_idx = index - 1\n",
    "\n",
    "assert idx != len(list_dates)\n",
    "\n",
    "if index >= len(list_dates) - 1:\n",
    "    save_idx = -2\n",
    "    after = df[\"date\"] > list_dates[-window - 1]\n",
    "    data_slice = df.loc[after]\n",
    "else:\n",
    "    after = df[\"date\"] > list_dates[idx - window]\n",
    "    before = df[\"date\"] <= list_dates[idx]\n",
    "    data_slice = df.loc[after & before]\n",
    "\n",
    "# select appropriate timeframe\n",
    "# print('len of data slice: ', len(data_slice), 'head: ', data_slice.head())\n",
    "tf_series = []\n",
    "\n",
    "# group dataslices by timeframe\n",
    "for freq in timeframes:\n",
    "    if freq == download_interval:\n",
    "        tf_series.append(data_slice[\"close\"].tail(20))\n",
    "        continue\n",
    "    group_dt = util.group_by(data_slice, freq)\n",
    "    tf_series.append(group_dt[\"close\"].tail(20))\n",
    "series_dict[list_dates[save_idx]] = tf_series\n",
    "index += 1"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "outputs": [
    {
     "data": {
      "text/plain": "                         date  open  high   low  close  volume\n1   2022-03-13 01:00:00+00:00 38.9K 39.3K 38.9K    39K   1.29K\n2   2022-03-13 02:00:00+00:00   39K 39.2K   39K  39.2K 649.294\n3   2022-03-13 03:00:00+00:00 39.2K 39.2K 39.1K  39.2K 604.576\n4   2022-03-13 04:00:00+00:00 39.2K 39.3K 39.1K  39.1K 547.678\n5   2022-03-13 05:00:00+00:00 39.1K 39.2K 39.1K  39.1K 318.244\n..                        ...   ...   ...   ...    ...     ...\n475 2022-04-01 19:00:00+00:00 46.3K 46.4K 46.1K  46.4K   1.36K\n476 2022-04-01 20:00:00+00:00 46.4K 46.4K 46.1K  46.1K 889.165\n477 2022-04-01 21:00:00+00:00 46.1K 46.4K 46.1K  46.3K 704.819\n478 2022-04-01 22:00:00+00:00 46.3K 46.4K 46.2K  46.4K   1.04K\n479 2022-04-01 23:00:00+00:00 46.4K 46.4K 46.1K  46.3K 737.491\n\n[479 rows x 6 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>date</th>\n      <th>open</th>\n      <th>high</th>\n      <th>low</th>\n      <th>close</th>\n      <th>volume</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>1</th>\n      <td>2022-03-13 01:00:00+00:00</td>\n      <td>38.9K</td>\n      <td>39.3K</td>\n      <td>38.9K</td>\n      <td>39K</td>\n      <td>1.29K</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2022-03-13 02:00:00+00:00</td>\n      <td>39K</td>\n      <td>39.2K</td>\n      <td>39K</td>\n      <td>39.2K</td>\n      <td>649.294</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>2022-03-13 03:00:00+00:00</td>\n      <td>39.2K</td>\n      <td>39.2K</td>\n      <td>39.1K</td>\n      <td>39.2K</td>\n      <td>604.576</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>2022-03-13 04:00:00+00:00</td>\n      <td>39.2K</td>\n      <td>39.3K</td>\n      <td>39.1K</td>\n      <td>39.1K</td>\n      <td>547.678</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>2022-03-13 05:00:00+00:00</td>\n      <td>39.1K</td>\n      <td>39.2K</td>\n      <td>39.1K</td>\n      <td>39.1K</td>\n      <td>318.244</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>475</th>\n      <td>2022-04-01 19:00:00+00:00</td>\n      <td>46.3K</td>\n      <td>46.4K</td>\n      <td>46.1K</td>\n      <td>46.4K</td>\n      <td>1.36K</td>\n    </tr>\n    <tr>\n      <th>476</th>\n      <td>2022-04-01 20:00:00+00:00</td>\n      <td>46.4K</td>\n      <td>46.4K</td>\n      <td>46.1K</td>\n      <td>46.1K</td>\n      <td>889.165</td>\n    </tr>\n    <tr>\n      <th>477</th>\n      <td>2022-04-01 21:00:00+00:00</td>\n      <td>46.1K</td>\n      <td>46.4K</td>\n      <td>46.1K</td>\n      <td>46.3K</td>\n      <td>704.819</td>\n    </tr>\n    <tr>\n      <th>478</th>\n      <td>2022-04-01 22:00:00+00:00</td>\n      <td>46.3K</td>\n      <td>46.4K</td>\n      <td>46.2K</td>\n      <td>46.4K</td>\n      <td>1.04K</td>\n    </tr>\n    <tr>\n      <th>479</th>\n      <td>2022-04-01 23:00:00+00:00</td>\n      <td>46.4K</td>\n      <td>46.4K</td>\n      <td>46.1K</td>\n      <td>46.3K</td>\n      <td>737.491</td>\n    </tr>\n  </tbody>\n</table>\n<p>479 rows × 6 columns</p>\n</div>"
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[(df[\"date\"] > list_dates[20 - 20]) & (df[\"date\"] < list_dates[20])]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}