eve_flare_prediction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c83fbe20-349f-4214-859a-5b568196412c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import datetime\n",
    "#from astropy.time import Time\n",
    "import datetime as dt\n",
    "import time\n",
    "from IPython.display import clear_output\n",
    "from matplotlib import gridspec\n",
    "\n",
    "#from scipy.interpolate import interp1d\n",
    "debug = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c495ae6-0a61-4c66-9279-fbaab99f40da",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_latest_eve(debug=debug):\n",
    "    # Add try to escape if error\n",
    "\n",
    "    error = [0]\n",
    "\n",
    "    eve_fn = \"https://lasp.colorado.edu/eve/data_access/eve_data/quicklook/L0CS/LATEST_EVE_L0CS_DIODES_10s_counts.json\"\n",
    "\n",
    "    try:        \n",
    "        df_eve = pd.read_json(eve_fn)\n",
    "    except ValueError:\n",
    "        error = [-10]\n",
    "#        return [-1], [-1], [-1], [-1], [-1], [-1]\n",
    "        return [error]\n",
    "    \n",
    "    #TaiOffset = 378691200 + 37\n",
    "    \n",
    "    #tt = Time(df_eve['TAI'] - TaiOffset, format='unix_tai', scale='tai')\n",
    "    # can we do TAI conversion just in datetime?\n",
    "    leap_seconds = 37\n",
    "    ta0 = dt.datetime(1958,1,1,0,0,0,0) # TAI base time\n",
    "    tt = []\n",
    "    for ttime in df_eve['TAI']:\n",
    "        tmp = ttime - leap_seconds\n",
    "        ss = int(tmp)\n",
    "        us = 1e6*(tmp-ss)\n",
    "        tt.append(ta0 + dt.timedelta(seconds = ss, microseconds = us))\n",
    "    \n",
    "    #edate = np.asarray(tt.datetime)\n",
    "    edate = np.asarray(tt)\n",
    "    df_eve['dt'] = edate # add dates column\n",
    "    df_eve['dt'] = pd.to_datetime(df_eve['dt']) # force it to datetime\n",
    "\n",
    "    e0 = np.asarray(df_eve['ESP_0_7_COUNTS'])\n",
    "    e30 = np.asarray(df_eve['ESP_30_COUNTS'])\n",
    "    \n",
    "    # Make averaged dataframe\n",
    "    df_slow=df_eve.groupby(pd.Grouper(key='dt',freq='1min')).mean() \n",
    "    stt = []\n",
    "    for ttime in df_slow['TAI']:\n",
    "        tmp = ttime - leap_seconds\n",
    "        ss = int(tmp)\n",
    "        us = 1e6*(tmp-ss)\n",
    "        stt.append(ta0 + dt.timedelta(seconds = ss, microseconds = us))\n",
    "    sdate = np.asarray(stt)\n",
    "    #stt = Time(df_slow['TAI'] - TaiOffset, format='unix_tai', scale='tai')\n",
    "    #sdate = np.asarray(stt.datetime)\n",
    "    \n",
    "    s0 = np.asarray(df_slow['ESP_0_7_COUNTS'])\n",
    "    s30 = np.asarray(df_slow['ESP_30_COUNTS'])\n",
    "    del df_eve\n",
    "    del df_slow\n",
    "    \n",
    "    e_data = [edate, e0, e30, sdate, s0, s30]\n",
    "    \n",
    "    return [error, e_data]\n",
    "#    return edate, e0, e30, sdate, s0, s30\n",
    "    \n",
    "def get_latest_goes(debug=debug):\n",
    "    error = [0]\n",
    "    goes_fn_root = \"https://services.swpc.noaa.gov/json/goes/\"\n",
    "    p6h = \"primary/xrays-6-hour.json\"\n",
    "    s6h = \"secondary/xrays-6-hour.json\"\n",
    "    \n",
    "    try:\n",
    "        df_tmp=pd.read_json(goes_fn_root+p6h)\n",
    "    except:\n",
    "        error = [-20]\n",
    "#        return [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1]\n",
    "        return error\n",
    "        \n",
    "    long_p = df_tmp[df_tmp[\"energy\"]==\"0.1-0.8nm\"]\n",
    "    short_p = df_tmp[df_tmp[\"energy\"]==\"0.05-0.4nm\"]    \n",
    "    tmp = df_tmp[\"satellite\"]\n",
    "    gp = np.asarray(tmp)[-1]\n",
    "\n",
    "    df_tmp=pd.read_json(goes_fn_root+s6h)\n",
    "    long_s = df_tmp[df_tmp[\"energy\"]==\"0.1-0.8nm\"]\n",
    "    short_s = df_tmp[df_tmp[\"energy\"]==\"0.05-0.4nm\"]\n",
    "    tmp = df_tmp[\"satellite\"]\n",
    "    gs = np.asarray(tmp)[-1]\n",
    "\n",
    "    times_p_tmp = np.asarray(long_p[\"time_tag\"])\n",
    "    long_p_tmp = np.asarray(long_p[\"observed_flux\"])\n",
    "    short_p_tmp =np.asarray(short_p[\"observed_flux\"])\n",
    "\n",
    "    times_s_tmp = np.asarray(long_s[\"time_tag\"])\n",
    "    long_s_tmp = np.asarray(long_s[\"observed_flux\"])\n",
    "    short_s_tmp =np.asarray(short_s[\"observed_flux\"])\n",
    "\n",
    "\n",
    "    gp_dat = []\n",
    "    gp_long = []\n",
    "    gp_short = []\n",
    "\n",
    "    for ii in range(times_p_tmp.size):\n",
    "        tmp2 = dt.datetime.strptime(times_p_tmp[ii], '%Y-%m-%dT%H:%M:%SZ')\n",
    "        gp_dat.append(tmp2)\n",
    "        gp_long.append(float(long_p_tmp[ii]))\n",
    "        gp_short.append(float(short_p_tmp[ii]))\n",
    "\n",
    "    gs_dat = []\n",
    "    gs_long = []\n",
    "    gs_short = []\n",
    "\n",
    "    for ii in range(times_s_tmp.size):\n",
    "        tmp2 = dt.datetime.strptime(times_s_tmp[ii], '%Y-%m-%dT%H:%M:%SZ')\n",
    "        gs_dat.append(tmp2)\n",
    "        gs_long.append(float(long_s_tmp[ii]))\n",
    "        gs_short.append(float(short_s_tmp[ii]))\n",
    "        \n",
    "    gp_dat = np.asarray(gp_dat)\n",
    "    gp_long = np.asarray(gp_long)\n",
    "    gp_short = np.asarray(gp_short)\n",
    "    \n",
    "    gs_dat = np.asarray(gs_dat)\n",
    "    gs_long = np.asarray(gs_long)\n",
    "    gs_short = np.asarray(gs_short)\n",
    "    \n",
    "    g_data = [gp_dat, gp_long, gp_short, gs_dat, gs_long, gs_short, gp, gs]\n",
    "    \n",
    "    return [error, g_data]\n",
    "    #return gp_dat, gp_long, gp_short, gs_dat, gs_long, gs_short, gp, gs\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efef243c-6f54-47cb-b01b-79b013c11c61",
   "metadata": {},
   "outputs": [],
   "source": [
    "eve_fn = \"https://lasp.colorado.edu/eve/data_access/eve_data/quicklook/L0CS/LATEST_EVE_L0CS_DIODES_10s_counts.json\"\n",
    "df_eve = pd.read_json(eve_fn)\n",
    "\n",
    "#df_eve.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "992eecfe-421f-46dd-8949-f0aedbaebeda",
   "metadata": {},
   "source": [
    "# New Version Stuff (see EVE_FastFlareCheckRC0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "95ebf28b-756a-4320-9c70-5bfe265ce1c3",
   "metadata": {},
   "source": [
    "## Fit EVE to GOES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "939dc31b-5de3-45c6-80f5-05b3511c1b81",
   "metadata": {},
   "outputs": [],
   "source": [
    "#https://stackoverflow.com/questions/32723150/rounding-up-to-nearest-30-minutes-in-python\n",
    "def find_end_time(tin, delta):\n",
    "    return tin + (dt.datetime.min - tin) % delta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6643cee-d516-4c27-aed6-9d7ead49b2c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fit GOES to EVE\n",
    "mm_a = 7.288781677851528e-09; cc_a = -4.698588594426906e-06 # first guess each fit will improve, OR check for bad fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d8b48807-1615-4b65-ba79-ce5e620a2c87",
   "metadata": {},
   "outputs": [],
   "source": [
    "def do_it(debug=debug):\n",
    "    error = [0]\n",
    "#    e_dat, e0, e30, s_dat, s0, s30 \n",
    "#    e_dat, e0, e30, s_dat, s0, s30 = get_latest_eve()\n",
    "\n",
    "    e_list = get_latest_eve()\n",
    "    error = e_list[0]\n",
    "    if error[0] < 0:\n",
    "        print('***')\n",
    "        return [e_list[0]]\n",
    "    \n",
    "    \n",
    "#    gp_dat, gp_long, gp_short, gs_dat, gs_long, gs_short, gp, gs  = get_latest_goes()\n",
    "    g_list  = get_latest_goes()\n",
    "    error = g_list[0]\n",
    "    if error[0] < 0:\n",
    "        return [g_list[0]]\n",
    "    \n",
    "# decompose goes data lists\n",
    "    e_data = e_list[1]\n",
    "    g_data = g_list[1]\n",
    "    \n",
    "    e_dat = e_data[0]; e0 = e_data[1]; e30 = e_data[2]\n",
    "    s_dat = e_data[3]; s0 = e_data[4]; s30 = e_data[5]\n",
    "    \n",
    "    gp_dat = g_data[0]; gp_long = g_data[1]; gp_short = g_data[2]\n",
    "    gs_dat = g_data[3]; gs_long = g_data[4]; gs_short = g_data[5]\n",
    "    gp = g_data[6]; gs = g_data[7]\n",
    "    \n",
    "    \n",
    "\n",
    "# Make common data grid\n",
    "    grid_length = 2 # hrs\n",
    "    grid_space = 30 # sec\n",
    "# Need to find end time that can have interpolation for all channels\n",
    "    latest = [np.max(e_dat),np.max(gp_dat), np.max(gs_dat) ]\n",
    "    end_tmp = np.min(latest)\n",
    "    end_tmp1 = end_tmp - dt.timedelta(seconds=grid_space)\n",
    "    end_time = find_end_time(end_tmp1, dt.timedelta(seconds=grid_space))\n",
    "    start_time = end_time - dt.timedelta(hours = grid_length)\n",
    "    \n",
    "# Find start time of GOES data\n",
    "    earliest = [np.min(gp_dat), np.min(gs_dat)]\n",
    "    g_tmp = np.max(earliest)\n",
    "    g_tmp1 = g_tmp + dt.timedelta(seconds=grid_space)\n",
    "    g_time = find_end_time(g_tmp1, dt.timedelta(seconds=grid_space))\n",
    "\n",
    "\n",
    "\n",
    "    if debug:\n",
    "        print(start_time)\n",
    "        print(end_time)\n",
    "        print(end_tmp)\n",
    "\n",
    "    time_grid = np.arange(start_time, end_time,dt.timedelta(seconds=grid_space) )\n",
    "    time_grid_all = np.arange(g_time, end_time,dt.timedelta(seconds=grid_space) ) \n",
    "\n",
    "#    if debug:\n",
    "#        for tt in time_grid:\n",
    "#            print(tt)\n",
    "\n",
    "\n",
    "# interpolations\n",
    "#https://stackoverflow.com/questions/54304423/why-does-a-conversion-from-np-datetime64-to-float-and-back-lead-to-a-time-differ\n",
    "    xx= time_grid.astype(\"float\")\n",
    "    xx_all = time_grid_all.astype(\"float\")\n",
    "\n",
    "# convert DTs\n",
    "# GOES Pri\n",
    "    xxtmp = []\n",
    "    for tt in gp_dat:\n",
    "        xxtmp.append(np.datetime64(tt))\n",
    "    xxtmp = np.asarray(xxtmp)\n",
    "    xgp = xxtmp.astype(\"float\")\n",
    "    gpl_i = np.interp(xx,xgp,gp_long )\n",
    "    gps_i = np.interp(xx,xgp,gp_short )\n",
    "    \n",
    "    gpl_ia = np.interp(xx_all,xgp,gp_long )\n",
    "    gps_ia = np.interp(xx_all,xgp,gp_short )\n",
    "\n",
    "\n",
    "# GOES Sec\n",
    "    xxtmp = []\n",
    "    for tt in gs_dat:\n",
    "        xxtmp.append(np.datetime64(tt))\n",
    "    xxtmp = np.asarray(xxtmp)\n",
    "    xgs = xxtmp.astype(\"float\")\n",
    "    gsl_i = np.interp(xx,xgs,gs_long )\n",
    "    gss_i = np.interp(xx,xgs,gs_short )\n",
    "    \n",
    "    gsl_ia = np.interp(xx_all,xgs,gs_long )\n",
    "    gss_ia = np.interp(xx_all,xgs,gs_short )\n",
    "    \n",
    "\n",
    "#EVE fast\n",
    "    xxtmp = []\n",
    "    for tt in e_dat:\n",
    "        xxtmp.append(np.datetime64(tt))\n",
    "    xxtmp = np.asarray(xxtmp)\n",
    "    xe = xxtmp.astype(\"float\")\n",
    "    e0_i = np.interp(xx,xe,e0 )\n",
    "\n",
    "\n",
    "#EVE slow\n",
    "    xxtmp = []\n",
    "    for tt in s_dat:\n",
    "        xxtmp.append(np.datetime64(tt))\n",
    "    xxtmp = np.asarray(xxtmp)\n",
    "    xs = xxtmp.astype(\"float\")\n",
    "    s0_i = np.interp(xx,xs,s0 )\n",
    "    \n",
    "    \n",
    "    gl = np.maximum(gpl_i, gsl_i)\n",
    "    gl_all =  np.maximum(gpl_ia, gsl_ia)\n",
    "    gs = np.maximum(gps_ia, gss_ia)\n",
    "    gr = gs/(gl_all+1e-15)\n",
    "    \n",
    "    if debug:\n",
    "        plt.plot(time_grid, gpl_i/np.max(gpl_i), 'r')\n",
    "        plt.plot(time_grid, gsl_i/np.max(gsl_i), 'b')\n",
    "        plt.plot(time_grid, e0_i/np.max(e0_i), 'g')\n",
    "        plt.plot(time_grid, s0_i/np.max(s0_i), '.')\n",
    "        plt.show()\n",
    "        \n",
    "# Make list to return data\n",
    "    g_raw_list = [gp_dat, gp_long, gp_short, gs_dat, gs_long, gs_short]\n",
    "    g_intp_list = [time_grid,time_grid_all, gl,gs, gr, gpl_i, gsl_i]\n",
    "    eve_raw_list = [e_dat, e0, e30, s_dat, s0, s30]\n",
    "    eve_intp_list = [e0_i, s0_i]\n",
    "    \n",
    "    r_list = [error, g_raw_list, g_intp_list, eve_raw_list, eve_intp_list ]\n",
    "\n",
    "    return r_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d3f77a6-1c16-403b-80bd-aff6a9af230a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def do_slope_pred(e_dat,s_dat, time_range, e0, s0):\n",
    "    ct = dt.datetime.utcnow()\n",
    "    st = ct - dt.timedelta(hours = float(time_range))\n",
    "    time_good = np.where(e_dat >= st)\n",
    "#    good = np.where((e0 > 100) & (e30 <2000))\n",
    "    te0 = e0[time_good]\n",
    "    xx = e_dat[time_good]\n",
    "    ft = []\n",
    "\n",
    "    npts = len(te0)\n",
    "    grad0=[]\n",
    "    if npts > 1: # do the gradients stuff\n",
    "        for jj in range(npts):\n",
    "            ft.append(xx[jj])\n",
    "            if jj==0:\n",
    "                grad0.append(0)\n",
    "            else:\n",
    "                grad0.append(e0[jj] - e0[jj-1])\n",
    "    grad0 = np.asarray(grad0)\n",
    "    ft = np.array(ft)\n",
    "    \n",
    "    grad_s=[]\n",
    "    \n",
    "    #slow gradient slope\n",
    "    npts_s = len(s0)\n",
    "    st = []\n",
    "    if npts_s > 1: # do the gradients stuff\n",
    "        for jj in range(npts_s):\n",
    "            st.append(s_dat[jj])\n",
    "            if jj==0:\n",
    "                grad_s.append(0)\n",
    "            else:\n",
    "                grad_s.append(s0[jj] - s0[jj-1])\n",
    "    grad_s = np.asarray(grad_s)\n",
    "    st=np.asarray(st)\n",
    "    ns_pred = -5\n",
    "    tt_s = np.arange(-1*ns_pred)\n",
    "    yy_s = grad_s[ns_pred:]\n",
    "    slope_s, intercept_s = np.polyfit(tt_s, yy_s, deg=1)\n",
    "\n",
    "  \n",
    "    # grad now for prediction\n",
    "    n_pred = -25\n",
    "    tt = np.arange(-1*n_pred)\n",
    "    yy = e0[n_pred:]\n",
    "    slope, intercept = np.polyfit(tt, yy, deg=1)\n",
    "    # forward \n",
    "    xxpred = np.arange(-1*n_pred,-5*n_pred)\n",
    "    yypred = []\n",
    "    ttpred = []\n",
    "    slope_av = 0.6*(0.018*slope_s + 0.982*slope)\n",
    "    for ii in range(len(xxpred)):\n",
    "        ttpred.append(xx[-1] + dt.timedelta(seconds = 10*ii))\n",
    "        yypred.append(e0[-1] + ii*slope_av)\n",
    "    ttpred=np.asarray(ttpred)\n",
    "    yypred=np.asarray(yypred)\n",
    "    error = [0]\n",
    "    pred_array = [ttpred, yypred]\n",
    "    grad_array = [ft, grad0, st, grad_s]\n",
    "    return [error, pred_array, grad_array]\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1067206d-d960-433f-9cdd-66732c92964e",
   "metadata": {},
   "outputs": [],
   "source": [
    "m_hist = []\n",
    "c_hist = []\n",
    "g_hist = [] # Goes L flux"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff4b65f0-2efa-4813-b8c1-3c5483c28dcf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# To Do\n",
    "# add latency times\n",
    "# scaling of esp plots to last 30 mins ?\n",
    "# consistent gradient calculations\n",
    "# organize into defines\n",
    "# Fast slope plot seems wrong\n",
    "# Add exception handling to do_it call\n",
    "# Set initial fit parameters based on measured DN so it gets the solution faster\n",
    "\n",
    "\n",
    "# From Tom E.'s email 10/3/2023 11:59am\n",
    "r2t_coef =  [2.74603592e+00, 1.29468042e+02, -9.66284279e+02, 5.51752562e+03, \\\n",
    "             -1.86637781e+04, 3.59506086e+04, -3.60993534e+04, 1.46869290e+04]\n",
    "\n",
    "time_range = 3\n",
    "loop_update_time = 20 #seconds to wait before looking for data again\n",
    "try:\n",
    "    while True:\n",
    "        #time_grid, time_grid_all, gl, gs, gr, gpl_i, gsl_i, e0_i, s0_i,e_dat, e0  = do_it(debug = False)\n",
    "        rl = do_it(debug = False)    #<<< Add try/except\n",
    "        # decompose return list\n",
    "        error = rl[0]\n",
    "        if error[0] == 0:\n",
    "            g_raw_list = rl[1];  g_intp_list= rl[2]; eve_raw_list= rl[3]; eve_intp_list = rl[4]\n",
    "        \n",
    "            time_grid = g_intp_list[0];  time_grid_all= g_intp_list[1]\n",
    "            gl= g_intp_list[2];  gs= g_intp_list[3]; gr= g_intp_list[4]\n",
    "            gpl_i = g_intp_list[5];  gsl_i = g_intp_list[6]\n",
    "            e0_i = eve_intp_list[0]; s0_i = eve_intp_list[1]\n",
    "            e_dat = eve_raw_list[0]; e0 = eve_raw_list[1] ; e30 = eve_raw_list[2] \n",
    "            s_dat = eve_raw_list[3]; s0 = eve_raw_list[4] ; s30 = eve_raw_list[5]\n",
    "            gp_dat = g_raw_list[0]\n",
    "        \n",
    "        \n",
    "        # Do Plots\n",
    "#        if (error == 0): # make sure we were able to get data\n",
    "\n",
    "            mm, cc = np.polyfit(s0_i, gl, deg=1 )\n",
    "# Time stuff\n",
    "            ct = dt.datetime.utcnow()\n",
    "            pt= ct + dt.timedelta(hours = float(0.25))\n",
    "            st = ct - dt.timedelta(hours = float(time_range))    \n",
    "\n",
    "    #### OLD stuff #######\n",
    "            # extract just the data we want\n",
    "            \n",
    "            time_good = np.where(e_dat >= st)\n",
    "#    good = np.where((e0 > 100) & (e30 <2000))\n",
    "            te0 = e0[time_good]\n",
    "            te30 = e30[time_good]\n",
    "    \n",
    "    \n",
    "    \n",
    "# Scaling\n",
    "            c30_max = np.max(te30)# 300\n",
    "            c30_min = np.min(te30)# 190\n",
    "            c30_scl = c30_max - c30_min\n",
    "     \n",
    "            c0_max = np.max(te0)\n",
    "            c0_min = np.min(te0)\n",
    "            c0_scl = c0_max - c0_min\n",
    "    \n",
    "    \n",
    "\n",
    "\n",
    "\n",
    "            pe0 = (te0 - c0_min)/c0_scl\n",
    "            ps0 = (s0 - c0_min)/c0_scl\n",
    "#           pe30 = (e30[good]- 1.0*np.min(e30[good]))/np.max(e30[good])\n",
    "            pe30 = (te30- c30_min)/c30_scl\n",
    "            ps30 = (s30- c30_min)/c30_scl\n",
    "        \n",
    "            xx =  e_dat[time_good]\n",
    "            nptso = len(te0)\n",
    "            grad0o=[]\n",
    "            if nptso > 1: # do the gradients stuff\n",
    "                for jj in range(nptso):\n",
    "                    if jj==0:\n",
    "                        grad0o.append(0)\n",
    "                    else:\n",
    "                        grad0o.append(e0[jj] - e0[jj-1])\n",
    "            grad0o = np.asarray(grad0o)\n",
    "    \n",
    "            grad_so=[]\n",
    "    \n",
    "    #slow gradient slope\n",
    "            npts_so = len(s0)\n",
    "            if npts_so > 1: # do the gradients stuff\n",
    "                for jj in range(npts_so):\n",
    "                    if jj==0:\n",
    "                        grad_so.append(0)\n",
    "                    else:\n",
    "                        grad_so.append(s0[jj] - s0[jj-1])\n",
    "            grad_so = np.asarray(grad_so)\n",
    "            ns_predo = -5\n",
    "            tt_s = np.arange(-1*ns_predo)\n",
    "            yy_s = grad_so[ns_predo:]\n",
    "            slope_s, intercept_s = np.polyfit(tt_s, yy_s, deg=1)\n",
    "\n",
    "    \n",
    "            # grad now for prediction\n",
    "            n_predo = -25\n",
    "            tt = np.arange(-1*n_predo)\n",
    "            yy = pe0[n_predo:]\n",
    "            slope, intercept = np.polyfit(tt, yy, deg=1)\n",
    "            # forward \n",
    "            xxpred = np.arange(-1*n_predo,-5*n_predo)\n",
    "            yypred = []\n",
    "            ttpred = []\n",
    "            slope_av = 0.6*(0.018*slope_s + 0.982*slope)\n",
    "            for ii in range(len(xxpred)):\n",
    "                ttpred.append(xx[-1] + dt.timedelta(seconds = 10*ii))\n",
    "                yypred.append(ps0[-1] + ii*slope_av)\n",
    "\n",
    "\n",
    "# Update fit coefficients, need to think of the best way to do this i.e. limits to update, start again, or keep existing\n",
    "            if ((np.abs((mm_a - mm)/mm_a) <0.99) and (np.abs((cc_a - cc)/cc_a) < 0.99)):\n",
    "                # print(\"updating m, c\")\n",
    "                mm_a = mm_a/2 + mm/2\n",
    "                cc_a = cc_a/2 +cc/2\n",
    "            else:\n",
    "                mm_a = mm_a\n",
    "                cc_a = cc_a\n",
    "            fig_test = plt.figure(figsize=(16,8))\n",
    "            #gs = gridspec.GridSpec(4, 1, height_ratios=[3, 1, 1, 1]) \n",
    "        \n",
    "            # Time stuff\n",
    "            #ct = dt.datetime.utcnow()\n",
    "            #pt= ct + dt.timedelta(hours = float(0.25))\n",
    "            #st = ct - dt.timedelta(hours = float(time_range))\n",
    "            \n",
    "            #    error = [0]\n",
    "            #    pred_array = [ttpred, yypred]\n",
    "            #    grad_array = [ft, grad0, st, grad_s]\n",
    "            #    return [error, pred_array, grad_array]\n",
    "        \n",
    "            #tp, yp, ft, fs, slt, ss = do_slope_pred(e_dat, s_dat, time_range, e0, s0)\n",
    "            rl = do_slope_pred(e_dat, s_dat, time_range, e0, s0)\n",
    "            error = rl[0]\n",
    "            if error[0] == 0: # need to find a better place for this, but this error is defined as 0 at the moment\n",
    "                pred_array = rl[1]; grad_array = rl[2]\n",
    "                tp = pred_array[0]\n",
    "                yp = pred_array[1]\n",
    "                ft = grad_array[0]\n",
    "                fs = grad_array[1]\n",
    "                slt = grad_array[2]\n",
    "                ss = grad_array[3]\n",
    "                \n",
    "        \n",
    "            g_t = r2t_coef[7] * gr**7 + r2t_coef[6] * gr**6 + r2t_coef[5] * gr**5 + \\\n",
    "              r2t_coef[4] * gr**4  +r2t_coef[3] * gr**3 + r2t_coef[2] * gr**2 + \\\n",
    "              r2t_coef[1]* gr + r2t_coef[0]\n",
    "        \n",
    "\n",
    "            fig_test.add_subplot(4,1,1) # <<<<<<<<<<<<<<<<<<<<<<<<<< XRS EVE Plot\n",
    "            #plt.axhline(y= 1e-5,  color='red')\n",
    "            #plt.axhline(y= 1e-6,  color='yellow')\n",
    "\n",
    "\n",
    "            plt.plot(time_grid, gl, color='red', label = 'GOES Long (fit length)')\n",
    "            plt.plot(time_grid_all, gs, color='blue', label = 'GOES Short')\n",
    "        \n",
    "            plt.plot(e_dat, mm_a*e0 + cc_a, color='orange', linestyle = ' ', marker = '+', markersize = 0.5, label = 'EVE 0 fit')\n",
    "            plt.plot(tp, mm_a*yp + cc_a, 'g:', label = 'EVE Predict')\n",
    "            \n",
    "            plt.yscale('log')\n",
    "            plt.ylabel('GOES XRS (W/m2)')\n",
    "\n",
    "            plt.ylim(1e-9, 1e-4)\n",
    "            plt.axhline(y=1e-5, linestyle = '--',color='green')\n",
    "            plt.xlim(st,pt)\n",
    "            plt.xticks(color='w')\n",
    "            plt.axvline(x= ct, linestyle = '--', color='black')\n",
    "            tnow = dt.datetime.strftime(ct, \"%H:%M:%S\")\n",
    "            plt.text(ct, 1.3e-4, tnow, horizontalalignment='center')\n",
    "            #plt.title('------ Combined XRS - EVE ------')\n",
    "            plt.grid()\n",
    "            plt.legend(loc = 'lower left')\n",
    "            \n",
    "            fig_test.add_subplot(4,1,2) # <<<<<<<<<<<<<<<<<<<<<<<<<< Plot I like\n",
    "            # set scaling time for normalization\n",
    "            t_scale = 60 # minutes\n",
    "            \n",
    "            #fast\n",
    "            plt.plot(xx,pe0, color='cyan')\n",
    "            plt.plot(ttpred, yypred, 'b:', label = 'Simple Predict') # <<<< prediction part\n",
    "            plt.plot(xx,(pe30 - np.min(pe30)), color='orange')    \n",
    "            #slow\n",
    "            plt.plot(s_dat, ps0, 'b', label = 'ESP 0-7 nm')\n",
    "            plt.plot(s_dat, ps30 - np.min(pe30), 'r', label = 'ESP 30 nm')\n",
    "    \n",
    "\n",
    "\n",
    "\n",
    "            plt.ylim(-0.25,1.25)\n",
    "    \n",
    "    #plt.xlabel('Time (UTC)')\n",
    "            #tnow = dt.datetime.strftime(ct, \"%H:%M:%S\")\n",
    "            #plt.text(ct, 1.3, tnow, horizontalalignment='center')\n",
    "    \n",
    "    \n",
    "            plt.xticks(color='w')\n",
    "            plt.ylabel('Normalized Counts')\n",
    "            plt.xlim(st,pt)\n",
    "            plt.axvline(x= ct, linestyle = '--', color='black') \n",
    "            plt.legend(loc = 'upper left')\n",
    "            plt.grid()\n",
    "\n",
    "        \n",
    "            fig_test.add_subplot(4,1,3) # <<<<<<<<<<<<<<<<<<<<<<<<<< XRS Temperature\n",
    "\n",
    "            plt.plot(time_grid_all,g_t, 'g-')\n",
    "            plt.xlim(st,pt)\n",
    "            plt.axvline(x= ct, linestyle = '--', color='black')\n",
    "            plt.ylabel('T (MK)')\n",
    "            plt.xticks(color='w')\n",
    "            \n",
    "            xmin, xmax, ymin, ymax = plt.axis()\n",
    "            td = int(time_range *20)\n",
    "            #plt.text(ct - dt.timedelta(minutes=td), 0.9*ymax, delay_g, color='k', fontsize = 12, bbox=dict(facecolor='white'))\n",
    "            plt.grid()\n",
    " \n",
    "            #plt.plot(e_dat, e30, color='orange', label = '30 nm 10 s' )\n",
    "            #plt.plot(s_dat, s30, color='red', label = '30 nm 1 min' )\n",
    "            #plt.xlim(st,pt)\n",
    "            #plt.axvline(x= ct, linestyle = '--', color='black')\n",
    "            #plt.grid()\n",
    "            #plt.legend(loc = 'lower left')\n",
    "            ##plt.xlabel('Time (UTC)')\n",
    "            #plt.xticks(color='w')\n",
    "            #plt.ylabel('ESP 30 nm CPS')\n",
    "            \n",
    "            fig_test.add_subplot(4,1,4) # <<<<<<<<<<<<<<<<<<<<<<<<<< ESP 30 nm \n",
    "            plt.plot(ft, fs, color='grey', label = 'Fast Slope' )# < this is delayed I think\n",
    "            plt.plot(slt, ss, color='black', label = 'Slow Slope' )\n",
    "            plt.xlim(st,pt)\n",
    "            plt.axvline(x= ct, linestyle = '--', color='black')\n",
    "            plt.grid()\n",
    "            plt.legend(loc = 'lower left')\n",
    "            plt.xlabel('Time (UTC)')\n",
    "            plt.ylabel('Delta CPS')\n",
    "        \n",
    "        \n",
    "            plt.show()\n",
    "\n",
    "\n",
    "            m_hist.append(mm_a)\n",
    "            c_hist.append(cc_a)\n",
    "            # print(mm_a, cc_a)\n",
    "            #time.sleep(60)\n",
    "            \n",
    "    \n",
    "# Wait loop so we don't spam the servers\n",
    "            print('Waiting: ', end=' ')\n",
    "            for jj in range(int(loop_update_time/5)):\n",
    "                tn = dt.datetime.utcnow()\n",
    "                print(loop_update_time - jj*5, end='s ')\n",
    "                time.sleep(5)\n",
    "            print(\" Fetching New Data \", end=' ')\n",
    "            clear_output(wait=True)\n",
    "    \n",
    "except KeyboardInterrupt: \n",
    "    print('interrupted!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "92ef1d33-1842-414f-b2dd-48ad137ad5fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Need to stop data plots to do this step\n",
    "\n",
    "m_ave = sum(m_hist) / len(m_hist)\n",
    "c_ave = sum(c_hist) / len(c_hist)\n",
    "\n",
    "fig_fit_history = plt.figure(figsize=(16,4))\n",
    "\n",
    "fig_fit_history.add_subplot(1,2,1)\n",
    "plt.plot(m_hist)\n",
    "plt.axhline(y= m_ave)\n",
    "plt.title = (\"slope\")\n",
    "\n",
    "fig_fit_history.add_subplot(1,2,2)\n",
    "plt.plot(c_hist)\n",
    "plt.axhline(y= c_ave)\n",
    "plt.title = (\"offset\")\n",
    "\n",
    "plt.show()\n",
    "\n",
    "print('Slope  avereage: {}'.format(m_ave))\n",
    "print('Offset avereage: {}'.format(c_ave))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2164515c-47eb-4213-b28d-be7cbdccf04d",
   "metadata": {},
   "outputs": [],
   "source": [
    "tl = get_latest_eve(debug=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "163aca65-6353-42cf-baea-8bf11133d41b",
   "metadata": {},
   "outputs": [],
   "source": [
    "error = tl[0]\n",
    "if error[0] < 0:\n",
    "    print(error[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c1efef43-d041-4194-93c5-0f2133dacc33",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(rl[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "29c37420-6b6f-4cb1-90be-2e9fd4e384cb",
   "metadata": {},
   "source": [
    "# Old"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ffbe947f-b792-4abf-828f-caf850c33d15",
   "metadata": {},
   "outputs": [],
   "source": [
    "tai_max = (np.max(df_eve['TAI']))\n",
    "#TaiOffset = 378691200 + 37\n",
    "#TaiOffset = 37\n",
    "leap_seconds = 37\n",
    "    \n",
    "# tt = Time(tai_max - TaiOffset, format='unix_tai', scale='tai')\n",
    "# Don't use astropy for Juno\n",
    "ta0 = dt.datetime(1958,1,1,0,0,0,0) # TAI base time\n",
    "tt = []\n",
    "for ttime in df_eve['TAI']:\n",
    "    tmp = ttime - leap_seconds\n",
    "    ss = int(tmp)\n",
    "    us = 1e6*(tmp-ss)\n",
    "    tt.append(ta0 + dt.timedelta(seconds = ss, microseconds = us))\n",
    "tt = np.asarray(tt)\n",
    "\n",
    "print(tai_max, tt, tt.datetime)\n",
    "\n",
    "tmp = (tai_max - TaiOffset)\n",
    "print(tmp)\n",
    "\n",
    "ss = int(tmp)\n",
    "us = 1e6*(tmp-ss)\n",
    "print(ss, us)\n",
    "\n",
    "dt0 = datetime.datetime(1958,1,1,0,0,0,0)\n",
    "print(dt0)\n",
    "\n",
    "dt1 = dt0 + datetime.timedelta(seconds = ss, microseconds = us)\n",
    "print(dt1)\n",
    "\n",
    "leap_seconds = 37\n",
    "dt0 = datetime.datetime(1958,1,1,0,0,0,0) # TAI base time\n",
    "tt = []\n",
    "for ttime in df_eve['TAI']:\n",
    "    tmp = ttime - leap_seconds\n",
    "    ss = int(tmp)\n",
    "    us = 1e6*(tmp-ss)\n",
    "    tt.append(dt0 + datetime.timedelta(seconds = ss, microseconds = us))\n",
    "eve_time = np.asarray(tt)\n",
    "\n",
    "\n",
    "e0 =  df_eve['ESP_0_7_COUNTS'].to_numpy()\n",
    "\n",
    "if True:\n",
    "    npts = len(e0)\n",
    "    grad0=[]\n",
    "    if npts > 1: # do the gradients stuff\n",
    "        for jj in range(npts):\n",
    "            if jj==0:\n",
    "                grad0.append(0)\n",
    "            else:\n",
    "                grad0.append(e0[jj] - e0[jj-1])\n",
    "    grad0 = np.asarray(grad0)\n",
    "\n",
    "\n",
    "plt.plot(eve_time, e0)\n",
    "plt.plot(eve_time, 50*grad0)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "72e27235-0e45-42da-b724-8d2701605ade",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}