scheduler_v0.3.0.py

#!/usr/bin/env python2

'''
scheduler_v0.3.0 is a copy of v0.2.9:
    Added pseudo color processing of channel 3, 5 & 6, whichever is available

    expected formatting of email_config.txt:
        displayed user name
        e-mail address (gmail)
        password

    expected formatting of user_location.txt:
        Longitude (East is positive)
        Latitude (North is positive)

    expected formatting of mail_list.txt:
        e-mail address on every line
'''
from __future__ import print_function
import sys
import os
import subprocess
import time
import datetime
import ephem
import urllib2
import math
# import xmlrpclib
import logging
from logging import handlers
# import psutil
import smtplib
import platform
from os import listdir
from os.path import isfile, join, getmtime, basename
from PIL import Image, ImageOps
from email.mime.image import MIMEImage
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText

# general
script_name = os.path.splitext(os.path.basename(__file__))[0]
log_filename = script_name + '.log'
deg_per_rad = 180 / math.pi
speed_of_light = 299792458  # m/s
main_folder = os.path.join(os.path.expanduser('~'),'Projects/Python_projects/Meteor_receiver')
# main_folder = os.path.dirname(__file__)	somehow, this doesn't seem to return the path on the rpi, so we have to find another way...

# configure logging
log = logging.getLogger('')
log.setLevel(logging.DEBUG)
format = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
# print logging to console
ch = logging.StreamHandler(sys.stdout)
ch.setFormatter(format)
log.addHandler(ch)
# save logging to file
fh = handlers.RotatingFileHandler(log_filename, maxBytes=(1048576*5), backupCount=7)
fh.setFormatter(format)
log.addHandler(fh)
# first logging lines
logging.debug("-----------------------------------------------------------------------------------------------")
logging.debug(datetime.datetime.now().strftime(script_name + ' (no doppler) started on %d/%m/%y at local %H:%M'))

# Check if all expected directories exist & create them if not
if not os.path.exists(os.path.join(main_folder, 'Data')):
    os.makedirs(os.path.join(main_folder, 'Data'))
    logging.debug('Created Data directory')

if not os.path.exists(os.path.join(main_folder, 'Images')):
    os.makedirs(os.path.join(main_folder, 'Images'))
    logging.debug('Created Images directory')

if not os.path.exists(os.path.join(main_folder, 'TLE')):
    os.makedirs(os.path.join(main_folder, 'TLE'))
    logging.debug('Created TLE directory')

# GRC script to launch
# grc_script = './GRC/METEOR_M2_v05_noGUI.py'
grc_script = 'GRC/METEOR_M2_v04_noGUI.py'
grc_script = os.path.join(main_folder, grc_script)
SDR_capture = [sys.executable, grc_script]
logging.debug('GRC script: ' + grc_script)

# doppler server address & port
# grc_server = xmlrpclib.Server("http://localhost:8080")
# logging.debug(grc_server)

# set observer location
with open(os.path.join(main_folder, 'user_location.txt')) as f:
    user_loc = f.read().splitlines()
location = ephem.Observer()
location.lon = user_loc[0]		# +E
location.lat = user_loc[1]		# +N
location.elevation = 10
logging.debug('Location: ' + str(location.lat) + 'N, ' + str(location.lon) + 'E, ' + str(location.elevation) + 'm elevation')

# url to fetch TLE info from
TLE_url = 'http://www.celestrak.com/NORAD/elements/weather.txt'
TLE_offline_path = os.path.join(main_folder, 'TLE/weather.txt')

# NORAD sat name
sat_name = 'METEOR-M 2 '
logging.debug('Sat name: ' + sat_name)


def retrieve_tle(satellite):
    logging.debug('trying TLE download...')
    try:
        tle_file = urllib2.urlopen(TLE_url, timeout=10).readlines()
    except:
        logging.debug('No internet access, using offline TLE file')
        with open(TLE_offline_path) as f:
            tle_file = f.readlines()
    else:
        logging.debug('Download successful, storing TLE file offline')
        logging.debug(datetime.datetime.now().strftime('TLE file downloaded & saved %d/%m/%y %H:%M'))
        with open(TLE_offline_path, 'w') as f:
            for line in tle_file:
                f.write(line)

    for i in range(0, len(tle_file)):
        line = tle_file[i]
        if line[:11] == satellite:
            print('TLE data: \n')
            print(tle_file[i])
            print(tle_file[i + 1])
            print(tle_file[i + 2])
            line_1 = tle_file[i]
            line_2 = tle_file[i + 1]
            line_3 = tle_file[i + 2]

    return ephem.readtle(line_1, line_2, line_3)


def doppler(velocity):
    frequency = 137900000
    return (speed_of_light/(speed_of_light + velocity)) * frequency


# decodes the .s file that was generated in the last 30 minutes,
# removes older .s files, returns the BMP file name
def decode_capture():
    mypath = os.path.join(main_folder, 'Data')

    onlyfiles = [file for file in listdir(mypath) if isfile(join(mypath, file))]

    valid_file = False

    for file in onlyfiles:
        if str(file).endswith('.s'):
            mod_time = getmtime(join(mypath, file))
            date = datetime.datetime.fromtimestamp(mod_time)
            if date > (datetime.datetime.now() - datetime.timedelta(minutes=30)):
            # if file == s_file:

                valid_file = True

                LRPT_soft = os.path.join(mypath, file)

                output_file_1 = os.path.join(mypath, file[:-2])	 # remove .s extension
                output_file_2 = os.path.join(mypath, file[:-2]) + "_2"  # remove .s extension
                os.chdir(main_folder)
                # since the rpi3 has 4 cores, we can run 2 medet instances to cover all 6 APIDs
                if platform.machine() == 'x86_64':
                    medet_command = './Decoder/medet' + " " + LRPT_soft + " " + \
                        output_file_1 + " -Q -b 64 -g 65 -r 68"	 # if on linux64
                elif platform.machine().startswith('arm'):
                    medet_command = './Decoder/medet_arm' + " " + LRPT_soft + " " + \
                        output_file_1 + " -Q -b 64 -g 65 -r 68"	 # if on rpi
                else:
                    logging.error('platform unsupported by decoder')
                    return ('none', 'none')		# I'm not sure what else to do here...
                logging.debug('running: ' + medet_command)
                medet_process_1 = subprocess.Popen([medet_command], shell=True)

                if platform.machine() == 'x86_64':
                    medet_command = './Decoder/medet' + " " + LRPT_soft + " " + \
                        output_file_2 + " -Q -b 66 -g 67 -r 69"  # if on linux64
                elif platform.machine().startswith('arm'):
                    medet_command = './Decoder/medet_arm' + " " + LRPT_soft + " " + \
                        output_file_2 + " -Q -b 66 -g 67 -r 69"  # if on rpi
                else:
                    logging.debug('platform unsupported by decoder')
                    return ('none', 'none')       # I'm not sure what else to do here...
                logging.debug('running: ' + medet_command)
                medet_process_2 = subprocess.Popen([medet_command], shell=True)

                returncode_1 = medet_process_1.wait()
                logging.debug('1st medet ran with returncode: %s' % returncode_1)

                returncode_2 = medet_process_2.wait()
                logging.debug('2nd medet ran with returncode: %s' % returncode_2)

            else:
                # remove old .s files
                os.remove(os.path.join(mypath, file))
                logging.debug('removed old file %s' % os.path.join(mypath, file))

    if valid_file is True:
        file_name_1 = os.path.basename(output_file_1)
        file_name_2 = os.path.basename(output_file_2)
        logging.debug('decode_capture returned file name: %s & %s' % (file_name_1, file_name_2))
        return (file_name_1, file_name_2)
    else:
        logging.debug('decoding: no recent file was found')
        return ('none', 'none')


def detect_blanks(image):
    channels = [0, 0, 0]
    image_dir = os.path.join(main_folder, 'Data', image + '.bmp')
    try:
        image_raw = Image.open(image_dir)
    except:
        logging.debug('detecting blanks: no raw image was found')
        return 'none'
    else:
        r, g, b = image_raw.split()

        # red channel
        image = Image.merge("RGB", (r, r, r))
        extrema = image.convert("L").getextrema()
        if extrema != (0, 0):
            channels[2] += 1

        # green channel
        image = Image.merge("RGB", (g, g, g))
        extrema = image.convert("L").getextrema()
        if extrema != (0, 0):
            channels[1] += 1

        # blue channel
        image = Image.merge("RGB", (b, b, b))
        extrema = image.convert("L").getextrema()
        if extrema != (0, 0):
            channels[0] += 1

        image.close()

    return channels


def process_image(image_1, image_2):
    if image_1 != 'none':
        # NS or SN pass?
        evening_pass = False
        ignore_evening_passes = False
        pass_time = int(image_1[-4:])
        if ((pass_time > 1600) or (pass_time < 0400)):
            evening_pass = True
            ignore_evening_passes = True

        # first check which channels were active
        active_channels = [0, 0, 0, 0, 0, 0]

        active_decoded_channels = detect_blanks(image_1)
        if active_decoded_channels == 'none':
            return [], [0, 0, 0, 0, 0, 0]   # no image was decoded from recording

        active_channels[0] = active_decoded_channels[0]
        active_channels[1] = active_decoded_channels[1]
        active_channels[4] = active_decoded_channels[2]

        active_decoded_channels = detect_blanks(image_2)
        active_channels[2] = active_decoded_channels[0]
        active_channels[3] = active_decoded_channels[1]
        active_channels[5] = active_decoded_channels[2]

        logging.debug('Detected active channels: ' + str(active_channels))
        if (sum(active_channels) != 3):
            logging.debug('Amount of active channels incorrect: channels ' + str(active_channels))

        image_1_dir = os.path.join(main_folder, 'Data', image_1 + '.bmp')
        image_2_dir = os.path.join(main_folder, 'Data', image_2 + '.bmp')
        logging.debug('processing file %s' % image_1_dir)

        image_125 = Image.open(image_1_dir)
        ch_4, ch_1, ch_0 = image_125.split()
        image_346 = Image.open(image_2_dir)
        ch_5, ch_3, ch_2 = image_346.split()

        list_of_images = []

        if (active_channels[:2] == [1, 1]) and (ignore_evening_passes is False):
            image_dir = os.path.join(main_folder, 'Images', image_1 + '_122.jpg')
            image = Image.merge("RGB", (ch_1, ch_1, ch_0))
            image.save(image_dir, quality=90, subsampling=0)
            list_of_images.append(image_dir)

        if (active_channels == [1, 1, 1, 0, 0, 0]) and (ignore_evening_passes is False):
            image_dir = os.path.join(main_folder, 'Images', image_1 + '_123.jpg')
            image = Image.merge("RGB", (ch_2, ch_1, ch_0))
            image.save(image_dir, quality=90, subsampling=0)
            list_of_images.append(image_dir)

        if (active_channels == [1, 1, 0, 1, 0, 0]) and (ignore_evening_passes is False):
            image_dir = os.path.join(main_folder, 'Images', image_1 + '_124.jpg')
            image = Image.merge("RGB", (ch_3, ch_1, ch_0))
            image.save(image_dir, quality=90, subsampling=0)
            list_of_images.append(image_dir)

        if (active_channels[4] == 1):
            image_dir = os.path.join(main_folder, 'Images', image_1 + '_555.jpg')
            image = Image.merge("RGB", (ch_4, ch_4, ch_4))
            image = ImageOps.invert(image)
            image = ImageOps.autocontrast(image)
            # thermal information gets lost here, but we get better looking images
            image.save(image_dir, quality=90, subsampling=0)
            list_of_images.append(image_dir)

        if (active_channels[5] == 1):
            image_dir = os.path.join(main_folder, 'Images', image_1 + '_666.jpg')
            image = Image.merge("RGB", (ch_5, ch_5, ch_5))
            image = ImageOps.invert(image)
            image = ImageOps.autocontrast(image)
            # thermal information gets lost here, but we get better looking images
            image.save(image_dir, quality=90, subsampling=0)
            list_of_images.append(image_dir)

        # Pseudo color images
        # Water ice seems to reflect in bands 1 & 2, but not in band 3
        # None of these bands are visible at night (especially in winter time)
        if (active_channels == [1, 1, 1, 0, 0, 0]) and (ignore_evening_passes is False):
            # combine useful channels
            image_dir = os.path.join(main_folder, 'Images', image_1 + '_123_pseudo.bmp')
            image = Image.merge("RGB", (ch_2, ch_1, ch_0))
            image.save(image_dir)

            temp_im_dir = os.path.join(main_folder, 'Images', image_1 + '_temp.bmp')
            pseudo_color(image_dir, temp_im_dir, -1, 1, 1)  # G+B-R

            image_pseudo = Image.open(temp_im_dir)
            image_dir = os.path.join(main_folder, 'Images', image_1 + '_ice.jpg')
            image_pseudo.save(image_dir, quality=90, subsampling=0)
            list_of_images.append(image_dir)

        # Thermal infrared band 5
        if (active_channels[4] == 1):
            # combine useful channels
            image_dir = os.path.join(main_folder, 'Images', image_1 + '_555_pseudo.bmp')
            image = Image.merge("RGB", (ch_4, ch_4, ch_4))
            image = ImageOps.autocontrast(image)
            image.save(image_dir)

            temp_im_dir = os.path.join(main_folder, 'Images', image_1 + '_temp.bmp')
            pseudo_color(image_dir, temp_im_dir, 1, 0, 0)  # G

            image_pseudo = Image.open(temp_im_dir)
            image_dir = os.path.join(main_folder, 'Images', image_1 + '_thermal.jpg')
            image_pseudo.save(image_dir, quality=90, subsampling=0)
            list_of_images.append(image_dir)

        for image_dir in list_of_images:
            if (evening_pass):
                image = Image.open(image_dir)
                image = image.rotate(180)
                image.save(image_dir)
            logging.debug('Saved %s' % image_dir)

        if not list_of_images:
            return [], [0, 0, 0, 0, 0, 0]   # no image was decoded from recording
        else:
            return list_of_images, active_channels
    else:
        return [], [0, 0, 0, 0, 0, 0]


def pseudo_color(input, output, rCh, bCh, gCh):
    # launch c program to apply pseudo color to a weighted combination of RGB
    if platform.machine() == 'x86_64':
        command = './Pseudo_color/pseudo ' + input + " " + output + " " + \
            str(rCh) + " " + str(gCh) + " " + str(bCh)
    elif platform.machine().startswith('arm'):
        command = './Pseudo_color/pseudo_arm ' + input + " " + output + " " + \
            str(rCh) + " " + str(gCh) + " " + str(bCh)
    else:
        logging.error('platform unsupported by pseudo color converter')
        return -1		# I'm not sure what else to do here...
    logging.debug('running: ' + command)
    pseudo_color_process = subprocess.Popen([command], shell=True)
    return_code = pseudo_color_process.wait()
    logging.debug('pseudo_color ran with returncode: %s' % return_code)


def email_image(list_of_images, active_channels):

    if list_of_images:      # if there are images to be e-mailed
        with open(os.path.join(main_folder, 'mail_list.txt')) as f:
            mail_list = f.read().splitlines()

        with open(os.path.join(main_folder, 'email_config.txt')) as f:
            email_credentials = f.read().splitlines()

        msg = MIMEMultipart()
        msg['Subject'] = 'METEOR-M2 image'
        msg['From'] = email_credentials[0]
        msg.preamble = 'METEOR-M2 image: %s' % basename(list_of_images[0].split('_')[0])

        channel_comment = []
        channel_comment.append('Band No 1 (VIS= Visible)						0.50 +- 0.2 - 0.70 +- 0.2 um\n')
        channel_comment.append('Band No 2 (VNIR=Visible Near Infrared)			0.70 +- 0.2 - 1.10 +- 0.2 um\n')
        channel_comment.append('Band No 3 (SWIR= Short Wave Infrared)			1.60 +- 0.50 - 1.80 +- 0.50 um\n')
        channel_comment.append('Band No 4 (MWIR= Mid Wave Infrared)		    	3.50 +- 0.50 - 4.10 +- 0.50 um\n')
        channel_comment.append('Band No 5 (TIR = Thermal Infrared)				10.5 +- 0.50 - 11.5 +- 0.50 um\n')
        channel_comment.append('Band No 6 (TIR = Thermal Infrared)				11.5 +- 0.50 - 12.5 +- 0.50 um\n')

        # Add info about active channels to test body of e-mail
        s = 'Satellite is transmitting channels: \n'
        for i in range(0, 6):
            if active_channels[i] == 1:
                s += channel_comment[i]
        msg.attach(MIMEText(s))

        # Add images
        for image_dir in list_of_images:
            fp = open(image_dir, 'rb')
            img = MIMEImage(fp.read())
            img.add_header('Content-Disposition', 'attachment', filename=basename(image_dir))
            fp.close()
            msg.attach(img)

        try:
            mail_server = smtplib.SMTP_SSL('smtp.gmail.com')
            logging.debug('logging into mail server')

            mail_server.login(email_credentials[1], email_credentials[2])
            logging.debug('logged in, sending e-mail')

            mail_server.sendmail(email_credentials[1], mail_list, msg.as_string())
            logging.debug('e-mail sent, logging out')

            mail_server.quit()
            logging.debug('logged out of mail server')
        except:
            logging.debug('There was a problem sending the e-mail')
        else:
            pass
    else:
        logging.debug('no e-mail sent as there was no decoded image')


try:
    os.system('setterm -cursor off')    # doesn't work in windows though...
except:
    print("Couldn't switch cursor off, elevation & azimuth might blink")

while True:
    # 1) download & extract TLE
    meteor_M2 = retrieve_tle(sat_name)

    # 2a) wait for positive elevation
    location.date = datetime.datetime.utcnow()
    meteor_M2.compute(location)

    logging.debug('waiting for positive satellite elevation')

    while meteor_M2.alt < 0:
        location.date = datetime.datetime.utcnow()
        meteor_M2.compute(location)
        print('elevation %4.2f deg, azimuth %5.2f deg                              ' \
            % (meteor_M2.alt * deg_per_rad, meteor_M2.az * deg_per_rad), end="\r", file=sys.stderr)
        # sys.stdout.flush()
        # print('velocity %4.2f m/s' % meteor_M2.range_velocity)
        # doppler_shift = doppler(meteor_M2.range_velocity)

        time.sleep(1)

    # 2b) Stop network
    os.system("sudo ifconfig eth0 down")

    # 3) start GRC script
    logging.debug('start GRC script')

    grc_process = subprocess.Popen(SDR_capture, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    logging.debug('started ' + grc_script + ' with PID:' + str(grc_process.pid)\
        + ' ' + str(SDR_capture)+ ' started at ' + str(datetime.datetime.now()))
    # however, no indication is given if the script was succesfully started...

    # 4) calculate & update Doppler while positive elevation
    # server_error_counter = 0
    maximum_elevation = 0.0
    while meteor_M2.alt > 0:
        location.date = datetime.datetime.utcnow()
        meteor_M2.compute(location)

        # doppler_shift = doppler(meteor_M2.range_velocity)
        # print(doppler_shift)
        # try:
        # 	grc_server.set_doppler_freq(doppler_shift)
        # except:
        # 	print('error setting doppler shift')	# we will see errors when grc script is not running
        # 	server_error_counter = server_error_counter + 1
        # 	print(server_error_counter)		# we're curious when things go wrong
        # 	pass
        # time.sleep(0.1)		# why do we crash after a while when sleeping only 100ms but not when sleeping 1s?
        # time.sleep(0.3)		# crashes with 300ms as well
        # moreover, why does rpi crash and PC doesn't?

        print('elevation %4.2f deg, azimuth %5.2f deg                              ' \
            % (meteor_M2.alt * deg_per_rad, meteor_M2.az * deg_per_rad), end="\r", file=sys.stderr)
        # sys.stdout.flush()
        # print('velocity %4.2f m/s' % meteor_M2.range_velocity

        time.sleep(1)

    # 5) stop GRC script
    logging.debug('stopping GRC script')

    try:
        grc_process.send_signal(subprocess.signal.SIGINT)		# don't use shell !
    except:
        logging.debug("script didn't seem to be running anymore")
    else:
        returncode = grc_process.wait()
        logging.debug('GRC script ended with return code: %s' % returncode)

    # 6) decode image with medet
    # we already waited for the GRC process to finish, no need to wait some more, right?
    time.sleep(20)

    image_1, image_2 = decode_capture()	 # since we don't know what file name the GRC script generated, we look for a file generated in the last 30min

    # 6a) Restart network
    os.system("sudo ifconfig eth0 up")
    time.sleep(60)

    # 7) process decoded image
    image_list, act_channels = process_image(image_1, image_2)	# by default no wine on ARM devices (to use e.g. LrptImageProcessor.exe, so only basic RGB channel processing

    # 8) e-mail final images to mail list
    email_image(image_list, act_channels)

    logging.debug('all done, start over for next pass')