v2_dataset_creator.py

import json
import logging
import os
import sys
import argparse

import urllib3
from influxdb import DataFrameClient

from classes.artificial_features import ArtificialFeatures
from classes.features_analyzer_v2 import FeaturesAnalyzerV2
from classes.inputs_gatherer_v2 import InputsGathererV2

urllib3.disable_warnings()

def get_data_file_path(region, main_cfg):
    str_input_folder = '%s%s_%s%s_%s%s%s' % (main_cfg['outputFolder'], region,
                                             main_cfg['dataset']['startYear'],
                                             main_cfg['dataset']['startDay'],
                                             main_cfg['dataset']['endYear'],
                                             main_cfg['dataset']['endDay'], os.sep)
    return '%s%s_dataset.csv' % (str_input_folder, str_input_folder.split(os.sep)[1])

if __name__ == "__main__":
    # --------------------------------------------------------------------------- #
    # Configuration file
    # --------------------------------------------------------------------------- #
    arg_parser = argparse.ArgumentParser()
    arg_parser.add_argument("-c", help="configuration file")
    # arg_parser.add_argument("-t", help="type (MOR | EVE)")
    arg_parser.add_argument("-l", help="log file (optional, if empty log redirected on stdout)")
    args = arg_parser.parse_args()

    # Load the main parameters
    config_file = args.c
    if os.path.isfile(config_file) is False:
        print('\nATTENTION! Unable to open configuration file %s\n' % config_file)
        sys.exit(1)

    cfg = json.loads(open(args.c).read())

    # Load the connections parameters and update the config dict with the related values
    cfg_conns = json.loads(open(cfg['connectionsFile']).read())
    cfg.update(cfg_conns)

    # Define the forecast type
    # forecast_type = args.t

    # --------------------------------------------------------------------------- #
    # Set logging object
    # --------------------------------------------------------------------------- #
    if not args.l:
        log_file = None
    else:
        log_file = args.l

    logger = logging.getLogger()
    logging.basicConfig(format='%(asctime)-15s::%(levelname)s::%(funcName)s::%(message)s', level=logging.INFO,
                        filename=log_file)

    logger.info('Starting program')

    logger.info('Connection to InfluxDb server on socket [%s:%s]' % (cfg['influxDB']['host'], cfg['influxDB']['port']))
    try:
        influx_client = DataFrameClient(host=cfg['influxDB']['host'], port=cfg['influxDB']['port'],
                                        password=cfg['influxDB']['password'], username=cfg['influxDB']['user'],
                                        database=cfg['influxDB']['database'], ssl=cfg['influxDB']['ssl'])
    except Exception as e:
        logger.error('EXCEPTION: %s' % str(e))
        sys.exit(3)
    logger.info('Connection successful')

    # --------------------------------------------------------------------------- #
    # Start calculations
    # --------------------------------------------------------------------------- #

    af = ArtificialFeatures(influx_client, None, cfg, logger)
    ig = InputsGathererV2(influx_client, cfg, logger, af)
    fa = FeaturesAnalyzerV2(ig, cfg, logger)
    fa.dataset_creator()
    logger.info('Ending program')