Merge pull request #108 from pysat/lasp_mgii

New Instrument: LASP MgII

CHANGELOG.md

@@ -3,6 +3,12 @@ Change Log
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](https://semver.org/).
 
+[0.0.9] - 2023-01-18
+--------------------
+* Enhancements
+   * Added an instrument for the LASP MgII core-to-wing index
+   * Added functions for general LISIRD downloads
+
 [0.0.8] - 2022-11-29
 --------------------
 * Bugs

docs/methods.rst

@@ -45,3 +45,12 @@ filter for other data sets.
 
 .. automodule:: pysatSpaceWeather.instruments.methods.kp_ap
    :members:
+
+LISIRD
+------
+Contains functions to support LASP Interactive Solar IRradiance Datacenter
+(LISIRD) data sets and downloads.
+
+
+.. automodule:: pysatSpaceWeather.instruments.methods.lisird
+   :members:

docs/supported_instruments.rst

@@ -116,3 +116,18 @@ are available from the German Research Centre for Geosciences at Potsdam,
 
 .. automodule:: pysatSpaceWeather.instruments.sw_kp
    :members:
+
+
+.. _mgii-inst:
+
+MgII Core-to-Wing Ratio
+-----------------------
+
+The core-to-wing ratio of the solar MgII line is a proxy for solar chromospheric
+variability.  It has been used to extract a precise measurement of solar
+activity at Earth.  The two data sets provided by LASP together provide index
+values from 1978 through 2020.
+
+
+.. automodule:: pysatSpaceWeather.instruments.sw_mgii
+   :members:

pysatSpaceWeather/instruments/__init__.py

@@ -1,7 +1,7 @@
 from pysatSpaceWeather.instruments import methods  # noqa F401
 
 __all__ = ['ace_epam', 'ace_mag', 'ace_sis', 'ace_swepam',
-           'sw_dst', 'sw_f107', 'sw_kp']
+           'sw_dst', 'sw_f107', 'sw_kp', 'sw_mgii']
 
 for inst in __all__:
-    exec("from pysatSpaceWeather.instruments import {x}".format(x=inst))
+    exec("from pysatSpaceWeather.instruments import {inst}".format(inst=inst))

pysatSpaceWeather/instruments/methods/__init__.py

@@ -1,3 +1,6 @@
+from pysatSpaceWeather.instruments.methods import ace  # noqa F401
+from pysatSpaceWeather.instruments.methods import dst  # noqa F401
+from pysatSpaceWeather.instruments.methods import f107  # noqa F401
 from pysatSpaceWeather.instruments.methods import general  # noqa F401
-from pysatSpaceWeather.instruments.methods import ace, dst  # noqa F401
-from pysatSpaceWeather.instruments.methods import f107, kp_ap  # noqa F401
+from pysatSpaceWeather.instruments.methods import kp_ap  # noqa F401
+from pysatSpaceWeather.instruments.methods import lisird # noqa F401

pysatSpaceWeather/instruments/methods/lisird.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-.
+# Full license can be found in License.md
+# Full author list can be found in .zenodo.json file
+# DOI:10.5281/zenodo.3986138
+# ----------------------------------------------------------------------------
+"""Provides support functions for the LASP LISIRD data base."""
+
+import datetime as dt
+import json
+import numpy as np
+import os
+import pandas as pds
+import requests
+
+import pysat
+
+ackn = "".join(["LASP Interactive Solar Irradiance Data Center provides ",
+                "access to many solar datasets generated by researchers at ",
+                "LASP and other institutions."])
+
+
+def references(platform, name, tag, inst_id):
+    """Provide references for different Instrument data products.
+
+    Parameters
+    ----------
+    platform : str
+        Instrument platform
+    name : str
+        Instrument name
+    tag : str
+        Instrument tag
+    inst_id : str
+        Instrument ID
+
+    Returns
+    -------
+    refs : str
+        String of references
+
+    """
+
+    refs = {'sw': {'mgii': {
+        'composite': {
+            '': ''.join(["Viereck, R. A., Floyd, L. E., Crane, P. C., Woods, ",
+                         "T. N., Knapp, B. G., Rottman, G., Weber, M., Puga,",
+                         " L. C., and DeLand, M. T. (2004), A composite Mg ",
+                         "II index spanning from 1978 to 2003, Space Weather",
+                         ", 2, S10005, doi:10.1029/2004SW000084."])},
+        'sorce': {
+            '': "\n".join([
+                "".join(["Snow, M, William E. McClintock, Thomas N. Woods, ",
+                         "Oran R. White, Jerald W. Harder, and Gary Rottman ",
+                         "(2005). The Mg II Index from SORCE, Solar Phys., ",
+                         "230, 1, 325-344."]),
+                "".join(["Heath, D. and Schlesinger, B. (1986). The Mg 280-nm ",
+                         "doublet as a monitor of changes in solar ",
+                         "ultraviolet irradiance, JGR, 91, 8672-8682."])])}}}}
+
+    return refs[platform][name][tag][inst_id]
+
+
+def build_lisird_url(lisird_data_name, start, stop):
+    """Build a LASP LISIRD direct download URL.
+
+    Parameters
+    ----------
+    lisird_data_name : str
+        Name of the data set on the LISARD server
+    start : dt.datetime
+        Start time
+    stop : dt.datetime
+        Stop time
+
+    Returns
+    -------
+    url : str
+        URL that will download the desired data
+
+    """
+
+    # Define the formatting for the start and stop times
+    tfmt = "%Y-%m-%dT%H:%M:%S.000Z"
+
+    url = "".join(["https://lasp.colorado.edu/lisird/latis/dap/",
+                   lisird_data_name, ".json?&time>=", start.strftime(tfmt),
+                   '&time<=', stop.strftime(tfmt),
+                   "&format_time(yyyy-MM-dd'T'HH:mm:ss.SSS)"])
+
+    return url
+
+
+def download(date_array, data_path, local_file_prefix, local_date_fmt,
+             lisird_data_name, freq, update_files=True, fill_vals=None):
+    """Download routine for LISIRD data.
+
+    Parameters
+    ----------
+    date_array : array-like
+        Sequence of dates for which files will be downloaded.
+    data_path : str
+        Path to data directory.
+    local_file_prefix : str
+        Prefix for local files, e.g., 'tag_' or 'tag_monthly_'
+    local_date_fmt : str
+        String format for the local filename, e.g., '%Y-%m-%d' or '%Y-%m'
+    lisird_data_name : str
+        Name of the data set on the LISARD server
+    freq : pds.DateOffset or dt.timedelta
+        Offset to add to the start date to ensure all data is downloaded
+        (inclusive)
+    update_files : bool
+        Re-download data for files that already exist if True (default=False)
+    fill_vals : dict or NoneType
+        Dict of fill values to replace with NaN by variable name or None to
+        leave alone
+
+    Raises
+    ------
+    IOError
+        If there is a gateway timeout when downloading data
+    KeyError
+        If the `fill_vals` input does not match the downloaded data
+
+    """
+    # Initialize the fill_vals dict, if necessary
+    if fill_vals is None:
+        fill_vals = {}
+
+    # Cycle through all the dates
+    for dl_date in date_array:
+        # Build the local filename
+        local_file = os.path.join(
+            data_path, ''.join([local_file_prefix,
+                                dl_date.strftime(local_date_fmt), '.txt']))
+
+        # Determine if the download should occur
+        if update_files or not os.path.isfile(local_file):
+            # Get the URL for the desired data
+            url = build_lisird_url(lisird_data_name, dl_date, dl_date + freq)
+
+            # The data is returned as a JSON file
+            req = requests.get(url)
+
+            # Process the JSON file
+            if req.text.find('Gateway Timeout') >= 0:
+                raise IOError(''.join(['Gateway timeout when requesting ',
+                                       'file using command: ', url]))
+
+            if req.ok:
+                raw_dict = json.loads(req.text)[lisird_data_name]
+                data = pds.DataFrame.from_dict(raw_dict['samples'])
+                if data.empty:
+                    pysat.logger.warning("no data for {:}".format(dl_date))
+                else:
+                    # The URL specifies the time format, so break it down
+                    frac_sec = [int(tval.split('.')[-1])
+                                for tval in data['time']]
+                    times = [dt.datetime.strptime(tval.split('.')[0],
+                                                  '%Y-%m-%dT%H:%M:%S')
+                             + dt.timedelta(microseconds=frac_sec[i] * 6)
+                             for i, tval in enumerate(data.pop('time'))]
+                    data.index = times
+
+                    # Replace fill value with NaNs
+                    for var in fill_vals.keys():
+                        if var in data.columns:
+                            idx, = np.where(data[var] == fill_vals[var])
+                            data.iloc[idx, :] = np.nan
+                        else:
+                            raise KeyError(''.join(['unknown fill value ',
+                                                    'variable name supplied: ',
+                                                    var]))
+
+                    # Create a local CSV file
+                    data.to_csv(local_file, header=True)
+            else:
+                pysat.logger.info("".join(["Data not downloaded for ",
+                                           dl_date.strftime("%d %b %Y"),
+                                           ", date may be out of range ",
+                                           "for the database."]))
+    return

pysatSpaceWeather/instruments/sw_f107.py

@@ -57,17 +57,16 @@
 
 import datetime as dt
 import ftplib
-import json
 import numpy as np
 import os
 import pandas as pds
 import pysat
 import requests
 import sys
-import warnings
 
 from pysatSpaceWeather.instruments.methods import f107 as mm_f107
 from pysatSpaceWeather.instruments.methods import general
+from pysatSpaceWeather.instruments.methods import lisird
 
 logger = pysat.logger
 
@@ -464,60 +463,10 @@ def download(date_array, tag, inst_id, data_path, update_files=False):
                 date_array[-1], freq='MS')
 
         # Download from LASP, by month
-        for dl_date in date_array:
-            # Create the name to which the local file will be saved
-            str_date = dl_date.strftime('%Y-%m')
-            data_file = os.path.join(data_path,
-                                     'f107_monthly_{:s}.txt'.format(str_date))
-
-            if update_files or not os.path.isfile(data_file):
-                # Set the download webpage
-                dstr = ''.join(['http://lasp.colorado.edu/lisird/latis/dap/',
-                                'noaa_radio_flux.json?time%3E=',
-                                dl_date.strftime('%Y-%m-%d'),
-                                'T00:00:00.000Z&time%3C=',
-                                (dl_date + pds.DateOffset(months=1)
-                                 - pds.DateOffset(days=1)).strftime('%Y-%m-%d'),
-                                'T00:00:00.000Z'])
-
-                # The data is returned as a JSON file
-                req = requests.get(dstr)
-
-                # Process the JSON file
-                if req.text.find('Gateway Timeout') >= 0:
-                    raise IOError(''.join(['Gateway timeout when requesting ',
-                                           'file using command: ', dstr]))
-
-                if req.ok:
-                    raw_dict = json.loads(req.text)['noaa_radio_flux']
-                    data = pds.DataFrame.from_dict(raw_dict['samples'])
-                    if data.empty:
-                        warnings.warn("no data for {:}".format(dl_date),
-                                      UserWarning)
-                    else:
-                        # The file format changed over time
-                        try:
-                            # This is the new data format
-                            times = [dt.datetime.strptime(time, '%Y%m%d')
-                                     for time in data.pop('time')]
-                        except ValueError:
-                            # Accepts old file formats
-                            times = [dt.datetime.strptime(time, '%Y %m %d')
-                                     for time in data.pop('time')]
-                        data.index = times
-
-                        # Replace fill value with NaNs
-                        for var in data.columns:
-                            idx, = np.where(data[var] == -99999.0)
-                            data.iloc[idx, :] = np.nan
-
-                        # Create a local CSV file
-                        data.to_csv(data_file, header=True)
-                else:
-                    pysat.logger.info("".join(["Data not downloaded for ",
-                                               dl_date.strftime("%d %b %Y"),
-                                               ", date may be out of range ",
-                                               "for the database."]))
+        freq = pds.DateOffset(months=1, seconds=-1)
+        lisird.download(date_array, data_path, 'f107_monthly_', '%Y-%m',
+                        'noaa_radio_flux', freq, update_files,
+                        {'f107_adjusted': -99999.0, 'f107_observed': -99999.0})
 
     elif tag == 'prelim':
         ftp = ftplib.FTP('ftp.swpc.noaa.gov')  # Connect to host, default port