diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml new file mode 100644 index 0000000..ba9c31f --- /dev/null +++ b/.github/workflows/publish-to-pypi.yml @@ -0,0 +1,113 @@ +name: Publish Python 🐍 distribution 📦 to PyPI + +on: push + +jobs: + build: + name: Build distribution 📦 + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + - name: Install pypa/build + run: python3 -m pip install build --user + - name: Build a binary wheel and a source tarball + run: python3 -m build + - name: Store the distribution packages + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: dist/ + + publish-to-pypi: + name: >- + Publish Python 🐍 distribution 📦 to PyPI + if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes + needs: + - build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/pyglider + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution 📦 to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + github-release: + name: >- + Sign the Python 🐍 distribution 📦 with Sigstore + and upload them to GitHub Release + needs: + - publish-to-pypi + runs-on: ubuntu-latest + + permissions: + contents: write # IMPORTANT: mandatory for making GitHub Releases + id-token: write # IMPORTANT: mandatory for sigstore + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Sign the dists with Sigstore + uses: sigstore/gh-action-sigstore-python@v2.1.1 + with: + inputs: >- + ./dist/*.tar.gz + ./dist/*.whl + - name: Create GitHub Release + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + gh release create + '${{ github.ref_name }}' + --repo '${{ github.repository }}' + --notes "" + - name: Upload artifact signatures to GitHub Release + env: + GITHUB_TOKEN: ${{ github.token }} + # Upload to GitHub Release using the `gh` CLI. + # `dist/` contains the built packages, and the + # sigstore-produced signatures and certificates. + run: >- + gh release upload + '${{ github.ref_name }}' dist/** + --repo '${{ github.repository }}' + + publish-to-testpypi: + name: Publish Python 🐍 distribution 📦 to TestPyPI + needs: + - build + runs-on: ubuntu-latest + + environment: + name: testpypi + url: https://test.pypi.org/p/pyglider + + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution 📦 to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ \ No newline at end of file diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 23cabab..f9641ed 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,42 +7,33 @@ jobs: strategy: matrix: os: ["ubuntu-latest"] - python-version: ["3.9", "3.10"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v2 - - name: Cache conda - uses: actions/cache@v2 - env: - # Increase this value to reset cache if etc/example-environment.yml has not changed - CACHE_NUMBER: 0 + - uses: actions/checkout@v4 + - name: mamba setup enviroment + uses: mamba-org/setup-micromamba@v1.9.0 with: - path: ~/conda_pkgs_dir - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{hashFiles('environment.yml') }} - - uses: conda-incubator/setup-miniconda@v2 - with: - activate-environment: pyglider-test + environment-name: test-env environment-file: tests/environment.yml - python-version: ${{ matrix.python-version }} - channel-priority: strict - use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly! + create-args: >- + python=${{ matrix.python-version }} - name: Conda info - shell: bash -l {0} + shell: micromamba-shell {0} run: conda info; conda list - name: install pyglider source - shell: bash -l {0} + shell: micromamba-shell {0} run: which pip; pip install -e . - name: Process seaexplorer - shell: bash -l {0} + shell: micromamba-shell {0} run: which python; cd tests/example-data/example-seaexplorer; make clean-all; python process_deploymentRealTime.py - name: Process slocum - shell: bash -l {0} + shell: micromamba-shell {0} run: which python; cd tests/example-data/example-slocum; make clean-all; python process_deploymentRealTime.py - name: Process seaexplorer-legato-flntu-arod-ad2cp - shell: bash -l {0} + shell: micromamba-shell {0} run: which python; cd tests/example-data/example-seaexplorer-legato-flntu-arod-ad2cp; make clean-all; python process_deploymentRealTime.py - name: Run tests - shell: bash -l {0} + shell: micromamba-shell {0} run: which python; pytest --cov --cov-report xml - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 \ No newline at end of file + uses: codecov/codecov-action@v4 diff --git a/.readthedocs.yaml b/.readthedocs.yaml index dfb1063..a6835ca 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,12 +1,42 @@ +# Read the Docs configuration file for Sphinx projects +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required version: 2 -python: - version: 3.8 - install: - - method: pip - path: . - extra_requirements: - - docs - -# Build documentation in the docs/ directory with Sphinx + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.12" + # You can also specify other tool versions: + # nodejs: "20" + # rust: "1.70" + # golang: "1.20" + +# Build documentation in the "docs/" directory with Sphinx sphinx: - configuration: docs/conf.py \ No newline at end of file + configuration: docs/conf.py + # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs + # builder: "dirhtml" + # Fail on all warnings to avoid broken references + # fail_on_warning: true + +python: + install: + - requirements: docs-requirements.txt + # Install our python package before building the docs + - method: pip + path: . + +# Optionally build your docs in additional formats such as PDF and ePub +# formats: +# - pdf +# - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +# python: +# install: +# - requirements: docs/requirements.txt \ No newline at end of file diff --git a/docs/getting-started-slocum.md b/docs/getting-started-slocum.md index 14f3407..e7dfaa6 100644 --- a/docs/getting-started-slocum.md +++ b/docs/getting-started-slocum.md @@ -2,19 +2,12 @@ ## Gather data -Slocum gliders have 4 types of files. For telemetry data there is `*.sbd` files for sensor data, and `*.tbd` for the glider's attitude and position data. These are called `*.dbd` and `*.ebd` respectively, when retrieved from the gliders' payload post deployment. These need to be made available in a single directory for `pyglider` to process. +Slocum gliders have 4 types of files. For telemetry data there are `*.tbd` files for sensor data, and `*.sbd` for the glider's attitude and position data. These are called `*.ebd` and `*.tbd` respectively, when retrieved from the gliders' payload post deployment. Modern gliders have compressed version of these, eg `*.tcd`, `*.scd` that *pyglider* should be able to parse. These data files need to be made available in a _single_ directory for *pyglider* to process. Note that on the glider they are often separated into `science/logs` and `flight/logs`. -Slocums also have a sensor cache file `*.cac`, all of which have randomized names. These are needed by the processing, and are usually stored in a separate cache directory. +Slocum gliders also have a sensor cache file `*.cac`, all of which have randomized names. These are needed by the processing, and are usually stored in a separate cache directory. -You can download and expand example data using `.get_example_data`: +You can download example data at which will add a local directory `example-data` to your current directory. -```python -import pyglider.example_data as pexamp - -pexamp.get_example_data('./') -``` - -which will add a local directory `example-data` to your current directory. ## Make a deployment configuration file @@ -33,8 +26,9 @@ The example script is relatively straight forward if there is no intermediate pr Data comes from an input directory, and is translated into a single CF-compliant netCDF timeseries file using the package [dbdreader](https://dbdreader.readthedocs.io/en/latest/). Finally individual profiles are saved and a 2-D 1-m grid in time-depth is saved. -```{note} There is a version that does not require `dbdreader` to do the initial conversion from the Dinkum format to netCDF. However it is quite slow, particularly for full-resolution datasets, and less robust. We suggest using the `slocum.raw_to_timeseries`. -``` +:::{note} +There is a version that does not require `dbdreader` to do the initial conversion from the Dinkum format to netCDF. However it is quite slow, particularly for full-resolution datasets, and less robust. We suggest using the `slocum.raw_to_timeseries`. +::: It is possible that between these steps the user will want to add any screening steps, or adjustments to the calibrations. PyGlider does not provide those steps, but is designed so they are easy to add. diff --git a/environment.yml b/environment.yml index 4c73e6a..31db1a8 100644 --- a/environment.yml +++ b/environment.yml @@ -9,9 +9,9 @@ dependencies: - dask - netcdf4 - gsw + - polars>=1.1 - scipy - bitstring - pooch - pip: - dbdreader - - polars diff --git a/pyglider/_version.py b/pyglider/_version.py index cba8e59..0fd1318 100644 --- a/pyglider/_version.py +++ b/pyglider/_version.py @@ -1 +1 @@ -__version__ = '0.0.4' \ No newline at end of file +__version__ = '0.0.7' \ No newline at end of file diff --git a/pyglider/ncprocess.py b/pyglider/ncprocess.py index b330dc0..9d379e4 100644 --- a/pyglider/ncprocess.py +++ b/pyglider/ncprocess.py @@ -14,7 +14,7 @@ _log = logging.getLogger(__name__) -def extract_timeseries_profiles(inname, outdir, deploymentyaml): +def extract_timeseries_profiles(inname, outdir, deploymentyaml, force=False): """ Extract and save each profile from a timeseries netCDF. @@ -29,16 +29,18 @@ def extract_timeseries_profiles(inname, outdir, deploymentyaml): deploymentyaml : str or Path location of deployment yaml file for the netCDF file. This should be the same yaml file that was used to make the timeseries file. + + force : bool, default False + Force an overwite even if profile netcdf already exists """ try: os.mkdir(outdir) except FileExistsError: pass - with open(deploymentyaml) as fin: - deployment = yaml.safe_load(fin) - meta = deployment['metadata'] + deployment = utils._get_deployment(deploymentyaml) + meta = deployment['metadata'] with xr.open_dataset(inname) as ds: _log.info('Extracting profiles: opening %s', inname) profiles = np.unique(ds.profile_index) @@ -49,7 +51,7 @@ def extract_timeseries_profiles(inname, outdir, deploymentyaml): dss = ds.isel(time=ind) outname = outdir + '/' + 'mission_grid.nc' _log.info('Checking %s', outname) - if not os.path.exists(outname): + if force or (not os.path.exists(outname)): # this is the id for the whole file, not just this profile.. dss['trajectory'] = utils.get_file_id(ds).encode() trajlen = len(utils.get_file_id(ds).encode()) @@ -68,16 +70,31 @@ def extract_timeseries_profiles(inname, outdir, deploymentyaml): dss['v'] = dss.water_velocity_northward.mean() dss['v'].attrs = profile_meta['v'] elif 'u' in profile_meta: - dss['u'] = profile_meta['u'].get('_FillValue', np.NaN) + dss['u'] = profile_meta['u'].get('_FillValue', np.nan) dss['u'].attrs = profile_meta['u'] - dss['v'] = profile_meta['v'].get('_FillValue', np.NaN) + dss['v'] = profile_meta['v'].get('_FillValue', np.nan) dss['v'].attrs = profile_meta['v'] + else: + dss['u'] = np.nan + dss['v'] = np.nan - dss['profile_id'] = np.array(p*1.0) + + dss['profile_id'] = np.int32(p) dss['profile_id'].attrs = profile_meta['profile_id'] + if '_FillValue' not in dss['profile_id'].attrs: + dss['profile_id'].attrs['_FillValue'] = -1 + dss['profile_id'].attrs['valid_min'] = np.int32(dss['profile_id'].attrs['valid_min']) + dss['profile_id'].attrs['valid_max'] = np.int32(dss['profile_id'].attrs['valid_max']) + dss['profile_time'] = dss.time.mean() dss['profile_time'].attrs = profile_meta['profile_time'] + # remove units so they can be encoded later: + try: + del dss.profile_time.attrs['units'] + del dss.profile_time.attrs['calendar'] + except KeyError: + pass dss['profile_lon'] = dss.longitude.mean() dss['profile_lon'].attrs = profile_meta['profile_lon'] dss['profile_lat'] = dss.latitude.mean() @@ -85,7 +102,7 @@ def extract_timeseries_profiles(inname, outdir, deploymentyaml): dss['lat'] = dss['latitude'] dss['lon'] = dss['longitude'] - dss['platform'] = np.NaN + dss['platform'] = np.int32(1) comment = (meta['glider_model'] + ' operated by ' + meta['institution']) dss['platform'].attrs['comment'] = comment @@ -96,40 +113,62 @@ def extract_timeseries_profiles(inname, outdir, deploymentyaml): meta['glider_model'] + dss['platform'].attrs['id']) dss['platform'].attrs['type'] = 'platform' dss['platform'].attrs['wmo_id'] = meta['wmo_id'] + if '_FillValue' not in dss['platform'].attrs: + dss['platform'].attrs['_FillValue'] = -1 - dss['lat_uv'] = np.NaN + + dss['lat_uv'] = np.nan dss['lat_uv'].attrs = profile_meta['lat_uv'] - dss['lon_uv'] = np.NaN + dss['lon_uv'] = np.nan dss['lon_uv'].attrs = profile_meta['lon_uv'] - dss['time_uv'] = np.NaN + dss['time_uv'] = np.nan dss['time_uv'].attrs = profile_meta['time_uv'] - dss['instrument_ctd'] = np.NaN + dss['instrument_ctd'] = np.int32(1.0) dss['instrument_ctd'].attrs = profile_meta['instrument_ctd'] + if '_FillValue' not in dss['instrument_ctd'].attrs: + dss['instrument_ctd'].attrs['_FillValue'] = -1 dss.attrs['date_modified'] = str(np.datetime64('now')) + 'Z' - # ancillary variables:: + # ancillary variables: link and create with values of 2. If + # we dont' want them all 2, then create these variables in the + # time series to_fill = ['temperature', 'pressure', 'conductivity', - 'salinity', 'density', 'lon', 'lat', 'depth'] + 'salinity', 'density', 'lon', 'lat', 'depth'] for name in to_fill: - dss[name].attrs['ancillary_variables'] = name + '_qc' + qcname = name + '_qc' + dss[name].attrs['ancillary_variables'] = qcname + if qcname not in dss.keys(): + dss[qcname] = ('time', 2 * np.ones(len(dss[name]), np.int8)) + dss[qcname].attrs = utils.fill_required_qcattrs({}, name) + # 2 is "not eval" # outname = outdir + '/' + utils.get_file_id(dss) + '.nc' _log.info('Writing %s', outname) - timeunits = 'nanoseconds since 1970-01-01T00:00:00Z' + timeunits = 'seconds since 1970-01-01T00:00:00Z' timecalendar = 'gregorian' + try: + del dss.profile_time.attrs['_FillValue'] + del dss.profile_time.attrs['units'] + except KeyError: + pass dss.to_netcdf(outname, encoding={'time': {'units': timeunits, - 'calendar': timecalendar}, + 'calendar': timecalendar, + 'dtype': 'float64'}, 'profile_time': - {'units': timeunits}}) + {'units': timeunits, + '_FillValue': -99999.0, + 'dtype': 'float64'}, + } + + ) # add traj_strlen using bare ntcdf to make IOOS happy with netCDF4.Dataset(outname, 'r+') as nc: nc.renameDimension('string%d' % trajlen, 'traj_strlen') - -def make_gridfiles(inname, outdir, deploymentyaml, *, fnamesuffix='', dz=1): +def make_gridfiles(inname, outdir, deploymentyaml, *, fnamesuffix='', dz=1, starttime='1970-01-01'): """ Turn a timeseries netCDF file into a vertically gridded netCDF. @@ -160,11 +199,12 @@ def make_gridfiles(inname, outdir, deploymentyaml, *, fnamesuffix='', dz=1): except FileExistsError: pass - with open(deploymentyaml) as fin: - deployment = yaml.safe_load(fin) + deployment = utils._get_deployment(deploymentyaml) + profile_meta = deployment['profile_variables'] ds = xr.open_dataset(inname, decode_times=True) + ds = ds.where(ds.time > np.datetime64(starttime), drop=True) _log.info(f'Working on: {inname}') _log.debug(str(ds)) _log.debug(str(ds.time[0])) @@ -174,16 +214,22 @@ def make_gridfiles(inname, outdir, deploymentyaml, *, fnamesuffix='', dz=1): profiles = [p for p in profiles if (~np.isnan(p) and not (p % 1) and (p > 0))] profile_bins = np.hstack((np.array(profiles) - 0.5, [profiles[-1]+0.5])) - + _log.debug(profile_bins) Nprofiles = len(profiles) _log.info(f'Nprofiles {Nprofiles}') depth_bins = np.arange(0, 1100.1, dz) depths = depth_bins[:-1] + 0.5 - + xdimname = 'time' dsout = xr.Dataset( coords={'depth': ('depth', depths), - 'profile': ('time', profiles)}) - print('Booo', ds.time, ds.temperature) + 'profile': (xdimname, profiles)}) + dsout['depth'].attrs = {'units': 'm', + 'long_name': 'Depth', + 'standard_name': 'depth', + 'positive': 'down', + 'coverage_content_type': 'coordinate', + 'comment': 'center of depth bins'} + ds['time_1970'] = ds.temperature.copy() ds['time_1970'].values = ds.time.values.astype(np.float64) for td in ('time_1970', 'longitude', 'latitude'): @@ -201,12 +247,12 @@ def make_gridfiles(inname, outdir, deploymentyaml, *, fnamesuffix='', dz=1): good = np.where(~np.isnan(ds['time']) & (ds['profile_index'] % 1 == 0))[0] _log.info(f'Done times! {len(dat)}') dsout['profile_time_start'] = ( - ('time'), dat, profile_meta['profile_time_start']) + (xdimname), dat, profile_meta['profile_time_start']) dsout['profile_time_end'] = ( - ('time'), dat, profile_meta['profile_time_end']) + (xdimname), dat, profile_meta['profile_time_end']) for k in ds.keys(): - if k in ['time', 'longitude', 'latitude', 'depth'] or 'time' in k: + if k in ['time', 'profile', 'longitude', 'latitude', 'depth'] or 'time' in k: continue _log.info('Gridding %s', k) good = np.where(~np.isnan(ds[k]) & (ds['profile_index'] % 1 == 0))[0] @@ -223,7 +269,6 @@ def make_gridfiles(inname, outdir, deploymentyaml, *, fnamesuffix='', dz=1): "scipy.stats.gmean") else: average_method = "mean" - dat, xedges, yedges, binnumber = stats.binned_statistic_2d( ds['profile_index'].values[good], ds['depth'].values[good], @@ -231,7 +276,7 @@ def make_gridfiles(inname, outdir, deploymentyaml, *, fnamesuffix='', dz=1): bins=[profile_bins, depth_bins]) _log.debug(f'dat{np.shape(dat)}') - dsout[k] = (('depth', 'time'), dat.T, ds[k].attrs) + dsout[k] = (('depth', xdimname), dat.T, ds[k].attrs) # fill gaps in data: dsout[k].values = utils.gappy_fill_vertical(dsout[k].values) @@ -247,11 +292,45 @@ def make_gridfiles(inname, outdir, deploymentyaml, *, fnamesuffix='', dz=1): dsout = dsout.drop(['water_velocity_eastward', 'water_velocity_northward']) dsout.attrs = ds.attrs + dsout.attrs.pop('cdm_data_type') + # fix to be ISO parsable: + if len(dsout.attrs['deployment_start']) > 18: + dsout.attrs['deployment_start'] = dsout.attrs['deployment_start'][:19] + dsout.attrs['deployment_end'] = dsout.attrs['deployment_end'][:19] + dsout.attrs['time_coverage_start'] = dsout.attrs['time_coverage_start'][:19] + dsout.attrs['time_coverage_end'] = dsout.attrs['time_coverage_end'][:19] + # fix standard_name so they don't overlap! + try: + dsout['waypoint_latitude'].attrs.pop('standard_name') + dsout['waypoint_longitude'].attrs.pop('standard_name') + dsout['profile_time_start'].attrs.pop('standard_name') + dsout['profile_time_end'].attrs.pop('standard_name') + except: + pass + # set some attributes for cf guidance + # see H.6.2. Profiles along a single trajectory + # https://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/build/aphs06.html + dsout.attrs['featureType'] = 'trajectoryProfile' + dsout['profile'].attrs['cf_role'] = 'profile_id' + dsout['mission_number'] = np.int32(1) + dsout['mission_number'].attrs['cf_role'] = 'trajectory_id' + dsout = dsout.set_coords(['latitude', 'longitude', 'time']) + for k in dsout: + if k in ['profile', 'depth', 'latitude', 'longitude', 'time', 'mission_number']: + dsout[k].attrs['coverage_content_type'] = 'coordinate' + else: + dsout[k].attrs['coverage_content_type'] = 'physicalMeasurement' + outname = outdir + '/' + ds.attrs['deployment_name'] + '_grid' + fnamesuffix + '.nc' _log.info('Writing %s', outname) # timeunits = 'nanoseconds since 1970-01-01T00:00:00Z' - dsout.to_netcdf(outname) + dsout.to_netcdf( + outname, + encoding={'time': {'units': 'seconds since 1970-01-01T00:00:00Z', + '_FillValue': np.nan, + 'calendar': 'gregorian', + 'dtype': 'float64'}}) _log.info('Done gridding') return outname diff --git a/pyglider/seaexplorer.py b/pyglider/seaexplorer.py index e86212b..28468f6 100644 --- a/pyglider/seaexplorer.py +++ b/pyglider/seaexplorer.py @@ -41,7 +41,7 @@ def _sort(ds): def raw_to_rawnc(indir, outdir, deploymentyaml, incremental=True, min_samples_in_file=5, dropna_subset=None, dropna_thresh=1): """ - Convert seaexplorer text files to raw parquet files. + Convert seaexplorer text files to raw parquet pandas files. Parameters ---------- @@ -268,8 +268,8 @@ def merge_parquet(indir, outdir, deploymentyaml, incremental=False, kind='raw'): Only add new files.... """ - with open(deploymentyaml) as fin: - deployment = yaml.safe_load(fin) + deployment = utils._get_deployment(deploymentyaml) + metadata = deployment['metadata'] id = metadata['glider_name'] outgli = outdir + '/' + id + '-rawgli.parquet' @@ -339,13 +339,14 @@ def _remove_fill_values(df, fill_value=-9999): def raw_to_timeseries(indir, outdir, deploymentyaml, kind='raw', - profile_filt_time=100, profile_min_time=300, maxgap=300, interpolate=False, fnamesuffix=''): + profile_filt_time=100, profile_min_time=300, + maxgap=10, interpolate=False, fnamesuffix=''): """ A little different than above, for the 4-file version of the data set. """ - with open(deploymentyaml) as fin: - deployment = yaml.safe_load(fin) + deployment = utils._get_deployment(deploymentyaml) + metadata = deployment['metadata'] ncvar = deployment['netcdf_variables'] device_data = deployment['glider_devices'] @@ -425,7 +426,9 @@ def raw_to_timeseries(indir, outdir, deploymentyaml, kind='raw', val = np.interp(time_timebase.astype(float), time_var.astype(float), var_non_nan) # interpolate only over those gaps that are smaller than 'maxgap' - tg_ind = utils.find_gaps(time_var.astype(float), time_timebase.astype(float), maxgap) + # apparently maxgap is to be in somethng like seconds, and this data is in ms. Certainly + # the default of 0.3 s was not intended. Changing default to 10 s: + tg_ind = utils.find_gaps(time_var.astype(float), time_timebase.astype(float), maxgap*1000) val[tg_ind] = np.nan else: val = val[indctd] @@ -524,7 +527,8 @@ def raw_to_timeseries(indir, outdir, deploymentyaml, kind='raw', if 'units' in ds.ad2cp_time.attrs.keys(): ds.ad2cp_time.attrs.pop('units') ds.to_netcdf(outname, 'w', - encoding={'time': {'units': 'milliseconds since 1970-01-01T00:00:00Z'}}) + encoding={'time': {'units': 'seconds since 1970-01-01T00:00:00Z', + 'dtype': 'float64'}}) return outname diff --git a/pyglider/slocum.py b/pyglider/slocum.py index 1ebd883..83dfea3 100644 --- a/pyglider/slocum.py +++ b/pyglider/slocum.py @@ -13,10 +13,11 @@ import logging import numpy as np import os +import re import time import xarray as xr import xml.etree.ElementTree as ET -import yaml +from collections.abc import Iterable import pyglider.utils as utils @@ -345,8 +346,8 @@ def dbd_to_dict(dinkum_file, cachedir, keys=None): _log.debug('Diagnostic check passed. Endian is %s', endian) nsensors = int(meta['sensors_per_cycle']) - currentValues = np.zeros(int(meta['sensors_per_cycle'])) + np.NaN - data = np.zeros((DINKUMCHUNKSIZE, nsensors)) + np.NaN + currentValues = np.zeros(int(meta['sensors_per_cycle'])) + np.nan + data = np.zeros((DINKUMCHUNKSIZE, nsensors)) + np.nan # Then there's a data cycle with every sensor marked as updated, giving # us our initial values. # 01 means updated with 'same value', 10 means updated with a new value, @@ -370,7 +371,7 @@ def dbd_to_dict(dinkum_file, cachedir, keys=None): binaryData.bytealign() for i, code in enumerate(updatedCode): if code == '00': # No new value - currentValues[i] = np.NaN + currentValues[i] = np.nan elif code == '01': # Same value as before. continue elif code == '10': # New value. @@ -404,7 +405,7 @@ def dbd_to_dict(dinkum_file, cachedir, keys=None): if ndata % DINKUMCHUNKSIZE == 0: # need to allocate more data! data = np.concatenate( - (data, np.NaN + np.zeros((DINKUMCHUNKSIZE, nsensors))), + (data, np.nan + np.zeros((DINKUMCHUNKSIZE, nsensors))), axis=0) elif d == 'X': # End of file cycle tag. We made it through. @@ -496,7 +497,7 @@ def add_times_flight_sci(fdata, sdata=None): sdata['m_present_time_sci'] = np.interp( sdata['sci_m_present_time'], tf, pt, np.nan, np.nan) else: - sdata['m_present_time_sci'] = np.NaN * sdata['sci_m_present_time'] + sdata['m_present_time_sci'] = np.nan * sdata['sci_m_present_time'] return fdata, sdata @@ -621,8 +622,8 @@ def merge_rawnc(indir, outdir, deploymentyaml, scisuffix = scisuffix.lower() glidersuffix = glidersuffix.lower() - with open(deploymentyaml) as fin: - deployment = yaml.safe_load(fin) + deployment = utils._get_deployment(deploymentyaml) + metadata = deployment['metadata'] id = metadata['glider_name'] + metadata['glider_serial'] @@ -684,8 +685,7 @@ def raw_to_timeseries(indir, outdir, deploymentyaml, *, name of the new merged netcdf file. """ - with open(deploymentyaml) as fin: - deployment = yaml.safe_load(fin) + deployment = utils._get_deployment(deploymentyaml) metadata = deployment['metadata'] ncvar = deployment['netcdf_variables'] device_data = deployment['glider_devices'] @@ -717,7 +717,6 @@ def raw_to_timeseries(indir, outdir, deploymentyaml, *, if atts != 'coordinates': attr[atts] = ncvar[name][atts] ds[name] = (('time'), ebd[name].values, attr) - for name in thenames: _log.info('working on %s', name) if 'method' in ncvar[name].keys(): @@ -733,7 +732,7 @@ def raw_to_timeseries(indir, outdir, deploymentyaml, *, _log.debug('EBD sensorname %s', sensorname) val = ebd[sensorname] val = utils._zero_screen(val) - # val[val==0] = np.NaN + # val[val==0] = np.nan val = convert(val) else: _log.debug('DBD sensorname %s', sensorname) @@ -791,7 +790,8 @@ def raw_to_timeseries(indir, outdir, deploymentyaml, *, def binary_to_timeseries(indir, cachedir, outdir, deploymentyaml, *, search='*.[D|E]BD', fnamesuffix='', time_base='sci_water_temp', profile_filt_time=100, - profile_min_time=300, maxgap=300): + profile_min_time=300, maxgap=300, + replace_attrs=None): """ Convert directly from binary files to netcdf timeseries file. Requires dbdreader to be installed. @@ -807,16 +807,31 @@ def binary_to_timeseries(indir, cachedir, outdir, deploymentyaml, *, outdir : string Directory to put the merged timeseries files. - deploymentyaml : str - YAML text file with deployment information for this glider. + deploymentyaml : str or list + Name of YAML text file with deployment information for this glider. - profile_filt_time : float + If a list, then the YAML files are read in order, and any top-level dictionaries + are overwritten from the previous YAMLs. The advantage of this is that it allows + metadata that is common to multiple ways of processing the data come from the + first file, and then subsequent files change "netcdf_variables" if desired. + + profile_filt_time : float or None time in seconds over which to smooth the pressure time series for finding up and down profiles (note, doesn't filter the data that is - saved) + saved). If None, then do not find profiles. - profile_min_time : float - minimum time to consider a profile an actual profile (seconds) + profile_min_time : float or None + minimum time to consider a profile an actual profile (seconds). If None, + then do not find profiles. + + maxgap : float + Longest gap in seconds to interpolate over when matching instrument + timeseries. + + replace_attrs : dict or None + replace global attributes in the metadata after reading the metadata + file in. Helpful when processing runs with only a couple things that + change. Returns ------- @@ -827,8 +842,11 @@ def binary_to_timeseries(indir, cachedir, outdir, deploymentyaml, *, if not have_dbdreader: raise ImportError('Cannot import dbdreader') - with open(deploymentyaml) as fin: - deployment = yaml.safe_load(fin) + deployment = utils._get_deployment(deploymentyaml) + if replace_attrs: + for att in replace_attrs: + deployment['metadata'][att] = replace_attrs[att] + ncvar = deployment['netcdf_variables'] device_data = deployment['glider_devices'] thenames = list(ncvar.keys()) @@ -863,6 +881,8 @@ def binary_to_timeseries(indir, cachedir, outdir, deploymentyaml, *, # get the time: time = data.pop(0) ds['time'] = (('time'), time, attr) + ds['latitude'] = (('time'), np.zeros(len(time))) + ds['longitude'] = (('time'), np.zeros(len(time))) # get the time_base data: basedata = data.pop(0) # slot the time_base variable into the right place in the @@ -886,8 +906,9 @@ def binary_to_timeseries(indir, cachedir, outdir, deploymentyaml, *, val = data[nn] # interpolate only over those gaps that are smaller than 'maxgap' + # in seconds _t, _ = dbd.get(ncvar[name]['source']) - tg_ind = utils.find_gaps(_t,time,maxgap) + tg_ind = utils.find_gaps(_t, time, maxgap) val[tg_ind] = np.nan val = utils._zero_screen(val) @@ -901,11 +922,12 @@ def binary_to_timeseries(indir, cachedir, outdir, deploymentyaml, *, ValueError(f'{sensorname} not in science or eng parameter names') # make the attributes: - ncvar[name].pop('coordinates', None) + ncvar[name]['coordinates'] = 'time' attrs = ncvar[name] attrs = utils.fill_required_attrs(attrs) ds[name] = (('time'), val, attrs) + _log.info(f'Getting glider depths, {ds}') _log.debug(f'HERE, {ds.pressure[0:100]}') @@ -913,27 +935,28 @@ def binary_to_timeseries(indir, cachedir, outdir, deploymentyaml, *, ds = utils.get_distance_over_ground(ds) ds = utils.get_derived_eos_raw(ds) - ds = ds.assign_coords(longitude=ds.longitude) - ds = ds.assign_coords(latitude=ds.latitude) - ds = ds.assign_coords(depth=ds.depth) # screen out-of-range times; these won't convert: - ds['time'] = ds.time.where((ds.time>0) & (ds.time<6.4e9), np.NaN) - ds['time'] = (('time'), (ds.time.values.astype('timedelta64[s]') + - np.datetime64('1970-01-01T00:00:00')).astype('datetime64[ns]'), attr) + ds['time'] = ds.time.where((ds.time>0) & (ds.time<6.4e9), np.nan) + # convert time to datetime64: + ds['time'] = (ds.time*1e9).astype('datetime64[ns]') + ds['time'].attrs = attr ds = utils.fill_metadata(ds, deployment['metadata'], device_data) - start = ds['time'].values[0] - end = ds['time'].values[-1] + + start = ds.time.values[0] + end = ds.time.values[0] _log.debug('Long') _log.debug(ds.longitude.values[-2000:]) - ds.attrs['deployment_start'] = str(start) - ds.attrs['deployment_end'] = str(end) + # make sure this is ISO readable.... + ds.attrs['deployment_start'] = str(start)[:18] + ds.attrs['deployment_end'] = str(end)[:18] _log.debug(ds.depth.values[:100]) _log.debug(ds.depth.values[2000:2100]) - ds = utils.get_profiles_new( - ds, filt_time=profile_filt_time, profile_min_time=profile_min_time) + if (profile_filt_time is not None) and (profile_min_time is not None): + ds = utils.get_profiles_new( + ds, filt_time=profile_filt_time, profile_min_time=profile_min_time) _log.debug(ds.depth.values[:100]) _log.debug(ds.depth.values[2000:2100]) @@ -943,8 +966,12 @@ def binary_to_timeseries(indir, cachedir, outdir, deploymentyaml, *, pass outname = (outdir + '/' + ds.attrs['deployment_name'] + fnamesuffix + '.nc') _log.info('writing %s', outname) + # convert time back to float64 seconds for ERDDAP etc happiness, as they won't take ns + # as a unit: ds.to_netcdf(outname, 'w', - encoding={'time': {'units': 'seconds since 1970-01-01T00:00:00Z'}}) + encoding={'time': {'units': 'seconds since 1970-01-01T00:00:00Z', + '_FillValue': np.nan, + 'dtype': 'float64'}}) return outname @@ -1044,6 +1071,9 @@ def parse_logfiles(files): times = [''] * 10000 gps = [''] * 10000 amph = [''] * 10000 + relcharge = np.zeros(10000) * np.nan + volts = np.zeros(10000) * np.nan + ntimes = 0 for fn in files: found_time = False @@ -1058,16 +1088,28 @@ def parse_logfiles(files): gps[ntimes - 1] = ll if found_time and "sensor:m_coulomb_amphr_total" in ll: amph[ntimes-1] = ll + if ll.startswith(' sensor:m_lithium_battery_relative_charge'): + pattern = r'=(\d+\.\d+)' + match = re.search(pattern, ll) + relcharge[ntimes-1] = float(match.group(1)) + if ll.startswith(' sensor:m_battery(volts)='): + pattern = r'=(\d+\.\d+)' + match = re.search(pattern, ll) + volts[ntimes-1] = float(match.group(1)) amph = amph[:ntimes] gps = gps[:ntimes] times = times[:ntimes] - + volts = volts[:ntimes] + relcharge = relcharge[:ntimes] # now parse them out = xr.Dataset( coords={'time': ('surfacing', np.zeros(ntimes, dtype='datetime64[ns]'))}) - out['ampH'] = ('surfacing', np.zeros(ntimes) * np.NaN) - out['lon'] = ('surfacing', np.zeros(ntimes) * np.NaN) - out['lat'] = ('surfacing', np.zeros(ntimes) * np.NaN) + out['ampH'] = ('surfacing', np.zeros(ntimes) * np.nan) + out['lon'] = ('surfacing', np.zeros(ntimes) * np.nan) + out['lat'] = ('surfacing', np.zeros(ntimes) * np.nan) + # these don't need to be parsed: + out['volts'] = ('surfacing', volts[:ntimes]) + out['relcharge'] = ('surfacing', relcharge[:ntimes]) for i in range(ntimes): timestring = times[i][11:-13] @@ -1086,5 +1128,94 @@ def parse_logfiles(files): return out +def parse_logfiles_maybe(files): + """ + Parse time, lat, lon, and amph_total from glider logfiles. + + Parameters + ---------- + files : list of strings or Paths + List of logfiles to parse. Should be sorted. + + Returns + ------- + out : xarray + xarray data set with fields time, lon, lat, ampH indexed by surfacing. + More could be added. + """ + + times = [''] * 10000 + gps = [''] * 10000 + amph = [''] * 10000 + surfacereason = '' + missionnum = [''] * 10000 + abortsegment = 0 + abortcause = 0 + + ntimes = 0 + for fn in files: + found_time = False + + with open(fn, 'r') as fin: + for l in fin: + if 'Curr Time:' in l: + times[ntimes] = l + ntimes += 1 + found_time=True + elif found_time and 'GPS Location' in l: + gps[ntimes - 1] = l + elif found_time and "sensor:m_coulomb_amphr_total" in l: + amph[ntimes-1] = l + elif found_time and "Because:" in l: + surfacereason = l + elif found_time and "MissionNum" in l: + missionnum[ntimes-1] = l + elif found_time and "abort segment:" in l: + abortsegment = l + elif found_time and "abort cause:" in l: + abortcause = l + + amph = amph[:ntimes] + gps = gps[:ntimes] + times = times[:ntimes] + missionnum = missionnum[:ntimes] + + # now parse them + out = xr.Dataset(coords={'time': ('surfacing', np.zeros(ntimes, dtype='datetime64[ns]'))}) + out['ampH'] = ('surfacing', np.zeros(ntimes) * np.nan) + out['lon'] = ('surfacing', np.zeros(ntimes) * np.nan) + out['lat'] = ('surfacing', np.zeros(ntimes) * np.nan) + out['missionnum'] = ('surfacing', np.zeros(ntimes) * np.nan) + out.attrs['surfacereason'] = surfacereason + # ABORT HISTORY: last abort segment: hal_1002-2024-183-0-0 (0171.0000) + out.attrs['abortsegment'] = float(abortsegment[-11:-2]) + out.attrs['abortcause'] = abortcause + + for i in range(ntimes): + timestring = times[i][11:-13] + out['time'][i] = np.datetime64(datetime.strptime(timestring, '%a %b %d %H:%M:%S %Y'), 'ns') + try: + if '=' in amph[i]: + st = amph[i].index('=') + en = amph[i][st:].index(' ') + st + out['ampH'][i] = float(amph[i][(st+1):en]) + + # GPS Location: 4912.737 N -12357.253 E measured 110.757 secs ago + sp = gps[i].split() + out['lat'][i] = utils.nmea2deg(float(sp[2])) + out['lon'][i] = utils.nmea2deg(float(sp[4])) + + # MissionName:calvert.mi MissionNum:hal_1002-2024-183-4-41 (0175.0041) + if len(missionnum[i]) > 12: + out['missionnum'][i] = float(missionnum[i][-11:-2]) + except: + pass + + return out + + + + + __all__ = ['binary_to_rawnc', 'merge_rawnc', 'raw_to_timeseries', 'parse_gliderState', 'parse_logfiles'] diff --git a/pyglider/utils.py b/pyglider/utils.py index 146e841..e55350c 100644 --- a/pyglider/utils.py +++ b/pyglider/utils.py @@ -7,6 +7,9 @@ from scipy.signal import argrelextrema import gsw import logging +from pathlib import Path +import yaml + _log = logging.getLogger(__name__) @@ -26,11 +29,15 @@ def get_distance_over_ground(ds): ds : `.xarray.Dataset` With ``distance_over_ground`` key. """ + good = ~np.isnan(ds.latitude + ds.longitude) - dist = gsw.distance(ds.longitude[good].values, ds.latitude[good].values)/1000 - dist = np.roll(np.append(dist, 0), 1) - dist = np.cumsum(dist) - dist = np.interp(ds.time, ds.time[good], dist) + if np.any(good): + dist = gsw.distance(ds.longitude[good].values, ds.latitude[good].values)/1000 + dist = np.roll(np.append(dist, 0), 1) + dist = np.cumsum(dist) + dist = np.interp(ds.time, ds.time[good], dist) + else: + dist = 0 * ds.latitude.values attr = {'long_name': 'distance over ground flown since mission start', 'method': 'get_distance_over_ground', 'units': 'km', @@ -56,9 +63,13 @@ def get_glider_depth(ds): """ good = np.where(~np.isnan(ds.pressure))[0] - ds['depth'] = ds.pressure * 0. - ds['depth'].values = -gsw.z_from_p(ds.pressure.values, - ds.latitude.values) + ds['depth'] = ds.pressure + try: + meanlat = ds.latitude.mean(skipna=True) + ds['depth'].values = -gsw.z_from_p(ds.pressure.values, + ds.latitude.fillna(meanlat).values) + except AttributeError: + pass # now we really want to know where it is, so interpolate: if len(good) > 0: ds['depth'].values = np.interp( @@ -69,9 +80,10 @@ def get_glider_depth(ds): 'comment': 'from science pressure and interpolated', 'instrument': 'instrument_ctd', 'observation_type': 'calulated', - 'accuracy': '1', 'precision': '2', 'resolution': '0.02', + 'accuracy': 1.0, + 'precision': 2.0, 'resolution': 0.02, 'platform': 'platform', - 'valid_min': '0', 'valid_max': '2000', + 'valid_min': 0.0, 'valid_max': 2000.0, 'reference_datum': 'surface', 'positive': 'down'} ds['depth'].attrs = attr return ds @@ -85,8 +97,11 @@ def get_profiles(ds, min_dp=10.0, inversion=3., filt_length=7, make two variables: profile_direction and profile_index; this version is good for lots of data. Less good for sparse data """ - profile = ds.pressure.values * np.NaN - direction = ds.pressure.values * np.NaN + if 'pressure' not in ds: + _log.warning('No "pressure" variable in the data set; not searching for profiles') + return ds + profile = ds.pressure.values * np.nan + direction = ds.pressure.values * np.nan pronum = 1 lastpronum = 0 @@ -150,6 +165,10 @@ def get_profiles_new(ds, min_dp=10.0, filt_time=100, profile_min_time=300): Minimum time length of profile in s. """ + if 'pressure' not in ds: + _log.warning('No "pressure" variable in the data set; not searching for profiles') + return ds + profile = ds.pressure.values * 0 direction = ds.pressure.values * 0 pronum = 1 @@ -218,7 +237,6 @@ def get_profiles_new(ds, min_dp=10.0, filt_time=100, profile_min_time=300): direction[ins] = -1 pronum += 1 - _log.debug('Doing this...') attrs = collections.OrderedDict([ ('long_name', 'profile index'), ('units', '1'), @@ -300,15 +318,16 @@ def get_derived_eos_raw(ds): ('method', 'get_derived_eos_raw'), ('observation_type', 'calulated'), ('instrument', 'instrument_ctd'), - ('valid_max', '40.0'), - ('valid_min', '0.0'), - ('accuracy', '0.01'), - ('precision', '0.01'), - ('resolution', '0.001')]) + ('valid_max', 40.0), + ('valid_min', 0.0), + ('accuracy', 0.01), + ('precision', 0.01), + ('resolution', 0.001)]) attrs = fill_required_attrs(attrs) ds['salinity'].attrs = attrs - sa = gsw.SA_from_SP(ds['salinity'], ds['pressure'], ds['longitude'], - ds['latitude']) + long = ds.longitude.fillna(ds.longitude.mean(skipna=True)) + lat = ds.latitude.fillna(ds.latitude.mean(skipna=True)) + sa = gsw.SA_from_SP(ds['salinity'], ds['pressure'], long, lat) ct = gsw.CT_from_t(sa, ds['temperature'], ds['pressure']) ds['potential_density'] = (('time'), 1000 + gsw.density.sigma0(sa, ct).values) attrs = collections.OrderedDict([ @@ -320,9 +339,9 @@ def get_derived_eos_raw(ds): ('method', 'get_derived_eos_raw'), ('observation_type', 'calulated'), ('instrument', 'instrument_ctd'), - ('accuracy', '0.01'), - ('precision', '0.01'), - ('resolution', '0.001') + ('accuracy', 0.01), + ('precision', 0.01), + ('resolution', 0.001) ]) attrs = fill_required_attrs(attrs) ds['potential_density'].attrs = attrs @@ -338,11 +357,11 @@ def get_derived_eos_raw(ds): ('sources', 'salinity temperature pressure'), ('instrument', 'instrument_ctd'), ('method', 'get_derived_eos_raw'), - ('valid_min', '1000.0'), - ('valid_max', '1040.0'), - ('accuracy', '0.01'), - ('precision', '0.01'), - ('resolution', '0.001') + ('valid_min', 990.0), + ('valid_max', 1040.0), + ('accuracy', 0.01), + ('precision', 0.01), + ('resolution', 0.001) ]) attrs = fill_required_attrs(attrs) ds['density'].attrs = attrs @@ -357,9 +376,9 @@ def get_derived_eos_raw(ds): ('observation_type', 'calulated'), ('method', 'get_derived_eos_raw'), ('instrument', 'instrument_ctd'), - ('accuracy', '0.002'), - ('precision', '0.001'), - ('resolution', '0.0001') + ('accuracy', 0.002), + ('precision', 0.001), + ('resolution', 0.0001) ]) attrs = fill_required_attrs(attrs) ds['potential_temperature'].attrs = attrs @@ -390,6 +409,22 @@ def fill_required_attrs(attrs): return attrs +def fill_required_qcattrs(attrs, varname): + required = { + "units": "1", + "flag_values": np.array([1, 2, 3, 4, 9], dtype=np.int8), + "valid_min": np.int8(1), + "valid_max": np.int8(9), + "flag_meanings": "PASS NOT_EVALUATED SUSPECT FAIL MISSING", + "standard_name": "quality_flag", + "long_name": "Initial flag for {varname}" + } + for k in required.keys(): + if not (k in attrs.keys()): + attrs[k] = required[k] + return attrs + + def get_file_id(ds): """ Make a file id for a Dataset @@ -440,19 +475,27 @@ def fill_metadata(ds, metadata, sensor_data): """ good = ~np.isnan(ds.latitude.values + ds.longitude.values) - ds.attrs['geospatial_lat_max'] = np.max(ds.latitude.values[good]) - ds.attrs['geospatial_lat_min'] = np.min(ds.latitude.values[good]) - ds.attrs['geospatial_lon_max'] = np.max(ds.longitude.values[good]) - ds.attrs['geospatial_lon_min'] = np.min(ds.longitude.values[good]) + if np.any(good): + ds.attrs['geospatial_lat_max'] = np.max(ds.latitude.values[good]) + ds.attrs['geospatial_lat_min'] = np.min(ds.latitude.values[good]) + ds.attrs['geospatial_lon_max'] = np.max(ds.longitude.values[good]) + ds.attrs['geospatial_lon_min'] = np.min(ds.longitude.values[good]) + else: + ds.attrs['geospatial_lat_max'] = np.nan + ds.attrs['geospatial_lat_min'] = np.nan + ds.attrs['geospatial_lon_max'] = np.nan + ds.attrs['geospatial_lon_min'] = np.nan + ds.attrs['geospatial_lat_units'] = 'degrees_north' ds.attrs['geospatial_lon_units'] = 'degrees_east' ds.attrs['netcdf_version'] = '4.0' # TODO get this somehow... ds.attrs['history'] = 'CPROOF glider toolbox version: pre-tag' for k, v in metadata.items(): ds.attrs[k] = v - ds.attrs['featureType'] = 'timeseries' + ds.attrs['featureType'] = 'trajectory' ds.attrs['cdm_data_type'] = 'Trajectory' - ds.attrs['Conventions'] = 'CF-1.6' + ds.attrs['Conventions'] = 'CF-1.8' + ds.attrs['standard_name_vocabulary'] = 'CF STandard Name Table v72' ds.attrs['date_created'] = str(np.datetime64('now')) + 'Z' ds.attrs['date_issued'] = str(np.datetime64('now')) + 'Z' ds.attrs['date_modified'] = " " @@ -475,7 +518,7 @@ def fill_metadata(ds, metadata, sensor_data): def _zero_screen(val): - val[val == 0] = np.NaN + val[val == 0] = np.nan return val @@ -568,6 +611,7 @@ def gappy_fill_vertical(data): data[:, j][ind[0]:ind[-1]] = np.interp(int, ind, data[ind, j]) return data + def find_gaps(sample_time, timebase, maxgap): """ Return an index into *timebase* where True are times in gaps of *sample_time* larger @@ -576,18 +620,19 @@ def find_gaps(sample_time, timebase, maxgap): # figure out which sample each time in time base belongs to: time_index = np.searchsorted(sample_time, timebase, side='right') time_index = np.clip(time_index, 0, len(sample_time)-1) - + # figure out the space between sample pairs dt = np.concatenate(([0], np.diff(sample_time))) # get the gap size for each timebase data point: ddt = dt[time_index] - - # get the indices of timebase that are too large and account for the - # degenerate case when a timebase point falls directly on a sample time. - index = ~np.logical_or((ddt <= maxgap),(np.isin(timebase,sample_time))) - + + # get the indices of timebase that are too large and account for the + # degenerate case when a timebase point falls directly on a sample time. + index = ~np.logical_or((ddt <= maxgap), (np.isin(timebase,sample_time))) + return index + def _parse_gliderxml_pos(fname): """ DEPRECATED: use slocum.parse_gliderState instead @@ -666,6 +711,58 @@ def example_gridplot(filename, outname, fig.savefig(outname, dpi=dpi) +def _get_deployment(deploymentyaml): + """ + Take the list of files in *deploymentyaml* and parse them + for deployment information, with subsequent files overwriting + previous files. + """ + if isinstance(deploymentyaml, str): + deploymentyaml = [deploymentyaml,] + deployment = {} + for nn, d in enumerate(deploymentyaml): + with open(d) as fin: + deployment_ = yaml.safe_load(fin) + for k in deployment_: + deployment[k] = deployment_[k] + + return deployment + + +def _any_newer(dirname, filename): + """ + Check if any files in dirname are newer than filename + """ + filename = Path(filename) + dirname = Path(dirname) + print(filename, filename.exists()) + if not filename.exists(): + return True + + mod_time = filename.stat().st_mtime + is_newer = False + for file_path in dirname.iterdir(): + if file_path.is_file(): + if file_path.stat().st_mtime > mod_time: + is_newer = True + break + + return is_newer + + +def _get_glider_name_slocum(current_directory): + glider = current_directory.parts[-2] + mission = current_directory.parts[-1] + print(f'Glider {glider} and mission: {mission}') + slocum_glider = glider[4:] + if slocum_glider[-4:-3].isnumeric(): + slocum_glider = slocum_glider[:-4] + '_' + slocum_glider[-4:] + else: + slocum_glider = slocum_glider[:-3] + '_' + slocum_glider[-3:] + + return glider, mission, slocum_glider + + __all__ = ['get_distance_over_ground', 'get_glider_depth', 'get_profiles_new', 'get_derived_eos_raw', "fill_metadata", "nmea2deg", "gappy_fill_vertical", "oxygen_concentration_correction"] diff --git a/requirements.txt b/requirements.txt index a15c091..a46e8ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,4 @@ numpy pooch scipy xarray -polars>0.16 +polars>=1.1 diff --git a/setup.py b/setup.py index 7394694..6f7ee00 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,8 @@ "gsw", "scipy", "bitstring", - "pooch" + "pooch", + "polars" ], license='Apache', extras_require={ @@ -25,5 +26,7 @@ "docs": ["pydata-sphinx-theme", "numpydoc", "autoapi", "myst-parser", "sphinx"] }, - zip_safe=True + zip_safe=True, + long_description=open('README.md').read(), + long_description_content_type='text/markdown', ) diff --git a/tests/_copyresultstoexpected.py b/tests/_copyresultstoexpected.py new file mode 100644 index 0000000..7dd1b77 --- /dev/null +++ b/tests/_copyresultstoexpected.py @@ -0,0 +1,17 @@ +""" +If we run the tests and decide we want all the test results to be copied over, this is faster... +""" +from pathlib import Path +from shutil import copy + +todo = {'example-data/example-seaexplorer/L0-timeseries-test/dfo-eva035-20190718.nc': + 'expected/example-seaexplorer/L0-timeseries', + 'example-data/example-seaexplorer-raw/L0-timeseries-test/dfo-bb046-20200908.nc': 'expected/example-seaexplorer-raw/L0-timeseries', + 'example-data/example-slocum/L0-timeseries/dfo-rosie713-20190615.nc': 'expected/example-slocum/L0-timeseries', + 'example-data/example-slocum-littleendian/L0-timeseries-test/dfo-maria997-20220614.nc': 'expected/example-slocum-littleendian/L0-timeseries' + + } + +for td in todo: + copy(td, todo[td]) + diff --git a/tests/environment.yml b/tests/environment.yml index 5edbc77..c3e117b 100644 --- a/tests/environment.yml +++ b/tests/environment.yml @@ -11,10 +11,12 @@ dependencies: - gsw - scipy - bitstring + - polars>=1.1 - pytest - pytest-cov - pooch - matplotlib + - compliance-checker + - cc-plugin-glider - pip: - dbdreader - - polars diff --git a/tests/example-data/example-seaexplorer-raw/deployment.yml b/tests/example-data/example-seaexplorer-raw/deployment.yml index ed82130..1809b97 100644 --- a/tests/example-data/example-seaexplorer-raw/deployment.yml +++ b/tests/example-data/example-seaexplorer-raw/deployment.yml @@ -104,8 +104,8 @@ netcdf_variables: observation_type: measured platform: platform reference: WGS84 - valid_max: "90.0" - valid_min: "-90.0" + valid_max: 90.0 + valid_min: -90.0 coordinate_reference_frame: urn:ogc:crs:EPSG::4326 longitude: @@ -120,29 +120,29 @@ netcdf_variables: observation_type: measured platform: platform reference: WGS84 - valid_max: "180.0" - valid_min: "-180.0" + valid_max: 180.0 + valid_min: -180.0 coordinate_reference_frame: urn:ogc:crs:EPSG::4326 heading: source: Heading long_name: glider heading angle standard_name: platform_orientation - units: rad + units: degrees coordinates: time depth latitude longitude pitch: source: Pitch long_name: glider pitch angle standard_name: platform_pitch_angle - units: rad + units: degrees coordinates: time depth latitude longitude roll: source: Roll long_name: glider roll angle standard_name: platform_roll_angle - units: rad + units: degrees coordinates: time depth latitude longitude # data parameters @@ -153,12 +153,12 @@ netcdf_variables: units: S m-1 coordinates: time depth latitude longitude instrument: instrument_ctd - valid_min: "0." - valid_max: "10." + valid_min: 0 + valid_max: 10 observation_type: "measured" - accuracy: "0.0003" - precision: "0.0001" - resolution: "0.00002" + accuracy: 0.0003 + precision: 0.0001 + resolution: 0.00002 temperature: source: GPCTD_TEMPERATURE @@ -167,12 +167,12 @@ netcdf_variables: units: Celsius coordinates: time depth latitude longitude instrument: instrument_ctd - valid_min: "-5" - valid_max: "50" + valid_min: -5 + valid_max: 50 observation_type: "measured" - accuracy: "0.002" - precision: "0.001" - resolution: "0.0002" + accuracy: 0.002 + precision: 0.001 + resolution: 0.0002 pressure: source: GPCTD_PRESSURE @@ -180,15 +180,15 @@ netcdf_variables: standard_name: sea_water_pressure units: dbar coordinates: time depth latitude longitude - valid_min: "0" - valid_max: "2000" + valid_min: 0 + valid_max: 2000 positive: "down" reference_datum: "sea-surface" instrument: "instrument_ctd" observation_type: "measured" - accuracy: "1" - precision: "2" - resolution: "0.02" + accuracy: 1 + precision: 2 + resolution: 0.02 comment: "ctd pressure sensor" # optics: @@ -231,8 +231,8 @@ profile_variables: profile_id: comment: Sequential profile number within the trajectory. This value is unique in each file that is part of a single trajectory/deployment. long_name: 'Profile ID' - valid_max: '2147483647' - valid_min: '1' + valid_max: 2147483647 + valid_min: 1 profile_time: comment: Timestamp corresponding to the mid-point of the profile @@ -265,8 +265,8 @@ profile_variables: platform: platform standard_name: latitude units: degrees_north - valid_max: "90.0" - valid_min: "-90.0" + valid_max: 90.0 + valid_min: -90.0 profile_lon: comment: Value is interpolated to provide an estimate of the latitude at the mid-point of the profile @@ -275,8 +275,8 @@ profile_variables: platform: platform standard_name: longitude units: degrees_east - valid_max: "180.0" - valid_min: "-180.0" + valid_max: 180.0 + valid_min: -180.0 u: comment: The depth-averaged current is an estimate of the net current measured while the glider is underwater. The value is calculated over the entire underwater segment, which may consist of 1 or more dives. @@ -285,8 +285,8 @@ profile_variables: platform: platform standard_name: eastward_sea_water_velocity units: m s-1 - valid_max: "10.0" - valid_min: "-10.0" + valid_max: 10.0 + valid_min: -10.0 v: comment: The depth-averaged current is an estimate of the net current measured while the glider is underwater. The value is calculated over the entire underwater segment, which may consist of 1 or more dives. @@ -295,8 +295,8 @@ profile_variables: platform: platform standard_name: northward_sea_water_velocity units: m s-1 - valid_max: "10.0" - valid_min: "-10.0" + valid_max: 10.0 + valid_min: -10.0 lon_uv: comment: Not computed @@ -305,8 +305,8 @@ profile_variables: platform: platform standard_name: longitude units: degrees_east - valid_max: "180.0" - valid_min: "-180.0" + valid_max: 180.0 + valid_min: -180.0 lat_uv: comment: Not computed @@ -315,8 +315,8 @@ profile_variables: platform: platform standard_name: latitude units: degrees_north - valid_max: "90.0" - valid_min: "-90.0" + valid_max: 90.0 + valid_min: -90.0 time_uv: comment: Not computed @@ -329,7 +329,7 @@ profile_variables: instrument_ctd: comment: pumped CTD calibration_date: "2018-11-02" - calibration_report: + calibration_report: factory_calibrated: "yes" long_name: Seabird Glider Payload CTD make_model: Seabird GPCTD diff --git a/tests/example-data/example-seaexplorer/deploymentRealtime.yml b/tests/example-data/example-seaexplorer/deploymentRealtime.yml index 50a7cf9..3893c68 100644 --- a/tests/example-data/example-seaexplorer/deploymentRealtime.yml +++ b/tests/example-data/example-seaexplorer/deploymentRealtime.yml @@ -100,8 +100,8 @@ netcdf_variables: observation_type: measured platform: platform reference: WGS84 - valid_max: "90.0" - valid_min: "-90.0" + valid_max: 90.0 + valid_min: -90.0 coordinate_reference_frame: urn:ogc:crs:EPSG::4326 longitude: @@ -116,29 +116,29 @@ netcdf_variables: observation_type: measured platform: platform reference: WGS84 - valid_max: "180.0" - valid_min: "-180.0" + valid_max: 180.0 + valid_min: -180.0 coordinate_reference_frame: urn:ogc:crs:EPSG::4326 heading: source: Heading long_name: glider heading angle standard_name: platform_orientation - units: rad + units: degrees coordinates: time depth latitude longitude pitch: source: Pitch long_name: glider pitch angle standard_name: platform_pitch_angle - units: rad + units: degrees coordinates: time depth latitude longitude roll: source: Roll long_name: glider roll angle standard_name: platform_roll_angle - units: rad + units: degrees coordinates: time depth latitude longitude # data parameters @@ -149,12 +149,12 @@ netcdf_variables: units: S m-1 coordinates: time depth latitude longitude instrument: instrument_ctd - valid_min: "0." - valid_max: "10." + valid_min: 0 + valid_max: 10 observation_type: "measured" - accuracy: "0.0003" - precision: "0.0001" - resolution: "0.00002" + accuracy: 0.0003 + precision: 0.0001 + resolution: 0.00002 temperature: source: GPCTD_TEMPERATURE @@ -163,12 +163,12 @@ netcdf_variables: units: Celsius coordinates: time depth latitude longitude instrument: instrument_ctd - valid_min: "-5" - valid_max: "50" + valid_min: -5 + valid_max: 50 observation_type: "measured" - accuracy: "0.002" - precision: "0.001" - resolution: "0.0002" + accuracy: 0.002 + precision: 0.001 + resolution: 0.0002 pressure: source: GPCTD_PRESSURE @@ -176,15 +176,15 @@ netcdf_variables: standard_name: sea_water_pressure units: dbar coordinates: time depth latitude longitude - valid_min: "0" - valid_max: "2000" + valid_min: 0 + valid_max: 2000 positive: "down" reference_datum: "sea-surface" instrument: "instrument_ctd" observation_type: "measured" - accuracy: "1" - precision: "2" - resolution: "0.02" + accuracy: 1 + precision: 2 + resolution: 0.02 comment: "ctd pressure sensor" # optics: @@ -244,8 +244,8 @@ profile_variables: profile_id: comment: Sequential profile number within the trajectory. This value is unique in each file that is part of a single trajectory/deployment. long_name: 'Profile ID' - valid_max: '2147483647' - valid_min: '1' + valid_max: 2147483647 + valid_min: 1 profile_time: comment: Timestamp corresponding to the mid-point of the profile @@ -275,8 +275,8 @@ profile_variables: platform: platform standard_name: latitude units: degrees_north - valid_max: "90.0" - valid_min: "-90.0" + valid_max: 90.0 + valid_min: -90.0 profile_lon: comment: Value is interpolated to provide an estimate of the latitude at the mid-point of the profile @@ -285,8 +285,8 @@ profile_variables: platform: platform standard_name: longitude units: degrees_east - valid_max: "180.0" - valid_min: "-180.0" + valid_max: 180.0 + valid_min: -180.0 u: comment: The depth-averaged current is an estimate of the net current measured while the glider is underwater. The value is calculated over the entire underwater segment, which may consist of 1 or more dives. @@ -295,8 +295,8 @@ profile_variables: platform: platform standard_name: eastward_sea_water_velocity units: m s-1 - valid_max: "10.0" - valid_min: "-10.0" + valid_max: 10.0 + valid_min: -10.0 v: comment: The depth-averaged current is an estimate of the net current measured while the glider is underwater. The value is calculated over the entire underwater segment, which may consist of 1 or more dives. @@ -305,8 +305,8 @@ profile_variables: platform: platform standard_name: northward_sea_water_velocity units: m s-1 - valid_max: "10.0" - valid_min: "-10.0" + valid_max: 10.0 + valid_min: -10.0 lon_uv: comment: Not computed @@ -315,8 +315,8 @@ profile_variables: platform: platform standard_name: longitude units: degrees_east - valid_max: "180.0" - valid_min: "-180.0" + valid_max: 180.0 + valid_min: -180.0 lat_uv: comment: Not computed @@ -325,8 +325,8 @@ profile_variables: platform: platform standard_name: latitude units: degrees_north - valid_max: "90.0" - valid_min: "-90.0" + valid_max: 90.0 + valid_min: -90.0 time_uv: comment: Not computed diff --git a/tests/example-data/example-slocum-littleendian/deployment.yml b/tests/example-data/example-slocum-littleendian/deployment.yml index a5d258e..346587b 100755 --- a/tests/example-data/example-slocum-littleendian/deployment.yml +++ b/tests/example-data/example-slocum-littleendian/deployment.yml @@ -105,8 +105,8 @@ netcdf_variables: observation_type: measured platform: platform reference: WGS84 - valid_max: "90.0" - valid_min: "-90.0" + valid_max: 90.0 + valid_min: -90.0 coordinate_reference_frame: urn:ogc:crs:EPSG::4326 longitude: @@ -120,29 +120,29 @@ netcdf_variables: observation_type: measured platform: platform reference: WGS84 - valid_max: "180.0" - valid_min: "-180.0" + valid_max: 180.0 + valid_min: -180.0 coordinate_reference_frame: urn:ogc:crs:EPSG::4326 heading: source: m_heading long_name: glider heading angle standard_name: platform_orientation - units: rad + units: degrees coordinates: time depth latitude longitude pitch: source: m_pitch long_name: glider pitch angle standard_name: platform_pitch_angle - units: rad + units: degrees coordinates: time depth latitude longitude roll: source: m_roll long_name: glider roll angle standard_name: platform_roll_angle - units: rad + units: degrees coordinates: time depth latitude longitude # profile info: @@ -168,12 +168,12 @@ netcdf_variables: units: S m-1 coordinates: time depth latitude longitude instrument: instrument_ctd - valid_min: "0." - valid_max: "10." + valid_min: 0 + valid_max: 10 observation_type: "measured" - accuracy: "0.0003" + accuracy: 0.0003 precision: " " - resolution: "0.0001" + resolution: 0.0001 temperature: source: sci_water_temp @@ -182,12 +182,12 @@ netcdf_variables: units: Celsius coordinates: time depth latitude longitude instrument: instrument_ctd - valid_min: "-5" - valid_max: "50" + valid_min: -5 + valid_max: 50 observation_type: "measured" - accuracy: "0.002" + accuracy: 0.002 precision: " " - resolution: "0.00005" + resolution: 0.00005 pressure: source: sci_water_pressure @@ -196,15 +196,15 @@ netcdf_variables: units: dbar coordinates: time depth latitude longitude conversion: bar2dbar - valid_min: "0" - valid_max: "2000" + valid_min: 0 + valid_max: 2000 positive: "down" reference_datum: "sea-surface" instrument: "instrument_ctd" observation_type: "measured" - accuracy: "0.5" + accuracy: 0.5 precision: " " - resolution: "0.01" + resolution: 0.01 comment: "ctd pressure sensor" # optics: @@ -247,8 +247,8 @@ profile_variables: profile_id: comment: Sequential profile number within the trajectory. This value is unique in each file that is part of a single trajectory/deployment. long_name: 'Profile ID' - valid_max: '2147483647' - valid_min: '1' + valid_max: 2147483647 + valid_min: 1 profile_time: comment: Timestamp corresponding to the mid-point of the profile @@ -281,8 +281,8 @@ profile_variables: platform: platform standard_name: latitude units: degrees_north - valid_max: "90.0" - valid_min: "-90.0" + valid_max: 90.0 + valid_min: -90.0 profile_lon: comment: Value is interpolated to provide an estimate of the latitude at the mid-point of the profile @@ -291,8 +291,8 @@ profile_variables: platform: platform standard_name: longitude units: degrees_east - valid_max: "180.0" - valid_min: "-180.0" + valid_max: 180.0 + valid_min: -180.0 u: comment: The depth-averaged current is an estimate of the net current measured while the glider is underwater. The value is calculated over the entire underwater segment, which may consist of 1 or more dives. @@ -301,8 +301,8 @@ profile_variables: platform: platform standard_name: eastward_sea_water_velocity units: m s-1 - valid_max: "10.0" - valid_min: "-10.0" + valid_max: 10.0 + valid_min: -10.0 v: comment: The depth-averaged current is an estimate of the net current measured while the glider is underwater. The value is calculated over the entire underwater segment, which may consist of 1 or more dives. @@ -311,8 +311,8 @@ profile_variables: platform: platform standard_name: northward_sea_water_velocity units: m s-1 - valid_max: "10.0" - valid_min: "-10.0" + valid_max: 10.0 + valid_min: -10.0 lon_uv: comment: Not computed @@ -321,8 +321,8 @@ profile_variables: platform: platform standard_name: longitude units: degrees_east - valid_max: "180.0" - valid_min: "-180.0" + valid_max: 180.0 + valid_min: -180.0 lat_uv: comment: Not computed @@ -331,8 +331,8 @@ profile_variables: platform: platform standard_name: latitude units: degrees_north - valid_max: "90.0" - valid_min: "-90.0" + valid_max: 90.0 + valid_min: -90.0 time_uv: comment: Not computed diff --git a/tests/example-data/example-slocum/deploymentRealtime.yml b/tests/example-data/example-slocum/deploymentRealtime.yml index b9f2dcb..3487811 100644 --- a/tests/example-data/example-slocum/deploymentRealtime.yml +++ b/tests/example-data/example-slocum/deploymentRealtime.yml @@ -32,7 +32,7 @@ metadata: license: "This data may be redistributed and used without restriction or warranty" metadata_link: "https://cproof.uvic.ca" - Metadata_Conventions: CF-1.6, Unidata Dataset Discovery v1.0 + Metadata_Conventions: CF-1.8, Unidata Dataset Discovery v1.0 naming_authority: "ca.uvic.cproof" platform_type: "Slocum Glider" processing_level: "Data provided as is with no expressed or implied @@ -44,9 +44,9 @@ metadata: publisher_url: http://cproof.uvic.ca references: cproof toolbox URL # https://www.nodc.noaa.gov/General/NODC-Archive/seanamelist.txt - sea_name: Coastal Waters of British Columbia + sea_name: Coastal Waters of Southeast Alaska and British Columbia source: Observational data from a profiling glider. - standard_name_vocabulary: CF STandard Name Table v49 + standard_name_vocabulary: CF STandard Name Table v72 summary: Manufacturer test in Saanich Inlet. transmission_system: IRRIDIUM wmo_id: "999999" @@ -88,7 +88,6 @@ netcdf_variables: units: seconds since 1970-01-01T00:00:00Z axis: T observation_type: "measured" - coordinates: time depth latitude longitude latitude: source: m_lat @@ -96,13 +95,12 @@ netcdf_variables: standard_name: latitude units: degrees_north axis: Y - coordinates: time depth latitude longitude comment: "Estimated between surface fixes" observation_type: measured platform: platform reference: WGS84 - valid_max: "90.0" - valid_min: "-90.0" + valid_max: 90.0 + valid_min: -90.0 coordinate_reference_frame: urn:ogc:crs:EPSG::4326 longitude: @@ -111,35 +109,31 @@ netcdf_variables: standard_name: longitude units: degrees_east axis: X - coordinates: time depth latitude longitude comment: "Estimated between surface fixes" observation_type: measured platform: platform reference: WGS84 - valid_max: "180.0" - valid_min: "-180.0" + valid_max: 180.0 + valid_min: -180.0 coordinate_reference_frame: urn:ogc:crs:EPSG::4326 heading: source: m_heading long_name: glider heading angle standard_name: platform_orientation - units: rad - coordinates: time depth latitude longitude + units: degrees pitch: source: m_pitch long_name: glider pitch angle standard_name: platform_pitch_angle - units: rad - coordinates: time depth latitude longitude + units: degrees roll: source: m_roll long_name: glider roll angle standard_name: platform_roll_angle - units: rad - coordinates: time depth latitude longitude + units: degrees # profile info: waypoint_latitude: @@ -147,14 +141,12 @@ netcdf_variables: long_name: waypoint latitude standard_name: latitude units: degree_north - coordinates: time depth latitude longitude waypoint_longitude: source: c_wpt_lon long_name: waypoint longitude standard_name: longitude units: degree_east - coordinates: time depth latitude longitude # data parameters conductivity: @@ -162,45 +154,42 @@ netcdf_variables: long_name: water conductivity standard_name: sea_water_electrical_conductivity units: S m-1 - coordinates: time depth latitude longitude instrument: instrument_ctd - valid_min: "0." - valid_max: "10." + valid_min: 0. + valid_max: 10. observation_type: "measured" - accuracy: "0.0003" - precision: "0.0001" - resolution: "0.00002" + accuracy: 0.0003 + precision: 0.0001 + resolution: 0.00002 temperature: source: sci_water_temp long_name: water temperature standard_name: sea_water_temperature units: Celsius - coordinates: time depth latitude longitude instrument: instrument_ctd - valid_min: "-5" - valid_max: "50" + valid_min: -5.0 + valid_max: 50.0 observation_type: "measured" - accuracy: "0.002" - precision: "0.001" - resolution: "0.0002" + accuracy: 0.002 + precision: 0.001 + resolution: 0.0002 pressure: source: sci_water_pressure long_name: water pressure standard_name: sea_water_pressure units: dbar - coordinates: time depth latitude longitude conversion: bar2dbar - valid_min: "0" - valid_max: "2000" + valid_min: 0.0 + valid_max: 2000.0 positive: "down" reference_datum: "sea-surface" instrument: "instrument_ctd" observation_type: "measured" - accuracy: "1" - precision: "2" - resolution: "0.02" + accuracy: 1.0 + precision: 2.0 + resolution: 0.02 comment: "ctd pressure sensor" @@ -210,19 +199,16 @@ netcdf_variables: long_name: chlorophyll standard_name: concentration_of_chlorophyll_in_sea_water units: mg m-3 - coordinates: time depth latitude longitude cdom: source: sci_flbbcd_cdom_units long_name: CDOM units: ppb - coordinates: time depth latitude longitude backscatter_700: source: sci_flbbcd_bb_units long_name: 700 nm wavelength backscatter units: "1" - coordinates: time depth latitude longitude # Oxygen oxygen_concentration: @@ -230,7 +216,6 @@ netcdf_variables: long_name: oxygen concentration standard_name: mole_concentration_of_dissolved_molecular_oxygen_in_sea_water units: umol l-1 - coordinates: time depth latitude longitude # derived water speed: # water_velocity_eastward: @@ -252,8 +237,8 @@ profile_variables: profile_id: comment: Sequential profile number within the trajectory. This value is unique in each file that is part of a single trajectory/deployment. long_name: 'Profile ID' - valid_max: '2147483647' - valid_min: '1' + valid_max: 2147483646 + valid_min: 1 profile_time: comment: Timestamp corresponding to the mid-point of the profile @@ -283,8 +268,8 @@ profile_variables: platform: platform standard_name: latitude units: degrees_north - valid_max: "90.0" - valid_min: "-90.0" + valid_max: 90.0 + valid_min: -90.0 profile_lon: comment: Value is interpolated to provide an estimate of the latitude at the mid-point of the profile @@ -293,8 +278,8 @@ profile_variables: platform: platform standard_name: longitude units: degrees_east - valid_max: "180.0" - valid_min: "-180.0" + valid_max: 180.0 + valid_min: -180.0 u: comment: The depth-averaged current is an estimate of the net current measured while the glider is underwater. The value is calculated over the entire underwater segment, which may consist of 1 or more dives. @@ -303,8 +288,8 @@ profile_variables: platform: platform standard_name: eastward_sea_water_velocity units: m s-1 - valid_max: "10.0" - valid_min: "-10.0" + valid_max: 10.0 + valid_min: -10.0 v: comment: The depth-averaged current is an estimate of the net current measured while the glider is underwater. The value is calculated over the entire underwater segment, which may consist of 1 or more dives. @@ -313,8 +298,8 @@ profile_variables: platform: platform standard_name: northward_sea_water_velocity units: m s-1 - valid_max: "10.0" - valid_min: "-10.0" + valid_max: 10.0 + valid_min: -10.0 lon_uv: comment: Not computed @@ -323,8 +308,8 @@ profile_variables: platform: platform standard_name: longitude units: degrees_east - valid_max: "180.0" - valid_min: "-180.0" + valid_max: 180.0 + valid_min: -180.0 lat_uv: comment: Not computed @@ -333,8 +318,8 @@ profile_variables: platform: platform standard_name: latitude units: degrees_north - valid_max: "90.0" - valid_min: "-90.0" + valid_max: 90.0 + valid_min: -90.0 time_uv: comment: Not computed diff --git a/tests/example-data/example-slocum/process_deploymentRealTime.py b/tests/example-data/example-slocum/process_deploymentRealTime.py index d242949..35db32b 100644 --- a/tests/example-data/example-slocum/process_deploymentRealTime.py +++ b/tests/example-data/example-slocum/process_deploymentRealTime.py @@ -51,7 +51,7 @@ rawdir, l1tsdir, deploymentyaml, profile_filt_time=100, profile_min_time=300) -if False: +if True: # make profile netcdf files for ioos gdac... ncprocess.extract_timeseries_profiles(outname, profiledir, deploymentyaml) diff --git a/tests/results/example-seaexplorer-raw/L0-timeseries/dfo-bb046-20200908.nc b/tests/expected/example-seaexplorer-raw/L0-timeseries/dfo-bb046-20200908.nc similarity index 77% rename from tests/results/example-seaexplorer-raw/L0-timeseries/dfo-bb046-20200908.nc rename to tests/expected/example-seaexplorer-raw/L0-timeseries/dfo-bb046-20200908.nc index 55d56cd..c923c62 100644 Binary files a/tests/results/example-seaexplorer-raw/L0-timeseries/dfo-bb046-20200908.nc and b/tests/expected/example-seaexplorer-raw/L0-timeseries/dfo-bb046-20200908.nc differ diff --git a/tests/results/example-seaexplorer/L0-timeseries/dfo-eva035-20190718.nc b/tests/expected/example-seaexplorer/L0-timeseries/dfo-eva035-20190718.nc similarity index 70% rename from tests/results/example-seaexplorer/L0-timeseries/dfo-eva035-20190718.nc rename to tests/expected/example-seaexplorer/L0-timeseries/dfo-eva035-20190718.nc index e159748..b57e1e9 100644 Binary files a/tests/results/example-seaexplorer/L0-timeseries/dfo-eva035-20190718.nc and b/tests/expected/example-seaexplorer/L0-timeseries/dfo-eva035-20190718.nc differ diff --git a/tests/results/example-slocum-littleendian/L0-timeseries/dfo-maria997-20220614.nc b/tests/expected/example-slocum-littleendian/L0-timeseries/dfo-maria997-20220614.nc similarity index 89% rename from tests/results/example-slocum-littleendian/L0-timeseries/dfo-maria997-20220614.nc rename to tests/expected/example-slocum-littleendian/L0-timeseries/dfo-maria997-20220614.nc index d76694c..52f3432 100644 Binary files a/tests/results/example-slocum-littleendian/L0-timeseries/dfo-maria997-20220614.nc and b/tests/expected/example-slocum-littleendian/L0-timeseries/dfo-maria997-20220614.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0636.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0636.nc new file mode 100644 index 0000000..5a6ee51 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0636.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0646.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0646.nc new file mode 100644 index 0000000..cccd05b Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0646.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0657.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0657.nc new file mode 100644 index 0000000..3d4a4f2 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0657.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0708.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0708.nc new file mode 100644 index 0000000..925006c Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0708.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0718.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0718.nc new file mode 100644 index 0000000..049c743 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0718.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0729.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0729.nc new file mode 100644 index 0000000..f9d6262 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0729.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0739.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0739.nc new file mode 100644 index 0000000..5f68d78 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0739.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0749.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0749.nc new file mode 100644 index 0000000..9c88ca8 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0749.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0800.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0800.nc new file mode 100644 index 0000000..2928cad Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0800.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0831.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0831.nc new file mode 100644 index 0000000..4b64629 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0831.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0842.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0842.nc new file mode 100644 index 0000000..a6e9a23 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0842.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0853.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0853.nc new file mode 100644 index 0000000..925dca1 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0853.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0902.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0902.nc new file mode 100644 index 0000000..5f019d9 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0902.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0913.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0913.nc new file mode 100644 index 0000000..319efb8 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0913.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0923.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0923.nc new file mode 100644 index 0000000..063072c Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0923.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0933.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0933.nc new file mode 100644 index 0000000..92a45b5 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0933.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0944.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0944.nc new file mode 100644 index 0000000..4d1473f Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0944.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0954.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0954.nc new file mode 100644 index 0000000..4237187 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T0954.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1004.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1004.nc new file mode 100644 index 0000000..391072c Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1004.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1033.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1033.nc new file mode 100644 index 0000000..4737f89 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1033.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1044.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1044.nc new file mode 100644 index 0000000..4892f2a Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1044.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1055.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1055.nc new file mode 100644 index 0000000..026c825 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1055.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1105.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1105.nc new file mode 100644 index 0000000..dfd080f Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1105.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1115.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1115.nc new file mode 100644 index 0000000..1b6affa Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1115.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1125.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1125.nc new file mode 100644 index 0000000..b3d7bef Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1125.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1209.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1209.nc new file mode 100644 index 0000000..34d2ad0 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1209.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1220.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1220.nc new file mode 100644 index 0000000..c3c3a30 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1220.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1231.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1231.nc new file mode 100644 index 0000000..c6b262a Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1231.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1241.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1241.nc new file mode 100644 index 0000000..c46e680 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1241.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1252.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1252.nc new file mode 100644 index 0000000..15ef422 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1252.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1302.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1302.nc new file mode 100644 index 0000000..6176837 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1302.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1313.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1313.nc new file mode 100644 index 0000000..0c9eaf6 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1313.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1323.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1323.nc new file mode 100644 index 0000000..5af6612 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1323.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1335.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1335.nc new file mode 100644 index 0000000..5ecb21f Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1335.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1345.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1345.nc new file mode 100644 index 0000000..94d92d4 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1345.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1415.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1415.nc new file mode 100644 index 0000000..727b99c Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1415.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1425.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1425.nc new file mode 100644 index 0000000..9db57b0 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1425.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1435.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1435.nc new file mode 100644 index 0000000..8ac4a7e Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1435.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1445.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1445.nc new file mode 100644 index 0000000..20067bd Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1445.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1455.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1455.nc new file mode 100644 index 0000000..595ec1e Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1455.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1506.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1506.nc new file mode 100644 index 0000000..f1aacda Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1506.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1516.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1516.nc new file mode 100644 index 0000000..2348ff8 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1516.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1526.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1526.nc new file mode 100644 index 0000000..658a62b Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1526.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1536.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1536.nc new file mode 100644 index 0000000..811174a Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1536.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1546.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1546.nc new file mode 100644 index 0000000..d7a40c4 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1546.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1607.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1607.nc new file mode 100644 index 0000000..9ba7a80 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1607.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1618.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1618.nc new file mode 100644 index 0000000..a459f35 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1618.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1627.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1627.nc new file mode 100644 index 0000000..375a391 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1627.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1637.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1637.nc new file mode 100644 index 0000000..821d880 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1637.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1646.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1646.nc new file mode 100644 index 0000000..bc3a2fc Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1646.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1656.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1656.nc new file mode 100644 index 0000000..b99394a Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1656.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1705.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1705.nc new file mode 100644 index 0000000..6029c70 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1705.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1714.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1714.nc new file mode 100644 index 0000000..f756490 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1714.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1723.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1723.nc new file mode 100644 index 0000000..1f14a47 Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1723.nc differ diff --git a/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1732.nc b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1732.nc new file mode 100644 index 0000000..a37b8aa Binary files /dev/null and b/tests/expected/example-slocum/L0-profiles/dfo-rosie713-20190620T1732.nc differ diff --git a/tests/results/example-slocum/L0-timeseries/dfo-rosie713-20190615.nc b/tests/expected/example-slocum/L0-timeseries/dfo-rosie713-20190615.nc similarity index 71% rename from tests/results/example-slocum/L0-timeseries/dfo-rosie713-20190615.nc rename to tests/expected/example-slocum/L0-timeseries/dfo-rosie713-20190615.nc index 2513d0f..b390195 100644 Binary files a/tests/results/example-slocum/L0-timeseries/dfo-rosie713-20190615.nc and b/tests/expected/example-slocum/L0-timeseries/dfo-rosie713-20190615.nc differ diff --git a/tests/test_pyglider.py b/tests/test_pyglider.py index e5663ca..0c003bd 100644 --- a/tests/test_pyglider.py +++ b/tests/test_pyglider.py @@ -4,6 +4,7 @@ import numpy as np import yaml +import pyglider.ncprocess as ncprocess import pyglider.seaexplorer as seaexplorer import pyglider.slocum as slocum @@ -24,7 +25,7 @@ # Open test data file test_data = xr.open_dataset( library_dir / - 'tests/results/example-seaexplorer/L0-timeseries/dfo-eva035-20190718.nc') + 'tests/expected/example-seaexplorer/L0-timeseries/dfo-eva035-20190718.nc') variables = list(output.variables) @@ -84,7 +85,6 @@ def test_example_seaexplorer_interp_nrt(var): # Test raw (full resolution) seaexplorer data. - rawdir = str(example_dir / 'example-seaexplorer-raw/delayed_raw/') + '/' rawncdir = str(example_dir / 'example-seaexplorer-raw/delayed_rawnc/') + '/' deploymentyaml_raw = str(example_dir / 'example-seaexplorer-raw/deployment.yml') @@ -97,7 +97,7 @@ def test_example_seaexplorer_interp_nrt(var): # Open test data file test_data_raw = xr.open_dataset( library_dir / - 'tests/results/example-seaexplorer-raw/L0-timeseries/dfo-bb046-20200908.nc') + 'tests/expected/example-seaexplorer-raw/L0-timeseries/dfo-bb046-20200908.nc') @pytest.mark.parametrize("var", variables) def test_example_seaexplorer_raw(var): @@ -148,135 +148,3 @@ def test_example_seaexplorer_interp_raw(var): assert np.allclose( np.array(dt0, dtype='float64'), np.array(dt1, dtype='float64')) - - -# Create an L0 timeseries from slocum data and test that the resulting netcdf is -# identical to the test data -cacdir = str(example_dir / 'example-slocum/cac/') + '/' -sensorlist = str(example_dir / 'example-slocum/dfo-rosie713_sensors.txt') -binarydir = str(example_dir / 'example-slocum/realtime_raw/') + '/' -rawdir_slocum = str(example_dir / 'example-slocum/realtime_rawnc/') + '/' -deploymentyaml_slocum = str(example_dir / 'example-slocum/deploymentRealtime.yml') -l1tsdir = str(example_dir / 'example-slocum/L0-timeseries-test/') + '/' -scisuffix = 'tbd' -glidersuffix = 'sbd' -do_direct = True - -if do_direct: - # turn *.sbd and *.tbd into timeseries netcdf files - outname_slocum = slocum.binary_to_timeseries( - binarydir, cacdir, l1tsdir, deploymentyaml_slocum, search='*.[s|t]bd', - profile_filt_time=20, profile_min_time=20) -else: - slocum.binary_to_rawnc( - binarydir, rawdir_slocum, cacdir, sensorlist, deploymentyaml_slocum, - incremental=False, scisuffix=scisuffix, glidersuffix=glidersuffix) - - slocum.merge_rawnc(rawdir_slocum, rawdir_slocum, deploymentyaml_slocum, - scisuffix=scisuffix, glidersuffix=glidersuffix) - outname_slocum = slocum.raw_to_timeseries( - rawdir_slocum, l1tsdir, deploymentyaml_slocum, - profile_filt_time=100, profile_min_time=300) - -output_slocum = xr.open_dataset(outname_slocum) -# Open test data file -test_data_slocum = xr.open_dataset( - library_dir / - 'tests/results/example-slocum/L0-timeseries/dfo-rosie713-20190615.nc') -variables_slocum = list(output_slocum.variables) - - -def test_variables_slocum(): - test_variables = list(test_data_slocum.variables) - test_variables.sort() - variables_slocum.sort() - assert variables_slocum == test_variables - - -@pytest.mark.parametrize("var", variables_slocum) -def test_example_slocum(var): - # Test that variables and coordinates match - assert output_slocum[var].attrs == test_data_slocum[var].attrs - if var not in ['time']: - np.testing.assert_allclose(output_slocum[var].values, - test_data_slocum[var].values, rtol=1e-6) - else: - dt0 = output_slocum[var].values - np.datetime64('2000-01-01') - dt1 = test_data_slocum[var].values - np.datetime64('2000-01-01') - assert np.allclose( - np.array(dt0, dtype='float64'), - np.array(dt1, dtype='float64')) - - -def test_example_slocum_metadata(): - # Test that attributes match. Have to remove creation and issue - # dates first - output_slocum.attrs.pop('date_created') - output_slocum.attrs.pop('date_issued') - test_data_slocum.attrs.pop('date_created') - test_data_slocum.attrs.pop('date_issued') - assert output_slocum.attrs == test_data_slocum.attrs - - -# Create an L0 timeseries from slocum data and test that the resulting netcdf is -# identical to the test data -cacdir = str(example_dir / 'example-slocum-littleendian/cac/') + '/' -sensorlist = str( - example_dir / 'example-slocum-littleendian/dfo-maria997_sensors.txt') -binarydir = str( - example_dir / 'example-slocum-littleendian/realtime_raw/') + '/' -rawdir_slocum = str( - example_dir / 'example-slocum-littleendian/realtime_rawnc/') + '/' -deploymentyaml_slocum = str( - example_dir / 'example-slocum-littleendian/deployment.yml') -l1tsdir = str( - example_dir / 'example-slocum-littleendian/L0-timeseries-test/') + '/' -scisuffix = 'tbd' -glidersuffix = 'sbd' - -slocum.binary_to_rawnc( - binarydir, rawdir_slocum, cacdir, sensorlist, deploymentyaml_slocum, - incremental=True, scisuffix=scisuffix, glidersuffix=glidersuffix) -slocum.merge_rawnc(rawdir_slocum, rawdir_slocum, deploymentyaml_slocum, - scisuffix=scisuffix, glidersuffix=glidersuffix) -outname_slocum_le = slocum.raw_to_timeseries( - rawdir_slocum, l1tsdir, deploymentyaml_slocum, - profile_filt_time=400, profile_min_time=100) -output_slocum_le = xr.open_dataset(outname_slocum_le) -# Open test data file -test_data_slocum_le = xr.open_dataset( - library_dir / - ('tests/results/example-slocum-littleendian/' + - 'L0-timeseries/dfo-maria997-20220614.nc')) -variables_slocum_le = list(output_slocum.variables) - - -def test_variables_slocum_littleendian(): - test_variables = list(test_data_slocum_le.variables) - test_variables.sort() - variables_slocum.sort() - assert variables_slocum == test_variables - - -@pytest.mark.parametrize("var", variables_slocum) -def test_example_slocum_littleendian(var): - # Test that variables and coordinates match - assert output_slocum_le[var].attrs == test_data_slocum_le[var].attrs - if var not in ['time']: - np.testing.assert_allclose(output_slocum_le[var].values, - test_data_slocum_le[var].values, rtol=1e-6) - else: - dt0 = output_slocum_le[var].values - np.datetime64('2000-01-01') - dt1 = test_data_slocum_le[var].values - np.datetime64('2000-01-01') - assert np.allclose( - np.array(dt0, dtype='float64'), - np.array(dt1, dtype='float64')) - - -def test_example_slocum_littleendian_metadata(): - # Test that attributes match. Have to remove creation and issue dates first - output_slocum_le.attrs.pop('date_created') - output_slocum_le.attrs.pop('date_issued') - test_data_slocum_le.attrs.pop('date_created') - test_data_slocum_le.attrs.pop('date_issued') - assert output_slocum_le.attrs == test_data_slocum_le.attrs diff --git a/tests/test_seaexplorer.py b/tests/test_seaexplorer.py index 4519a71..4eb3d94 100644 --- a/tests/test_seaexplorer.py +++ b/tests/test_seaexplorer.py @@ -58,12 +58,12 @@ def test_merge_rawnc(): result_default = seaexplorer.merge_parquet( 'tests/data/realtime_rawnc/', 'tests/data/realtime_rawnc/', - example_dir / 'example-seaexplorer/deploymentRealtime.yml') + str(example_dir / 'example-seaexplorer/deploymentRealtime.yml')) result_sub = seaexplorer.merge_parquet( 'tests/data/realtime_rawnc/', 'tests/data/realtime_rawnc/', - example_dir / 'example-seaexplorer/deploymentRealtime.yml', + str(example_dir / 'example-seaexplorer/deploymentRealtime.yml'), kind='sub') assert result_default is False assert result_sub is True @@ -91,11 +91,11 @@ def test_raw_to_timeseries(): with pytest.raises(FileNotFoundError) as missing_file_exc: result_default = seaexplorer.raw_to_timeseries('tests/data/realtime_rawnc/', 'tests/data/l0-profiles/', - example_dir / 'example-seaexplorer/deploymentRealtime.yml', + str(example_dir / 'example-seaexplorer/deploymentRealtime.yml'), ) result_sub = seaexplorer.raw_to_timeseries('tests/data/realtime_rawnc/', 'tests/data/l0-profiles/', - example_dir / 'example-seaexplorer/deploymentRealtime.yml', + str(example_dir / 'example-seaexplorer/deploymentRealtime.yml'), kind='sub') assert 'No such file or directory' in str(missing_file_exc) assert result_sub == 'tests/data/l0-profiles/dfo-eva035-20190718.nc' @@ -114,12 +114,12 @@ def test_missing_bad_timebase(): with pytest.raises(ValueError) as bad_timebase_exc: result_bad_timebase = seaexplorer.raw_to_timeseries('tests/data/realtime_rawnc/', 'tests/data/l0-profiles/', - example_dir / 'example-seaexplorer/bad_timebase.yml', + str(example_dir / 'example-seaexplorer/bad_timebase.yml'), kind='sub') with pytest.raises(ValueError) as no_timebase_exc: result_no_timebase = seaexplorer.raw_to_timeseries('tests/data/realtime_rawnc/', 'tests/data/l0-profiles/', - example_dir / 'example-seaexplorer/no_timebase.yml', + str(example_dir / 'example-seaexplorer/no_timebase.yml'), kind='sub') assert "sensor not found in pld1 columns" in str(bad_timebase_exc) assert "Must specify timebase" in str(no_timebase_exc) diff --git a/tests/test_slocum.py b/tests/test_slocum.py new file mode 100644 index 0000000..bc6beaa --- /dev/null +++ b/tests/test_slocum.py @@ -0,0 +1,162 @@ +import xarray as xr + +from compliance_checker.runner import ComplianceChecker, CheckSuite +import json +from pathlib import Path +import pytest +import numpy as np +import yaml + +import pyglider.ncprocess as ncprocess +import pyglider.seaexplorer as seaexplorer +import pyglider.slocum as slocum + + + +library_dir = Path(__file__).parent.parent.absolute() +example_dir = library_dir / 'tests/example-data/' + +# Create an L0 timeseries from slocum data and test that the resulting netcdf is +# identical to the test data +cacdir = example_dir / 'example-slocum/cac/' +sensorlist = str(example_dir / 'example-slocum/dfo-rosie713_sensors.txt') +binarydir = str(example_dir / 'example-slocum/realtime_raw/') + '/' +rawdir_slocum = str(example_dir / 'example-slocum/realtime_rawnc/') + '/' +deploymentyaml_slocum = str(example_dir / 'example-slocum/deploymentRealtime.yml') +tsdir = str(example_dir / 'example-slocum/L0-timeseries/') + '/' +scisuffix = 'tbd' +glidersuffix = 'sbd' +profiledir = str(example_dir / 'example-slocum/L0-profiles/') +do_direct = True + +# This needs to get run every time the tests are run, so do at top level: + +# turn *.sbd and *.tbd into timeseries netcdf files +outname_slocum = slocum.binary_to_timeseries(binarydir, cacdir, tsdir, deploymentyaml_slocum, + search='*.[s|t]bd', profile_filt_time=20, + profile_min_time=20) +# make profiles... +ncprocess.extract_timeseries_profiles(outname_slocum, profiledir, deploymentyaml_slocum, + force=True) + +output_slocum = xr.open_dataset(outname_slocum) +# Open test data file +test_data_slocum = xr.open_dataset( + library_dir / + 'tests/expected/example-slocum/L0-timeseries/dfo-rosie713-20190615.nc') +variables_slocum = list(output_slocum.variables) + + +def test_variables_slocum(): + test_variables = list(test_data_slocum.variables) + test_variables.sort() + variables_slocum.sort() + assert variables_slocum == test_variables + + +@pytest.mark.parametrize("var", variables_slocum) +def test_example_slocum(var): + # Test that variables and coordinates match + assert output_slocum[var].attrs == test_data_slocum[var].attrs + if var not in ['time']: + np.testing.assert_allclose(output_slocum[var].values, + test_data_slocum[var].values, rtol=1e-6) + else: + dt0 = output_slocum[var].values - np.datetime64('2000-01-01') + dt1 = test_data_slocum[var].values - np.datetime64('2000-01-01') + assert np.allclose( + np.array(dt0, dtype='float64'), + np.array(dt1, dtype='float64')) + + +def test_example_slocum_metadata(): + # Test that attributes match. Have to remove creation and issue + # dates first + output_slocum.attrs.pop('date_created') + output_slocum.attrs.pop('date_issued') + test_data_slocum.attrs.pop('date_created') + test_data_slocum.attrs.pop('date_issued') + assert output_slocum.attrs == test_data_slocum.attrs + + +# test the profiles with compliance_checker... + +def test_profiles_compliant(): + # Load all available checker classes + check_suite = CheckSuite() + check_suite.load_all_available_checkers() + # Run cf and adcc checks + path = profiledir + '/dfo-rosie713-20190620T1313.nc' + checker_names = ['gliderdac', 'cf:1.8'] + verbose = 0 + criteria = 'normal' + output_filename = example_dir / 'report.json' + output_format = 'json' + """ + Inputs to ComplianceChecker.run_checker + + path Dataset location (url or file) + checker_names List of string names to run, should match keys of checkers dict (empty list means run all) + verbose Verbosity of the output (0, 1, 2) + criteria Determines failure (lenient, normal, strict) + output_filename Path to the file for output + output_format Format of the output + + @returns If the tests failed (based on the criteria) + """ + return_value, errors = ComplianceChecker.run_checker(path, + checker_names, + verbose, + criteria, + output_filename=output_filename, + output_format=output_format) + # Open the JSON output and get the compliance scores + with open(output_filename, 'r') as fp: + cc_data = json.load(fp) + test = cc_data['gliderdac'] + assert test['high_count'] == 0 + assert test['medium_count'] == 0 + assert test['low_count'] == 0 + test = cc_data['cf:1.8'] + assert test['high_count'] == 0 + assert test['medium_count'] == 0 + assert test['low_count'] == 0 + + +def test_timeseries_compliant(): + # Load all available checker classes + check_suite = CheckSuite() + check_suite.load_all_available_checkers() + # Run cf and adcc checks + path = tsdir + '/dfo-rosie713-20190615.nc' + checker_names = ['cf:1.8'] + verbose = 0 + criteria = 'normal' + output_filename = example_dir / 'report.json' + output_format = 'json' + """ + Inputs to ComplianceChecker.run_checker + + path Dataset location (url or file) + checker_names List of string names to run, should match keys of checkers dict (empty list means run all) + verbose Verbosity of the output (0, 1, 2) + criteria Determines failure (lenient, normal, strict) + output_filename Path to the file for output + output_format Format of the output + + @returns If the tests failed (based on the criteria) + """ + return_value, errors = ComplianceChecker.run_checker(path, + checker_names, + verbose, + criteria, + output_filename=output_filename, + output_format=output_format) + # Open the JSON output and get the compliance scores + with open(output_filename, 'r') as fp: + cc_data = json.load(fp) + test = cc_data['cf:1.8'] + assert test['high_count'] == 0 + # somehow the checker is confused by our trajectory variables. + assert test['medium_count'] == 1 + assert test['low_count'] == 0 diff --git a/tests/test_utils.py b/tests/test_utils.py index 8fc0c86..69439e4 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -10,7 +10,7 @@ library_dir = Path(__file__).parent.parent.absolute() example_dir = library_dir / 'tests/example-data/' -test_data = xr.open_dataset(library_dir / 'tests/results/example-seaexplorer/L0-timeseries/dfo-eva035-20190718.nc') +test_data = xr.open_dataset(library_dir / 'tests/expected/example-seaexplorer/L0-timeseries/dfo-eva035-20190718.nc') deploymentyaml = example_dir / 'example-seaexplorer-legato-flntu-arod-ad2cp/deploymentRealtime.yml' with open(deploymentyaml) as fin: deployment = yaml.safe_load(fin)