Merge development to main branch

.github/workflows/deploy-docs.yaml

@@ -0,0 +1,40 @@
+name: Build documentation
+
+on:
+  push:
+    branches:
+      - main
+      - development
+
+jobs:
+  deploy_docs:
+    environment: 
+      name: ${{github.ref_name}}
+      url: ${{ github.ref == 'main' && 'https://cioos-siooc.github.io/ocean-data-parser/main/' || 'https://cioos-siooc.github.io/ocean-data-parser/dev/' }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV 
+      - uses: actions/cache@v3
+        with:
+          key: mkdocs-material-${{ env.cache_id }}
+          path: .cache
+          restore-keys: |
+            mkdocs-material-
+      - run: |
+          python -m pip install --upgrade pip
+          pip install ".[dev]"
+          pip install -r docs-requirements.txt
+      - run: |
+          git config --global user.name "${GITHUB_ACTOR}"
+          git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com"
+      - run: |
+          if [[ "${{github.ref_name}}" == "main" ]]; then
+              mike deploy main --push --force
+          else
+              mike deploy dev --push --force
+          fi
+          mike set-default dev --push

.github/workflows/test-docs.yaml

@@ -0,0 +1,32 @@
+name: Test docs generation 
+
+on:
+  pull_request:
+    branches:
+      - main
+      - development
+
+jobs:
+  test_docs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV 
+      - uses: actions/cache@v3
+        with:
+          key: mkdocs-material-${{ env.cache_id }}
+          path: .cache
+          restore-keys: |
+            mkdocs-material-
+      - run: |
+        python -m pip install --upgrade pip
+          pip install -r docs-requirements.txt
+          pip install ".[dev]"
+      - run: |
+          git config --global user.name "${GITHUB_ACTOR}"
+          git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com"
+      - run: |
+          mike deploy test

.github/workflows/python-package.yml → .github/workflows/test-pytest.yaml

@@ -1,35 +1,45 @@
-name: Python package
+name: Test linter and full pytest
 
 on:
   pull_request:
     branches:
       - main
       - development
+    paths:
+      - setup.py
+      - MANIFEST.in
+      - ocean_data_parser/**
+      - tests/**
+
+  push:
+    branches:
+      - main
+      - development
+    paths:
+      - setup.py
+      - MANIFEST.in
+      - ocean_data_parser/**
+      - tests/**
 
 jobs:
-  build:
+  testing:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10']
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v3
+      - uses: actions/setup-python@v4
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: '3.10'
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install wheel
-          pip install .[dev]
+          pip install ".[dev]"
       - name: Lint with flake8
         run: |
           # stop the build if there are Python syntax errors or undefined names
           flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
       - name: Test with pytest parsers
-        run: pytest
+        run: pytest -W error::UserWarning
       - name: Run benchmark
-        run: pytest tests/run_benchmark.py  --benchmark-json output.json
+        run: pytest tests/run_benchmark.py  --benchmark-json output.json

.gitignore

@@ -19,4 +19,29 @@ dist/
 pdc-amundsen-conversion.log
 
 tests/parsers_test_files/amundsen/12713/**/*
-output/
+output/
+docs/read/**/*-hook.md
+*.log
+file_registry.csv
+temp/**
+tests/test_file_registry_temp.csv
+
+temp/**
+*.log
+.env
+docs/notebooks/*.ipynb
+docs/user_guide/vocabularies/**
+
+# test files generated by notebooks to ignore 
+notebooks/CTD_HUD2018030_017_01_UP.ODF
+notebooks/NAV_20130726.int
+notebooks/1801_001.int
+notebooks/nmeadata-*.txt
+notebooks/MI18MHDR.btl
+notebooks/56001001.p2022
+
+#ignore docs that are automatically generated
+docs/user_guide/vocabularies/*.md
+docs/user_guide/parsers/parser-list.md
+
+site/**

.releaserc.yaml

@@ -1,4 +1,4 @@
-repositoryUrl: https://github.com/HakaiInstitute/ocean-data-parser.git
+repositoryUrl: https://github.com/cioos-siooc/ocean-data-parser.git
 branches:
   - main
   - name: development

MANIFEST.in

@@ -1,4 +1,5 @@
-recursive-include ocean_data_parser/read/vocabularies *
-include ocean_data_parser/read/dfo/odf_source/config.json 
-recursive-include ocean_data_parser/read/dfo/odf_source/references *
-recursive-include ocean_data_parser/read/dfo/odf_source/references/geographical_areas *
+recursive-include ocean_data_parser/vocabularies *
+include ocean_data_parser/parsers/dfo/odf_source/*config.json 
+recursive-include ocean_data_parser/parsers/dfo/odf_source/references *
+recursive-include ocean_data_parser/parsers/dfo/odf_source/references/geographical_areas *
+include ocean_data_parser/batch/default-batch-config.yaml

README.md

@@ -1,40 +1,70 @@
-# Ocean Data Parser
+<!-- NOTE: All sections are placeholders. Use the relevant ones-->
+
+![Logo](docs/images/logo_EN_FR-1024x208.png#gh-light-mode-only)
+![Logo](docs/images/cioos-national_EN_FR_W-01.png#gh-dark-mode-only)
+
+<!-- Make a favicon/logo using something like:
+
+* https://favicon.io/
+* https://www.shopify.com/tools/logo-maker/open-source-software
+* https://primitive.lol/ -->
+
+# ocean-data-parser
+
+<!-- You can get project-relevant badges from: [shields.io](https://shields.io/) -->
+
+[![Build documentation](https://github.com/cioos-siooc/ocean-data-parser/actions/workflows/deploy-docs.yaml/badge.svg)](https://github.com/cioos-siooc/ocean-data-parser/actions/workflows/deploy-docs.yaml)
+
+
+`ocean-data-parser` - a Python package for parsing oceanographic proprietary data formats to [xarray Dataset](https://docs.xarray.dev/en/stable/). Documentation [here](https://cioos-siooc.github.io/ocean-data-parser/).
+
 ## Installation
-With miniconda, create a new environement:
-```console
-conda create --name ocean_parser 
-````
+Install the package with the following command, ideally within a virtual environment:
 
-Get in the environment:
 ```console
-conda activate ocean_parser
+pip install git+https://github.com/cioos-siooc/ocean-data-parser.git
 ```
 
-Install the present package:
+### How to
+#### odpy cli
+Once installed, the package is usable via the command line interface:
 ```console
-pip install git+https://github.com/HakaiInstitute/ocean-data-parser.git
+odpy --help
 ```
 
-
-For development, clone locally the package :
-```console
-git clone git+https://github.com/HakaiInstitute/ocean-data-parser.git
+To batch convert a series of files to NetCDF:
 ```
-and install the package:
-```console
-cd ocean-data-parser
-pip install -e .
-```
-For development purposes, it is recommended to install the development requirements:
-```console
-pip install -e ".[dev]"
+odpy convert -i '**/*.cnv' -p 'seabird.cnv'
 ```
+#### Format auto-detection
+Load a compatible file with the automated parser detection method:
 
-## How to
-To parse seabird file:
 ```python
-from ocean_data_parser.read import searbird
+import ocean_data_parser.parsers
 
-PATH_TO_SEABIRD_CNV = "PATH_TO_SEABIRD_CNV"
-ds = seabird.cnv(PATH_TO_SEABIRD_CNV)
+# Load a file to an xarray object
+ds = ocean_data_parser.parsers.file('Path to file')
+
+# Save to netcdf
+ds.to_netcdf('save-path.nc')
 ```
+!!!warning
+    The parser detection method relies on the file extension and the first few lines present within the given file.
+
+Or specify the specific parser to use for this file format:
+``` python
+from ocean_data_parser.parsers import seabird
+
+# Load a seabird cnv file as an xarray dataset
+ds = seabird.cnv('Path to seabird cnv file')
+
+# Save to netcdf
+ds.to_netcdf('save-path.nc')
+```
+The `ocean-data-parser` can then be used within either a Python package, script or jupyter notebook. See the [documentation Notebook section](https://cioos-siooc.github.io/ocean-data-parser) for examples of how to use the package within a jupyter notebook.
+
+## Contributions
+
+All contributions are welcome! 
+
+Please create a new [discussion](https://github.com/cioos-siooc/ocean-data-parser/discussions) or [issue](https://github.com/cioos-siooc/ocean-data-parser/issues) within the GitHub repository for any questions, ideas and suggestions. 

docs-requirements.txt

@@ -0,0 +1,12 @@
+mkdocs
+mkdocs-click
+mkdocs-gen-files
+mkdocs-jupyter
+mkdocs-material
+mkdocs-simple-hooks
+mkdocstrings[python]
+mkdocs-section-index
+mike
+tabulate
+pandas
+material-plausible-plugin

docs/get_started/contributions.md

@@ -0,0 +1,3 @@
+All contributions are welcome! 
+
+Please create a new [discussion](https://github.com/cioos-siooc/ocean-data-parser/discussions) or [issue](https://github.com/cioos-siooc/ocean-data-parser/issues) within the github repository for any questions, ideas and suggestions. 

docs/get_started/development.md

@@ -0,0 +1,45 @@
+
+### Installation
+Clone the project locally
+
+```shell
+  git clone git+https://github.com/cioos-siooc/ocean-data-parser.git
+```
+
+Go to the project directory
+
+```shell
+  cd ocean-data-parser
+```
+
+Install dependencies
+
+```shell
+  pip install -e ".[dev]"
+```
+
+### Testing
+The package use pytest to run a series of tests in order to help the development of the different aspect of the package. Within a developping environment, to run the different tests, run the pytest commmand through your terminal within the base directory of the repository. Pytest can also be integrated with different IDE and is run on any pushes and PR to the `main` and `development` branches.
+
+### Parsers Tests
+The package contains a number of different parsers compatible with different standard file formats. Each parser is tested on a series of test files made available within the [test file directory](https://github.com/cioos-siooc/ocean-data-parser/blob/main/tests/parsers_test_files) The different tests executed on each individual parsers can be subdivided in 3 main categories:
+1. Parse test file to an xarray dataset
+2. Parse test file to an xarray dataset and save to a NetCDF4 file.
+3. Parse test file to an xarray dataset and compare to a reference file ('*_reference.nc) if made available. Any differences are flagged
+4. *(in development)* Assess parsed xarray object compliance with the different convention by using the ioos-compliance checker, resulting objects should be to a minimum compliante to ACDD 1.3 and CF 1.6. Other conventions can be added by adding them to the xarray object global attribute `Convention`.
+
+
+### Documentation
+To run a local instance of the documentation webpage. Install the dependancies:
+```console
+pip install -r docs-requirements.txt
+```
+
+And run the command:
+
+```shell
+mike serve
+```
+
+Any documentation changes to the main and development branches will automatically update respectively the main and dev document pages.
+