diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index c6cf394..093f833 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -11,6 +11,7 @@ Please tag them using `#`, e.g. `Fixed #42`. - [ ] New code includes dedicated tests. - [ ] New code has been linted. - [ ] New code follows the project's style. -- [ ] New code is compatible with a 3-Clause BSD license. +- [ ] New code is compatible with the 3-Clause BSD license. - [ ] CHANGELOG has been updated. - [ ] AUTHORS has been updated. +- [ ] Copyright years in module docstrings have been updated. diff --git a/.github/workflows/CI_changelog.yml b/.github/workflows/CI_changelog.yml index 0a64155..7ff4351 100644 --- a/.github/workflows/CI_changelog.yml +++ b/.github/workflows/CI_changelog.yml @@ -10,7 +10,7 @@ name: CI_changelog on: pull_request: - branches: [ master, develop ] + branches: [ master, develop, develop_vof ] jobs: changelog: diff --git a/.github/workflows/CI_check_version.yml b/.github/workflows/CI_check_version.yml new file mode 100644 index 0000000..e993f8f --- /dev/null +++ b/.github/workflows/CI_check_version.yml @@ -0,0 +1,55 @@ +# This workflow will check that the code version was increased. +# +# It is triggered for any PR to the master branch. +# +# Warning: the code version must *absolutely* follow a semantic approach, i.e.: +# 10.0.1, 10.0.2.dev0 is valid +# 10.0, 10, is NOT valid ! +# +# Copyright (c) 2022 fpavogt; frederic.vogt@meteoswiss.ch + +name: CI_check_version + +on: + pull_request: + branches: [ master ] + +jobs: + version: + + runs-on: ubuntu-latest + steps: + + - name: Checkout current repository + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.10' + + - name: Install dependancies + run: | + python -m pip install --upgrade pip + pip install setuptools + shell: bash + + - name: wget/cp the BASE and HEAD version files + # Here, I could get both the BASE and HEAD files with wget, but cp-ing the HEAD is safer, + # since we have cloned the repo anyway. + run: | + export VERSION_LOC=src/ampycloud/version.py + echo $GITHUB_HEAD_REF + echo $GITHUB_BASE_REF + wget -O base_version.tmp https://raw.githubusercontent.com/$GITHUB_REPOSITORY/$GITHUB_BASE_REF/$VERSION_LOC + cp ./$VERSION_LOC head_version.tmp + shell: bash + + - name: Check if the version was increased + run: | + HEAD_VERSION="$(grep VERSION head_version.tmp | grep -o '[[:alnum:]]*\.[[:alnum:]]*\.[[:alnum:]]*\.\?[[:alnum:]]*')" + echo "HEAD_VERSION:" $HEAD_VERSION + BASE_VERSION="$(grep VERSION base_version.tmp | grep -o '[[:alnum:]]*\.[[:alnum:]]*\.[[:alnum:]]*\.\?[[:alnum:]]*')" + echo "BASE_VERSION:" $BASE_VERSION + python ./.github/workflows/check_version.py $HEAD_VERSION $BASE_VERSION + shell: bash diff --git a/.github/workflows/CI_docs_build_and_check.yml b/.github/workflows/CI_docs_build_and_check.yml index b8d2acc..56deff8 100644 --- a/.github/workflows/CI_docs_build_and_check.yml +++ b/.github/workflows/CI_docs_build_and_check.yml @@ -7,7 +7,7 @@ on: push: branches: [ master ] pull_request: - branches: [ master, develop ] + branches: [ master, develop, develop_vof ] jobs: docs: diff --git a/.github/workflows/CI_pylinter.yml b/.github/workflows/CI_pylinter.yml index 0e46303..479f760 100644 --- a/.github/workflows/CI_pylinter.yml +++ b/.github/workflows/CI_pylinter.yml @@ -9,7 +9,7 @@ on: #push: # branches: [ master ] pull_request: - branches: [ master, develop ] + branches: [ master, develop, develop_vof ] jobs: pylinter: diff --git a/.github/workflows/CI_pypi.yml b/.github/workflows/CI_pypi.yml new file mode 100644 index 0000000..449ca4e --- /dev/null +++ b/.github/workflows/CI_pypi.yml @@ -0,0 +1,57 @@ +# This workflow will push the code onto pypi. +# It assumes that TESTPYPI_API_TOKEN and PYPI_API_TOKEN secrets from GITHUB have been defined +# at the repo or organization levels to upload the package via API authentification. +# +# It will trigger the moment a new release or pre-release is being published. +# +# Copyright (c) 2022 fpavogt; frederic.vogt@meteoswiss.ch + +name: CI_pypi + +on: + release: + types: [published] + +jobs: + pypi: + + runs-on: ubuntu-latest + steps: + + - name: Checkout current repository + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.10' + + - name: Install dependancies + run: | + python -m pip install --upgrade pip + pip install setuptools + pip install wheel + pip install twine + shell: bash + + - name: Build the wheels + run: | + python setup.py sdist bdist_wheel + shell: bash + + - name: Deploy to testpypi + # Let's make use of Github secrets to avoid spelling out secret stuff + env: + TESTPYPI_TOKEN: ${{ secrets.TESTPYPI_API_TOKEN }} + # We first go to testpypi to make sure nothing blows up. + run: | + twine upload -r testpypi dist/* --verbose --skip-existing -u __token__ -p "$TESTPYPI_TOKEN" + shell: bash + + - name: Deploy to pypi + # Let's make use of Github secrets to avoid spelling out secret stuff + env: + PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} + run: | + twine upload dist/* --verbose --skip-existing -u __token__ -p "$PYPI_TOKEN" + shell: bash diff --git a/.github/workflows/CI_pytest.yml b/.github/workflows/CI_pytest.yml index 82808fc..3ec2bd9 100644 --- a/.github/workflows/CI_pytest.yml +++ b/.github/workflows/CI_pytest.yml @@ -12,7 +12,7 @@ on: #push: # branches: [ master ] pull_request: - branches: [ master, develop ] + branches: [ master, develop, develop_vof ] jobs: pytest: diff --git a/.github/workflows/check_version.py b/.github/workflows/check_version.py new file mode 100644 index 0000000..6901f93 --- /dev/null +++ b/.github/workflows/check_version.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +''' +Copyright (c) 2022 MeteoSwiss, contributors listed in AUTHORS. + +Distributed under the terms of the 3-Clause BSD License. + +SPDX-License-Identifier: BSD-3-Clause + +This script can be used together with a Github Action to check whether a code version has been +properly incremented. + +Created January 2022; F.P.A. Vogt; frederic.vogt@meteoswiss.ch +''' + +import argparse +from pkg_resources import parse_version + +def main(): + ''' The main function. ''' + + # Use argparse to allow feeding parameters to this script + parser = argparse.ArgumentParser(description='''Compare the versions between the head and base + branches of the PR. Fails is Head <= Base.''', + epilog='Feedback, questions, comments: \ + frederic.vogt@meteoswiss.ch \n', + formatter_class=argparse.RawTextHelpFormatter) + + parser.add_argument('head', action='store', metavar='version', + help='Head version.') + + parser.add_argument('base', action='store', metavar='version', + help='Base version.') + + # What did the user type in ? + args = parser.parse_args() + + # Print the versions fed by the user, for monitoring + print("Head:", args.head) + print("Base:", args.base) + + if parse_version(args.head) > parse_version(args.base): + print("Version was increased. Well done.") + return True + + raise Exception('Ouch ! Version was not increased ?!') + +if __name__ == '__main__': + + main() diff --git a/.github/workflows/docs_check.py b/.github/workflows/docs_check.py index db5aa69..38428da 100644 --- a/.github/workflows/docs_check.py +++ b/.github/workflows/docs_check.py @@ -2,9 +2,9 @@ ''' Copyright (c) 2020-2021 MeteoSwiss, contributors listed in AUTHORS. -Distributed under the terms of the GNU General Public License v3.0 or later. +Distributed under the terms of the 3-Clause BSD License. -SPDX-License-Identifier: GPL-3.0-or-later +SPDX-License-Identifier: BSD-3-Clause This script can be used together with a Github Action to build the docs, and check for errors and warnings. This script assumes that it is being executed from within the "docs" folder. diff --git a/.github/workflows/pylinter.py b/.github/workflows/pylinter.py index f948590..168bcce 100644 --- a/.github/workflows/pylinter.py +++ b/.github/workflows/pylinter.py @@ -2,9 +2,9 @@ ''' Copyright (c) 2020-2021 MeteoSwiss, created by F.P.A. Vogt; frederic.vogt@meteoswiss.ch -Distributed under the terms of the GNU General Public License v3.0 or later. +Distributed under the terms of the 3-Clause BSD License. -SPDX-License-Identifier: GPL-3.0-or-later +SPDX-License-Identifier: BSD-3-Clause This script can be used together with a Github Action to run pylint on all the .py files in a repository. Command line arguments can be used to search for a specific subset of errors (if any are @@ -12,7 +12,8 @@ will print all the errors found, but will not raise any Exception). If a score is specified, the script will raise an Exception if it is not met. -Created May 2020; F.P.A. Vogt; frederic.vogt@meteoswiss.ch +Created May 2020; fpavogt; frederic.vogt@meteoswiss.ch +Adapted Jan 2022; fpavogt; frederic.vogt@meteoswiss.ch ''' import argparse diff --git a/.pylintrc b/.pylintrc index abd50e1..6c1ccdb 100644 --- a/.pylintrc +++ b/.pylintrc @@ -10,4 +10,4 @@ [BASIC] # Good variable names which should always be accepted, separated by a comma. -good-names=a, ax, b, c, fn, i, j, k, _, w, x, y, z +good-names=a, ax, b, c, dt, fn, i, j, k, _, w, x, y, z diff --git a/CHANGELOG b/CHANGELOG index 588a0da..2fc2ddb 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -4,7 +4,7 @@ The format is inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [v0.3.0] ### Added: ### Fixed: ### Changed: @@ -12,6 +12,23 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ### Removed: ### Security: +## [v0.2.0.dev] +### Added: + - [fpavogt, 2022-01-21] New CI Actions to automate the pypi releases. + - [fpavogt, 2022-01-17] Added new utils.utils.temp_seed() function to set random seed temporarily only. + - [fpavogt, 2022-01-13] New icao module, for easy/clean access to the significant_cloud() function. + - [fpavogt, 2022-01-13] Doc cleanup/improvement, and use of :py:func: (etc...) function to ease user navigation. +### Fixed: + - [fpavogt, 2022-01-17] Fix #49 - mock cloud layers now have proper type values. + - [fpavogt, 2022-01-10] Fix issues #40 and #41. +### Changed: + - [fpavogt, 2022-01-14] Fix #47. + - [fpavogt, 2022-01-10] Update copyright years. +### Deprecated: +### Removed: + - [fpavogt, 2022-01-14] ampycloud.core.synop(), and all synop references. +### Security: + ## [v0.1.0] ### Added: - [fpavogt, 2021-12-21] Add tests to check real data for scientific stability (fixes #33). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8caa18a..c2c0c87 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,9 +2,9 @@ If you: -* have a **question** about ampycloud: [jump here.](https://github.com/MeteoSwiss/ampycloud/discussions) -* want to **report a bug** with ampycloud: [jump here instead.](https://github.com/MeteoSwiss/ampycloud/issues) -* are considering to **contribute** to ampycloud (:heart_eyes: :tada:): read on ! +* :boom: want to **report a bug** with ampycloud: [jump here.](https://github.com/MeteoSwiss/ampycloud/issues) +* :question: have a **question** about ampycloud: [jump here instead.](https://github.com/MeteoSwiss/ampycloud/discussions) +* :construction_worker: want to **contribute** to ampycloud (:heart_eyes: :tada:): read on ! ## Table of contents @@ -14,14 +14,19 @@ If you: - [Essential things to know about ampycloud for dev work](#essential-things-to-know-about-ampycloud-for-dev-work) - [Branching model](#branching-model) - [Installing from source](#installing-from-source) - - [CI/CD](#ci/cd) + - [CI/CD](#cicd) - [Linting](#linting) - [Logging](#logging) - [Exceptions and Warnings](#exceptions-and-warnings) + - [Type hints](#type-hints-) - [Docstrings](#docstrings) - [Documentation](#documentation) - [Testing](#testing) - [Plotting](#plotting) + - [Release mechanisms](#release-mechanisms) +- [Less-Essential things to know about ampycloud for dev work](#less-essential-things-to-know-about-ampycloud-for-dev-work) + - [Updating the copyright years](#updating-the-copyright-years) + ## Code of conduct @@ -78,12 +83,12 @@ Automated CI/CD checks are triggered upon Pull Requests being issued towards the * code testing using `pytest` * check that the CHANGELOG was updated * check that the Sphinx docs compile -* automatic publication of the Sphinx docs - -Additional CI/CD tasks will be added eventually, including: - -* automatic release mechanism, incl. pypi upload +* automatic publication of the Sphinx docs (for a PR to `master` only) +* check that the code version was incremented (for PR to `master` only) +There is another Github action responsible for publishing the code onto pypi, that gets triggered +upon a new release or pre-release being published. See the ampycloud +[release mechanisms](#release-mechanisms) for details. ### Linting: @@ -152,15 +157,26 @@ from .errors import AmpycloudWarning warnings.warn('...', AmpycloudWarning) ``` +### Type hints ... + +... should be used in ampycloud. Here's an example: +``` +from typing import Union +from pathlib import Path + + +def set_prms(pth : Union[str, Path]) -> None: + """ ... """ +``` +See [the official Python documentation](https://docs.python.org/3/library/typing.html) for more info. + ### Docstrings -Google Style ! Please try to stick to the following example: +Google Style ! Please try to stick to the following example. Note the use of `:py:class:...` +([or `:py:func:...`, `py:mod:...` etc ...](https://www.sphinx-doc.org/en/master/usage/restructuredtext/domains.html#cross-referencing-python-objects)) with relative import to cleanly link to our own +functions, classes, etc ... : ``` """ A brief one-liner description in present tense, that finishes with a dot. -Use some -multi-line space for -more detailed info. - Args: x (float|int): variable x could be of 2 types ... note the use of `|` to say that ! - *float*: x could be a float @@ -169,10 +185,16 @@ Args: y (list[str]|str, optional): variable y info Returns: - bool: some grand Truth about the World. + :py:class:`.data.CeiloChunk`: more lorem ipsum ... Raises: - AmpycloudError: if blah and blah occurs. + :py:exc:`.errors.AmpycloudError`: if blah and blah occurs. + + +Use some +multi-line space for +more detailed info. Refer to the whole module as :py:mod:`ampycloud`. +Do all this **after** the Args, Returns, and Raises sections ! Example: If needed, you can specify chunks of code using code blocks:: @@ -184,8 +206,11 @@ Note: `Source `__ Please note the double _ _ after the link ! +Important: + Something you're hoping users will read ... + Caution: - Something to be careful about. + Something you're hoping users will read carefully ... """ ``` @@ -209,7 +234,8 @@ sh build_docs.sh This will create the `.html` pages of the compiled documentation under `./build`. In particular, this bash script will automatically update the help message from the high-level ampycloud entry point ``ampycloud_speed_test``, create the demo figure for the main page, compile and ingest all the -docstrings, etc ... +docstrings, etc ... . See the ampycloud[release mechanisms](#release-mechansims) for more info about +the automated publication of the documentation upon new releases. ### Testing @@ -236,12 +262,13 @@ this list of scientific tests *as short as possible, but as complete as necessar If one of these tests fail, it is possible to generate the corresponding diagnostic plot with the following fixture-argument: ``` -pytest --do_SCIPLOTS +pytest --DO_SCIPLOTS ``` ### Plotting -Because the devs care about the look of plots, ampycloud ships with specific matplotlib styles that will get used by default. For this to work as intended, any plotting function must be wrapped with +Because the devs care about the look of plots, ampycloud ships with specific matplotlib styles that +will get used by default. For this to work as intended, any plotting function must be wrapped with the `plots.utils.set_mplstyle` decorator, as follows: ``` # Import from Python @@ -263,3 +290,52 @@ def some_plot_function(...): With this decorator, all functions will automatically deploy the effects associated to the value of `dynamic.AMPYCLOUD_PRMS.MPL_STYLE` which can take one of the following values: `['base', 'latex', 'metsymb']`. + +### Release mechanisms + +When changes merged in the `develop` branch are stable and deemed *worthy*, follow these steps to +create a new release of ampycloud: + +1) Create a PR from `develop` to `master`. + + :warning: Merge only if all checks pass, **including the version check !** + + :white_check_mark: The [live ampycloud documentation](https://MeteoSwiss.github.io/ampycloud) + will be automatically updated (via the `CI_docs_build_and_publish.yml` Action) when the PR to + `master` is merged. + +2) Manually create a new release from Github. + + :warning: **Make sure to issue it from the `master` branch !** + + :warning: **Make sure to set the same version number as set in the code !** + + :white_check_mark: The code will be automatically pushed onto pypi (via the `CI_pypi.yml` Action) + when the release is *published*. This will work the same for pre-releases. + + :smirk: *Side note for (test)pypi: ampycloud will be published under the + [MeteoSwiss](https://pypi.org/user/MeteoSwiss/) account using an + [API token](https://pypi.org/help/#apitoken). The token is stored as an organization-level + Github secret.* + +3) That's it ! Wait a few seconds/minutes, and you'll see the updates: + + - on the [release page](https://github.com/MeteoSwiss/ampycloud/releases), + - in the [README](https://github.com/MeteoSwiss/ampycloud/blob/develop/README.md) tags, + - on [testpypi](https://test.pypi.org/project/ampycloud/) and [pypi](https://pypi.org/project/ampycloud/), + - on the [`gh-pages` branch](https://github.com/MeteoSwiss/ampycloud/tree/gh-pages), and + - in the [live documentation](https://MeteoSwiss.github.io/ampycloud). + +## Less-Essential things to know about ampycloud for dev work + +### Updating the copyright years +The ampycloud copyright years may need to be updated if the development goes on beyond 2022. If so, +the copyright years will need to be manually updated in the following locations: + +* `docs/source/substitutions.rst` (the copyright tag) +* `docs/source/conf.py` (the `copyright` variable) +* `docs/source/license.rst` +* `README.md` (the copyright section) + +The copyright years are also present in all the docstring modules. These can be updated individually +if/when a modification is made to a given module. diff --git a/MANIFEST.in b/MANIFEST.in index 32bc978..d397482 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,8 @@ -include LICENSE include AUTHORS +include CHANGELOG +include CODE_OF_CONDUCT.md +include CONTRIBUTING.md +include LICENSE include README.md include src/ampycloud/prms/*.yml include src/ampycloud/plots/mpl_styles/*.mplstyle diff --git a/README.md b/README.md index 7846556..f05437c 100644 --- a/README.md +++ b/README.md @@ -11,11 +11,11 @@ ampycloud refers to both this Python package and the algorithm at its core, desi characterize cloud layers (i.e. height and sky coverage fraction) using ceilometer measurements (i.e. automatic cloud base *hits* measurements), and derive the corresponding METAR-like message. -For the full documentation, installation instructions, etc ..., go to: https://MeteoSwiss.github.io/ampycloud +For the full documentation, installation instructions, etc ..., go to: https://meteoswiss.github.io/ampycloud ### License & Copyright -ampycloud is released under the terms of **the 3-clause BSD license**. The copyright belongs to MeteoSwiss. +ampycloud is released under the terms of **the 3-Clause BSD license**. The copyright (2021-2022) belongs to MeteoSwiss. ### Contributing to ampycloud diff --git a/docs/source/conf.py b/docs/source/conf.py index 91c6125..a21171b 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -28,7 +28,7 @@ # -- Project information ----------------------------------------------------- project = 'ampycloud' -copyright = '2021, MeteoSwiss' +copyright = '2021-2022, MeteoSwiss' author = 'Frédéric P.A. Vogt' version = vers @@ -48,7 +48,7 @@ # Specify the parameters of the autodoc, in order to autodoc_default_options = { # 'members': 'var1, var2', - 'member-order': 'bysource', + 'member-order': 'bysource', # List fcts and classes in the same order they are in the files # 'special-members': '__init__', # 'undoc-members': False, # 'exclude-members': '__weakref__' diff --git a/docs/source/doc_todo.rst b/docs/source/doc_todo.rst index 4fb7a1a..f1a68f2 100644 --- a/docs/source/doc_todo.rst +++ b/docs/source/doc_todo.rst @@ -1,12 +1,8 @@ -Documentation todos -=================== +TODO list +========= -.. note:: - - All the following items need to be dealt with at some point ... - - When this page is finally empty ... delete it! +What follows is a list of all the ``TODO`` left in the docs and the code. diff --git a/docs/source/index.rst b/docs/source/index.rst index c9f0836..3af572c 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -8,7 +8,7 @@ ampycloud |version| |stars| |watch| .. todo:: - Tags for the latest pypi release and associated DOI should be added when releasing the code + Tags for the latest DOI should be added when releasing the code for the first time. These should also be added to the :ref:`acknowledge:Acknowledging ampycloud` page. @@ -23,19 +23,23 @@ ampycloud |version| |stars| |watch| Welcome to the ampycloud documentation -------------------------------------- -**What:** ampycloud refers to both a Python package and the algorithm at its core, designed to -characterize cloud layers (i.e. height and sky coverage fraction) using ceilometer measurements -(i.e. automatic cloud base *hits*), and derive the corresponding METAR-like message. -A visual illustration of the algorithm is visible in :numref:`fig-demo`. +* **What:** ampycloud refers to both a **Python package** and the **algorithm** at its core, designed + to characterize cloud layers (i.e. height and sky coverage fraction) using ceilometer measurements + (i.e. automatic cloud base *hits*), and derive a corresponding METAR-like message. + A visual illustration of the algorithm is visible in :numref:`fig-demo`. -**Where:** ampycloud lives in `a dedicated repository `_ -under the `MeteoSwiss organization `_ on Github, where you can -submit all your `questions `_ and -`bug reports `_. See -:ref:`troubleshooting:Troubleshooting` for more details. +* **Where:** ampycloud lives in `a dedicated repository `_ + under the `MeteoSwiss organization `_ on Github, where you can + submit all your `questions `_ and + `bug reports `_. See + :ref:`troubleshooting:Troubleshooting` for more details. -**Who:** ampycloud is being developed at MeteoSwiss. See also the code's -:ref:`license & copyright ` information. +* **Who:** ampycloud is being developed at `MeteoSwiss `__. + See also the code's :ref:`license & copyright ` information. + +* **How:** a scientific article describing the ampycloud **algorithm** is currently in preparation. + This article will be complemented by these webpages that contain the official documentation of the + ampycloud **Python package**. Scope of ampycloud ------------------ diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 026064f..422558d 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -5,28 +5,31 @@ Installation ============ -.. todo:: - - Include a link to the pypi page in the very next sentence. - -ampycloud may, one day, be available on pypi, which should make its installation straightforward. -In a terminal, you would be able to type: +ampycloud is available on pypi, which should make its installation straightforward. +Typing the following in a terminal should take care of things: .. code-block:: python pip install ampycloud -And that would take care of things. ampycloud uses `semantic versioning `_. -The latest stable version is |version|. +ampycloud uses `semantic versioning `_. The latest stable version is |version|. -The most recent release of ampycloud is available for download/cloning from its -`Github repository `_, in which case -the install command becomes: +The different releases of ampycloud are also available for download from its +`Github repository `_. + +If you plan to do dev-work with ampycloud, you should instead clone the `develop` branch +`of the ampycloud Github repository `__, in +which case the install command becomes: .. code-block:: python cd ./where/you/stored/ampycloud/ - pip install -e . + pip install -e .[dev] + +.. note:: + If you plan to do dev-work with ampycloud, you ought to read the + `contributing guidelines `__ + first. Requirements ------------ @@ -34,14 +37,14 @@ ampycloud is compatible with the following python versions: .. literalinclude:: ../../setup.py :language: python - :lines: 39 + :lines: 44 Furthermore, ampycloud relies on a few external modules, which will be automatically installed by ``pip`` if required: .. literalinclude:: ../../setup.py :language: python - :lines: 41-46 + :lines: 45-52 Testing the installation & Speed benchmark ------------------------------------------ diff --git a/docs/source/license.rst b/docs/source/license.rst index 69d3edc..4dfdb93 100644 --- a/docs/source/license.rst +++ b/docs/source/license.rst @@ -7,7 +7,7 @@ License & Copyright ampycloud is distributed under the terms of the 3-Clause BSD License terms, that are as follows: -Copyright 2021 MeteoSwiss, contributors listed under AUTHORS +Copyright 2021-2022 MeteoSwiss, contributors listed under AUTHORS Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/docs/source/running.rst b/docs/source/running.rst index 1e55774..55cea6b 100644 --- a/docs/source/running.rst +++ b/docs/source/running.rst @@ -10,8 +10,8 @@ Using ampycloud Running the algorithm --------------------- -ampycloud.core.run -.................. +The :py:func:`ampycloud.core.run` function +.......................................... Applying the ampycloud algorithm to a given set of ceilometer cloud base hits is done via the following function, that is also directly accessible as ``ampycloud.run()``. @@ -19,24 +19,45 @@ following function, that is also directly accessible as ``ampycloud.run()``. .. autofunction:: ampycloud.core.run :noindex: +The :py:class:`ampycloud.data.CeiloChunk` class +............................................... -The no-plots-required shortcuts -............................... +The function :py:func:`ampycloud.core.run` returns a :py:class:`ampycloud.data.CeiloChunk` class +instance, which is at the core of ampycloud. This class is used to load and format the +user-supplied data, execute the different ampycloud algorithm steps, and format their outcomes. -The following two functions, also accessible as ``ampycloud.metar() and ampycloud.synop()``, -will directly provide interested users with the ampycloud-METAR/synop messages for a given dataset. +The properties of the slices/groups/layers identified by the different steps of the ampycloud +algorithm are accessible, as :py:class:`pandas.DataFrame` instances, via the class properties +:py:attr:`ampycloud.data.CeiloChunk.slices`, :py:attr:`ampycloud.data.CeiloChunk.groups`, and +:py:attr:`ampycloud.data.CeiloChunk.layers`. -.. autofunction:: ampycloud.core.metar - :noindex: +.. note:: + :py:meth:`ampycloud.data.CeiloChunk.metar_msg` relies on + :py:attr:`ampycloud.data.CeiloChunk.layers` to derive the corresponding METAR-like message. -.. autofunction:: ampycloud.core.synop +All these slices/groups/layer parameters are being compiled/computed by +:py:meth:`ampycloud.data.CeiloChunk.metarize`, which contains all the info about the different +parameters. + +.. autofunction:: ampycloud.data.CeiloChunk.metarize :noindex: +The no-plots-required shortcut +.............................. + +The following function, also accessible as ``ampycloud.metar()``, +will directly provide interested users with the ampycloud METAR-like message for a given dataset. +It is a convenience function intended for users that do not want to generate diagnostic plots, but +only seek the outcome of the ampycloud algorithm formatted as a METAR-like ``str``. + +.. autofunction:: ampycloud.core.metar + :noindex: + Adjusting the default algorithm parameters .......................................... -.. caution:: +.. important:: It is highly recommended to adjust any scientific parameters **before** executing any of the ampycloud routines. Doing otherwise may have un-expected consequences (i.e. parameters may not @@ -44,20 +65,20 @@ Adjusting the default algorithm parameters The ampycloud parameters with a **scientific** impact on the outcome of the algorithm (see :ref:`here for the complete list `) -are accessible in the ``ampycloud.dynamic`` module. From there, users can easily adjust them as they -see fit. For example: +are accessible in the :py:mod:`ampycloud.dynamic` module. From there, users can easily adjust them +as they see fit. For example: :: from ampycloud import dynamic dynamic.AMPYCLOUD_PRMS.OKTA_LIM8 = 95 -Note that it is important to always import the entire ``dynamic`` module and stick to the above -structure if the updated parameters are to be *seen* by all the ampycloud modules. +Note that it is important to always import the entire :py:mod:`ampycloud.dynamic` module and stick to the +above structure if the updated parameters are to be *seen* by all the ampycloud modules. Alternatively, all the scientific parameters can be adjusted and fed to ampycloud via a YAML file, -in which case the routines ```ampycloud.copy_prm_file()`` and ``ampycloud.set_prms()`` may be of -interest. +in which case the following routines, also accessible as ``ampycloud.copy_prm_file()`` and +``ampycloud.set_prms()``, may be of interest: .. autofunction:: ampycloud.core.copy_prm_file :noindex: @@ -67,8 +88,8 @@ interest. :noindex: -If all hope is lost and you wish to revert to the original (default) values of the all the -ampycloud scientific parameters, you can use ```ampycloud.reset_prms()``. +If all hope is lost and you wish to revert to the original (default) values of all the +ampycloud scientific parameters, you can use :py:func:`ampycloud.core.reset_prms()`. .. autofunction:: ampycloud.core.reset_prms :noindex: @@ -77,29 +98,33 @@ ampycloud scientific parameters, you can use ```ampycloud.reset_prms()``. Advanced info for advanced users ******************************** -The majority of parameters present in ``dynamic.AMPYCLOUD_PRMS`` are fetched directly by the -methods of the ``CeiloChunk`` class when they are called. As a result, modifying a specific -parameter in ``dynamic.AMPYCLOUD_PRMS`` (e.g. ``dynamic.AMPYCLOUD_PRMS.OKTA_LIM8``) will be seen -by any ``CeiloChunk`` instance already in existence. +The majority of parameters present in :py:data:`ampycloud.dynamic.AMPYCLOUD_PRMS` are fetched +directly by the methods of the :py:class:`ampycloud.data.CeiloChunk` class when they are required. +As a result, modifying a specific entry in :py:data:`ampycloud.dynamic.AMPYCLOUD_PRMS` (e.g. +``OKTA_LIM8``) will be seen by any :py:class:`ampycloud.data.CeiloChunk` instance already in +existence. -The ``MSA`` and ``MSA_HIT_BUFFER`` are the only exception to this rule ! These two -parameters are being applied (and deep-copied as ``CeiloChunk`` class variables) immediately at the -initialization of any ``CeiloChunk`` instance. This implies that: +The ``MSA`` and ``MSA_HIT_BUFFER`` entries are the only exception to this rule ! These two +parameters are being applied (and deep-copied as :py:class:`ampycloud.data.CeiloChunk` instance +attributes) immediately at the initialization of any :py:class:`ampycloud.data.CeiloChunk` instance. +This implies that: 1. any cloud hits above ``MSA + MSA_HIT_BUFFER`` in the data will be cropped immediately in the - ``CeiloChunk.__init__()`` routine, and thus cannot be recovered by subsequently changing the - value of ``dynamic.AMPYCLOUD_PRMS.MSA``, and - 2. any METAR/SYNOP message issued will always be subject to the Minimum Sector Altitude - value that was specified in ``dynamic.AMPYCLOUD_PRMS.MSA`` at the time the class - instance was initialized. This is to ensure consistency with the cropped data at all times. + :py:meth:`ampycloud.data.CeiloChunk.__init__` routine, and thus cannot be recovered by + subsequently changing the value of ``MSA`` in :py:data:`ampycloud.dynamic.AMPYCLOUD_PRMS`, + and + 2. any METAR-like message issued will always be subject to the Minimum Sector Altitude + value that was specified in :py:data:`ampycloud.dynamic.AMPYCLOUD_PRMS` at the time the + :py:class:`ampycloud.data.CeiloChunk` instance was initialized. This is to ensure + consistency with the cropped data at all times. .. _logging: Logging ------- -A ``NullHandler()`` is being set by ampycloud, such that no logging will be apparent to the users -unless they explicitly set it up +A :py:class:`logging.NullHandler` instance is being created by ampycloud, such that no logging will +be apparent to the users unless they explicitly set it up themselves (`see here `_ for more details). @@ -113,7 +138,7 @@ make the following call before running ampycloud functions: logging.getLogger('ampycloud').setLevel('DEBUG') -Each ampycloud module has a dedicated ``logger`` based on the module ``__name__``. Hence, users +Each ampycloud module has a dedicated logger based on the module ``__name__``. Hence, users can adjust the logging level of each ampycloud module however they desire, e.g.: :: @@ -155,7 +180,7 @@ system-wide, and set: dynamic.AMPYCLOUD_PRMS.MPL_STYLE = 'metsymb' -.. warning:: +.. important:: Using a system-wide LaTeX installation to create matplotlib figures **is not officially supported by matplotib**, and thus **not officially supported by ampycloud** either. diff --git a/docs/source/scope.rst b/docs/source/scope.rst index bd4efe6..4142dad 100644 --- a/docs/source/scope.rst +++ b/docs/source/scope.rst @@ -8,8 +8,10 @@ This has the following implications for ampycloud: Depending on the ceilometer type, the user will need to decide how to treat VV hits *before* passing them to ampycloud, e.g. by removing them or by converting them to cloud base heights. + * ampycloud can evidently be used for R&D work, but the code itself should not be seen as an R&D platform. + * Contributions via Pull Requests are always welcome (and appreciated !), but will only be considered if they: @@ -20,3 +22,11 @@ This has the following implications for ampycloud: * The ingestion of external contributions may be delayed to allow for careful, internal verification that they do not affect the MeteoSwiss operational chain that relies on ampycloud. + + * ampycloud is designed (and intended to be used) as a regular Python module. As such, it is + not meant to interact (directly) with other programming languages. The implementation of a + complex API to interact with "the outside World" (e.g. to feed data to ampycloud using the + JSON format) is not foreseen. + + * ampycloud is not meant to handle/generate/derive ``CB/TCU`` codes. The module deals with + "basic" cloud layers only. diff --git a/docs/source/substitutions.rst b/docs/source/substitutions.rst index 406d5e9..2ed739f 100644 --- a/docs/source/substitutions.rst +++ b/docs/source/substitutions.rst @@ -18,7 +18,7 @@ :target: https://github.com/MeteoSwiss/ampycloud/releases .. |pypi| image:: https://img.shields.io/pypi/v/ampycloud.svg?colorB= - :target: https://pypi.python.org/pypi/ampycloud/ + :target: https://pypi.org/project/ampycloud/ -.. |copyright| image:: https://img.shields.io/badge/MeteoSwiss-%C2%A9_2021-black?logo=data%3Aimage%2Fsvg%2Bxml%3Bbase64%2CPHN2ZyAgIHhtbG5zOmRjPSJodHRwOi8vcHVybC5vcmcvZGMvZWxlbWVudHMvMS4xLyIgICB4bWxuczpjYz0iaHR0cDovL2NyZWF0aXZlY29tbW9ucy5vcmcvbnMjIiAgIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyIgICB4bWxuczpzdmc9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiAgIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgICB4bWxuczpzb2RpcG9kaT0iaHR0cDovL3NvZGlwb2RpLnNvdXJjZWZvcmdlLm5ldC9EVEQvc29kaXBvZGktMC5kdGQiICAgeG1sbnM6aW5rc2NhcGU9Imh0dHA6Ly93d3cuaW5rc2NhcGUub3JnL25hbWVzcGFjZXMvaW5rc2NhcGUiICAgd2lkdGg9Ijc3LjI2Njk5OHB0IiAgIGhlaWdodD0iODUuNTQ1NDcxcHQiICAgdmlld0JveD0iMCAwIDc3LjI2Njk5NCA4NS41NDU0NzMiICAgdmVyc2lvbj0iMS4yIiAgIGlkPSJzdmcyODciICAgc29kaXBvZGk6ZG9jbmFtZT0iU2hpZWxkX3JnYl9wb3NfRU4uc3ZnIiAgIGlua3NjYXBlOnZlcnNpb249IjAuOTIuMiA1YzNlODBkLCAyMDE3LTA4LTA2IiAgIGlua3NjYXBlOmV4cG9ydC1maWxlbmFtZT0iL1VzZXJzL2Z2b2d0L1Byb2plY3RzL2N1cnJlbnQvTUNIL1NFVC9TaGllbGRfcmdiX3Bvc19FTi5wbmciICAgaW5rc2NhcGU6ZXhwb3J0LXhkcGk9IjI5OTkuNjY3NyIgICBpbmtzY2FwZTpleHBvcnQteWRwaT0iMjk5OS42Njc3Ij4gIDxtZXRhZGF0YSAgICAgaWQ9Im1ldGFkYXRhMjkxIj4gICAgPHJkZjpSREY+ICAgICAgPGNjOldvcmsgICAgICAgICByZGY6YWJvdXQ9IiI+ICAgICAgICA8ZGM6Zm9ybWF0PmltYWdlL3N2Zyt4bWw8L2RjOmZvcm1hdD4gICAgICAgIDxkYzp0eXBlICAgICAgICAgICByZGY6cmVzb3VyY2U9Imh0dHA6Ly9wdXJsLm9yZy9kYy9kY21pdHlwZS9TdGlsbEltYWdlIiAvPiAgICAgICAgPGRjOnRpdGxlPjwvZGM6dGl0bGU+ICAgICAgPC9jYzpXb3JrPiAgICA8L3JkZjpSREY+ICA8L21ldGFkYXRhPiAgPHNvZGlwb2RpOm5hbWVkdmlldyAgICAgcGFnZWNvbG9yPSIjZmZmZmZmIiAgICAgYm9yZGVyY29sb3I9IiM2NjY2NjYiICAgICBib3JkZXJvcGFjaXR5PSIxIiAgICAgb2JqZWN0dG9sZXJhbmNlPSIxMCIgICAgIGdyaWR0b2xlcmFuY2U9IjEwIiAgICAgZ3VpZGV0b2xlcmFuY2U9IjEwIiAgICAgaW5rc2NhcGU6cGFnZW9wYWNpdHk9IjAiICAgICBpbmtzY2FwZTpwYWdlc2hhZG93PSIyIiAgICAgaW5rc2NhcGU6d2luZG93LXdpZHRoPSIxMzMyIiAgICAgaW5rc2NhcGU6d2luZG93LWhlaWdodD0iNzkwIiAgICAgaWQ9Im5hbWVkdmlldzI4OSIgICAgIHNob3dncmlkPSJmYWxzZSIgICAgIGZpdC1tYXJnaW4tdG9wPSIwIiAgICAgZml0LW1hcmdpbi1sZWZ0PSIwIiAgICAgZml0LW1hcmdpbi1yaWdodD0iMCIgICAgIGZpdC1tYXJnaW4tYm90dG9tPSIwIiAgICAgaW5rc2NhcGU6em9vbT0iMi4yMDMwNTA3IiAgICAgaW5rc2NhcGU6Y3g9IjEzNS42OTUzMyIgICAgIGlua3NjYXBlOmN5PSIyNi4yMjE0NDQiICAgICBpbmtzY2FwZTp3aW5kb3cteD0iNDkiICAgICBpbmtzY2FwZTp3aW5kb3cteT0iMTIiICAgICBpbmtzY2FwZTp3aW5kb3ctbWF4aW1pemVkPSIwIiAgICAgaW5rc2NhcGU6Y3VycmVudC1sYXllcj0ic3ZnMjg3IiAgICAgaW5rc2NhcGU6bWVhc3VyZS1zdGFydD0iMCwwIiAgICAgaW5rc2NhcGU6bWVhc3VyZS1lbmQ9IjAsMCIgLz4gIDxkZWZzICAgICBpZD0iZGVmczM4Ij4gICAgPGNsaXBQYXRoICAgICAgIGlkPSJjbGlwMSI+ICAgICAgPHBhdGggICAgICAgICBkPSJtIDE1MywyIGggMi45ODgyOCBWIDggSCAxNTMgWiBtIDAsMCIgICAgICAgICBpZD0icGF0aDIiICAgICAgICAgaW5rc2NhcGU6Y29ubmVjdG9yLWN1cnZhdHVyZT0iMCIgLz4gICAgPC9jbGlwUGF0aD4gICAgPGNsaXBQYXRoICAgICAgIGlkPSJjbGlwMiI+ICAgICAgPHBhdGggICAgICAgICBkPSJtIDI5LDUzIGggNCB2IDYuMDUwNzgxIGggLTQgeiBtIDAsMCIgICAgICAgICBpZD0icGF0aDUiICAgICAgICAgaW5rc2NhcGU6Y29ubmVjdG9yLWN1cnZhdHVyZT0iMCIgLz4gICAgPC9jbGlwUGF0aD4gICAgPGNsaXBQYXRoICAgICAgIGlkPSJjbGlwMyI+ICAgICAgPHBhdGggICAgICAgICBkPSJtIDQyLDU1IGggNCB2IDQuMDUwNzgxIGggLTQgeiBtIDAsMCIgICAgICAgICBpZD0icGF0aDgiICAgICAgICAgaW5rc2NhcGU6Y29ubmVjdG9yLWN1cnZhdHVyZT0iMCIgLz4gICAgPC9jbGlwUGF0aD4gICAgPGNsaXBQYXRoICAgICAgIGlkPSJjbGlwNCI+ICAgICAgPHBhdGggICAgICAgICBkPSJtIDQ2LDU1IGggMyB2IDQuMDUwNzgxIGggLTMgeiBtIDAsMCIgICAgICAgICBpZD0icGF0aDExIiAgICAgICAgIGlua3NjYXBlOmNvbm5lY3Rvci1jdXJ2YXR1cmU9IjAiIC8+ICAgIDwvY2xpcFBhdGg+ICAgIDxjbGlwUGF0aCAgICAgICBpZD0iY2xpcDUiPiAgICAgIDxwYXRoICAgICAgICAgZD0ibSA1Miw1MyBoIDUgdiA2LjA1MDc4MSBoIC01IHogbSAwLDAiICAgICAgICAgaWQ9InBhdGgxNCIgICAgICAgICBpbmtzY2FwZTpjb25uZWN0b3ItY3VydmF0dXJlPSIwIiAvPiAgICA8L2NsaXBQYXRoPiAgICA8Y2xpcFBhdGggICAgICAgaWQ9ImNsaXA2Ij4gICAgICA8cGF0aCAgICAgICAgIGQ9Im0gNTcsNTUgaCA1IHYgNC4wNTA3ODEgaCAtNSB6IG0gMCwwIiAgICAgICAgIGlkPSJwYXRoMTciICAgICAgICAgaW5rc2NhcGU6Y29ubmVjdG9yLWN1cnZhdHVyZT0iMCIgLz4gICAgPC9jbGlwUGF0aD4gICAgPGNsaXBQYXRoICAgICAgIGlkPSJjbGlwNyI+ICAgICAgPHBhdGggICAgICAgICBkPSJtIDcwLDU1IGggNCB2IDQuMDUwNzgxIGggLTQgeiBtIDAsMCIgICAgICAgICBpZD0icGF0aDIwIiAgICAgICAgIGlua3NjYXBlOmNvbm5lY3Rvci1jdXJ2YXR1cmU9IjAiIC8+ICAgIDwvY2xpcFBhdGg+ICAgIDxjbGlwUGF0aCAgICAgICBpZD0iY2xpcDgiPiAgICAgIDxwYXRoICAgICAgICAgZD0ibSA3NCw1MyBoIDUgdiA2LjA1MDc4MSBoIC01IHogbSAwLDAiICAgICAgICAgaWQ9InBhdGgyMyIgICAgICAgICBpbmtzY2FwZTpjb25uZWN0b3ItY3VydmF0dXJlPSIwIiAvPiAgICA8L2NsaXBQYXRoPiAgICA8Y2xpcFBhdGggICAgICAgaWQ9ImNsaXA5Ij4gICAgICA8cGF0aCAgICAgICAgIGQ9Im0gNzksNTUgaCA0IHYgNC4wNTA3ODEgaCAtNCB6IG0gMCwwIiAgICAgICAgIGlkPSJwYXRoMjYiICAgICAgICAgaW5rc2NhcGU6Y29ubmVjdG9yLWN1cnZhdHVyZT0iMCIgLz4gICAgPC9jbGlwUGF0aD4gICAgPGNsaXBQYXRoICAgICAgIGlkPSJjbGlwMTAiPiAgICAgIDxwYXRoICAgICAgICAgZD0ibSA4Nyw1NSBoIDQgdiA0LjA1MDc4MSBoIC00IHogbSAwLDAiICAgICAgICAgaWQ9InBhdGgyOSIgICAgICAgICBpbmtzY2FwZTpjb25uZWN0b3ItY3VydmF0dXJlPSIwIiAvPiAgICA8L2NsaXBQYXRoPiAgICA8Y2xpcFBhdGggICAgICAgaWQ9ImNsaXAxMSI+ICAgICAgPHBhdGggICAgICAgICBkPSJtIDkxLDU0IGggMyB2IDUuMDUwNzgxIGggLTMgeiBtIDAsMCIgICAgICAgICBpZD0icGF0aDMyIiAgICAgICAgIGlua3NjYXBlOmNvbm5lY3Rvci1jdXJ2YXR1cmU9IjAiIC8+ICAgIDwvY2xpcFBhdGg+ICAgIDxjbGlwUGF0aCAgICAgICBpZD0iY2xpcDEyIj4gICAgICA8cGF0aCAgICAgICAgIGQ9Im0gOTYsNTUgaCA1IHYgNC4wNTA3ODEgaCAtNSB6IG0gMCwwIiAgICAgICAgIGlkPSJwYXRoMzUiICAgICAgICAgaW5rc2NhcGU6Y29ubmVjdG9yLWN1cnZhdHVyZT0iMCIgLz4gICAgPC9jbGlwUGF0aD4gIDwvZGVmcz4gIDxnICAgICBpZD0iZzQzNyIgICAgIHRyYW5zZm9ybT0ic2NhbGUoMy42MjY5NjkzKSI+ICAgIDxwYXRoICAgICAgIHN0eWxlPSJmaWxsOiNlYjIxMmI7ZmlsbC1vcGFjaXR5OjE7ZmlsbC1ydWxlOm5vbnplcm87c3Ryb2tlOm5vbmUiICAgICAgIGQ9Ik0gMjEuMjY2OTY1LDEuNzM4MjgxOCBDIDIxLjI2Njk2NSwxLjczODI4MTggMTcuMjk4MjE1LDAgMTAuNjUzNjg0LDAgNC4wMDkxNTE5LDAgMC4wMzY0OTYwNywxLjczODI4MTggMC4wMzY0OTYwNywxLjczODI4MTggYyAwLDAgLTAuMzU1NDY4Miw3LjY5NTMxMiAxLjE1MjM0MzgzLDEyLjA0Njg3NDIgMi42NjAxNTYsNy42NTIzNDQgOS40NjA5MzgxLDkuODAwNzgyIDkuNDYwOTM4MSw5LjgwMDc4MiBoIDAuMDAzOSBjIDAsMCA2LjgwMDc4MSwtMi4xNDg0MzggOS40NjA5MzcsLTkuODAwNzgyIDEuNTA3ODE5LC00LjM1MTU2MjIgMS4xNTIzNSwtMTIuMDQ2ODc0MyAxLjE1MjM1LC0xMi4wNDY4NzQzIiAgICAgICBpZD0icGF0aDI4MiIgICAgICAgaW5rc2NhcGU6Y29ubmVjdG9yLWN1cnZhdHVyZT0iMCIgLz4gICAgPHBhdGggICAgICAgc3R5bGU9ImZpbGw6I2ZmZmZmZjtmaWxsLW9wYWNpdHk6MTtmaWxsLXJ1bGU6bm9uemVybztzdHJva2U6bm9uZSIgICAgICAgZD0iTSAxNy43MDgzNzEsOC4zNzQ5OTk4IFYgMTIuNjEzMjgyIEggMTIuNzU5MTUyIFYgMTcuNTYyNSBIIDguNTIwODcwOSBWIDEyLjYxMzI4MiBIIDMuNTcxNjUxOSBWIDguMzc0OTk5OCBoIDQuOTQ5MjE5IHYgLTQuOTQ5MjE4IGggNC4yMzgyODExIHYgNC45NDkyMTggaCA0Ljk0OTIxOSIgICAgICAgaWQ9InBhdGgyODQiICAgICAgIGlua3NjYXBlOmNvbm5lY3Rvci1jdXJ2YXR1cmU9IjAiIC8+ICA8L2c+PC9zdmc+ +.. |copyright| image:: https://img.shields.io/badge/MeteoSwiss-%C2%A9_2021--2022-black?logo=data%3Aimage%2Fsvg%2Bxml%3Bbase64%2CPHN2ZyAgIHhtbG5zOmRjPSJodHRwOi8vcHVybC5vcmcvZGMvZWxlbWVudHMvMS4xLyIgICB4bWxuczpjYz0iaHR0cDovL2NyZWF0aXZlY29tbW9ucy5vcmcvbnMjIiAgIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyIgICB4bWxuczpzdmc9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiAgIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgICB4bWxuczpzb2RpcG9kaT0iaHR0cDovL3NvZGlwb2RpLnNvdXJjZWZvcmdlLm5ldC9EVEQvc29kaXBvZGktMC5kdGQiICAgeG1sbnM6aW5rc2NhcGU9Imh0dHA6Ly93d3cuaW5rc2NhcGUub3JnL25hbWVzcGFjZXMvaW5rc2NhcGUiICAgd2lkdGg9Ijc3LjI2Njk5OHB0IiAgIGhlaWdodD0iODUuNTQ1NDcxcHQiICAgdmlld0JveD0iMCAwIDc3LjI2Njk5NCA4NS41NDU0NzMiICAgdmVyc2lvbj0iMS4yIiAgIGlkPSJzdmcyODciICAgc29kaXBvZGk6ZG9jbmFtZT0iU2hpZWxkX3JnYl9wb3NfRU4uc3ZnIiAgIGlua3NjYXBlOnZlcnNpb249IjAuOTIuMiA1YzNlODBkLCAyMDE3LTA4LTA2IiAgIGlua3NjYXBlOmV4cG9ydC1maWxlbmFtZT0iL1VzZXJzL2Z2b2d0L1Byb2plY3RzL2N1cnJlbnQvTUNIL1NFVC9TaGllbGRfcmdiX3Bvc19FTi5wbmciICAgaW5rc2NhcGU6ZXhwb3J0LXhkcGk9IjI5OTkuNjY3NyIgICBpbmtzY2FwZTpleHBvcnQteWRwaT0iMjk5OS42Njc3Ij4gIDxtZXRhZGF0YSAgICAgaWQ9Im1ldGFkYXRhMjkxIj4gICAgPHJkZjpSREY+ICAgICAgPGNjOldvcmsgICAgICAgICByZGY6YWJvdXQ9IiI+ICAgICAgICA8ZGM6Zm9ybWF0PmltYWdlL3N2Zyt4bWw8L2RjOmZvcm1hdD4gICAgICAgIDxkYzp0eXBlICAgICAgICAgICByZGY6cmVzb3VyY2U9Imh0dHA6Ly9wdXJsLm9yZy9kYy9kY21pdHlwZS9TdGlsbEltYWdlIiAvPiAgICAgICAgPGRjOnRpdGxlPjwvZGM6dGl0bGU+ICAgICAgPC9jYzpXb3JrPiAgICA8L3JkZjpSREY+ICA8L21ldGFkYXRhPiAgPHNvZGlwb2RpOm5hbWVkdmlldyAgICAgcGFnZWNvbG9yPSIjZmZmZmZmIiAgICAgYm9yZGVyY29sb3I9IiM2NjY2NjYiICAgICBib3JkZXJvcGFjaXR5PSIxIiAgICAgb2JqZWN0dG9sZXJhbmNlPSIxMCIgICAgIGdyaWR0b2xlcmFuY2U9IjEwIiAgICAgZ3VpZGV0b2xlcmFuY2U9IjEwIiAgICAgaW5rc2NhcGU6cGFnZW9wYWNpdHk9IjAiICAgICBpbmtzY2FwZTpwYWdlc2hhZG93PSIyIiAgICAgaW5rc2NhcGU6d2luZG93LXdpZHRoPSIxMzMyIiAgICAgaW5rc2NhcGU6d2luZG93LWhlaWdodD0iNzkwIiAgICAgaWQ9Im5hbWVkdmlldzI4OSIgICAgIHNob3dncmlkPSJmYWxzZSIgICAgIGZpdC1tYXJnaW4tdG9wPSIwIiAgICAgZml0LW1hcmdpbi1sZWZ0PSIwIiAgICAgZml0LW1hcmdpbi1yaWdodD0iMCIgICAgIGZpdC1tYXJnaW4tYm90dG9tPSIwIiAgICAgaW5rc2NhcGU6em9vbT0iMi4yMDMwNTA3IiAgICAgaW5rc2NhcGU6Y3g9IjEzNS42OTUzMyIgICAgIGlua3NjYXBlOmN5PSIyNi4yMjE0NDQiICAgICBpbmtzY2FwZTp3aW5kb3cteD0iNDkiICAgICBpbmtzY2FwZTp3aW5kb3cteT0iMTIiICAgICBpbmtzY2FwZTp3aW5kb3ctbWF4aW1pemVkPSIwIiAgICAgaW5rc2NhcGU6Y3VycmVudC1sYXllcj0ic3ZnMjg3IiAgICAgaW5rc2NhcGU6bWVhc3VyZS1zdGFydD0iMCwwIiAgICAgaW5rc2NhcGU6bWVhc3VyZS1lbmQ9IjAsMCIgLz4gIDxkZWZzICAgICBpZD0iZGVmczM4Ij4gICAgPGNsaXBQYXRoICAgICAgIGlkPSJjbGlwMSI+ICAgICAgPHBhdGggICAgICAgICBkPSJtIDE1MywyIGggMi45ODgyOCBWIDggSCAxNTMgWiBtIDAsMCIgICAgICAgICBpZD0icGF0aDIiICAgICAgICAgaW5rc2NhcGU6Y29ubmVjdG9yLWN1cnZhdHVyZT0iMCIgLz4gICAgPC9jbGlwUGF0aD4gICAgPGNsaXBQYXRoICAgICAgIGlkPSJjbGlwMiI+ICAgICAgPHBhdGggICAgICAgICBkPSJtIDI5LDUzIGggNCB2IDYuMDUwNzgxIGggLTQgeiBtIDAsMCIgICAgICAgICBpZD0icGF0aDUiICAgICAgICAgaW5rc2NhcGU6Y29ubmVjdG9yLWN1cnZhdHVyZT0iMCIgLz4gICAgPC9jbGlwUGF0aD4gICAgPGNsaXBQYXRoICAgICAgIGlkPSJjbGlwMyI+ICAgICAgPHBhdGggICAgICAgICBkPSJtIDQyLDU1IGggNCB2IDQuMDUwNzgxIGggLTQgeiBtIDAsMCIgICAgICAgICBpZD0icGF0aDgiICAgICAgICAgaW5rc2NhcGU6Y29ubmVjdG9yLWN1cnZhdHVyZT0iMCIgLz4gICAgPC9jbGlwUGF0aD4gICAgPGNsaXBQYXRoICAgICAgIGlkPSJjbGlwNCI+ICAgICAgPHBhdGggICAgICAgICBkPSJtIDQ2LDU1IGggMyB2IDQuMDUwNzgxIGggLTMgeiBtIDAsMCIgICAgICAgICBpZD0icGF0aDExIiAgICAgICAgIGlua3NjYXBlOmNvbm5lY3Rvci1jdXJ2YXR1cmU9IjAiIC8+ICAgIDwvY2xpcFBhdGg+ICAgIDxjbGlwUGF0aCAgICAgICBpZD0iY2xpcDUiPiAgICAgIDxwYXRoICAgICAgICAgZD0ibSA1Miw1MyBoIDUgdiA2LjA1MDc4MSBoIC01IHogbSAwLDAiICAgICAgICAgaWQ9InBhdGgxNCIgICAgICAgICBpbmtzY2FwZTpjb25uZWN0b3ItY3VydmF0dXJlPSIwIiAvPiAgICA8L2NsaXBQYXRoPiAgICA8Y2xpcFBhdGggICAgICAgaWQ9ImNsaXA2Ij4gICAgICA8cGF0aCAgICAgICAgIGQ9Im0gNTcsNTUgaCA1IHYgNC4wNTA3ODEgaCAtNSB6IG0gMCwwIiAgICAgICAgIGlkPSJwYXRoMTciICAgICAgICAgaW5rc2NhcGU6Y29ubmVjdG9yLWN1cnZhdHVyZT0iMCIgLz4gICAgPC9jbGlwUGF0aD4gICAgPGNsaXBQYXRoICAgICAgIGlkPSJjbGlwNyI+ICAgICAgPHBhdGggICAgICAgICBkPSJtIDcwLDU1IGggNCB2IDQuMDUwNzgxIGggLTQgeiBtIDAsMCIgICAgICAgICBpZD0icGF0aDIwIiAgICAgICAgIGlua3NjYXBlOmNvbm5lY3Rvci1jdXJ2YXR1cmU9IjAiIC8+ICAgIDwvY2xpcFBhdGg+ICAgIDxjbGlwUGF0aCAgICAgICBpZD0iY2xpcDgiPiAgICAgIDxwYXRoICAgICAgICAgZD0ibSA3NCw1MyBoIDUgdiA2LjA1MDc4MSBoIC01IHogbSAwLDAiICAgICAgICAgaWQ9InBhdGgyMyIgICAgICAgICBpbmtzY2FwZTpjb25uZWN0b3ItY3VydmF0dXJlPSIwIiAvPiAgICA8L2NsaXBQYXRoPiAgICA8Y2xpcFBhdGggICAgICAgaWQ9ImNsaXA5Ij4gICAgICA8cGF0aCAgICAgICAgIGQ9Im0gNzksNTUgaCA0IHYgNC4wNTA3ODEgaCAtNCB6IG0gMCwwIiAgICAgICAgIGlkPSJwYXRoMjYiICAgICAgICAgaW5rc2NhcGU6Y29ubmVjdG9yLWN1cnZhdHVyZT0iMCIgLz4gICAgPC9jbGlwUGF0aD4gICAgPGNsaXBQYXRoICAgICAgIGlkPSJjbGlwMTAiPiAgICAgIDxwYXRoICAgICAgICAgZD0ibSA4Nyw1NSBoIDQgdiA0LjA1MDc4MSBoIC00IHogbSAwLDAiICAgICAgICAgaWQ9InBhdGgyOSIgICAgICAgICBpbmtzY2FwZTpjb25uZWN0b3ItY3VydmF0dXJlPSIwIiAvPiAgICA8L2NsaXBQYXRoPiAgICA8Y2xpcFBhdGggICAgICAgaWQ9ImNsaXAxMSI+ICAgICAgPHBhdGggICAgICAgICBkPSJtIDkxLDU0IGggMyB2IDUuMDUwNzgxIGggLTMgeiBtIDAsMCIgICAgICAgICBpZD0icGF0aDMyIiAgICAgICAgIGlua3NjYXBlOmNvbm5lY3Rvci1jdXJ2YXR1cmU9IjAiIC8+ICAgIDwvY2xpcFBhdGg+ICAgIDxjbGlwUGF0aCAgICAgICBpZD0iY2xpcDEyIj4gICAgICA8cGF0aCAgICAgICAgIGQ9Im0gOTYsNTUgaCA1IHYgNC4wNTA3ODEgaCAtNSB6IG0gMCwwIiAgICAgICAgIGlkPSJwYXRoMzUiICAgICAgICAgaW5rc2NhcGU6Y29ubmVjdG9yLWN1cnZhdHVyZT0iMCIgLz4gICAgPC9jbGlwUGF0aD4gIDwvZGVmcz4gIDxnICAgICBpZD0iZzQzNyIgICAgIHRyYW5zZm9ybT0ic2NhbGUoMy42MjY5NjkzKSI+ICAgIDxwYXRoICAgICAgIHN0eWxlPSJmaWxsOiNlYjIxMmI7ZmlsbC1vcGFjaXR5OjE7ZmlsbC1ydWxlOm5vbnplcm87c3Ryb2tlOm5vbmUiICAgICAgIGQ9Ik0gMjEuMjY2OTY1LDEuNzM4MjgxOCBDIDIxLjI2Njk2NSwxLjczODI4MTggMTcuMjk4MjE1LDAgMTAuNjUzNjg0LDAgNC4wMDkxNTE5LDAgMC4wMzY0OTYwNywxLjczODI4MTggMC4wMzY0OTYwNywxLjczODI4MTggYyAwLDAgLTAuMzU1NDY4Miw3LjY5NTMxMiAxLjE1MjM0MzgzLDEyLjA0Njg3NDIgMi42NjAxNTYsNy42NTIzNDQgOS40NjA5MzgxLDkuODAwNzgyIDkuNDYwOTM4MSw5LjgwMDc4MiBoIDAuMDAzOSBjIDAsMCA2LjgwMDc4MSwtMi4xNDg0MzggOS40NjA5MzcsLTkuODAwNzgyIDEuNTA3ODE5LC00LjM1MTU2MjIgMS4xNTIzNSwtMTIuMDQ2ODc0MyAxLjE1MjM1LC0xMi4wNDY4NzQzIiAgICAgICBpZD0icGF0aDI4MiIgICAgICAgaW5rc2NhcGU6Y29ubmVjdG9yLWN1cnZhdHVyZT0iMCIgLz4gICAgPHBhdGggICAgICAgc3R5bGU9ImZpbGw6I2ZmZmZmZjtmaWxsLW9wYWNpdHk6MTtmaWxsLXJ1bGU6bm9uemVybztzdHJva2U6bm9uZSIgICAgICAgZD0iTSAxNy43MDgzNzEsOC4zNzQ5OTk4IFYgMTIuNjEzMjgyIEggMTIuNzU5MTUyIFYgMTcuNTYyNSBIIDguNTIwODcwOSBWIDEyLjYxMzI4MiBIIDMuNTcxNjUxOSBWIDguMzc0OTk5OCBoIDQuOTQ5MjE5IHYgLTQuOTQ5MjE4IGggNC4yMzgyODExIHYgNC45NDkyMTggaCA0Ljk0OTIxOSIgICAgICAgaWQ9InBhdGgyODQiICAgICAgIGlua3NjYXBlOmNvbm5lY3Rvci1jdXJ2YXR1cmU9IjAiIC8+ICA8L2c+PC9zdmc+ :target: https://www.meteoswiss.admin.ch diff --git a/setup.py b/setup.py index 6e6ae4c..1994976 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. @@ -30,7 +30,12 @@ # Tell setuptools packages are under src package_dir={"": "src"}, - url="https://github.com/MeteoSwiss/ampycloud", + url="https://meteoswiss.github.io/ampycloud", + project_urls={ + 'Source': 'https://github.com/MeteoSwiss/ampycloud/', + 'Changelog': 'https://meteoswiss.github.io/ampycloud/changelog.html', + 'Issues': 'https://github.com/MeteoSwiss/ampycloud/issues' + }, author="Frédéric P.A. Vogt", author_email="frederic.vogt@meteoswiss.ch", description="Characterization of cloud layers from ceilometer measurements", @@ -65,6 +70,7 @@ # Indicate who your project is intended for 'Intended Audience :: Science/Research', 'Topic :: Scientific/Engineering :: Meteorology', + 'Topic :: Scientific/Engineering :: Atmospheric Science', # Pick your license as you wish (should match "license" above) 'License :: OSI Approved :: BSD License', diff --git a/src/ampycloud/__init__.py b/src/ampycloud/__init__.py index 7b12248..0ee226a 100644 --- a/src/ampycloud/__init__.py +++ b/src/ampycloud/__init__.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. diff --git a/src/ampycloud/__main__.py b/src/ampycloud/__main__.py index 4192c29..5895f08 100644 --- a/src/ampycloud/__main__.py +++ b/src/ampycloud/__main__.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. diff --git a/src/ampycloud/cluster.py b/src/ampycloud/cluster.py index bb2f67b..3c75ff2 100644 --- a/src/ampycloud/cluster.py +++ b/src/ampycloud/cluster.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. @@ -25,18 +25,18 @@ def agglomerative_cluster(data : np.ndarray, n_clusters : int = None, affinity : str = 'euclidean', linkage : str = 'single', distance_threshold : Union[int, float] = 1) -> tuple: - """ Function that wraps arround sklearn.cluster.AgglomerativeClustering. + """ Function that wraps arround :py:class:`sklearn.cluster.AgglomerativeClustering`. Args: data (ndarray): array of [x, y] pairs to run the clustering on. - n_clusters (int, optional): see sklearn.cluster.AgglomerativeClustering for details. - Defaults to None. - affinity (str, optional): see sklearn.cluster.AgglomerativeClustering for details. - Defaults to 'euclidian'. - linkage (str, optional): see sklearn.cluster.AgglomerativeClustering for details. - Defaults to 'single'. - distance_threshold (int|float, optional): see sklearn.cluster.AgglomerativeClustering for - details. Defaults to 1. + n_clusters (int, optional): see :py:class:`sklearn.cluster.AgglomerativeClustering` + for details. Defaults to None. + affinity (str, optional): see :py:class:`sklearn.cluster.AgglomerativeClustering` for + details. Defaults to 'euclidian'. + linkage (str, optional): see :py:class:`sklearn.cluster.AgglomerativeClustering` for + details. Defaults to 'single'. + distance_threshold (int|float, optional): see + :py:class:`sklearn.cluster.AgglomerativeClustering` for details. Defaults to 1. Returns: int, ndarray: number of clusters found, and corresponding clustering labels for each data diff --git a/src/ampycloud/core.py b/src/ampycloud/core.py index 6e5bd73..57aa478 100644 --- a/src/ampycloud/core.py +++ b/src/ampycloud/core.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the BSD-3-Clause license. @@ -31,11 +31,11 @@ @log_func_call(logger) def copy_prm_file(save_loc : str = './', which : str = 'defaults') -> None: - """ Save a local copy of a specific ampycloud parameter file. + """ Create a local copy of a specific ampycloud parameter file. Args: save_loc (str, optional): location to save the YML file to. Defaults to './'. - which (str, optional): name of thee parameter file to copy. Defaults to 'defaults'. + which (str, optional): name of the parameter file to copy. Defaults to 'defaults'. Example: :: @@ -86,10 +86,11 @@ def set_prms(pth : Union[str, Path]) -> None: pth (str|Path): path+filename to a YAML parameter file for ampycloud. .. note:: - It is recommended to first get a copy of the default ampycloud parameter file using - ``ampycloud.copy_prm_file()``, and edit its content as required. + It is recommended to first get a copy of the default ampycloud parameter file + using :py:func:`.copy_prm_file()`, and edit its content as required. - Doing so should ensure full compliance with default structure of ``dynamic.AMPYCLOUD_PRMS``. + Doing so should ensure full compliance with the default structure of + :py:data:`.dynamic.AMPYCLOUD_PRMS`. Example: :: @@ -155,24 +156,26 @@ def reset_prms() -> None: @log_func_call(logger) def run(data : pd.DataFrame, geoloc : str = None, ref_dt : str = None) -> CeiloChunk: - """ Run the ampycloud algorithm on a given dataset. + """ Runs the ampycloud algorithm on a given dataset. Args: - data (pd.DataFrame): the data to be processed, as a pandas DataFrame. + data (pd.DataFrame): the data to be processed, as a py:class:`pandas.DataFrame`. geoloc (str, optional): the name of the geographic location where the data was taken. Defaults to None. ref_dt (str, optional): reference date and time of the observations, corresponding to Delta t = 0. Defaults to None. Returns: - CeiloChunk: the data chunk with all the processing outcome bundled cleanly. + :py:class:`.data.CeiloChunk`: the data chunk with all the processing outcome bundled + cleanly. - All that is required to run ampycloud is a properly formatted dataset. At the moment, - specifying ``geoloc`` and ``ref_dt`` serves no purpose other than to enhance plots (should they - be created). There is no special requirements for ``geoloc`` and ``ref_dt``: so long as they are - strings, you can set them to whatever you please. + All that is required to run the ampycloud algorithm is a properly formatted dataset. At the + moment, specifying ``geoloc`` and ``ref_dt`` serves no purpose other than to enhance plots + (should they be created). There is no special requirements for ``geoloc`` and ``ref_dt``: as + long as they are strings, you can set them to whatever you please. - The input ``data`` must be a ``pandas.DataFrame`` with the following column names (types): + The input ``data`` must be a :py:class:`pandas.DataFrame` with the following column names + (types): :: 'ceilo' (str), 'dt' (float), 'alt' (float), 'type' (int) @@ -190,43 +193,45 @@ def run(data : pd.DataFrame, geoloc : str = None, ref_dt : str = None) -> CeiloC cloud level 2, cloud level 3, etc ...), the ``type`` of these measurements could be ``1``, ``2``, ``3``, ... Any data point with a ``type`` of ``-1`` will be flagged in the ampycloud plots as a vertical Visibility (VV) hits, **but it will not be treated any differently than any - other regular hit**. + other regular hit**. Type ``0`` corresponds to no (cloud) detection. It is possible to obtain an example of the required ``data`` format from the - ``ampycloud.utils.mocker.canonical_demo_dataset()`` routine of the package, like so: + :py:func:`.utils.mocker.canonical_demo_data` routine of the package, like so: :: from ampycloud.utils import mocker - mock_data = mocker.canonical_demo_dataset() + mock_data = mocker.canonical_demo_data() - Caution: - ampycloud treats Vertical Visibility hits just like any other hit. Hence, it is up to the - user to adjust the Vertical Visibility hit altitude (and/or ignore some of them) prior to - feeding them to ampycloud. + .. important :: + ampycloud treats Vertical Visibility hits no differently than any other hit. Hence, it is up + to the user to adjust the Vertical Visibility hit altitude (and/or ignore some of them, for + example) prior to feeding them to ampycloud, so that it can be used as a cloud hit. - Caution: - ampycloud uses the ``dt`` and ``ceilo`` values to decide if two hits simultaenous, or not. - It is thus important that the values of ``dt`` be sufficiently precise to distinguish + .. important:: + ampycloud uses the ``dt`` and ``ceilo`` values to decide if two hits are simultaenous, or + not. It is thus important that the values of ``dt`` be sufficiently precise to distinguish between different measurements. Essentially, each *measurement* (which may be comprised of several hits) should be associated to a unique ``(ceilo; dt)`` set of values. Failure to do so may result in incorrect estimations of the cloud layer densities. See - ``ampycloud.data.CeiloChunk.max_hits_per_layer`` for more details. + :py:attr:`.data.CeiloChunk.max_hits_per_layer` for more details. - All the scientific parameters of the algorithm are set dynamically in ampycloud.dynamic. - From within a Python session all these parameters can be changed directly. For example, + All the scientific parameters of the algorithm are set dynamically in the :py:mod:`.dynamic` + module. From within a Python session all these parameters can be changed directly. For example, to change the Minimum Sector Altitude, one would do: :: from ampycloud import dynamic dynamic.AMPYCLOUD_PRMS.MSA = 5000 - Alternatively, all the scientific parameters can also be defined and fed to ampycloud via a YML - file. See ``ampycloud.set_prms()`` for details. + Alternatively, all the scientific parameters can also be defined and fed to ampycloud via a YAML + file. See :py:func:`.set_prms()` for details. - The ``ampycloud.data.CeiloChunk`` instance returned by this function contain all the information + The :py:class:`.data.CeiloChunk` instance returned by this function contains all the information associated to the ampycloud algorithm, inclduing the raw data and slicing/grouping/layering - info. Its method `.metar_msg()` provides direct access to the resulting METAR-like message. + info. Its method :py:meth:`.data.CeiloChunk.metar_msg` provides direct access to the resulting + METAR-like message. Users that require the altitude, okta amount, and/or exact sky coverage + fraction of layers can get them via the :py:attr:`.data.CeiloChunk.layers` class property. Example: @@ -244,8 +249,11 @@ def run(data : pd.DataFrame, geoloc : str = None, ref_dt : str = None) -> CeiloC # Run the ampycloud algorithm on it chunk = ampycloud.run(mock_data, geoloc='Mock data', ref_dt=datetime.now()) - # Get the resulting METAR/SYNOP message - print(chunk.metar_msg(synop=False)) + # Get the resulting METAR message + print(chunk.metar_msg()) + + # Display the full information available for the layers found + print(chunk.layers) """ @@ -267,47 +275,15 @@ def run(data : pd.DataFrame, geoloc : str = None, ref_dt : str = None) -> CeiloC return chunk - -@log_func_call(logger) -def synop(data : pd.DataFrame) -> str: - """ Runs the ampycloud algorithm on a dataset and extract a synop report of the cloud layers. - - Args: - data (pd.DataFrame): the data to be processed, as a pandas DataFrame. - - Returns: - str: the synop message. - - Example: - :: - - import ampycloud - from ampycloud.utils import mocker - - # Generate the canonical demo dataset for ampycloud - mock_data = mocker.canonical_demo_data() - - # Compute the synop message - msg = ampycloud.synop(mock_data) - print(msg) - - """ - - # First, run the ampycloud algorithm - chunk = run(data) - - # Then, return the synop message - return chunk.metar_msg(synop=True, which='layers') - @log_func_call(logger) def metar(data : pd.DataFrame) -> str: """ Run the ampycloud algorithm on a dataset and extract a METAR report of the cloud layers. Args: - data (pd.DataFrame): the data to be processed, as a pandas DataFrame. + data (pd.DataFrame): the data to be processed, as a :py:class:`pandas.DataFrame`. Returns: - str: the metar message. + str: the METAR-like message. Example: :: @@ -327,16 +303,16 @@ def metar(data : pd.DataFrame) -> str: # First, run the ampycloud algorithm chunk = run(data) - # Then, return the synop message - return chunk.metar_msg(synop=False, which='layers') + # Then, return the METAR message + return chunk.metar_msg(which='layers') @log_func_call(logger) def demo() -> tuple: """ Run the ampycloud algorithm on a demonstration dataset. Returns: - pd.DataFrame, CeiloChunk: the mock dataset used for the demonstration, and the CeiloChunk - instance. + :py:class:`pandas.DataFrame`, :py:class:`.data.CeiloChunk`: the mock dataset used for the + demonstration, and the :py:class:`.data.CeiloChunk` instance. """ diff --git a/src/ampycloud/data.py b/src/ampycloud/data.py index 9900b73..eacaff0 100644 --- a/src/ampycloud/data.py +++ b/src/ampycloud/data.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. @@ -22,7 +22,7 @@ from . import scaler from . import cluster from . import layer -from . import wmo +from . import wmo, icao from . import dynamic # Instantiate the module logger @@ -91,6 +91,13 @@ def _cleanup_pdf(self, data : pd.DataFrame) -> pd.DataFrame: raise AmpycloudError('Ouch ! I was expecting data as a pandas DataFrame,'+ f' not: {type(data)}') + # Make sure the dataframe is not empty. + # Note: an empty dataframe = no measurements. This is NOT the same as "measuring" clear sky + # conditions, which would result in NaNs. + # If I have no measurements, I cannot issue a METAR. It would make no sense. + if len(data) == 0: + raise AmpycloudError("Ouch ! len(data) is 0. I can't work with no data !") + # Check that all the required columns are present in the data, with the correct format for (col, type_req) in self.DATA_COLS.items(): # If the requried column is missing, raise an Exception @@ -112,7 +119,12 @@ def _cleanup_pdf(self, data : pd.DataFrame) -> pd.DataFrame: if self.msa is not None: hit_alt_lim = self.msa + self.msa_hit_buffer logger.info('Cropping hits above MSA+buffer: %s ft', str(hit_alt_lim)) - data = data.drop(data[data.alt > hit_alt_lim].index) + # Type 1 or less hits above the cut threshold get turned to NaNs, to signal a + # non-detection below the MSA. Also change the hit type to 0 accordingly ! + data.loc[data[(data.alt > hit_alt_lim) & (data.type <= 1)].index, 'type'] = 0 + data.loc[data[(data.alt > hit_alt_lim) & (data.type <= 1)].index, 'alt'] = np.nan + # Type 2 or more hits get cropped (there should be only 1 non-detection per time-stamp). + data = data.drop(data[(data.alt > hit_alt_lim) & (data.type > 1)].index) return data @@ -252,13 +264,9 @@ def max_hits_per_layer(self) -> int: @log_func_call(logger) def metarize(self, which : int = 'slices', base_frac : float = 0.1, - lim0 : Union[int, float] = 2, lim8 : Union[int, float] = 98) -> None: - """ Assembles a dataframe of slice/group/layer METAR properties of interest, including: - number of hits, sky coverage percentage, okta count, base altitude, mean altitude, altitude - standard deviation, METAR code, significance, ... - - These properties get stored in a self._slices, self._groups, or self._layers class - attribute. + lim0 : Union[int, float] = 0, lim8 : Union[int, float] = 100) -> None: + """ Assembles a :py:class:`pandas.DataFrame` of slice/group/layer METAR properties of + interest. Args: which (str, optional): whether to process 'slices', 'groups', or 'layers'. @@ -268,9 +276,36 @@ def metarize(self, which : int = 'slices', base_frac : float = 0.1, deriving the slice/group/layer altitude (as a median). Defaults to 0.1, i.e. the cloud base is the median of the lowest 10% of cloud hits of that slice/group/layer. lim0 (int|float, optional): upper limit of the sky coverage percentage for the 0 okta - bin, in %. Defaults to 2. + bin, in %. Defaults to 0. lim8 (int|float, optional): lower limit of the sky coverage percentage for the 8 okta - bin, in %. Defaults to 98. + bin, in %. Defaults to 100. + + The :py:class:`pandas.DataFrame` generated by this method is subsequently available via the + the appropriate class property :py:attr:`.CeiloChunk.slices`, :py:attr:`.CeiloChunk.groups`, + or :py:attr:`.CeiloChunk.layers`, depending on the value of the argument ``which``. + + The slice/group/layer parameters computed/derived by this method include: + + * ``n_hits (int)``: duplicate-corrected number of hits + * ``perc (float)``: sky coverage percentage (between 0-100) + * ``okta (int)``: okta count + * ``alt_base (float)``: base altitude + * ``alt_mean (float)``: mean altitude + * ``alt_std (float)``: altitude standard deviation + * ``code (str)``: METAR-like code + * ``significant (bool)``: whether the layer is significant according to the ICAO rules. + See :py:func:`.icao.significant_cloud` for details. + * ``original_id (int)``: an ampycloud-internal identification number + * ``isolated (bool)``: isolation status (for slices only) + * ``ncomp (int)``: the number of subcomponents (for groups only) + + Important: + The value of ``n_hits`` is corrected for duplicate hits, to ensure a correct estimation + of the sky coverage fraction. Essentially, two (or more) *simultaneous hits from the + same ceilometer* are counted as one only. In other words, if a Type ``1`` and ``2`` hits + **from the same ceilometer, at the same observation time** are included in a given + slice/group/layer, they are counted as one hit only. This is a direct consequence of the + fact that clouds have a single base altitude at any given time [*citation needed*]. """ @@ -393,15 +428,7 @@ def metarize(self, which : int = 'slices', base_frac : float = 0.1, pdf.reset_index(drop=True, inplace=True) # Almost done ... I just need to figure out which levels are significant. - # This is just the basic WMO/ICAO selection rule ! - sig_level = 0 - for ind in range(len(oids)): - if pdf.at[ind, 'okta'] > sig_level: - sig_level += 2 - pdf.at[ind, 'significant'] = True - - else: - pdf.at[ind, 'significant'] = False + pdf.loc[:, 'significant'] = icao.significant_cloud(pdf['okta'].to_list()) # Finally, assign the outcome where it belongs. setattr(self, f'_{which}', pdf) @@ -411,9 +438,9 @@ def find_slices(self) -> None: """ Identify general altitude slices in the chunk data. Intended as the first stage towards the identification of cloud layers. - Note: - The "parameters" of this function are set in AMPYCLOUD_PRMS.SLICING_PRMS in the - dynamic.py module, which itself is defined in + Important: + The "parameters" of this function are set in the ``SLICING_PRMS`` entry of + :py:data:`.dynamic.AMPYCLOUD_PRMS`, which itself is (initially loaded from src/ampycloud.prms/ampycloud_default_prms.yml. """ @@ -457,9 +484,9 @@ def find_groups(self) -> None: """ Identifies groups of coherent hits accross overlapping slices. Intended as the second stage towards the identification of cloud layers. - Note: - The "parameters" of this function are set in AMPYCLOUD_PRMS.GROUPING_PRMS in the - dynamic.py module, which itself is defined in + Important: + The "parameters" of this function are set in the ``GROUPING_PRMS`` entry of + :py:data:`.dynamic.AMPYCLOUD_PRMS`, which itself is (initially loaded from src/ampycloud.prms/ampycloud_default_prms.yml. """ @@ -568,9 +595,9 @@ def find_layers(self) -> None: (if warranted) *significant* cloud sub-layers. Intended as the third stage towards the identification of cloud layers. - Note: - The "parameters" of this function are set in AMPYCLOUD_PRMS.LAYERING_PRMS in the - dynamic.py module, which itself is defined in + Important: + The "parameters" of this function are set in the ``LAYERING_PRMS`` entry of + :py:data:`.dynamic.AMPYCLOUD_PRMS`, which itself is (initially loaded from src/ampycloud.prms/ampycloud_default_prms.yml. """ @@ -659,8 +686,8 @@ def n_slices(self) -> Union[None, int]: @property def slices(self) -> pd.DataFrame: - """ Returns a pandas DataFrame with information regarding the different slices identified - by the slicing step. """ + """ Returns a :py:class:`pandas.DataFrame` with information regarding the different slices + identified by the slicing step. """ return self._slices @property @@ -679,8 +706,8 @@ def n_groups(self) -> Union[None, int]: @property def groups(self) -> pd.DataFrame: - """ Returns a pandas DataFrame with information regarding the different groups identified - by the grouping algorithm. """ + """ Returns a :py:class:`pandas.DataFrame` with information regarding the different groups + identified by the grouping algorithm. """ return self._groups @property @@ -699,27 +726,29 @@ def n_layers(self) -> Union[None, int]: @property def layers(self) -> pd.DataFrame: - """ Returns a pandas DataFrame with information regarding the different layers identified - by the layering algorithm. """ + """ Returns a :py:class:`pandas.DataFrame` with information regarding the different layers + identified by the layering algorithm. """ return self._layers - def metar_msg(self, synop : bool = False, which : str = 'layers') -> str: + def metar_msg(self, which : str = 'layers') -> str: """ Construct a METAR-like message for the identified cloud slices, groups, or layers. - The ICAO's cloud layer selection rules applicable to METARs will be applied, unless - synop = True. - - The Minimum Sector Altitude value set when the CeiloChunk instance was initialized will - be applied. - Args: - synop (bool optional): if True, all cloud layers will be reported. Else, the WMO's - cloud layer selection rules applicable to METARs will be applied. which (str, optional): whether to look at 'slices', 'groups', or 'layers'. Defaults to 'layers'. Returns: str: the METAR-like message. + + Important: + The ICAO's cloud layer selection rules applicable to METARs will be applied to create + the resulting ``str`` ! See :py:func:`.icao.significant_cloud` for details. + + .. Caution:: + The Minimum Sector Altitude value set when the :py:class:`.CeiloChunk` instance **was + initialized** will be applied ! If in doubt, the value used by this method is that set + in the (parent) class attribute :py:attr:`.AbstractChunk.msa`. + """ # Deal with the MSA: set it to infinity if None was specified @@ -738,14 +767,12 @@ def metar_msg(self, synop : bool = False, which : str = 'layers') -> str: # Deal with the situation where layers have been found ... msg = sligrolay['code'] - if synop: - report = (sligrolay['alt_base'] Config: return Config(paths=[str(Path(__file__).parent/ 'prms')], name='ampycloud_default_prms.yml') -# Load the defaults ampycloud parameters +#:dict: The ampycloud parameters, first set from a config file via :py:func:`.get_default_prms` AMPYCLOUD_PRMS = get_default_prms() diff --git a/src/ampycloud/errors.py b/src/ampycloud/errors.py index ade19e0..1260a7a 100644 --- a/src/ampycloud/errors.py +++ b/src/ampycloud/errors.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. @@ -10,7 +10,9 @@ class AmpycloudError(Exception): - """ The default error class for ampycloud, which is a child of the `Exception` class. """ + """ The default error class for ampycloud, which is a child of the :py:exc:`Exception` class. + """ class AmpycloudWarning(Warning): - """ The default warning class for ampycloud, which is a child of the `Warning` class. """ + """ The default warning class for ampycloud, which is a child of the :py:class:`Warning` class. + """ diff --git a/src/ampycloud/icao.py b/src/ampycloud/icao.py new file mode 100644 index 0000000..00b680d --- /dev/null +++ b/src/ampycloud/icao.py @@ -0,0 +1,55 @@ +""" +Copyright (c) 2022 MeteoSwiss, contributors listed in AUTHORS. + +Distributed under the terms of the 3-Clause BSD License. + +SPDX-License-Identifier: BSD-3-Clause + +Module contains: ICAO-related utilities +""" + +# Import from Python +import logging + +# Import from ampycloud +from .logger import log_func_call + +# Instantiate the module logger +logger = logging.getLogger(__name__) + +@log_func_call(logger) +def significant_cloud(oktas : list) -> list: + """ Assesses which cloud layers in a list are significant, according to the ICAO rules. + + Args: + oktas (list): the okta count of different cloud layers. **These are assumed to be sorted** + **from the lowest to the highest cloud layer !** + + Returns: + list of bool: whether a given layer is significant, or not. + + The ICAO rules applied are as follows: + + * first layer is always reported + * second layer must be SCT or more (i.e. 3 oktas or more) + * third layer must be BKN or more (i.e. 5 oktas or more) + * no more than 3 layers reported (since ampycloud does not deal with CB/TCU) + + **Source**: Sec. 4.5.4.3 e) & footnote #14 in Table A3-1, Meteorological Service for + International Air Navigation, Annex 3 to the Convention on International Civil Aviation, ICAO, + 20th edition, July 2018. + + """ + + sig_level = 0 + sig = [] + for okta in oktas: + # There can be no more than 3 significant cloud layers. + # See that footnote 14 in the ICAO doc ! + if okta > sig_level and sig.count(True) < 3: + sig_level += 2 + sig += [True] + else: + sig += [False] + + return sig diff --git a/src/ampycloud/layer.py b/src/ampycloud/layer.py index df628e9..35f29d5 100644 --- a/src/ampycloud/layer.py +++ b/src/ampycloud/layer.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-CLause BSD License. @@ -20,6 +20,7 @@ from .errors import AmpycloudError, AmpycloudWarning from .logger import log_func_call from .scaler import minmax_scaling +from .utils import utils # Instantiate the module logger logger = logging.getLogger(__name__) @@ -158,9 +159,9 @@ def ncomp_from_gmm(vals : np.ndarray, Akaike Information criterion scores. rescale_0_to_x (float, optional): if set, vals will be rescaled between 0 and this value before running the Gaussian Mixture Modelling. Defaults to None = no rescaling. - random_seed (int, optional): a value fed to numpy.random.seed to ensure repeatable - results. Defaults to 42, because it is the Answer to the Ultimate Question of Life, the - Universe, and Everything. + random_seed (int, optional): used to reset **temporarily** the value of + :py:func:`numpy.random.seed` to ensure repeatable results. Defaults to 42, because it + is the Answer to the Ultimate Question of Life, the Universe, and Everything. **kwargs (dict, optional): these will be fed to `best_gmm()`. Returns: @@ -172,8 +173,6 @@ def ncomp_from_gmm(vals : np.ndarray, ``_ """ - # To ensure repeatability of the results, let's define a seed here. - np.random.seed(random_seed) # If I get a 1-D array, deal with it. if np.ndim(vals) == 1: @@ -205,8 +204,9 @@ def ncomp_from_gmm(vals : np.ndarray, models = {} # Run the Gaussian Mixture fit for all cases ... should we do anything more fancy here ? - for n_val in ncomp: - models[n_val] = GaussianMixture(n_val, covariance_type='spherical').fit(vals) + with utils.tmp_seed(random_seed): + for n_val in ncomp: + models[n_val] = GaussianMixture(n_val, covariance_type='spherical').fit(vals) # Extract the AICS and BICS scores if scores == 'AIC': diff --git a/src/ampycloud/logger.py b/src/ampycloud/logger.py index 2fc9fbe..5a7500a 100644 --- a/src/ampycloud/logger.py +++ b/src/ampycloud/logger.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. diff --git a/src/ampycloud/plots/__init__.py b/src/ampycloud/plots/__init__.py index 83d1110..c5c082e 100644 --- a/src/ampycloud/plots/__init__.py +++ b/src/ampycloud/plots/__init__.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. diff --git a/src/ampycloud/plots/core.py b/src/ampycloud/plots/core.py index 8ac87ad..97602c9 100644 --- a/src/ampycloud/plots/core.py +++ b/src/ampycloud/plots/core.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. @@ -16,7 +16,7 @@ from ..data import CeiloChunk from ..logger import log_func_call from .diagnostics import DiagnosticPlot -from .utils import set_mplstyle +from .tools import set_mplstyle # Instantiate the module logger logger = logging.getLogger(__name__) @@ -83,7 +83,7 @@ def diagnostic(chunk : CeiloChunk, upto : str = 'layers', show_ceilos : bool = F adp.format_group_axes() if upto == 'layers': adp.show_layers() - adp.add_metar(synop=False) + adp.add_metar() # And add all the common stuff adp.add_ref_metar(ref_metar_origin, ref_metar) diff --git a/src/ampycloud/plots/diagnostics.py b/src/ampycloud/plots/diagnostics.py index ce8b98d..62fa7a2 100644 --- a/src/ampycloud/plots/diagnostics.py +++ b/src/ampycloud/plots/diagnostics.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. @@ -21,7 +21,7 @@ # Import from this package from ..scaler import scaling from .hardcoded import WIDTH_TWOCOL, MRKS -from .utils import texify +from .tools import texify from .. import wmo from ..data import CeiloChunk from .. import dynamic @@ -31,13 +31,13 @@ logger = logging.getLogger(__name__) class DiagnosticPlot: - """ Class used to create diagnsotic plots """ + """ Class used to create diagnostic plots. """ def __init__(self, chunk : CeiloChunk) -> None: """ The init function. Args: - chunk (CeiloChunk): A ceilometer Data Chunk. + :py:class:`ampycloud.data.CeiloChunk`: a ceilometer Data Chunk. """ @@ -91,12 +91,13 @@ def new_fig(self) -> None: def show_hits_only(self, show_ceilos : bool = False) -> None: """ Shows the ceilometer hits alone. - Note: - This will clear the plot first ! - Args: show_ceilos (bool, optional): whether to distinguish between the different ceilos, or not. + + Important: + This will clear the plot first ! + """ # Let's create an array of colors for *every* (sigh) point ... @@ -142,7 +143,7 @@ def show_hits_only(self, show_ceilos : bool = False) -> None: title='Ceilo. names') def show_slices(self) -> None: - """ Show the slices data. """ + """ Show the slice data. """ # Let's start by cleaning the plotting area self._axs[0].clear() @@ -390,15 +391,11 @@ def add_ref_metar(self, name : str, metar : str) -> None: # boxstyle='round, pad=0.25') ) - def add_metar(self, synop : bool = False) -> None: - """ Display the ampycloud METAR/SYNOP message. - - Args: - synop (bool, optional): If True, will display the full SYNOP message. Defaults to False. - """ + def add_metar(self) -> None: + """ Display the ampycloud METAR message.""" # Combine it all in one message - msg = r'\smaller \bf ampycloud: ' + self._chunk.metar_msg(synop=synop) + msg = r'\smaller \bf ampycloud: ' + self._chunk.metar_msg() if self._chunk.msa is not None: msg += '\n'+r'\smaller\smaller MSA: {} ft'.format(self._chunk.msa) @@ -418,7 +415,12 @@ def format_primary_axes(self) -> None: self._axs[0].set_ylabel(r'Alt. [ft]', labelpad=10) def format_slice_axes(self) -> None: - """ Format the duplicate axes related to the slicing part. """ + """ Format the duplicate axes related to the slicing part. + + Todo: + Cleanup the code once #25 is fixed. + + """ # First, get the scaling parameters, and switch them over to a 'descale' mode ... # TODO: Once issue #25 is fixed, maybe update these lines ... @@ -466,7 +468,12 @@ def format_slice_axes(self) -> None: secax_y.tick_params(axis='y', which='both', labelsize=rcParams['font.size']-2) def format_group_axes(self) -> None: - """ Format the duplicate axes related to the grouping part. """ + """ Format the duplicate axes related to the grouping part. + + Todo: + Cleanup the code once #25 is fixed. + + """ # First, get the scaling parameters, and switch them over to a 'descale' mode ... # TODO: once issue #25 is fixed, maybe update these lines ... diff --git a/src/ampycloud/plots/hardcoded.py b/src/ampycloud/plots/hardcoded.py index b824294..06d535e 100644 --- a/src/ampycloud/plots/hardcoded.py +++ b/src/ampycloud/plots/hardcoded.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. diff --git a/src/ampycloud/plots/secondary.py b/src/ampycloud/plots/secondary.py index 95d92fb..2d17f0a 100644 --- a/src/ampycloud/plots/secondary.py +++ b/src/ampycloud/plots/secondary.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. @@ -19,7 +19,7 @@ from ..scaler import scaling from .. import dynamic from .hardcoded import WIDTH_TWOCOL -from .utils import texify, set_mplstyle +from .tools import texify, set_mplstyle # Instantiate the module logger logger = logging.getLogger(__name__) diff --git a/src/ampycloud/plots/utils.py b/src/ampycloud/plots/tools.py similarity index 92% rename from src/ampycloud/plots/utils.py rename to src/ampycloud/plots/tools.py index 6d01854..66ac9f0 100644 --- a/src/ampycloud/plots/utils.py +++ b/src/ampycloud/plots/tools.py @@ -1,11 +1,11 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. SPDX-License-Identifier: BSD-3-Clause -Module contains: utilities functions for plots +Module contains: tools for plots """ # Import from Python @@ -39,13 +39,13 @@ def set_mplstyle(func : Callable) -> Callable: """ Intended to be used as a decorator around plotting functions, to set the plotting style. By defaults, the ``base`` ampycloud style will be enabled. Motivated users can tweak it further - by setting the ``dynamic.AMPYCLOUD_PRMS.MPL_STYLE`` keyword argument to: + by setting the ``MPL_STYLE`` entry of :py:data:`ampycloud.dynamic.AMPYCLOUD_PRMS` to: - ``latex``: to enable the use of a system-wide LaTeX engine, and the Computer Modern font. - ``metsymb``: to enable the use of a system-wide LaTeX engine, the Computer Modern font, and the ``metsymb`` LaTeX package to display proper okta symbols. - Note: + Important: The ``metsymb`` LaTeX package is NOT included with ampycloud, and must be installed separately. It is available at: https://github.com/MeteoSwiss/metsymb @@ -58,10 +58,11 @@ def set_mplstyle(func : Callable) -> Callable: - ``amsmath`` - ``amssymb`` - ``relsize`` - - ``metsymb`` (only if ``dynamic.AMPYCLOUD_PRMS.MPL_STYLE='metsymb'``) + - ``metsymb`` (only if the ``MPL_STYLE`` entry of + :py:data:`ampycloud.dynamic.AMPYCLOUD_PRMS` was set to ``'metsymb'``) Returns: - Callable: the decorator + Callable: the decorator. Todo: See https://github.com/MeteoSwiss/ampycloud/issues/18 diff --git a/src/ampycloud/prms/ampycloud_default_prms.yml b/src/ampycloud/prms/ampycloud_default_prms.yml index 9b1dc67..e5d54fa 100644 --- a/src/ampycloud/prms/ampycloud_default_prms.yml +++ b/src/ampycloud/prms/ampycloud_default_prms.yml @@ -23,7 +23,7 @@ OKTA_LIM8: 98 LAYER_BASE_FRAC: 0.1 # Minimum Sector Altitude, in ft. No cloud layers with a base above this value will be reported -# in the ampycloud METAR/SYNOP messages. Set it to null for no limit. +# in the ampycloud METAR messages. Set it to null for no limit. MSA: null # Additional distance above the MSA value (in ft) beyond which hits will be ignored (=cropped) by diff --git a/src/ampycloud/scaler.py b/src/ampycloud/scaler.py index 05e06bd..3455c60 100644 --- a/src/ampycloud/scaler.py +++ b/src/ampycloud/scaler.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. diff --git a/src/ampycloud/utils/__init__.py b/src/ampycloud/utils/__init__.py index b4bf6df..17e5924 100644 --- a/src/ampycloud/utils/__init__.py +++ b/src/ampycloud/utils/__init__.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. diff --git a/src/ampycloud/utils/mocker.py b/src/ampycloud/utils/mocker.py index 76e789a..c20306d 100644 --- a/src/ampycloud/utils/mocker.py +++ b/src/ampycloud/utils/mocker.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. @@ -17,41 +17,37 @@ # import from ampycloud from ..logger import log_func_call from ..errors import AmpycloudError +from . import utils # Instantiate the module logger logger = logging.getLogger(__name__) -# Define a proper random number generator -np.random.seed(42) - @log_func_call(logger) -def flat_layer(alt : float, alt_std : float, lookback_time : float, - hit_rate : float, sky_cov_frac : float) -> pd.DataFrame: +def flat_layer(dts : np.array, alt : float, alt_std : float, sky_cov_frac : float) -> pd.DataFrame: """ Generates a mock, flat, Gaussian cloud layer around a given altitude. Args: + dts (np.array of float): time deltas, in s, for the simulated ceilometer hits. alt (float): layer mean altitude, in ft. alt_std (float): layer altitude standard deviation, in ft. - lookback_time (float): length of the time interval, in s. - hit_rate (float): rate of data acquisition, in s. sky_cov_frac (float): Sky coverage fraction. Random hits will be set to NaN to reach this value. Must be 0 <= x <= 1. Returns: - pd.DataFrame: the simulated layer with columns ['alt', 'dt']. + :py:class:`pandas.DataFrame`: the simulated layer with columns ['dt', 'alt']. """ # How many points do I need to generate ? - n_pts = int(np.ceil(lookback_time/hit_rate)) + n_pts = len(dts) # Create the storage structure - out = pd.DataFrame(columns=['alt', 'dt'], dtype=float) + out = pd.DataFrame(columns=['dt', 'alt'], dtype=float) # Generate the random altitude data out['alt'] = np.random.normal(loc=alt, scale=alt_std, size=n_pts) # Cleanup any negative altitudes, if warranted. out.loc[out['alt']<=0, 'alt'] = np.nan - out['dt'] = np.random.random(n_pts) * -lookback_time + out['dt'] = dts # Empty hits to get the requested sky coverage fraction # First extract the hits I want to keep ... @@ -64,27 +60,25 @@ def flat_layer(alt : float, alt_std : float, lookback_time : float, return out @log_func_call(logger) -def sin_layer(alt : float, alt_std : float, lookback_time : float, - hit_rate : float, sky_cov_frac : float, +def sin_layer(dts : np.array, alt : float, alt_std : float, sky_cov_frac : float, period : Union[int, float], amplitude : Union[int, float]) -> pd.DataFrame: """ Generates a sinusoidal cloud layer. Args: + dts (np.array of float): time deltas, in s, for the simulated ceilometer hits. alt (float): layer mean altitude, in ft. alt_std (float): layer altitude standard deviation, in ft. - lookback_time (float): length of the time interval, in s. - hit_rate (float): rate of data acquisition. sky_cov_frac (float, optional): Sky coverage fraction. Random hits will be set to NaN to reach this value. Must be 0 <= x <= 1. period (int|float): period of the sine-wave, in s. amplitude (int|float): amplitude of the sine-wave, in ft. Returns: - pd.DataFrame: the simulated layer with columns ['alt', 'dt']. + :py:class:`pandas.DataFrame`: the simulated layer with columns ['alt', 'dt']. """ # First, get a flat layer - out = flat_layer(alt, alt_std, lookback_time, hit_rate, sky_cov_frac) + out = flat_layer(dts, alt, alt_std, sky_cov_frac) # And add to it a sinusoidal fluctuations. Note that nan should stay nan. out.loc[:, 'alt'] = out.loc[:, 'alt'] + np.sin(-np.pi/2 + out['dt']/period*2*np.pi) * amplitude @@ -92,57 +86,78 @@ def sin_layer(alt : float, alt_std : float, lookback_time : float, return out -def mock_layers(n_ceilos : int, layer_prms : list) -> pd.DataFrame: +def mock_layers(n_ceilos : int, lookback_time : float, hit_gap: float, + layer_prms : list) -> pd.DataFrame: """ Generate a mock set of cloud layers for a specified number of ceilometers. - TODO: - - add the possibility to set some VV hits in the mix - - add the possibility to have multiple hits at the same time steps - - all of this could be done much more professionally with classes ... - Args: n_ceilos (int): number of ceilometers to simulate. + lookback_time (float): length of the time interval, in s. + hit_gap (float): number of seconds between ceilometer measurements. layer_prms (list of dict): list of layer parameters, provided as a dict for each layer. - Each dict should specify all the parameters required to generate a sin layer, e.g.: + Each dict should specify all the parameters required to generate a + :py:func:`.sin_layer` (with the exception of ``dts`` that will be computed directly + from ``lookback_time`` and ``hit_gap``): :: - {'alt':1000, 'alt_std': 100, 'lookback_time' : 1200, - 'hit_rate': 60, 'sky_cov_frac': 1, + {'alt':1000, 'alt_std': 100, 'sky_cov_frac': 1, 'period': 100, 'amplitude': 0} Returns: - DataFrame: a pandas DataFrame with the mock data, ready to be fed to ampycloud. Columns - ['ceilo', 'dt', 'alt', 'type'] correspond to 1) ceilo names, 2) time deltas in s, - 3) hit altitudes in ft, and 4) hit type. + :py:class:`pandas.DataFrame`: a pandas DataFrame with the mock data, ready to be fed to + ampycloud. Columns ['ceilo', 'dt', 'alt', 'type'] correspond to 1) ceilo names, 2) time + deltas in s, 3) hit altitudes in ft, and 4) hit type. + + TODO: + - add the possibility to set some VV hits in the mix + - all of this could be done much more professionally with classes ... """ - # A simple sanity check of the input type, since it is a bit convoluted. + # A sanity check of the input type, since it is a bit convoluted. if not isinstance(layer_prms, list): raise AmpycloudError(f'Ouch ! layer_prms should be a list, not: {type(layer_prms)}') - for (ind, item) in enumerate(layer_prms): if not isinstance(item, dict): raise AmpycloudError(f'Ouch ! Element {ind} from layer_prms should be a dict,' + f' not: {type(item)}') - if not all(key in item.keys() for key in ['alt', 'alt_std', 'lookback_time', 'hit_rate', + if not all(key in item.keys() for key in ['alt', 'alt_std', 'sky_cov_frac', 'period', 'amplitude']): raise AmpycloudError('Ouch ! One or more of the following dict keys are missing in '+ - f"layer_prms[{ind}]: 'alt', 'alt_std', 'lookback_time',"+ - "'hit_rate','period', 'amplitude'.") + f"layer_prms[{ind}]: 'alt', 'alt_std', 'sky_cov_frac',"+ + "'period', 'amplitude'.") - # Let's create the layers individually for eahc ceilometer + # Let's create the layers individually for each ceilometer ceilos = [] for ceilo in range(n_ceilos): - # Let's now lopp through each cloud layer and generate them - layers = [sin_layer(**prms) for prms in layer_prms] + # Let's compute the time steps + n_pts = int(np.ceil(lookback_time/hit_gap)) + dts = np.random.random(n_pts) * -lookback_time + + # Let's now loop through each cloud layer and generate them + layers = [sin_layer(dts=dts, **prms) for prms in layer_prms] - # Merge them all into one ... + # Merge them all into one DataFrame ... layers = pd.concat(layers).reset_index(drop=True) + # Add the type column while I'm at it. Set it to None for now. + layers['type'] = None + + # Here, adjust the types so that it ranks lowest to highest for every dt step. + # This needs to be done on a point by point basis, given that layers can cross each other. + for dt in np.unique(layers['dt']): + # Get the hit altitudes, and sort them from lowest to highest + alts = layers[layers['dt']==dt]['alt'].sort_values(axis=0) - # Add the type column - layers['type'] = 99 + # Then deal with the other ones + for (a, alt) in enumerate(alts): + + # Except for the first one, any NaN hit gets dropped + if a>0 and np.isnan(alt): + layers.drop(index=alts.index[a], + inplace=True) + else: + layers.loc[alts.index[a], 'type'] = a+1 # Add the ceilo info as an int layers['ceilo'] = str(ceilo) @@ -153,7 +168,7 @@ def mock_layers(n_ceilos : int, layer_prms : list) -> pd.DataFrame: # Merge it all out = pd.concat(ceilos) # Sort the timesteps in order, and reset the index - out = out.sort_values('dt').reset_index(drop=True) + out = out.sort_values(['dt', 'alt']).reset_index(drop=True) # Fix the dtypes out.loc[:, 'dt'] = out['dt'].astype(float) @@ -166,26 +181,26 @@ def mock_layers(n_ceilos : int, layer_prms : list) -> pd.DataFrame: def canonical_demo_data() -> pd.DataFrame: """ This function creates the canonical ampycloud demonstration dataset, that can be used to illustrate the full behavior of the algorithm. + + Returns: + :py:class:`pandas.DataFrame`: the canonical mock dataset with properly-formatted columns. + """ # Create the "famous" mock dataset n_ceilos = 4 lookback_time = 1200 - hit_rate = 30 - - lyrs = [{'alt': 1000, 'alt_std': 100, 'lookback_time': lookback_time, 'hit_rate': hit_rate, - 'sky_cov_frac': 0.8, 'period': 10, 'amplitude': 0}, - {'alt': 2000, 'alt_std': 100, 'lookback_time': lookback_time, 'hit_rate': hit_rate, - 'sky_cov_frac': 0.5, 'period': 10, 'amplitude': 0}, - {'alt': 5000, 'alt_std': 300, 'lookback_time': lookback_time, 'hit_rate': hit_rate, - 'sky_cov_frac': 1, 'period': 2400, 'amplitude': 1400}, - {'alt': 5000, 'alt_std': 300, 'lookback_time': lookback_time, 'hit_rate': hit_rate, - 'sky_cov_frac': 1, 'period': 2400, 'amplitude': 1400}, - {'alt': 5100, 'alt_std': 500, 'lookback_time': lookback_time, 'hit_rate': hit_rate, - 'sky_cov_frac': 1, 'period': 10, 'amplitude': 0}, - {'alt': 5100, 'alt_std': 500, 'lookback_time': lookback_time, 'hit_rate': hit_rate, - 'sky_cov_frac': 1, 'period': 10, 'amplitude': 0} + hit_gap = 30 + + lyrs = [{'alt': 1000, 'alt_std': 100, 'sky_cov_frac': 0.1, 'period': 10, 'amplitude': 0}, + {'alt': 2000, 'alt_std': 100, 'sky_cov_frac': 0.5, 'period': 10, 'amplitude': 0}, + {'alt': 5000, 'alt_std': 200, 'sky_cov_frac': 1, 'period': 2400, 'amplitude': 1000}, ] - # Actually generate the mock data - return mock_layers(n_ceilos, lyrs) + + # Reset the random seed, but only do this temporarily, so as to not mess things up for the user. + with utils.tmp_seed(42): + # Actually generate the mock data + out = mock_layers(n_ceilos, lookback_time, hit_gap, lyrs) + + return out diff --git a/src/ampycloud/utils/performance.py b/src/ampycloud/utils/performance.py index 09b2c5a..80e4c34 100644 --- a/src/ampycloud/utils/performance.py +++ b/src/ampycloud/utils/performance.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. @@ -22,8 +22,8 @@ @log_func_call(logger) def get_speed_benchmark(niter : int = 10) -> tuple: - """ This function will run and time the ampycloud demo to assess the code's performance on - given machine. + """ This function will run and time :py:func:`ampycloud.core.demo` to assess the code's + performance on a given machine. For now, this is a rather dumb and uninspired way to do it. If the need ever arises, this could certainly be done better, and (for example) also with a finer step resolution to see @@ -31,7 +31,7 @@ def get_speed_benchmark(niter : int = 10) -> tuple: the mock dataset from its processing. Returns: - int, float, float, float, float, float: niter, mean, std, median, min, max + int, float, float, float, float, float: niter, mean, std, median, min, max: """ diff --git a/src/ampycloud/utils/utils.py b/src/ampycloud/utils/utils.py new file mode 100644 index 0000000..d64913a --- /dev/null +++ b/src/ampycloud/utils/utils.py @@ -0,0 +1,46 @@ +""" +Copyright (c) 2022 MeteoSwiss, contributors listed in AUTHORS. + +Distributed under the terms of the 3-Clause BSD License. + +SPDX-License-Identifier: BSD-3-Clause + +Module contains: generic utilities +""" + +# Import from Python +import logging +import contextlib +import numpy as np + +# Instantiate the module logger +logger = logging.getLogger(__name__) + +@contextlib.contextmanager +def tmp_seed(seed : int): + """ Temporarily reset the :py:func:`numpy.random.seed` value. + + Adapted from the reply of Paul Panzer on `SO `__. + + Example: + :: + + with temp_seed(42): + np.random.random(1) + + """ + + # Add a note in the logs about what is going on + logger.debug('Setting a temporary np.random.seed with value %i', seed) + + # Get the current seed + state = np.random.get_state() + + # Reset it with the temporary one + np.random.seed(seed) + + # Execute stuff, and reset the original seed once all is over. + try: + yield + finally: + np.random.set_state(state) diff --git a/src/ampycloud/version.py b/src/ampycloud/version.py index 4f9e2b0..5379b78 100644 --- a/src/ampycloud/version.py +++ b/src/ampycloud/version.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. @@ -8,4 +8,5 @@ Module contains: ampycloud version """ -VERSION = '0.1.0' +#:str: the one-and-only place where the ampycloud version is set. +VERSION = '0.2.0.dev0' diff --git a/src/ampycloud/wmo.py b/src/ampycloud/wmo.py index 7d6082f..1446c68 100644 --- a/src/ampycloud/wmo.py +++ b/src/ampycloud/wmo.py @@ -1,11 +1,11 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. SPDX-License-Identifier: BSD-3-Clause -Module contains: wmo-related utilities +Module contains: WMO-related utilities """ # Import from Python diff --git a/test/__init__.py b/test/__init__.py index fde7ae8..6982f03 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. diff --git a/test/ampycloud/__init__.py b/test/ampycloud/__init__.py index fde7ae8..6982f03 100644 --- a/test/ampycloud/__init__.py +++ b/test/ampycloud/__init__.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. diff --git a/test/ampycloud/plots/__init__.py b/test/ampycloud/plots/__init__.py new file mode 100644 index 0000000..6982f03 --- /dev/null +++ b/test/ampycloud/plots/__init__.py @@ -0,0 +1,7 @@ +""" +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. + +Distributed under the terms of the 3-Clause BSD License. + +SPDX-License-Identifier: BSD-3-Clause +""" diff --git a/test/ampycloud/plots/test_core.py b/test/ampycloud/plots/test_core.py index 3eb1cf6..58f0cfc 100644 --- a/test/ampycloud/plots/test_core.py +++ b/test/ampycloud/plots/test_core.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. @@ -26,6 +26,8 @@ def test_diagnostic(mpls): See conftest.py for details. """ + reset_prms() + if mpls: dynamic.AMPYCLOUD_PRMS.MPL_STYLE = mpls @@ -41,8 +43,8 @@ def test_diagnostic(mpls): # Create the diagnsotic plots at the four different upto levels for sufx in sufxs: diagnostic(chunk, upto=sufx, show_ceilos=True, show=False, - save_stem=base_name+sufx, save_fmts='pdf', - ref_metar_origin='Mock data', ref_metar='FEW008 BKN037') + save_stem=base_name+sufx, save_fmts='png', + ref_metar_origin='Mock data', ref_metar='FEW009 SCT018 BKN038') assert Path(base_name+sufx+'.pdf').exists diff --git a/test/ampycloud/plots/test_utils.py b/test/ampycloud/plots/test_tools.py similarity index 74% rename from test/ampycloud/plots/test_utils.py rename to test/ampycloud/plots/test_tools.py index 0009a46..9e0fb83 100644 --- a/test/ampycloud/plots/test_utils.py +++ b/test/ampycloud/plots/test_tools.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. @@ -9,7 +9,7 @@ """ # Import from ampycloud -from ampycloud.plots.utils import valid_styles +from ampycloud.plots.tools import valid_styles def test_valid_styles(): diff --git a/test/ampycloud/test_cluster.py b/test/ampycloud/test_cluster.py index eedacbc..34cd46a 100644 --- a/test/ampycloud/test_cluster.py +++ b/test/ampycloud/test_cluster.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. diff --git a/test/ampycloud/test_core.py b/test/ampycloud/test_core.py index 133d10b..00e5f94 100644 --- a/test/ampycloud/test_core.py +++ b/test/ampycloud/test_core.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. @@ -19,7 +19,7 @@ from ampycloud import dynamic, reset_prms from ampycloud.utils import mocker from ampycloud.data import CeiloChunk -from ampycloud.core import copy_prm_file, reset_prms, run, synop, metar, demo +from ampycloud.core import copy_prm_file, reset_prms, run, metar, demo def test_copy_prm_file(): """ Test the copy_prm_file routine.""" @@ -60,24 +60,19 @@ def test_run(): # Create some fake data to get started # 1 very flat layer with no gaps - mock_data = mocker.mock_layers(n_ceilos, - [{'alt':1000, 'alt_std': 10, 'lookback_time' : lookback_time, - 'hit_rate': rate, 'sky_cov_frac': 0.5, + mock_data = mocker.mock_layers(n_ceilos, lookback_time, rate, + [{'alt':1000, 'alt_std': 10, 'sky_cov_frac': 0.5, 'period': 100, 'amplitude': 0}, - {'alt':2000, 'alt_std': 10, 'lookback_time' : lookback_time, - 'hit_rate': rate, 'sky_cov_frac': 0.5, + {'alt':2000, 'alt_std': 10, 'sky_cov_frac': 0.5, 'period': 100, 'amplitude': 0}]) out = run(mock_data) assert isinstance(out, CeiloChunk) - assert out.metar_msg(synop=True) == 'FEW009 FEW019' - assert out.metar_msg(synop=False) == 'FEW009' + assert out.metar_msg() == 'SCT009 SCT019' - # While I'm at it, also check the metar and synop routines, that are so close - - assert synop(mock_data) == 'FEW009 FEW019' - assert metar(mock_data) == 'FEW009' + # While I'm at it, also check the metar routines, that are so close + assert metar(mock_data) == 'SCT009 SCT019' def test_run_single_point(): """ Test the code when a single data point is fed to it. """ diff --git a/test/ampycloud/test_data.py b/test/ampycloud/test_data.py index 5823104..8380a1d 100644 --- a/test/ampycloud/test_data.py +++ b/test/ampycloud/test_data.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. @@ -27,10 +27,11 @@ def test_ceilochunk_init(): # Create some fake data to get started # 1 very flat layer with no gaps - mock_data = mocker.mock_layers(n_ceilos, - [{'alt':1000, 'alt_std': 10, 'lookback_time' : lookback_time, - 'hit_rate': rate, 'sky_cov_frac': 1, + mock_data = mocker.mock_layers(n_ceilos, lookback_time, rate, + [{'alt':1000, 'alt_std': 10, 'sky_cov_frac': 1, 'period': 100, 'amplitude': 0}]) + # The following line is required as long as the mocker module issues mock data with type 99. + mock_data.iloc[-1, mock_data.columns.get_loc('type')] = 1 # Instantiate a CeiloChunk entity ... chunk = CeiloChunk(mock_data) @@ -41,7 +42,9 @@ def test_ceilochunk_init(): dynamic.AMPYCLOUD_PRMS.MSA = 0 dynamic.AMPYCLOUD_PRMS.MSA_HIT_BUFFER = 0 chunk = CeiloChunk(mock_data) - assert len(chunk.data) == 0 + # Applying the MSA should crop any Type 2 or more hits, and change Type 1 or less to Type 0. + assert len(chunk.data) == len(mock_data[mock_data.type <=1]) + assert not np.any(chunk.data.type>0) # Verify the class MSA value is correct too ... assert chunk.msa == 0 @@ -51,6 +54,10 @@ def test_ceilochunk_init(): chunk = CeiloChunk(mock_data) assert len(chunk.data) == len(mock_data) + # Check that feeding an empty dataframe crashes properly + with raises(AmpycloudError): + _ = CeiloChunk(mock_data[:0]) + # Let's not forget to reset the dynamic parameters to not mess up the other tests reset_prms() @@ -63,9 +70,8 @@ def test_ceilochunk_basic(): # Create some fake data to get started # 1 very flat layer with no gaps - mock_data = mocker.mock_layers(n_ceilos, - [{'alt':1000, 'alt_std': 10, 'lookback_time' : lookback_time, - 'hit_rate': rate, 'sky_cov_frac': 1, + mock_data = mocker.mock_layers(n_ceilos, lookback_time, rate, + [{'alt':1000, 'alt_std': 10, 'sky_cov_frac': 1, 'period': 100, 'amplitude': 0}]) # Instantiate a CeiloChunk entity ... @@ -111,7 +117,7 @@ def test_ceilochunk_basic(): # Assert the METAR-like message assert chunk.metar_msg() == 'OVC009' - assert chunk.metar_msg(which='groups', synop=True) == 'OVC009' + assert chunk.metar_msg(which='groups') == 'OVC009' def test_ceilochunk_nocld(): """ Test the methods of CeiloChunks when no clouds are seen in the interval. """ @@ -122,9 +128,8 @@ def test_ceilochunk_nocld(): # Create some fake data to get started # 1 very flat layer with no gaps - mock_data = mocker.mock_layers(n_ceilos, - [{'alt':1000, 'alt_std': 10, 'lookback_time' : lookback_time, - 'hit_rate': rate, 'sky_cov_frac': 0, + mock_data = mocker.mock_layers(n_ceilos, lookback_time, rate, + [{'alt':1000, 'alt_std': 10, 'sky_cov_frac': 0, 'period': 100, 'amplitude': 0}]) # Instantiate a CeiloChunk entity ... @@ -146,13 +151,10 @@ def test_ceilochunk_2lay(): rate = 30 # Create some fake data to get started - # 1 very flat layer with no gaps - mock_data = mocker.mock_layers(n_ceilos, - [{'alt':1000, 'alt_std': 10, 'lookback_time' : lookback_time, - 'hit_rate': rate, 'sky_cov_frac': 0.5, + mock_data = mocker.mock_layers(n_ceilos, lookback_time, rate, + [{'alt':1000, 'alt_std': 10, 'sky_cov_frac': 0.5, 'period': 100, 'amplitude': 0}, - {'alt':2000, 'alt_std': 10, 'lookback_time' : lookback_time, - 'hit_rate': rate, 'sky_cov_frac': 0.5, + {'alt':2000, 'alt_std': 10, 'sky_cov_frac': 0.5, 'period': 100, 'amplitude': 0}]) # Instantiate a CeiloChunk entity ... @@ -164,5 +166,4 @@ def test_ceilochunk_2lay(): chunk.find_layers() # Assert the final METAR code is correct - assert chunk.metar_msg() == 'FEW009' - assert chunk.metar_msg(synop=True) == 'FEW009 FEW019' + assert chunk.metar_msg() == 'SCT009 SCT019' diff --git a/test/ampycloud/test_dynamic.py b/test/ampycloud/test_dynamic.py index 7afd330..68457df 100644 --- a/test/ampycloud/test_dynamic.py +++ b/test/ampycloud/test_dynamic.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. @@ -35,14 +35,11 @@ def test_dynamic_module(): assert tmp == 'bad' # Also test with new dictionnary keys, that seems to behave differently - # TODO: if issue #24 gets fixed, this may need to be adjusted ... tmp = {} tmp.update(dynamic.AMPYCLOUD_PRMS.SLICING_PRMS.dt_scale_kwargs) tmp['new_entry'] = 'rubbish' - assert 'new_entry' in tmp.keys() assert 'new_entry' not in dynamic.AMPYCLOUD_PRMS.SLICING_PRMS.dt_scale_kwargs.keys() - # Reset everything so as to not break havoc with the other tests reset_prms() diff --git a/test/ampycloud/test_icao.py b/test/ampycloud/test_icao.py new file mode 100644 index 0000000..b6c5a89 --- /dev/null +++ b/test/ampycloud/test_icao.py @@ -0,0 +1,22 @@ +""" +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. + +Distributed under the terms of the 3-Clause BSD License. + +SPDX-License-Identifier: BSD-3-Clause + +Module content: tests for the icao module +""" + +# Import from this package +from ampycloud.icao import significant_cloud + +def test_significant_cloud(): + """ Test the significant_cloud function. """ + + assert isinstance(significant_cloud([0]), list) + assert significant_cloud([6, 6, 6]) == [True, True, True] + assert significant_cloud([6, 6, 6, 6, 6]) == [True, True, True, False, False] + assert significant_cloud([6, 1, 1, 6, 6]) == [True, False, False, True, True] + assert significant_cloud([6, 1, 3, 6, 6]) == [True, False, True, True, False] + assert significant_cloud([6, 5, 3]) == [True, True, False] diff --git a/test/ampycloud/test_layer.py b/test/ampycloud/test_layer.py index 6621013..5089683 100644 --- a/test/ampycloud/test_layer.py +++ b/test/ampycloud/test_layer.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. @@ -47,6 +47,8 @@ def test_ncomp_from_gmm(): # Generate random data with a normal distribution, and offset by 5-sigma each. # This is a level at which I should always be able to find the correct number of components. + # Set a random seed so I always get the same data + np.random.seed(42) comp1 = np.random.randn(100) comp2 = np.random.randn(100) + 5 comp3 = np.random.randn(100) + 10 diff --git a/test/ampycloud/test_scaler.py b/test/ampycloud/test_scaler.py index 2b5d36c..38f3cfe 100644 --- a/test/ampycloud/test_scaler.py +++ b/test/ampycloud/test_scaler.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. diff --git a/test/ampycloud/test_scientific_stability.py b/test/ampycloud/test_scientific_stability.py index 604418b..b672ad0 100644 --- a/test/ampycloud/test_scientific_stability.py +++ b/test/ampycloud/test_scientific_stability.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. diff --git a/test/ampycloud/test_version.py b/test/ampycloud/test_version.py index db8676b..c1a5ecd 100644 --- a/test/ampycloud/test_version.py +++ b/test/ampycloud/test_version.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. @@ -8,11 +8,19 @@ Module content: tests for the version module """ +# Import from Python +from pkg_resources import parse_version + # Import from this package from ampycloud.version import VERSION + def test_version(): """ Test the format of the code version.""" assert isinstance(VERSION, str) - assert len(VERSION.split('.'))==3 + # Here, let's make sure the version is valid. One way to check this is to make sure that it is + # not converted into a LegacyVersion. Any valid version should be greater than 0. + # Only LegacyVersion wouldn't. + # Not the most elegant, but better than nothing. + assert parse_version(VERSION) > parse_version('0') diff --git a/test/ampycloud/test_wmo.py b/test/ampycloud/test_wmo.py index 5a16ae4..48e763d 100644 --- a/test/ampycloud/test_wmo.py +++ b/test/ampycloud/test_wmo.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. diff --git a/test/ampycloud/utils/__init__.py b/test/ampycloud/utils/__init__.py new file mode 100644 index 0000000..6982f03 --- /dev/null +++ b/test/ampycloud/utils/__init__.py @@ -0,0 +1,7 @@ +""" +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. + +Distributed under the terms of the 3-Clause BSD License. + +SPDX-License-Identifier: BSD-3-Clause +""" diff --git a/test/ampycloud/utils/test_mocker.py b/test/ampycloud/utils/test_mocker.py index 212832f..bfa666c 100644 --- a/test/ampycloud/utils/test_mocker.py +++ b/test/ampycloud/utils/test_mocker.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. @@ -21,10 +21,11 @@ def test_mock_layers(): # Basic test with 1 ceilo and 1 flat layer n_ceilos = 1 - layer_prms =[{'alt':1000, 'alt_std': 100, 'lookback_time' : 1200, - 'hit_rate': 60, 'sky_cov_frac': 1, + lookback_time = 1200 + hit_gap = 60 + layer_prms =[{'alt':1000, 'alt_std': 100, 'sky_cov_frac': 1, 'period': 100, 'amplitude': 0}] - out = mock_layers(n_ceilos, layer_prms) + out = mock_layers(n_ceilos, lookback_time, hit_gap, layer_prms) # Correct type ? assert isinstance(out, pd.DataFrame) @@ -39,10 +40,11 @@ def test_mock_layers(): # Idem, but with holes n_ceilos = 2 - layer_prms =[{'alt':1000, 'alt_std': 100, 'lookback_time' : 1200, - 'hit_rate': 60, 'sky_cov_frac': 0.5, + lookback_time = 1200 + hit_gap = 60 + layer_prms =[{'alt':1000, 'alt_std': 100, 'sky_cov_frac': 0.5, 'period': 100, 'amplitude': 0}] - out = mock_layers(n_ceilos, layer_prms) + out = mock_layers(n_ceilos, lookback_time, hit_gap, layer_prms) # Correct number of points ? assert len(out) == 1200/60 * n_ceilos @@ -52,18 +54,34 @@ def test_mock_layers(): # In good numbers ? assert len(out[out['alt'].isna()]) == len(out)/2 - # And finally with more than 1 layer + # Now with more than 1 layer n_ceilos = 2 - layer_prms =[{'alt':1000, 'alt_std': 100, 'lookback_time' : 1200, - 'hit_rate': 60, 'sky_cov_frac': 0.5, + lookback_time = 1200 + hit_gap = 60 + layer_prms =[{'alt':1000, 'alt_std': 100, 'sky_cov_frac': 1, 'period': 100, 'amplitude': 0}, - {'alt':2000, 'alt_std': 200, 'lookback_time' : 600, - 'hit_rate': 60, 'sky_cov_frac': 1, + {'alt':10000, 'alt_std': 200, 'sky_cov_frac': 1, 'period': 100, 'amplitude': 0}, ] - out = mock_layers(n_ceilos, layer_prms) + out = mock_layers(n_ceilos, lookback_time, hit_gap, layer_prms) + # Correct number of points ? - assert len(out) == 1200/60 * n_ceilos + 600/60 * n_ceilos - # Holes present ? - assert np.any(out['alt'].isna()) + assert len(out) == 1200/60 * n_ceilos * len(layer_prms) + + # Now with an incomplete layers, to see if NaN's get handled properly + n_ceilos = 2 + lookback_time = 1200 + hit_gap = 60 + layer_prms =[{'alt':1000, 'alt_std': 100, 'sky_cov_frac': 1, + 'period': 100, 'amplitude': 0}, + {'alt':15000, 'alt_std': 200, 'sky_cov_frac': 1, + 'period': 100, 'amplitude': 0}, + ] + out = mock_layers(n_ceilos, lookback_time, hit_gap, layer_prms) + + # Holes present ? There should be None, since we have a second layer complete + assert np.any(~out['alt'].isna()) assert not np.any(out['dt'].isna()) + + # Make sur I have the correct number of timesteps + assert len(np.unique(out['dt'])) == n_ceilos * lookback_time/hit_gap diff --git a/test/ampycloud/utils/test_performance.py b/test/ampycloud/utils/test_performance.py index 19bcb32..24a1ef3 100644 --- a/test/ampycloud/utils/test_performance.py +++ b/test/ampycloud/utils/test_performance.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. diff --git a/test/ampycloud/utils/test_utils.py b/test/ampycloud/utils/test_utils.py new file mode 100644 index 0000000..c8db1c9 --- /dev/null +++ b/test/ampycloud/utils/test_utils.py @@ -0,0 +1,43 @@ +""" +Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. + +Distributed under the terms of the 3-Clause BSD License. + +SPDX-License-Identifier: BSD-3-Clause + +Module content: tests for the utils.utils module +""" + +# Import form Python +import numpy as np + +# Import from ampycloud +from ampycloud.utils.utils import tmp_seed + +def test_tmp_seed(): + """ This routine tests the tmp_seed. """ + + # Let's set a seed, get some random numbers, then check if I can reproduce this with tmp_seed. + np.random.seed(42) + a = np.random.random(100) + b = np.random.random(10) + c = np.random.random(1) + + # Make sure I get the concept of random numbers ... + np.random.seed(43) + a43 = np.random.random(100) + assert np.all(a != a43) + + np.random.seed(42) + a42 = np.random.random(100) + assert np.all(a == a42) + + # Now, actually check the function I am interested in. + np.random.seed(43) # Set a seed + with tmp_seed(42): # Temporarily set anopther one + assert np.all(a == np.random.random(100)) + assert np.all(b == np.random.random(10)) + assert np.all(c == np.random.random(1)) + + # Can I recover the original seed ? + assert np.all(a43 == np.random.random(100))