From a1553439d06a3ded758a8332697f2aa220cc317b Mon Sep 17 00:00:00 2001 From: Daniel Regenass Date: Mon, 8 Apr 2024 18:20:28 +0200 Subject: [PATCH 01/13] AMAROC-768 add heigh cloud flag and allow NSC --- src/ampycloud/data.py | 48 ++++++++++++++++++++++++++++---- test/ampycloud/test_core.py | 2 +- test/ampycloud/test_data.py | 55 +++++++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 6 deletions(-) diff --git a/src/ampycloud/data.py b/src/ampycloud/data.py index 6633dfa..644adf0 100644 --- a/src/ampycloud/data.py +++ b/src/ampycloud/data.py @@ -13,6 +13,7 @@ import logging import copy from abc import ABC, abstractmethod +import warnings import numpy as np import numpy.typing as npt import pandas as pd @@ -93,16 +94,31 @@ def _cleanup_pdf(self, data: pd.DataFrame) -> pd.DataFrame: # Begin with a thorough inspection of the dataset data = utils.check_data_consistency(data, req_cols=self.DATA_COLS) + # By default we set NSC to false, NOTE: attrs is still experimental in pandas! + try: + data.attrs['high_clouds_detected'] = False + except AttributeError: + # NOTE: Untestable as we cannot monkeypatch attrs as it would result in side-effects for __repr__ + # which break other tests + with warnings.catch_warnings(): + warnings.simplefilter("ignore") # pandas assumes we are trying to assign a column, which is not the case + data.attrs = {'high_clouds_detected': False} + # Then also drop any hits that is too high if self.msa is not None: hit_alt_lim = self.msa + self.msa_hit_buffer logger.info('Cropping hits above MSA+buffer: %s ft', str(hit_alt_lim)) # Type 1 or less hits above the cut threshold get turned to NaNs, to signal a # non-detection below the MSA. Also change the hit type to 0 accordingly ! - data.loc[data[(data.alt > hit_alt_lim) & (data.type <= 1)].index, 'type'] = 0 - data.loc[data[(data.alt > hit_alt_lim) & (data.type <= 1)].index, 'alt'] = np.nan + above_msa_t1_or_less = data[(data.alt > hit_alt_lim) & (data.type <= 1)].index + data.loc[above_msa_t1_or_less, 'type'] = 0 + data.loc[above_msa_t1_or_less, 'alt'] = np.nan # Type 2 or more hits get cropped (there should be only 1 non-detection per time-stamp). - data = data.drop(data[(data.alt > hit_alt_lim) & (data.type > 1)].index) + above_msa_t2_or_more = data[(data.alt > hit_alt_lim) & (data.type > 1)].index + data = data.drop(above_msa_t2_or_more) + if len(above_msa_t1_or_less) + len(above_msa_t2_or_more) > self._prms['MAX_HITS_OKTA0']: + logger.info("Hits above MSA exceeded threshold for okta 0, will add NSC flag") + data.attrs['high_clouds_detected'] = True return data @@ -992,6 +1008,28 @@ def layers(self) -> pd.DataFrame: identified by the layering algorithm. """ return self._layers + @property + def high_clouds_detected(self) -> bool: + """ Returns whether high clouds were detected in the data. + + Returns: + bool: whether high clouds were detected. + + """ + return self.data.attrs['high_clouds_detected'] + + def _ncd_or_nsc(self) -> str: + """ Return the METAR code for No Cloud Detected / No Significant Cloud based on the attribute set in data.attrs. + + Returns: + str: 'NCD' or 'NSC' + + """ + if self.high_clouds_detected: + return 'NSC' + else: + return 'NCD' + def metar_msg(self, which: str = 'layers') -> str: """ Construct a METAR-like message for the identified cloud slices, groups, or layers. @@ -1025,7 +1063,7 @@ def metar_msg(self, which: str = 'layers') -> str: # Deal with the 0 layer situation if getattr(self, f'n_{which}') == 0: - return 'NCD' + return self._ncd_or_nsc() # Deal with the situation where layers have been found ... msg = sligrolay['code'] @@ -1036,6 +1074,6 @@ def metar_msg(self, which: str = 'layers') -> str: # Here, deal with the situations when all clouds are above the MSA if len(msg) == 0: - return 'NCD' + return self._ncd_or_nsc() return msg diff --git a/test/ampycloud/test_core.py b/test/ampycloud/test_core.py index 75dc17c..1737ef4 100644 --- a/test/ampycloud/test_core.py +++ b/test/ampycloud/test_core.py @@ -82,7 +82,7 @@ def test_run(): # Test the ability to specific parameters locally only out = run(mock_data, prms={'MSA': 0}) - assert out.metar_msg() == 'NCD' + assert out.metar_msg() == 'NSC' assert dynamic.AMPYCLOUD_PRMS['MSA'] is None # Test that warnings are being raised if a bad parameter is being given diff --git a/test/ampycloud/test_data.py b/test/ampycloud/test_data.py index 75be198..fdc665d 100644 --- a/test/ampycloud/test_data.py +++ b/test/ampycloud/test_data.py @@ -80,6 +80,31 @@ def test_ceilochunk_init(): reset_prms() +@mark.parametrize('alt,expected_flag', [ + param(1000, False, id='low clouds'), + param(15000, True, id='high clouds'), +]) +def test_high_clouds_flag(alt: int, expected_flag: bool): + """ Test the high clouds flagging routine. """ + + dynamic.AMPYCLOUD_PRMS['MAX_HITS_OKTA0'] = 3 + dynamic.AMPYCLOUD_PRMS['MSA'] = 10000 + + n_ceilos = 4 + lookback_time = 1200 + rate = 30 + # Create some fake data to get started + # 1 very flat layer with no gaps + mock_data = mocker.mock_layers( + n_ceilos, lookback_time, rate, + [{'alt': alt, 'alt_std': 10, 'sky_cov_frac': 0.1, 'period': 100, 'amplitude': 0}] + ) + chunk = CeiloChunk(mock_data) + assert chunk.high_clouds_detected == expected_flag + + reset_prms() + + def test_ceilochunk_basic(): """ Test the basic methods of the CeiloChunk class. """ @@ -228,6 +253,36 @@ def test_ceilochunk_nocld(): assert chunk.metar_msg() == 'NCD' +def test_ceilochunk_highcld(): + """ Test the methods of CeiloChunks when high clouds are seen in the interval. """ + + dynamic.AMPYCLOUD_PRMS['MAX_HITS_OKTA0'] = 3 + dynamic.AMPYCLOUD_PRMS['MSA'] = 10000 + + n_ceilos = 4 + lookback_time = 1200 + rate = 30 + # Create some fake data to get started + # 1 very flat layer with no gaps + mock_data = mocker.mock_layers( + n_ceilos, lookback_time, rate, + [{'alt': 15000, 'alt_std': 10, 'sky_cov_frac': 0.1, 'period': 100, 'amplitude': 0}] + ) + + # Instantiate a CeiloChunk entity ... + chunk = CeiloChunk(mock_data) + + # Do the dance ... + chunk.find_slices() + chunk.find_groups() + chunk.find_layers() + + # Assert the final METAR code is correct + assert chunk.metar_msg() == 'NSC' + + reset_prms() + + def test_ceilochunk_2lay(): """ Test the methods of CeiloChunks when 2 layers are seen in the interval. """ From 2e2e2b0cb20dccd80071d833b2ccf11a9247e9b2 Mon Sep 17 00:00:00 2001 From: Daniel Regenass Date: Tue, 9 Apr 2024 09:51:39 +0200 Subject: [PATCH 02/13] AMAROC-768 formatting (easy fixes) --- src/ampycloud/data.py | 44 +++++++++++++++++++------------------ src/ampycloud/scaler.py | 12 +++++----- test/ampycloud/test_data.py | 5 +++-- 3 files changed, 32 insertions(+), 29 deletions(-) diff --git a/src/ampycloud/data.py b/src/ampycloud/data.py index 644adf0..ff704c7 100644 --- a/src/ampycloud/data.py +++ b/src/ampycloud/data.py @@ -98,10 +98,12 @@ def _cleanup_pdf(self, data: pd.DataFrame) -> pd.DataFrame: try: data.attrs['high_clouds_detected'] = False except AttributeError: - # NOTE: Untestable as we cannot monkeypatch attrs as it would result in side-effects for __repr__ - # which break other tests + # Catch for older versions of pandas that do not support DataFrame.attrs + # NOTE: Untestable as we cannot monkeypatch attrs as it would result + # in side-effects for __repr__ which break other tests with warnings.catch_warnings(): - warnings.simplefilter("ignore") # pandas assumes we are trying to assign a column, which is not the case + # pandas assumes we are trying to assign a column, which is not the case + warnings.simplefilter("ignore") data.attrs = {'high_clouds_detected': False} # Then also drop any hits that is too high @@ -202,8 +204,10 @@ def __init__(self, data: pd.DataFrame, prms: Optional[dict] = None, self._layers = None @log_func_call(logger) - def data_rescaled(self, dt_mode: Optional[str] = None, alt_mode: Optional[str] = None, - dt_kwargs: Optional[dict] = None, alt_kwargs: Optional[dict] = None) -> pd.DataFrame: + def data_rescaled( + self, dt_mode: Optional[str] = None, alt_mode: Optional[str] = None, + dt_kwargs: Optional[dict] = None, alt_kwargs: Optional[dict] = None + ) -> pd.DataFrame: """ Returns a copy of the data, rescaled according to the provided parameters. Args: @@ -312,13 +316,12 @@ def _get_min_sep_for_altitude(self, altitude: float) -> float: MIN_SEP_VALS """ - if len(self.prms['MIN_SEP_LIMS']) != \ - len(self.prms['MIN_SEP_VALS']) - 1: - raise AmpycloudError( - '"MIN_SEP_LIMS" must have one less item than "MIN_SEP_VALS".' - 'Got MIN_SEP_LIMS %i and MIN_SEP_VALS %i', - (self.prms['MIN_SEP_LIMS'], self.prms['MIN_SEP_VALS']) - ) + if len(self.prms['MIN_SEP_LIMS']) != len(self.prms['MIN_SEP_VALS']) - 1: + raise AmpycloudError( + '"MIN_SEP_LIMS" must have one less item than "MIN_SEP_VALS".' + f'Got MIN_SEP_LIMS {self.prms['MIN_SEP_LIMS']} ' + f'and MIN_SEP_VALS {self.prms['MIN_SEP_VALS']}.', + ) min_sep_val_id = np.searchsorted(self.prms['MIN_SEP_LIMS'], altitude) @@ -377,12 +380,11 @@ def _setup_sligrolay_pdf(self, which: str = 'slices') -> tuple[pd.DataFrame, npt ] # We want to raise early if 'which' is unknown. - if not which in ['slices', 'groups', 'layers']: + if which not in ['slices', 'groups', 'layers']: raise AmpycloudError( - 'Trying to initialize a data frame for %s ' + f'Trying to initialize a data frame for {which}, ' 'which is unknown. Keyword arg "which" must be one of' '"slices", "groups" or "layers"' - %which ) # If I am looking at the slices, also keep track of whether they are isolated, or not. @@ -535,9 +537,9 @@ def _calculate_sligrolay_base_height( # Which hits are in this sli/gro/lay ? in_sligrolay = self.data[which[:-1]+'_id'] == cid # Compute the base altitude - pdf.iloc[ind, pdf.columns.get_loc('alt_base')] = self._calculate_base_height_for_selection( - in_sligrolay, - ) + pdf.iloc[ + ind, pdf.columns.get_loc('alt_base') + ] = self._calculate_base_height_for_selection(in_sligrolay,) return pdf @log_func_call(logger) @@ -1019,7 +1021,8 @@ def high_clouds_detected(self) -> bool: return self.data.attrs['high_clouds_detected'] def _ncd_or_nsc(self) -> str: - """ Return the METAR code for No Cloud Detected / No Significant Cloud based on the attribute set in data.attrs. + """ Return the METAR code for No Cloud Detected / No Significant Cloud. + Decision based on the attribute set in data.attrs. Returns: str: 'NCD' or 'NSC' @@ -1027,8 +1030,7 @@ def _ncd_or_nsc(self) -> str: """ if self.high_clouds_detected: return 'NSC' - else: - return 'NCD' + return 'NCD' def metar_msg(self, which: str = 'layers') -> str: """ Construct a METAR-like message for the identified cloud slices, groups, or layers. diff --git a/src/ampycloud/scaler.py b/src/ampycloud/scaler.py index ee115d5..e752319 100644 --- a/src/ampycloud/scaler.py +++ b/src/ampycloud/scaler.py @@ -221,10 +221,10 @@ def convert_kwargs(vals: np.ndarray, fct: str, **kwargs: dict) -> dict: if fct == 'shift-and-scale': # In this case, the only data I may need to derive from the data is the shift. - if 'shift' in kwargs.keys(): + if 'shift' in kwargs: # Already set - do nothing return kwargs - if 'mode' in kwargs.keys(): + if 'mode' in kwargs: if kwargs['mode'] == 'do': kwargs['shift'] = np.nanmax(vals) elif kwargs['mode'] == 'undo': @@ -240,12 +240,12 @@ def convert_kwargs(vals: np.ndarray, fct: str, **kwargs: dict) -> dict: if fct == 'minmax-scale': # In this case, the challenge lies with identifying min_val and max_val, knowing that the # user may specify a min_range value. - if 'min_val' in kwargs.keys() and 'max_val' in kwargs.keys(): + if 'min_val' in kwargs and 'max_val' in kwargs: # Already specified ... do nothing return kwargs - if 'mode' in kwargs.keys(): + if 'mode' in kwargs: if kwargs['mode'] == 'do': - if 'min_range' in kwargs.keys(): + if 'min_range' in kwargs: min_range = kwargs['min_range'] kwargs.pop('min_range', None) else: @@ -260,7 +260,7 @@ def convert_kwargs(vals: np.ndarray, fct: str, **kwargs: dict) -> dict: raise AmpycloudError(f"Mode unknown: {kwargs['mode']}") # 'mode' not set -> will default to 'do' - if 'min_range' in kwargs.keys(): + if 'min_range' in kwargs: min_range = kwargs['min_range'] kwargs.pop('min_range', None) else: diff --git a/test/ampycloud/test_data.py b/test/ampycloud/test_data.py index fdc665d..8ac8b2b 100644 --- a/test/ampycloud/test_data.py +++ b/test/ampycloud/test_data.py @@ -96,8 +96,9 @@ def test_high_clouds_flag(alt: int, expected_flag: bool): # Create some fake data to get started # 1 very flat layer with no gaps mock_data = mocker.mock_layers( - n_ceilos, lookback_time, rate, - [{'alt': alt, 'alt_std': 10, 'sky_cov_frac': 0.1, 'period': 100, 'amplitude': 0}] + n_ceilos, lookback_time, rate, [ + {'alt': alt, 'alt_std': 10, 'sky_cov_frac': 0.1, 'period': 100, 'amplitude': 0} + ] ) chunk = CeiloChunk(mock_data) assert chunk.high_clouds_detected == expected_flag From 4ffcc62143c23156c57d0d369594b2519c0ec2ed Mon Sep 17 00:00:00 2001 From: Daniel Regenass Date: Tue, 9 Apr 2024 09:57:35 +0200 Subject: [PATCH 03/13] AMAROC-768 update copyright years --- src/ampycloud/data.py | 2 +- test/ampycloud/test_core.py | 2 +- test/ampycloud/test_data.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ampycloud/data.py b/src/ampycloud/data.py index ff704c7..feb4786 100644 --- a/src/ampycloud/data.py +++ b/src/ampycloud/data.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021-2023 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2024 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. diff --git a/test/ampycloud/test_core.py b/test/ampycloud/test_core.py index 1737ef4..1908d61 100644 --- a/test/ampycloud/test_core.py +++ b/test/ampycloud/test_core.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2024 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. diff --git a/test/ampycloud/test_data.py b/test/ampycloud/test_data.py index 8ac8b2b..b34721f 100644 --- a/test/ampycloud/test_data.py +++ b/test/ampycloud/test_data.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2021-2022 MeteoSwiss, contributors listed in AUTHORS. +Copyright (c) 2021-2024 MeteoSwiss, contributors listed in AUTHORS. Distributed under the terms of the 3-Clause BSD License. From cdbb945605b35e221621506784eb3fec851c7de4 Mon Sep 17 00:00:00 2001 From: Daniel Regenass Date: Tue, 9 Apr 2024 09:59:51 +0200 Subject: [PATCH 04/13] AMAROC-768 update changelog --- CHANGELOG | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 1c1a1b7..777da1c 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -3,6 +3,8 @@ All notable changes to ampycloud will be documented in this file. The format is inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [v1.1.0] + - [regDaniel, 2024-04-09] Add flag for high clouds and allow for NSC in METAR message ## [v1.0.0] ### Added - [regDaniel, 2023-11-09] Add minimum separation condition for grouping step From 8b180667dc00c1b6cc4661742c4f8f156aad35f0 Mon Sep 17 00:00:00 2001 From: Daniel Regenass Date: Tue, 9 Apr 2024 10:04:02 +0200 Subject: [PATCH 05/13] AMAROC-768 fix f-string --- src/ampycloud/data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ampycloud/data.py b/src/ampycloud/data.py index feb4786..57baf4a 100644 --- a/src/ampycloud/data.py +++ b/src/ampycloud/data.py @@ -319,8 +319,8 @@ def _get_min_sep_for_altitude(self, altitude: float) -> float: if len(self.prms['MIN_SEP_LIMS']) != len(self.prms['MIN_SEP_VALS']) - 1: raise AmpycloudError( '"MIN_SEP_LIMS" must have one less item than "MIN_SEP_VALS".' - f'Got MIN_SEP_LIMS {self.prms['MIN_SEP_LIMS']} ' - f'and MIN_SEP_VALS {self.prms['MIN_SEP_VALS']}.', + f'Got MIN_SEP_LIMS {self.prms["MIN_SEP_LIMS"]} ' + f'and MIN_SEP_VALS {self.prms["MIN_SEP_VALS"]}.', ) min_sep_val_id = np.searchsorted(self.prms['MIN_SEP_LIMS'], From fa4db2a631563870caebd20e0ead5975ef2dff43 Mon Sep 17 00:00:00 2001 From: Daniel Regenass Date: Tue, 9 Apr 2024 10:22:00 +0200 Subject: [PATCH 06/13] AMAROC-768 NSC if sligrolay in buffer --- src/ampycloud/data.py | 3 +++ test/ampycloud/test_data.py | 9 +++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/ampycloud/data.py b/src/ampycloud/data.py index 57baf4a..6846ed3 100644 --- a/src/ampycloud/data.py +++ b/src/ampycloud/data.py @@ -1076,6 +1076,9 @@ def metar_msg(self, which: str = 'layers') -> str: # Here, deal with the situations when all clouds are above the MSA if len(msg) == 0: + sligrolay_in_buffer = sligrolay['significant'] * (sligrolay['alt_base'] >= msa_val) + if sligrolay_in_buffer.any(): # clouds within the MSA buffer zone + return 'NSC' return self._ncd_or_nsc() return msg diff --git a/test/ampycloud/test_data.py b/test/ampycloud/test_data.py index b34721f..52bc644 100644 --- a/test/ampycloud/test_data.py +++ b/test/ampycloud/test_data.py @@ -254,11 +254,16 @@ def test_ceilochunk_nocld(): assert chunk.metar_msg() == 'NCD' -def test_ceilochunk_highcld(): +@mark.parametrize('alt', [ + param(10500, id='in buffer'), + param(15000, id='above buffer'), +]) +def test_ceilochunk_highcld(alt): """ Test the methods of CeiloChunks when high clouds are seen in the interval. """ dynamic.AMPYCLOUD_PRMS['MAX_HITS_OKTA0'] = 3 dynamic.AMPYCLOUD_PRMS['MSA'] = 10000 + dynamic.AMPYCLOUD_PRMS['MSA_HIT_BUFFER'] = 1000 n_ceilos = 4 lookback_time = 1200 @@ -267,7 +272,7 @@ def test_ceilochunk_highcld(): # 1 very flat layer with no gaps mock_data = mocker.mock_layers( n_ceilos, lookback_time, rate, - [{'alt': 15000, 'alt_std': 10, 'sky_cov_frac': 0.1, 'period': 100, 'amplitude': 0}] + [{'alt': alt, 'alt_std': 10, 'sky_cov_frac': 0.1, 'period': 100, 'amplitude': 0}] ) # Instantiate a CeiloChunk entity ... From 798f5b70753e0c34aeca76e0cd4cd332aa805731 Mon Sep 17 00:00:00 2001 From: Daniel Regenass Date: Tue, 9 Apr 2024 11:04:39 +0200 Subject: [PATCH 07/13] AMAROC-768 Restrict python versions for pylinter --- .github/workflows/CI_pylinter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI_pylinter.yml b/.github/workflows/CI_pylinter.yml index 9c0b2a5..3e7b014 100644 --- a/.github/workflows/CI_pylinter.yml +++ b/.github/workflows/CI_pylinter.yml @@ -18,7 +18,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: ['3.x'] + python-version: ['3.9', '3.10', '3.11'] steps: # Checkout our repository From 745a50ac3a0666e390600e3fddce6c1bf2d952a9 Mon Sep 17 00:00:00 2001 From: Daniel Regenass Date: Tue, 9 Apr 2024 11:14:19 +0200 Subject: [PATCH 08/13] AMAROC-768 install setuptools for pylinter --- .github/workflows/CI_pylinter.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI_pylinter.yml b/.github/workflows/CI_pylinter.yml index 3e7b014..9da6ac9 100644 --- a/.github/workflows/CI_pylinter.yml +++ b/.github/workflows/CI_pylinter.yml @@ -18,7 +18,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: ['3.9', '3.10', '3.11'] + python-version: ['3.x'] steps: # Checkout our repository @@ -32,9 +32,10 @@ jobs: python-version: ${{ matrix.python-version }} # Install any dependency we require - - name: Install dependancies + - name: Install dependencies run: | python -m pip install --upgrade pip + python -m pip install setuptools shell: bash # Here, let's install our module to make sure all the dependencies specified in setup.py are From c22cd99dd1d70fee81fea03f5b400a5cac240073 Mon Sep 17 00:00:00 2001 From: Daniel Regenass Date: Wed, 10 Apr 2024 13:38:05 +0200 Subject: [PATCH 09/13] AMAROC-768 add comments --- src/ampycloud/data.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/ampycloud/data.py b/src/ampycloud/data.py index 6846ed3..72ff39e 100644 --- a/src/ampycloud/data.py +++ b/src/ampycloud/data.py @@ -102,7 +102,8 @@ def _cleanup_pdf(self, data: pd.DataFrame) -> pd.DataFrame: # NOTE: Untestable as we cannot monkeypatch attrs as it would result # in side-effects for __repr__ which break other tests with warnings.catch_warnings(): - # pandas assumes we are trying to assign a column, which is not the case + # pandas assumes we are trying to assign a column, but as this is not the case + # we catch its warning and ignore it. warnings.simplefilter("ignore") data.attrs = {'high_clouds_detected': False} @@ -110,8 +111,9 @@ def _cleanup_pdf(self, data: pd.DataFrame) -> pd.DataFrame: if self.msa is not None: hit_alt_lim = self.msa + self.msa_hit_buffer logger.info('Cropping hits above MSA+buffer: %s ft', str(hit_alt_lim)) - # Type 1 or less hits above the cut threshold get turned to NaNs, to signal a - # non-detection below the MSA. Also change the hit type to 0 accordingly ! + # First layer and vervis hits above the cut threshold get turned to NaNs, to signal a + # non-detection below the MSA. Also change the hit type to 0 accordingly in order + # to create a "no hit detected" in the range of interest (i.e. below MSA). above_msa_t1_or_less = data[(data.alt > hit_alt_lim) & (data.type <= 1)].index data.loc[above_msa_t1_or_less, 'type'] = 0 data.loc[above_msa_t1_or_less, 'alt'] = np.nan @@ -119,7 +121,10 @@ def _cleanup_pdf(self, data: pd.DataFrame) -> pd.DataFrame: above_msa_t2_or_more = data[(data.alt > hit_alt_lim) & (data.type > 1)].index data = data.drop(above_msa_t2_or_more) if len(above_msa_t1_or_less) + len(above_msa_t2_or_more) > self._prms['MAX_HITS_OKTA0']: - logger.info("Hits above MSA exceeded threshold for okta 0, will add NSC flag") + logger.info( + "Hits above MSA + MSA_HIT_BUFFER exceeded threshold MAX_HITS_OKTA0. Will add " + "flag 'high_clouds_detected' to indicate the presence of high clouds." + ) data.attrs['high_clouds_detected'] = True return data @@ -1076,9 +1081,10 @@ def metar_msg(self, which: str = 'layers') -> str: # Here, deal with the situations when all clouds are above the MSA if len(msg) == 0: + # first check if any significant clouds are in the interval [MSA, MSA+MSA_HIT_BUFFER] sligrolay_in_buffer = sligrolay['significant'] * (sligrolay['alt_base'] >= msa_val) - if sligrolay_in_buffer.any(): # clouds within the MSA buffer zone - return 'NSC' - return self._ncd_or_nsc() + if sligrolay_in_buffer.any(): + return 'NSC' # and return a NSC as it implies that the cloud is above the MSA + return self._ncd_or_nsc() # else, check for CBH above MSA + MSA_HIT_BUFFER return msg From ba885da7e3b81f3134d5a68a6e40b392ac591a59 Mon Sep 17 00:00:00 2001 From: Daniel Regenass Date: Wed, 10 Apr 2024 14:55:21 +0200 Subject: [PATCH 10/13] AMAROC-768 drop attrs, add CeiloChunk attribute --- src/ampycloud/data.py | 32 ++++++++++++-------------------- test/ampycloud/test_data.py | 5 +++-- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/src/ampycloud/data.py b/src/ampycloud/data.py index 72ff39e..52ca514 100644 --- a/src/ampycloud/data.py +++ b/src/ampycloud/data.py @@ -94,20 +94,11 @@ def _cleanup_pdf(self, data: pd.DataFrame) -> pd.DataFrame: # Begin with a thorough inspection of the dataset data = utils.check_data_consistency(data, req_cols=self.DATA_COLS) - # By default we set NSC to false, NOTE: attrs is still experimental in pandas! - try: - data.attrs['high_clouds_detected'] = False - except AttributeError: - # Catch for older versions of pandas that do not support DataFrame.attrs - # NOTE: Untestable as we cannot monkeypatch attrs as it would result - # in side-effects for __repr__ which break other tests - with warnings.catch_warnings(): - # pandas assumes we are trying to assign a column, but as this is not the case - # we catch its warning and ignore it. - warnings.simplefilter("ignore") - data.attrs = {'high_clouds_detected': False} - - # Then also drop any hits that is too high + # By default we set this flag to false and overwrite if enough hits are present + self._clouds_above_msa_buffer = False + + # Drop any hits that are too high and check if they exceed the threshold for 1 OKTA + # if yes, set the flag clouds_above_msa_buffer to True if self.msa is not None: hit_alt_lim = self.msa + self.msa_hit_buffer logger.info('Cropping hits above MSA+buffer: %s ft', str(hit_alt_lim)) @@ -125,7 +116,7 @@ def _cleanup_pdf(self, data: pd.DataFrame) -> pd.DataFrame: "Hits above MSA + MSA_HIT_BUFFER exceeded threshold MAX_HITS_OKTA0. Will add " "flag 'high_clouds_detected' to indicate the presence of high clouds." ) - data.attrs['high_clouds_detected'] = True + self._clouds_above_msa_buffer = True return data @@ -1016,24 +1007,25 @@ def layers(self) -> pd.DataFrame: return self._layers @property - def high_clouds_detected(self) -> bool: - """ Returns whether high clouds were detected in the data. + def clouds_above_msa_buffer(self) -> bool: + """ Returns whether a number of hits exceeding the threshold for 1 okta is detected above + MSA + MSA_HIT_BUFFER. Returns: bool: whether high clouds were detected. """ - return self.data.attrs['high_clouds_detected'] + return self._clouds_above_msa_buffer def _ncd_or_nsc(self) -> str: """ Return the METAR code for No Cloud Detected / No Significant Cloud. - Decision based on the attribute set in data.attrs. + Decision based on the attribute self._clouds_above_msa_buffer. Returns: str: 'NCD' or 'NSC' """ - if self.high_clouds_detected: + if self._clouds_above_msa_buffer: return 'NSC' return 'NCD' diff --git a/test/ampycloud/test_data.py b/test/ampycloud/test_data.py index 52bc644..0cbe18f 100644 --- a/test/ampycloud/test_data.py +++ b/test/ampycloud/test_data.py @@ -84,11 +84,12 @@ def test_ceilochunk_init(): param(1000, False, id='low clouds'), param(15000, True, id='high clouds'), ]) -def test_high_clouds_flag(alt: int, expected_flag: bool): +def test_clouds_above_msa_buffer_flag(alt: int, expected_flag: bool): """ Test the high clouds flagging routine. """ dynamic.AMPYCLOUD_PRMS['MAX_HITS_OKTA0'] = 3 dynamic.AMPYCLOUD_PRMS['MSA'] = 10000 + dynamic.AMPYCLOUD_PRMS['MSA_HIT_BUFFER'] = 1000 n_ceilos = 4 lookback_time = 1200 @@ -101,7 +102,7 @@ def test_high_clouds_flag(alt: int, expected_flag: bool): ] ) chunk = CeiloChunk(mock_data) - assert chunk.high_clouds_detected == expected_flag + assert chunk.clouds_above_msa_buffer == expected_flag reset_prms() From 5aa7fae57d01a1432de0a8d51f449947698fe174 Mon Sep 17 00:00:00 2001 From: Daniel Regenass Date: Wed, 10 Apr 2024 15:00:02 +0200 Subject: [PATCH 11/13] AMAROC-768 remove unused import --- src/ampycloud/data.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ampycloud/data.py b/src/ampycloud/data.py index 52ca514..56e7126 100644 --- a/src/ampycloud/data.py +++ b/src/ampycloud/data.py @@ -13,7 +13,6 @@ import logging import copy from abc import ABC, abstractmethod -import warnings import numpy as np import numpy.typing as npt import pandas as pd From 456dbf559f3474539d36d57795aa1a72f7f33df7 Mon Sep 17 00:00:00 2001 From: Daniel Regenass Date: Wed, 10 Apr 2024 15:09:16 +0200 Subject: [PATCH 12/13] AMAROC-768 add NSC warning to scope.rst --- docs/source/scope.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/source/scope.rst b/docs/source/scope.rst index fe23e06..5825706 100644 --- a/docs/source/scope.rst +++ b/docs/source/scope.rst @@ -9,6 +9,16 @@ This has the following implications for ampycloud: passing them to ampycloud, e.g. by removing them or by converting them to cloud base heights. + * Note that regulation says that "if there are no clouds of operational significance + and no restriction on vertical visibility and the abbreviation 'CAVOK' is not + appropriate, the abbreviation 'NSC' should be used" (AMC1 MET.TR.205(e)(1)). + ampycloud cannot decide whether a 'CAVOK' is appropriate, and will therefore + always return 'NSC' if no clouds of operational significance are found. If no clouds + are detected at all by the ceilometers, ampycloud will return 'NCD'. Importantly, + users should bear in mind that ampycloud cannot handle CB and TCU cases, + such that any 'NCD'/'NSC' codes issued may need to be overwritten by the user in + certain situations. + * ampycloud can evidently be used for R&D work, but the code itself should not be seen as an R&D platform. From d09d1f53a1762a74ee28c09b98ee2d51b9e9ddc2 Mon Sep 17 00:00:00 2001 From: Dani Date: Wed, 10 Apr 2024 15:46:26 +0200 Subject: [PATCH 13/13] AMAROC-768 improve message in CHANGELOG --- CHANGELOG | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 777da1c..934390b 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -4,7 +4,7 @@ The format is inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [v1.1.0] - - [regDaniel, 2024-04-09] Add flag for high clouds and allow for NSC in METAR message + - [regDaniel, 2024-04-09] Add flag for clouds above (MSA + MSA_HIT_BUFFER) and allow for NSC in METAR message ## [v1.0.0] ### Added - [regDaniel, 2023-11-09] Add minimum separation condition for grouping step