Bufr ingest fixes (#210)

* BUFR: handle zero size Section3

* change getValue return value, when Missing

* BUFR: skip values when period is missing

* Improve missing values handling

* BUFR: fix int getValue

* BUFR: fix compressed text size

* BUFR: skip subset, if early or late

* BUFR: measure start/end is defined by [ 0 04 025 ]

* BUFR: skip duplicated pressure values

* BUFR: fix precipitation period sign

* BUFR: change the default HITIME

* datastore: change the default HITIME

* BUFR: increase logbuffer size

* BUFR: add bufr state id to ISO country code conversion

* BUFR: fix compressed missing string

* BUFR: iso country code to the gen wigos

* BUFR: removed unnecessary station name check

datastore/datastore/README.md

@@ -152,7 +152,7 @@ The following environment variables are supported:
 | `PGDBNAME`        | No  | `data`             | PostgreSQL database name. |
 | `DYNAMICTIME`     | No  | `true`             | Whether the valid time range is _dynamic_ or _static_ (defined below). |
 | `LOTIME`          | No  | `86400`            | The _earliest_ valid time as seconds to be either [1] subtracted from the current time (if the valid time range is _dynamic_) or [2] added to UNIX epoch (1970-01-01T00:00:00Z) (if the valid time range is _static_). In the case of a _static_ valid time range, the `LOTIME` can optionally be specified as an ISO-8601 datetime of the exact form `2023-10-10T00:00:00Z`. |
-| `HITIME`          | No  | `-2`               | Same as `LOTIME`, but for the _latest_ valid time. Note a default leeway of 2 seconds into the future to reduce risk of missing the newest observations. |
+| `HITIME`          | No  | `-600`               | Same as `LOTIME`, but for the _latest_ valid time. Note a default leeway of 10 minutes into the future to reduce risk of missing visual observations. |
 | `CLEANUPINTERVAL` | No  | `86400`            | The minimum time duration in seconds between automatic cleanups (like removing obsolete observations from the physical store). |
 | `PUTOBSLIMIT`     | No  | `100000`           | Maximum number of observations allowed in a single call to `PutObservations`. |
 

datastore/datastore/common/common.go

@@ -75,7 +75,7 @@ func initValidTimeRange() {
 	loTimeName := "LOTIME"
 	hiTimeName := "HITIME"
 	defaultLoTimeSecs := int64(86400)
-	defaultHiTimeSecs := int64(-2) // add leeway to reduce risk of missing the newest observations
+	defaultHiTimeSecs := int64(-600) // the reference time of visual observations is the next hour
 
 	if dynamicTime {
 		loTimeSecs = getSecs(loTimeName, defaultLoTimeSecs, false)

ingest/src/ingest/bufr/CountryCodes.h

@@ -308,4 +308,15 @@ static const std::list<cc_conv> country_codes = {
 
 };
 
+inline int bufrToIsocc(const int bufr_state_id) {
+  int ret = 0;
+  for (auto cc : country_codes) {
+    if (cc.bufr_code == bufr_state_id) {
+      ret = cc.iso_code;
+      break;
+    }
+  }
+  return ret;
+}
+
 #endif