BUFR: Fix membuffer copy (#193)

* BUFR: Fix membuffer copy * BUFR: Skip WIGOS generation, if it already exists
EURODEO · Oct 2, 2024 · 791c82a · 791c82a · github-actions · Oct 2, 2024
1 parent ea44877
commit 791c82a
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 12 deletions.
diff --git a/ingest/src/ingest/bufr/ESOHBufr.cpp b/ingest/src/ingest/bufr/ESOHBufr.cpp
@@ -246,7 +246,9 @@ std::list<std::string> ESOHBufr::msg() const {
                   LogEntry("Missing WMO Station number after WMO block",
                            LogLevel::WARN, __func__, bufr_id));
             }
-            wigos_id.setWmoId(wmo_block * 1000 + wmo_station);
+            if (!wigos_id.getWigosLocalId().size()) {
+              wigos_id.setWmoId(wmo_block * 1000 + wmo_station);
+            }
             skip_platform = false;
             // platform_check = true;
             break;

diff --git a/ingest/src/ingest/bufr/NorBufr.cpp b/ingest/src/ingest/bufr/NorBufr.cpp
@@ -640,14 +640,14 @@ uint64_t NorBufr::fromBuffer(char *ext_buf, u_int64_t ext_buf_pos,
   if (n >= ext_buf_size) {
     lb.addLogEntry(
         LogEntry("No more BUFR messages", LogLevel::WARN, __func__, bufr_id));
-    return 0;
+    return n;
   }
 
   // Section0 length
   uint64_t slen = 8;
   uint8_t sec0[slen];
-  if (ext_buf_pos + slen < ext_buf_size) {
-    memcpy(sec0, ext_buf + ext_buf_pos, slen);
+  if (ext_buf_pos + n + slen < ext_buf_size) {
+    memcpy(sec0, ext_buf + ext_buf_pos + n, slen);
   }
 
   len = NorBufrIO::getBytes(sec0 + 4, 3);
@@ -660,8 +660,8 @@ uint64_t NorBufr::fromBuffer(char *ext_buf, u_int64_t ext_buf_pos,
   buffer = new uint8_t[len];
   memcpy(buffer, sec0, slen);
 
-  if (ext_buf_pos + len <= ext_buf_size) {
-    memcpy(buffer + slen, ext_buf + ext_buf_pos + slen, len - slen);
+  if (ext_buf_pos + n + len <= ext_buf_size) {
+    memcpy(buffer + slen, ext_buf + ext_buf_pos + n + slen, len - slen);
   }
 
   int offset = checkBuffer();

diff --git a/ingest/src/ingest/bufr/NorBufrIO.cpp b/ingest/src/ingest/bufr/NorBufrIO.cpp
@@ -63,8 +63,9 @@ uint64_t NorBufrIO::findBytes(char *buf, unsigned int buf_size, const char *seq,
                               unsigned int size) {
   uint64_t j = 0;
   char c;
-  while (j < size && j < buf_size) {
-    c = buf[j];
+  uint64_t position = 0;
+  while (j < size && position < buf_size) {
+    c = buf[position];
     if (c == seq[j]) {
       ++j;
     } else {
@@ -74,8 +75,9 @@ uint64_t NorBufrIO::findBytes(char *buf, unsigned int buf_size, const char *seq,
         j = 0;
       }
     }
+    ++position;
   }
-  return (j < size ? ULONG_MAX : j);
+  return (j < size ? ULONG_MAX : position - size);
 }
 
 uint64_t NorBufrIO::getBytes(uint8_t *buffer, int size) {

diff --git a/ingest/src/ingest/bufr/bufresohmsg_py.cpp b/ingest/src/ingest/bufr/bufresohmsg_py.cpp
@@ -135,9 +135,9 @@ std::list<std::string> norbufr_bufresohmsg(std::string fname) {
 std::list<std::string> norbufr_bufresohmsgmem(char *api_buf, int api_size) {
 
   std::list<std::string> ret;
-  int position = 0;
+  uint64_t position = 0;
 
-  while (position < api_size) {
+  while (position < static_cast<uint64_t>(api_size)) {
 
     ESOHBufr *bufr = new ESOHBufr;
     // TODO:
@@ -146,7 +146,9 @@ std::list<std::string> norbufr_bufresohmsgmem(char *api_buf, int api_size) {
     bufr->setMsgTemplate(bufr_input_schema);
     bufr->setShadowWigos(default_shadow_wigos_py);
 
-    int n = bufr->fromBuffer(api_buf, position, api_size);
+    uint64_t n = bufr->fromBuffer(api_buf, position, api_size);
+    if (n == ULONG_MAX)
+      position = ULONG_MAX;
     if (n > position) {
       position = n;
       bufr->setTableB(
File	Stmts	Miss	Cover	Missing
\_\_init\_\_.py	0	0	100%
datastore_pb2.py	65	52	20%	34–85
datastore_pb2_grpc.py	54	27	50%	15–16, 19, 65–80, 121–123, 128–130, 135–137, 142–144, 148–173, 219, 246, 273, 300
export_metrics.py	10	0	100%
grpc_getter.py	20	11	45%	15–19, 23–26, 30–32, 36–38
locustfile.py	15	15	0%	1–31
main.py	43	7	84%	45, 50, 60, 70–71, 81–82
metadata_endpoints.py	65	31	52%	45–54, 58, 85, 100–219, 223
response_classes.py	5	0	100%
utilities.py	174	46	74%	20, 38, 45, 67–70, 78–89, 94–101, 121, 125, 127, 155, 161, 179, 193–194, 198, 214–218, 222–228, 232–234, 264, 268, 290, 295
custom_geo_json
edr_feature_collection.py	6	0	100%
formatters
\_\_init\_\_.py	11	0	100%
covjson.py	59	1	98%	91
geojson.py	21	2	90%	27, 52
openapi
custom_dimension_examples.py	4	0	100%
edr_query_parameter_descriptions.py	11	0	100%
openapi_examples.py	13	0	100%
routers
\_\_init\_\_.py	0	0	100%
edr.py	101	4	96%	348–349, 438–439
feature.py	47	19	60%	99–132, 148–153, 159–181
TOTAL	724	215	70%