Refactor nkv host

    Brief Changes:
    ==============
    * Fixed issue using resize and pushback at the same time
    * Fix listing_keys and data_cache proto-types
    * Gitlab upgrade, trigger:include now takes a string or list
    * Fix Sonar-Scanner download.
        - Get sonar-scanner from URL determined by CI/CD var
        - Determine sonar-scanner dir name by listing contents of zip

    Testing Done:
    =============
    * Run with gitlab pipeline

Signed-off-by: r.kavuluru <[email protected]>

.gitlab-ci.yml

@@ -18,9 +18,9 @@ deploy DSS:
     UPSTREAM_REF: $CI_MERGE_REQUEST_REF_PATH
   trigger:
     include:
-      project: dfs/dss/dss-ansible
-      ref: master
-      file: .gitlab-ci.yml
+      - project: dfs/dss/dss-ansible
+        ref: master
+        file: .gitlab-ci.yml
     strategy: depend
   resource_group: inv_$CI_PROJECT_NAME.ini
   rules:

.gitlab/sonar.yml

@@ -1,13 +1,15 @@
 sonar-scanner:
   stage: scan
   before_script:
-    # Download latest sonar-scanner from sonar-source
-    - rm -rf /sonar-scanner*
-    - wget --no-verbose --content-disposition -E -c "https://search.maven.org/remote_content?g=org.sonarsource.scanner.cli&a=sonar-scanner-cli&v=LATEST&c=linux&e=zip"
-    - unzip -q sonar-scanner-cli-*.zip -d /
+    # Download sonar-scanner from URL defined in CICD variable SONAR_SCANNER_URL
+    - export SONAR_SCANNER_FILENAME="${SONAR_SCANNER_URL##*/}"
+    - curl --silent --remote-name $SONAR_SCANNER_URL
+    # Get sonar-scanner root dir from printed contents of zip file
+    - export SONAR_SCANNER_DIR=$(unzip -l "$SONAR_SCANNER_FILENAME" | awk '{print $4}' | grep '/' | cut -d '/' -f 1 | sort | uniq -c | sort -nr | head -n 1 | awk '{print $2}')
+    - unzip -q $SONAR_SCANNER_FILENAME -d /
   script:
     # Scan with sonar-scanner
-    - /sonar-scanner-*-linux/bin/sonar-scanner  -Dsonar.qualitygate.wait=true -Dsonar.cfamily.build-wrapper-output=bw-output -Dsonar.coverageReportPaths=$SONAR_UNIT_TEST_REPORT
+    - /$SONAR_SCANNER_DIR/bin/sonar-scanner  -Dsonar.qualitygate.wait=true -Dsonar.cfamily.build-wrapper-output=bw-output -Dsonar.coverageReportPaths=$SONAR_UNIT_TEST_REPORT
   allow_failure: true
   needs:
     - build dss-sdk

host/src/include_private/nkv_framework.h

@@ -229,8 +229,10 @@
     int32_t core_to_pin;
     int32_t path_numa_node;
     std::atomic<uint32_t> nkv_async_path_cur_qd;
-    std::unordered_map<std::size_t, std::set<std::string> > *listing_keys; 
-    std::unordered_map<std::string, nkv_value_wrapper*> *data_cache; 
+    // This is a vector of maps with keys and set values
+    std::vector<std::unordered_map<std::size_t, std::set<std::string>>> listing_keys;
+    // This is a vector of maps with keys and wrapper object values
+    std::vector<std::unordered_map<std::string, nkv_value_wrapper*>> data_cache;
     std::atomic<uint32_t> nkv_outstanding_iter_on_path;
     std::atomic<uint32_t> nkv_path_stopping;
     std::atomic<uint64_t> nkv_num_key_prefixes;
@@ -244,7 +246,6 @@
     std::atomic<uint64_t> pending_io_size;
     std::atomic<uint64_t> pending_io_value;
     std::condition_variable cv_path;
-    //nkv_lruCache<std::string, nkv_value_wrapper> *cnt_cache;
     std::vector<nkv_lruCache<std::string, nkv_value_wrapper> *> cnt_cache;
     pthread_rwlock_t lru_rw_lock;
     std::mutex lru_lock;
@@ -282,16 +283,26 @@
         pthread_rwlock_init(&data_rw_lock_list[iter], NULL);
       }
 
-      listing_keys = new std::unordered_map<std::size_t, std::set<std::string> > (nkv_listing_cache_num_shards);
+      // Resize and initialize std::unordered_map at each index
+      listing_keys.resize(nkv_listing_cache_num_shards);
+      for (auto i =0; i<nkv_listing_cache_num_shards; i++) {
+          // Initialize map at index
+          listing_keys[i] = std::unordered_map<std::size_t, std::set<std::string>>();
+      }
       if (nkv_in_memory_exec) {
-        data_cache = new std::unordered_map<std::string, nkv_value_wrapper*> (nkv_listing_cache_num_shards);
+        // Resize and initialize data_cache
+        data_cache.resize(nkv_listing_cache_num_shards);
+        for (auto i=0; i<nkv_listing_cache_num_shards; i++) {
+            // Initialize map at index
+            data_cache[i] = std::unordered_map<std::string, nkv_value_wrapper*>();
+        }
       }
       cnt_cache.resize(nkv_read_cache_shard_size);
       for (auto i=0; i<nkv_read_cache_shard_size; i++) {
-          // Initialize class object
+          // Initialize object
           nkv_lruCache<std::string, nkv_value_wrapper> *cache_obj
               = new nkv_lruCache<std::string, nkv_value_wrapper>(nkv_read_cache_size);
-          cnt_cache.push_back(cache_obj);
+          cnt_cache[i] = cache_obj;
       }
       nkv_num_dc_keys = 0;
 

host/src/nkv_framework.cpp

@@ -2224,11 +2224,24 @@ NKVTargetPath::~NKVTargetPath() {
       }
 
       delete[] cache_rw_lock_list;
-      delete[] listing_keys;
+      // Explicit delete is not required with the STL container, clean up happens
+      // when container goes out of scope
+      // Just delete memory allocated outside STL container
       if (nkv_in_memory_exec) {
-        delete[] data_cache;
+          // All the previous memory allocated for nkv_value_wrapper
+          // needs to be freed
+          for (auto i=0; i<data_cache.size(); i++) {
+              // Each index at data_cache contains an
+              // unordered_map
+              for (auto it=data_cache[i].begin(); it!=data_cache[i].end(); it++) {
+                  // delete element only when not NULL
+                  if (it->second != NULL) {
+                      delete it->second;
+                  }
+              }
+          }
       }
-      //delete[] cnt_cache;
+      // Proceed to delete cnt_cache elements;
       for (int i=0; i<cnt_cache.size(); i++) {
           delete cnt_cache[i];
       }