add only unique stacks for training

fabric8-analytics · Mar 1, 2019 · 2aa9ff1 · 2aa9ff1
1 parent f431489
commit 2aa9ff1
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 2 deletions.
diff --git a/f8a_report/report_helper.py b/f8a_report/report_helper.py
@@ -228,12 +228,16 @@ def store_training_data(self, result):
         model_version = dt.now().strftime('%Y-%m-%d')
 
         for eco, stacks in result.items():
+            unique_stacks = {}
             obj_key = '{eco}/{depl_prefix}/{model_version}/data/manifest.json'.format(
                 eco=eco, depl_prefix=self.s3.deployment_prefix, model_version=model_version)
             package_list_for_eco = []
             for packages, reccurrence_count in stacks.items():
                 package_list = [x.strip().split(' ')[0] for x in packages.split(',')]
-                package_list_for_eco.append(package_list)
+                stack_str = "".join(package_list)
+                if stack_str not in unique_stacks:
+                    unique_stacks[stack_str] = 1
+                    package_list_for_eco.append(package_list)
 
             training_data = {
                 'ecosystem': eco,

diff --git a/tests/data/collateddata.json b/tests/data/collateddata.json
@@ -7,6 +7,7 @@
         "flask 1.2.3, sqlalchemy 1.4.3": 10
     },
     "maven": {
-        "io.vertx:vertx-core 3.4.2,io.vertx:vertx-web 3.4.2": 1
+        "io.vertx:vertx-core 3.4.2,io.vertx:vertx-web 3.4.2": 1,
+        "io.vertx:vertx-core 3.4.1,io.vertx:vertx-web 3.4.1": 1
     }
 }