[COST-4936] Add memory and operating system columns to aws trino table (

#5071) * [COST-4936] Add memory and operating system columns to aws trino table * fix typo
project-koku · May 2, 2024 · e48395e · e48395e
1 parent 5d8b4b4
commit e48395e
Show file tree

Hide file tree

Showing 6 changed files with 21 additions and 2 deletions.
diff --git a/dev/scripts/nise_ymls/ocp_on_aws/aws_static_data.yml b/dev/scripts/nise_ymls/ocp_on_aws/aws_static_data.yml
@@ -12,6 +12,7 @@ generators:
         resourceTags/user:dashed-key-on-aws: dashed-value
         resourceTags/user:Mapping: a1
         resourceTags/user:Map: a2
+        resourceTags/user:Name: instance-name-1
       cost_category:
         costCategory/env: prod
       instance_type:
@@ -40,6 +41,7 @@ generators:
         resourceTags/user:com_redhat_rhel_usage: disaster Recovery
         resourceTags/user:Mapping: b1
         resourceTags/user:Map: b2
+        resourceTags/user:Name: instance-name-2
       cost_category:
         costCategory/env: prod
       instance_type:
@@ -67,6 +69,7 @@ generators:
         resourceTags/user:CoM_RedHat_Rhel_varianT: Workstation
         resourceTags/user:Mapping: c1
         resourceTags/user:Map: c2
+        resourceTags/user:Name: instance_name_3
       cost_category:
         costCategory/env: stage
         costCategory/Organization: Ninja
@@ -94,6 +97,7 @@ generators:
         resourceTags/user:com_redhat_rhel_sla: self-support
         resourceTags/user:CoM_RedHat_Rhel_varianT: hPC
         resourceTags/user:com_redhat_rhel_usage: disaster Recovery
+        resourceTags/user:Name: instance name 4
       cost_category:
         costCategory/env: prod
       instance_type:
@@ -120,6 +124,7 @@ generators:
         resourceTags/user:com_redhat_rhel_sla: self-support
         resourceTags/user:CoM_RedHat_Rhel_varianT: hPC
         resourceTags/user:com_redhat_rhel_usage: disaster Recovery
+        resourceTags/user:Name: instance-name-5
       cost_category:
         costCategory/env: prod
       instance_type:

diff --git a/koku/masu/test/util/aws/test_aws_post_processor.py b/koku/masu/test/util/aws/test_aws_post_processor.py
@@ -49,6 +49,8 @@ def test_aws_generate_daily_data(self):
                 "product_instancetype": "t2.micro",
                 "product_region": "us-east-1",
                 "product_vcpu": "8",
+                "product_memory": "8 GiB",
+                "product_operatingsystem": "Linux",
                 "pricing_unit": "hours",
                 "resourcetags": '{"key": "value"}',
                 "costcategory": '{"cat": "egory"}',
@@ -81,6 +83,8 @@ def test_aws_generate_daily_data(self):
                 "product_instancetype": "t2.micro",
                 "product_region": "us-east-1",
                 "product_vcpu": "4",
+                "product_memory": "8 GiB",
+                "product_operatingsystem": "Linux",
                 "pricing_unit": "hours",
                 "resourcetags": '{"key": "value"}',
                 "costcategory": '{"cat": "egory"}',
@@ -114,6 +118,8 @@ def test_aws_generate_daily_data(self):
                 "product_region": "us-east-1",
                 "pricing_unit": "hours",
                 "product_vcpu": "8",
+                "product_memory": "8 GiB",
+                "product_operatingsystem": "Linux",
                 "resourcetags": '{"key": "value"}',
                 "costcategory": '{"cat": "egory"}',
                 "lineitem_usageamount": lineitem_usageamount,

diff --git a/koku/masu/util/aws/aws_post_processor.py b/koku/masu/util/aws/aws_post_processor.py
@@ -118,6 +118,7 @@ class AWSPostProcessor:
         "reservation_end_time": "reservation/EndTime",
         "product_physical_cores": "product/physicalCores",
         "identity_time_interval": "identity/TimeInterval",
+        "product_operating_system": "product/operatingSystem",
     }
 
     CSV_COLUMN_PREFIX = (
@@ -206,6 +207,7 @@ def _generate_daily_data(self, data_frame):
                 "lineitem_availabilityzone",
                 "lineitem_lineitemtype",
                 "product_productfamily",
+                "product_operatingsystem",
                 "product_instancetype",
                 "product_region",
                 "pricing_unit",
@@ -226,9 +228,10 @@ def _generate_daily_data(self, data_frame):
                 "pricing_publicondemandcost": ["sum"],
                 "pricing_publicondemandrate": ["max"],
                 "savingsplan_savingsplaneffectivecost": ["sum"],
-                "product_productname": ["max"],
                 "bill_invoiceid": ["max"],
+                "product_productname": ["max"],
                 "product_vcpu": ["max"],
+                "product_memory": ["max"],
             }
         )
         columns = daily_data_frame.columns.droplevel(1)

diff --git a/koku/masu/util/aws/common.py b/koku/masu/util/aws/common.py
@@ -139,13 +139,15 @@
     "product/instanceType",
     "product/vcpu",
     "product/memory",
+    "product/operatingSystem",
 }
 
 OPTIONAL_ALT_COLS = {
     "product_physical_cores",
     "product_instancetype",
     "product_vcpu",
     "product_memory",
+    "product_operating_system",
 }
 
 DATE_FMT = "%Y-%m-%d"

diff --git a/koku/reporting/provider/aws/models.py b/koku/reporting/provider/aws/models.py
@@ -48,6 +48,8 @@
     "product/instanceType": "",
     "product/physicalCores": "",
     "product/vcpu": "",
+    "product/memory": "",
+    "product/operatingSystem": "",
     "pricing/unit": "",
     "lineItem/UsageAmount": 0.0,
     "lineItem/NormalizationFactor": 0.0,

diff --git a/scripts/cji_scripts/migrate_trino.py b/scripts/cji_scripts/migrate_trino.py
@@ -163,7 +163,8 @@ def main():
     logging.info(schemas)
 
     tables_to_drop = [
-        "azure_line_items",
+        "aws_line_items",
+        "aws_line_items_daily",
     ]
     # columns_to_drop = ["ocp_matched"]
     # columns_to_add = {