From e48395e285252d010c97a35336091a29a4e4ba9b Mon Sep 17 00:00:00 2001 From: David Nakabaale Date: Thu, 2 May 2024 13:02:55 -0400 Subject: [PATCH] [COST-4936] Add memory and operating system columns to aws trino table (#5071) * [COST-4936] Add memory and operating system columns to aws trino table * fix typo --- dev/scripts/nise_ymls/ocp_on_aws/aws_static_data.yml | 5 +++++ koku/masu/test/util/aws/test_aws_post_processor.py | 6 ++++++ koku/masu/util/aws/aws_post_processor.py | 5 ++++- koku/masu/util/aws/common.py | 2 ++ koku/reporting/provider/aws/models.py | 2 ++ scripts/cji_scripts/migrate_trino.py | 3 ++- 6 files changed, 21 insertions(+), 2 deletions(-) diff --git a/dev/scripts/nise_ymls/ocp_on_aws/aws_static_data.yml b/dev/scripts/nise_ymls/ocp_on_aws/aws_static_data.yml index 7948e733b4..0d2efd2644 100644 --- a/dev/scripts/nise_ymls/ocp_on_aws/aws_static_data.yml +++ b/dev/scripts/nise_ymls/ocp_on_aws/aws_static_data.yml @@ -12,6 +12,7 @@ generators: resourceTags/user:dashed-key-on-aws: dashed-value resourceTags/user:Mapping: a1 resourceTags/user:Map: a2 + resourceTags/user:Name: instance-name-1 cost_category: costCategory/env: prod instance_type: @@ -40,6 +41,7 @@ generators: resourceTags/user:com_redhat_rhel_usage: disaster Recovery resourceTags/user:Mapping: b1 resourceTags/user:Map: b2 + resourceTags/user:Name: instance-name-2 cost_category: costCategory/env: prod instance_type: @@ -67,6 +69,7 @@ generators: resourceTags/user:CoM_RedHat_Rhel_varianT: Workstation resourceTags/user:Mapping: c1 resourceTags/user:Map: c2 + resourceTags/user:Name: instance_name_3 cost_category: costCategory/env: stage costCategory/Organization: Ninja @@ -94,6 +97,7 @@ generators: resourceTags/user:com_redhat_rhel_sla: self-support resourceTags/user:CoM_RedHat_Rhel_varianT: hPC resourceTags/user:com_redhat_rhel_usage: disaster Recovery + resourceTags/user:Name: instance name 4 cost_category: costCategory/env: prod instance_type: @@ -120,6 +124,7 @@ generators: resourceTags/user:com_redhat_rhel_sla: self-support resourceTags/user:CoM_RedHat_Rhel_varianT: hPC resourceTags/user:com_redhat_rhel_usage: disaster Recovery + resourceTags/user:Name: instance-name-5 cost_category: costCategory/env: prod instance_type: diff --git a/koku/masu/test/util/aws/test_aws_post_processor.py b/koku/masu/test/util/aws/test_aws_post_processor.py index 8a29ce3985..75dad5e745 100644 --- a/koku/masu/test/util/aws/test_aws_post_processor.py +++ b/koku/masu/test/util/aws/test_aws_post_processor.py @@ -49,6 +49,8 @@ def test_aws_generate_daily_data(self): "product_instancetype": "t2.micro", "product_region": "us-east-1", "product_vcpu": "8", + "product_memory": "8 GiB", + "product_operatingsystem": "Linux", "pricing_unit": "hours", "resourcetags": '{"key": "value"}', "costcategory": '{"cat": "egory"}', @@ -81,6 +83,8 @@ def test_aws_generate_daily_data(self): "product_instancetype": "t2.micro", "product_region": "us-east-1", "product_vcpu": "4", + "product_memory": "8 GiB", + "product_operatingsystem": "Linux", "pricing_unit": "hours", "resourcetags": '{"key": "value"}', "costcategory": '{"cat": "egory"}', @@ -114,6 +118,8 @@ def test_aws_generate_daily_data(self): "product_region": "us-east-1", "pricing_unit": "hours", "product_vcpu": "8", + "product_memory": "8 GiB", + "product_operatingsystem": "Linux", "resourcetags": '{"key": "value"}', "costcategory": '{"cat": "egory"}', "lineitem_usageamount": lineitem_usageamount, diff --git a/koku/masu/util/aws/aws_post_processor.py b/koku/masu/util/aws/aws_post_processor.py index a174b651ec..d3714c612b 100644 --- a/koku/masu/util/aws/aws_post_processor.py +++ b/koku/masu/util/aws/aws_post_processor.py @@ -118,6 +118,7 @@ class AWSPostProcessor: "reservation_end_time": "reservation/EndTime", "product_physical_cores": "product/physicalCores", "identity_time_interval": "identity/TimeInterval", + "product_operating_system": "product/operatingSystem", } CSV_COLUMN_PREFIX = ( @@ -206,6 +207,7 @@ def _generate_daily_data(self, data_frame): "lineitem_availabilityzone", "lineitem_lineitemtype", "product_productfamily", + "product_operatingsystem", "product_instancetype", "product_region", "pricing_unit", @@ -226,9 +228,10 @@ def _generate_daily_data(self, data_frame): "pricing_publicondemandcost": ["sum"], "pricing_publicondemandrate": ["max"], "savingsplan_savingsplaneffectivecost": ["sum"], - "product_productname": ["max"], "bill_invoiceid": ["max"], + "product_productname": ["max"], "product_vcpu": ["max"], + "product_memory": ["max"], } ) columns = daily_data_frame.columns.droplevel(1) diff --git a/koku/masu/util/aws/common.py b/koku/masu/util/aws/common.py index 4f6150697a..339564e804 100644 --- a/koku/masu/util/aws/common.py +++ b/koku/masu/util/aws/common.py @@ -139,6 +139,7 @@ "product/instanceType", "product/vcpu", "product/memory", + "product/operatingSystem", } OPTIONAL_ALT_COLS = { @@ -146,6 +147,7 @@ "product_instancetype", "product_vcpu", "product_memory", + "product_operating_system", } DATE_FMT = "%Y-%m-%d" diff --git a/koku/reporting/provider/aws/models.py b/koku/reporting/provider/aws/models.py index 68a5aaccac..2fcf2b471f 100644 --- a/koku/reporting/provider/aws/models.py +++ b/koku/reporting/provider/aws/models.py @@ -48,6 +48,8 @@ "product/instanceType": "", "product/physicalCores": "", "product/vcpu": "", + "product/memory": "", + "product/operatingSystem": "", "pricing/unit": "", "lineItem/UsageAmount": 0.0, "lineItem/NormalizationFactor": 0.0, diff --git a/scripts/cji_scripts/migrate_trino.py b/scripts/cji_scripts/migrate_trino.py index a4138f2db9..d71c049abc 100755 --- a/scripts/cji_scripts/migrate_trino.py +++ b/scripts/cji_scripts/migrate_trino.py @@ -163,7 +163,8 @@ def main(): logging.info(schemas) tables_to_drop = [ - "azure_line_items", + "aws_line_items", + "aws_line_items_daily", ] # columns_to_drop = ["ocp_matched"] # columns_to_add = {