From bc8eae4ad003596de64b2f149526d0d610c4a64b Mon Sep 17 00:00:00 2001 From: 1101-1 <70093559+1101-1@users.noreply.github.com> Date: Fri, 1 Nov 2024 15:28:46 +0500 Subject: [PATCH] [aws][feat] Make a collection of Ec2 Instance types only for existing instances (#2264) Co-authored-by: Matthias Veit --- plugins/aws/fix_plugin_aws/resource/base.py | 2 +- plugins/aws/fix_plugin_aws/resource/ec2.py | 46 ++++++- plugins/aws/fix_plugin_aws/resource/iam.py | 4 +- plugins/aws/test/graphbuilder_test.py | 1 + plugins/aws/test/resources/__init__.py | 3 +- ...es__instance_type_m4_large_mac2_metal.json | 128 ++++++++++++++++++ .../aws/test/resources/service_quotas_test.py | 7 +- 7 files changed, 182 insertions(+), 9 deletions(-) create mode 100644 plugins/aws/test/resources/files/ec2/describe-instance-types__instance_type_m4_large_mac2_metal.json diff --git a/plugins/aws/fix_plugin_aws/resource/base.py b/plugins/aws/fix_plugin_aws/resource/base.py index 75a7158f73..e5d0ff6794 100644 --- a/plugins/aws/fix_plugin_aws/resource/base.py +++ b/plugins/aws/fix_plugin_aws/resource/base.py @@ -198,7 +198,7 @@ def from_api(cls: Type[AwsResourceType], json: Json, builder: GraphBuilder) -> O return parse_json(json, cls, builder, cls.mapping) @classmethod - def collect_resources(cls: Type[AwsResource], builder: GraphBuilder) -> None: + def collect_resources(cls, builder: GraphBuilder) -> None: # Default behavior: in case the class has an ApiSpec, call the api and call collect. log.debug(f"Collecting {cls.__name__} in region {builder.region.name}") if spec := cls.api_spec: diff --git a/plugins/aws/fix_plugin_aws/resource/ec2.py b/plugins/aws/fix_plugin_aws/resource/ec2.py index 881b234308..36dbbb414b 100644 --- a/plugins/aws/fix_plugin_aws/resource/ec2.py +++ b/plugins/aws/fix_plugin_aws/resource/ec2.py @@ -6,6 +6,7 @@ from functools import partial from typing import ClassVar, Dict, Optional, List, Type, Any +from boto3.exceptions import Boto3Error from attrs import define, field from fix_plugin_aws.aws_client import AwsClient @@ -22,6 +23,7 @@ from fix_plugin_aws.resource.kms import AwsKmsKey from fix_plugin_aws.resource.s3 import AwsS3Bucket from fix_plugin_aws.utils import ToDict, TagsValue +from fix_plugin_aws.aws_client import AwsClient from fixlib.baseresources import ( BaseInstance, BaseKeyPair, @@ -384,6 +386,7 @@ class AwsEc2InferenceAcceleratorInfo: @define(eq=False, slots=False) class AwsEc2InstanceType(AwsResource, BaseInstanceType): + # collected via AwsEc2Instance kind: ClassVar[str] = "aws_ec2_instance_type" _kind_display: ClassVar[str] = "AWS EC2 Instance Type" _kind_description: ClassVar[str] = "AWS EC2 Instance Types are predefined virtual server configurations offered by Amazon Web Services. Each type specifies the compute, memory, storage, and networking capacity of the virtual machine. Users select an instance type based on their application's requirements, balancing performance and cost. EC2 instances can be launched, stopped, and terminated as needed for various computing workloads." # fmt: skip @@ -391,7 +394,6 @@ class AwsEc2InstanceType(AwsResource, BaseInstanceType): _kind_service: ClassVar[Optional[str]] = service_name _metadata: ClassVar[Dict[str, Any]] = {"icon": "type", "group": "compute"} _aws_metadata: ClassVar[Dict[str, Any]] = {"arn_tpl": "arn:{partition}:ec2:{region}:{account}:instance/{id}"} # fmt: skip - api_spec: ClassVar[AwsApiSpec] = AwsApiSpec(service_name, "describe-instance-types", "InstanceTypes") _reference_kinds: ClassVar[ModelReference] = { "successors": { "default": ["aws_ec2_instance"], @@ -456,6 +458,29 @@ class AwsEc2InstanceType(AwsResource, BaseInstanceType): auto_recovery_supported: Optional[bool] = field(default=None) supported_boot_modes: List[str] = field(factory=list) + @classmethod + def collect_resource_types(cls, builder: GraphBuilder, instance_types: List[str]) -> None: + spec = AwsApiSpec(service_name, "describe-instance-types", "InstanceTypes") + log.debug(f"Collecting {cls.__name__} in region {builder.region.name}") + try: + filters = [{"Name": "instance-type", "Values": instance_types}] + items = builder.client.list( + aws_service=spec.service, + action=spec.api_action, + result_name=spec.result_property, + expected_errors=spec.expected_errors, + Filters=filters, + ) + cls.collect(items, builder) + except Boto3Error as e: + msg = f"Error while collecting {cls.__name__} in region {builder.region.name}: {e}" + builder.core_feedback.error(msg, log) + raise + except Exception as e: + msg = f"Error while collecting {cls.__name__} in region {builder.region.name}: {e}" + builder.core_feedback.info(msg, log) + raise + @classmethod def collect(cls: Type[AwsResource], json: List[Json], builder: GraphBuilder) -> None: for js in json: @@ -467,6 +492,14 @@ def collect(cls: Type[AwsResource], json: List[Json], builder: GraphBuilder) -> # we collect instance types in all regions and make the data unique in the builder builder.global_instance_types[it.safe_name] = it + @classmethod + def service_name(cls) -> Optional[str]: + return service_name + + @classmethod + def called_collect_apis(cls) -> List[AwsApiSpec]: + return [AwsApiSpec(service_name, "describe-instance-types")] + # endregion @@ -1375,6 +1408,17 @@ class AwsEc2Instance(EC2Taggable, AwsResource, BaseInstance): instance_maintenance_options: Optional[str] = field(default=None) instance_user_data: Optional[str] = field(default=None) + @classmethod + def collect_resources(cls, builder: GraphBuilder) -> None: + super().collect_resources(builder) + ec2_instance_types = set() + for instance in builder.nodes(clazz=AwsEc2Instance): + ec2_instance_types.add(instance.instance_type) + if ec2_instance_types: + builder.submit_work( + service_name, AwsEc2InstanceType.collect_resource_types, builder, list(ec2_instance_types) + ) + @classmethod def collect(cls: Type[AwsResource], json: List[Json], builder: GraphBuilder) -> None: def fetch_user_data(instance: AwsEc2Instance) -> None: diff --git a/plugins/aws/fix_plugin_aws/resource/iam.py b/plugins/aws/fix_plugin_aws/resource/iam.py index bc6b65db99..bdf6b15a39 100644 --- a/plugins/aws/fix_plugin_aws/resource/iam.py +++ b/plugins/aws/fix_plugin_aws/resource/iam.py @@ -676,11 +676,11 @@ def called_collect_apis(cls) -> List[AwsApiSpec]: ] @classmethod - def collect_resources(cls: Type[AwsResource], builder: GraphBuilder) -> None: + def collect_resources(cls, builder: GraphBuilder) -> None: # start generation of the credentials resport and pick it up later builder.client.get(service_name, "generate-credential-report") # let super handle the rest (this will take some time for the report to be done) - super().collect_resources(builder) # type: ignore # mypy bug: https://github.com/python/mypy/issues/12885 + super().collect_resources(builder) @classmethod def collect(cls: Type[AwsResource], json_list: List[Json], builder: GraphBuilder) -> None: diff --git a/plugins/aws/test/graphbuilder_test.py b/plugins/aws/test/graphbuilder_test.py index e6bb29de3a..f5a769c3bd 100644 --- a/plugins/aws/test/graphbuilder_test.py +++ b/plugins/aws/test/graphbuilder_test.py @@ -17,6 +17,7 @@ def test_instance_type(builder: GraphBuilder) -> None: cloud_instance_data, ["aws", instance_type, "pricing", builder.region.id, "linux", "ondemand"] ) eu_builder = builder.for_region(AwsRegion(id="eu-central-1")) + builder.global_instance_types[instance_type] = AwsEc2InstanceType(id=instance_type) m4l_eu: AwsEc2InstanceType = eu_builder.instance_type(eu_builder.region, instance_type) # type: ignore assert m4l != m4l_eu assert m4l_eu == eu_builder.instance_type(eu_builder.region, instance_type) diff --git a/plugins/aws/test/resources/__init__.py b/plugins/aws/test/resources/__init__.py index 21847aee17..f2f3bab13f 100644 --- a/plugins/aws/test/resources/__init__.py +++ b/plugins/aws/test/resources/__init__.py @@ -180,7 +180,8 @@ def round_trip_for( to_collect = [cls] + collect_also if collect_also else [cls] builder = build_graph(to_collect, region_name=region_name) assert len(builder.graph.nodes) > 0 - for node, data in builder.graph.nodes(data=True): + nodes_to_process = list(builder.graph.nodes(data=True)) + for node, data in nodes_to_process: node.connect_in_graph(builder, data.get("source", {})) check_single_node(node) first = next(iter(builder.resources_of(cls))) diff --git a/plugins/aws/test/resources/files/ec2/describe-instance-types__instance_type_m4_large_mac2_metal.json b/plugins/aws/test/resources/files/ec2/describe-instance-types__instance_type_m4_large_mac2_metal.json new file mode 100644 index 0000000000..093544ce9a --- /dev/null +++ b/plugins/aws/test/resources/files/ec2/describe-instance-types__instance_type_m4_large_mac2_metal.json @@ -0,0 +1,128 @@ +{ + "InstanceTypes": [ + { + "InstanceType": "m4.large", + "CurrentGeneration": true, + "FreeTierEligible": false, + "SupportedUsageClasses": [ + "on-demand", + "spot" + ], + "SupportedRootDeviceTypes": [ + "ebs" + ], + "SupportedVirtualizationTypes": [ + "hvm" + ], + "BareMetal": false, + "Hypervisor": "nitro", + "ProcessorInfo": { + "SupportedArchitectures": [ + "x86_64" + ], + "SustainedClockSpeedInGhz": 3.5 + }, + "VCpuInfo": { + "DefaultVCpus": 8, + "DefaultCores": 4, + "DefaultThreadsPerCore": 2, + "ValidCores": [ + 2, + 4 + ], + "ValidThreadsPerCore": [ + 1, + 2 + ] + }, + "MemoryInfo": { + "SizeInMiB": 16384 + }, + "InstanceStorageSupported": false, + "InstanceStorageInfo": { + "EbsInfo": { + "EbsStorageSupported": false, + "EbsStorageInfo": { + "VolumeTypes": [ + "standard" + ], + "VolumeSizeInGiBMin": 1, + "VolumeSizeInGiBMax": 1024 + } + }, + "InstanceStorageSupported": false, + "InstanceStorageInfo": { + "VolumeTypes": [ + "standard" + ], + "VolumeSizeInGiBMin": 1, + "VolumeSizeInGiBMax": 1024 + } + }, + "GpuInfo": { + "GPUsSupported": false, + "GPUSupported": false, + "GPUSupportedOnDemand": false, + "GPUSupportedSpot": false + }, + "FpgaInfo": { + "FPGAsSupported": false, + "FPGASupported": false, + "FPGASupportedOnDemand": false, + "FPGASupportedSpot": false + }, + "InferenceAcceleratorInfo": { + "InferenceAcceleratorsSupported": false, + "InferenceAcceleratorsSupportedOnDemand": false, + "InferenceAcceleratorsSupportedSpot": false + }, + "EbsInfo": { + "EbsOptimizedSupport": "default", + "EncryptionSupport": "supported", + "EbsOptimizedInfo": { + "BaselineBandwidthInMbps": 2500, + "BaselineThroughputInMBps": 312.5, + "BaselineIops": 12000, + "MaximumBandwidthInMbps": 10000, + "MaximumThroughputInMBps": 1250, + "MaximumIops": 40000 + }, + "NvmeSupport": "required" + }, + "NetworkInfo": { + "NetworkPerformance": "Up to 12.5 Gigabit", + "MaximumNetworkInterfaces": 4, + "MaximumNetworkCards": 1, + "DefaultNetworkCardIndex": 0, + "NetworkCards": [ + { + "NetworkCardIndex": 0, + "NetworkPerformance": "Up to 12.5 Gigabit", + "MaximumNetworkInterfaces": 4 + } + ], + "Ipv4AddressesPerInterface": 15, + "Ipv6AddressesPerInterface": 15, + "Ipv6Supported": true, + "EnaSupport": "required", + "EfaSupported": false, + "EncryptionInTransitSupported": true + }, + "PlacementGroupInfo": { + "SupportedStrategies": [ + "cluster", + "partition", + "spread" + ] + }, + "HibernationSupported": false, + "BurstablePerformanceSupported": false, + "DedicatedHostsSupported": true, + "AutoRecoverySupported": true, + "SupportedBootModes": [ + "legacy-bios", + "uefi" + ] + } + ] +} \ No newline at end of file diff --git a/plugins/aws/test/resources/service_quotas_test.py b/plugins/aws/test/resources/service_quotas_test.py index f5153f662c..0da0b2a81c 100644 --- a/plugins/aws/test/resources/service_quotas_test.py +++ b/plugins/aws/test/resources/service_quotas_test.py @@ -6,7 +6,7 @@ from fix_plugin_aws.resource.base import AwsResource from fix_plugin_aws.aws_client import AwsClient from fix_plugin_aws.resource.base import GraphBuilder, AwsRegion -from fix_plugin_aws.resource.ec2 import AwsEc2InstanceType, AwsEc2Vpc +from fix_plugin_aws.resource.ec2 import AwsEc2InstanceType, AwsEc2Instance, AwsEc2Vpc from fix_plugin_aws.resource.elbv2 import AwsAlb from fix_plugin_aws.resource.iam import AwsIamServerCertificate from fix_plugin_aws.resource.service_quotas import AwsServiceQuota, RegionalQuotas @@ -20,11 +20,10 @@ def test_service_quotas() -> None: def test_instance_type_quotas() -> None: - _, builder = round_trip_for(AwsServiceQuota, "usage", "quota_type") - AwsEc2InstanceType.collect_resources(builder) + _, builder = round_trip_for(AwsServiceQuota, "usage", "quota_type", collect_also=[AwsEc2Instance]) for _, it in builder.global_instance_types.items(): builder.add_node(it, {}) - expect_quotas(builder, 3) + expect_quotas(builder, 5) def test_volume_type_quotas() -> None: