Skip to content

Commit

Permalink
[aws][feat] Make a collection of Ec2 Instance types only for existing…
Browse files Browse the repository at this point in the history
… instances (#2264)

Co-authored-by: Matthias Veit <[email protected]>
  • Loading branch information
1101-1 and aquamatthias authored Nov 1, 2024
1 parent ebb67be commit bc8eae4
Show file tree
Hide file tree
Showing 7 changed files with 182 additions and 9 deletions.
2 changes: 1 addition & 1 deletion plugins/aws/fix_plugin_aws/resource/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ def from_api(cls: Type[AwsResourceType], json: Json, builder: GraphBuilder) -> O
return parse_json(json, cls, builder, cls.mapping)

@classmethod
def collect_resources(cls: Type[AwsResource], builder: GraphBuilder) -> None:
def collect_resources(cls, builder: GraphBuilder) -> None:
# Default behavior: in case the class has an ApiSpec, call the api and call collect.
log.debug(f"Collecting {cls.__name__} in region {builder.region.name}")
if spec := cls.api_spec:
Expand Down
46 changes: 45 additions & 1 deletion plugins/aws/fix_plugin_aws/resource/ec2.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from functools import partial
from typing import ClassVar, Dict, Optional, List, Type, Any

from boto3.exceptions import Boto3Error
from attrs import define, field

from fix_plugin_aws.aws_client import AwsClient
Expand All @@ -22,6 +23,7 @@
from fix_plugin_aws.resource.kms import AwsKmsKey
from fix_plugin_aws.resource.s3 import AwsS3Bucket
from fix_plugin_aws.utils import ToDict, TagsValue
from fix_plugin_aws.aws_client import AwsClient
from fixlib.baseresources import (
BaseInstance,
BaseKeyPair,
Expand Down Expand Up @@ -384,14 +386,14 @@ class AwsEc2InferenceAcceleratorInfo:

@define(eq=False, slots=False)
class AwsEc2InstanceType(AwsResource, BaseInstanceType):
# collected via AwsEc2Instance
kind: ClassVar[str] = "aws_ec2_instance_type"
_kind_display: ClassVar[str] = "AWS EC2 Instance Type"
_kind_description: ClassVar[str] = "AWS EC2 Instance Types are predefined virtual server configurations offered by Amazon Web Services. Each type specifies the compute, memory, storage, and networking capacity of the virtual machine. Users select an instance type based on their application's requirements, balancing performance and cost. EC2 instances can be launched, stopped, and terminated as needed for various computing workloads." # fmt: skip
_docs_url: ClassVar[str] = "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html"
_kind_service: ClassVar[Optional[str]] = service_name
_metadata: ClassVar[Dict[str, Any]] = {"icon": "type", "group": "compute"}
_aws_metadata: ClassVar[Dict[str, Any]] = {"arn_tpl": "arn:{partition}:ec2:{region}:{account}:instance/{id}"} # fmt: skip
api_spec: ClassVar[AwsApiSpec] = AwsApiSpec(service_name, "describe-instance-types", "InstanceTypes")
_reference_kinds: ClassVar[ModelReference] = {
"successors": {
"default": ["aws_ec2_instance"],
Expand Down Expand Up @@ -456,6 +458,29 @@ class AwsEc2InstanceType(AwsResource, BaseInstanceType):
auto_recovery_supported: Optional[bool] = field(default=None)
supported_boot_modes: List[str] = field(factory=list)

@classmethod
def collect_resource_types(cls, builder: GraphBuilder, instance_types: List[str]) -> None:
spec = AwsApiSpec(service_name, "describe-instance-types", "InstanceTypes")
log.debug(f"Collecting {cls.__name__} in region {builder.region.name}")
try:
filters = [{"Name": "instance-type", "Values": instance_types}]
items = builder.client.list(
aws_service=spec.service,
action=spec.api_action,
result_name=spec.result_property,
expected_errors=spec.expected_errors,
Filters=filters,
)
cls.collect(items, builder)
except Boto3Error as e:
msg = f"Error while collecting {cls.__name__} in region {builder.region.name}: {e}"
builder.core_feedback.error(msg, log)
raise
except Exception as e:
msg = f"Error while collecting {cls.__name__} in region {builder.region.name}: {e}"
builder.core_feedback.info(msg, log)
raise

@classmethod
def collect(cls: Type[AwsResource], json: List[Json], builder: GraphBuilder) -> None:
for js in json:
Expand All @@ -467,6 +492,14 @@ def collect(cls: Type[AwsResource], json: List[Json], builder: GraphBuilder) ->
# we collect instance types in all regions and make the data unique in the builder
builder.global_instance_types[it.safe_name] = it

@classmethod
def service_name(cls) -> Optional[str]:
return service_name

@classmethod
def called_collect_apis(cls) -> List[AwsApiSpec]:
return [AwsApiSpec(service_name, "describe-instance-types")]


# endregion

Expand Down Expand Up @@ -1375,6 +1408,17 @@ class AwsEc2Instance(EC2Taggable, AwsResource, BaseInstance):
instance_maintenance_options: Optional[str] = field(default=None)
instance_user_data: Optional[str] = field(default=None)

@classmethod
def collect_resources(cls, builder: GraphBuilder) -> None:
super().collect_resources(builder)
ec2_instance_types = set()
for instance in builder.nodes(clazz=AwsEc2Instance):
ec2_instance_types.add(instance.instance_type)
if ec2_instance_types:
builder.submit_work(
service_name, AwsEc2InstanceType.collect_resource_types, builder, list(ec2_instance_types)
)

@classmethod
def collect(cls: Type[AwsResource], json: List[Json], builder: GraphBuilder) -> None:
def fetch_user_data(instance: AwsEc2Instance) -> None:
Expand Down
4 changes: 2 additions & 2 deletions plugins/aws/fix_plugin_aws/resource/iam.py
Original file line number Diff line number Diff line change
Expand Up @@ -676,11 +676,11 @@ def called_collect_apis(cls) -> List[AwsApiSpec]:
]

@classmethod
def collect_resources(cls: Type[AwsResource], builder: GraphBuilder) -> None:
def collect_resources(cls, builder: GraphBuilder) -> None:
# start generation of the credentials resport and pick it up later
builder.client.get(service_name, "generate-credential-report")
# let super handle the rest (this will take some time for the report to be done)
super().collect_resources(builder) # type: ignore # mypy bug: https://github.com/python/mypy/issues/12885
super().collect_resources(builder)

@classmethod
def collect(cls: Type[AwsResource], json_list: List[Json], builder: GraphBuilder) -> None:
Expand Down
1 change: 1 addition & 0 deletions plugins/aws/test/graphbuilder_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ def test_instance_type(builder: GraphBuilder) -> None:
cloud_instance_data, ["aws", instance_type, "pricing", builder.region.id, "linux", "ondemand"]
)
eu_builder = builder.for_region(AwsRegion(id="eu-central-1"))
builder.global_instance_types[instance_type] = AwsEc2InstanceType(id=instance_type)
m4l_eu: AwsEc2InstanceType = eu_builder.instance_type(eu_builder.region, instance_type) # type: ignore
assert m4l != m4l_eu
assert m4l_eu == eu_builder.instance_type(eu_builder.region, instance_type)
Expand Down
3 changes: 2 additions & 1 deletion plugins/aws/test/resources/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,8 @@ def round_trip_for(
to_collect = [cls] + collect_also if collect_also else [cls]
builder = build_graph(to_collect, region_name=region_name)
assert len(builder.graph.nodes) > 0
for node, data in builder.graph.nodes(data=True):
nodes_to_process = list(builder.graph.nodes(data=True))
for node, data in nodes_to_process:
node.connect_in_graph(builder, data.get("source", {}))
check_single_node(node)
first = next(iter(builder.resources_of(cls)))
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
{
"InstanceTypes": [
{
"InstanceType": "m4.large",
"CurrentGeneration": true,
"FreeTierEligible": false,
"SupportedUsageClasses": [
"on-demand",
"spot"
],
"SupportedRootDeviceTypes": [
"ebs"
],
"SupportedVirtualizationTypes": [
"hvm"
],
"BareMetal": false,
"Hypervisor": "nitro",
"ProcessorInfo": {
"SupportedArchitectures": [
"x86_64"
],
"SustainedClockSpeedInGhz": 3.5
},
"VCpuInfo": {
"DefaultVCpus": 8,
"DefaultCores": 4,
"DefaultThreadsPerCore": 2,
"ValidCores": [
2,
4
],
"ValidThreadsPerCore": [
1,
2
]
},
"MemoryInfo": {
"SizeInMiB": 16384
},
"InstanceStorageSupported": false,
"InstanceStorageInfo": {
"EbsInfo": {
"EbsStorageSupported": false,
"EbsStorageInfo": {
"VolumeTypes": [
"standard"
],
"VolumeSizeInGiBMin": 1,
"VolumeSizeInGiBMax": 1024
}
},
"InstanceStorageSupported": false,
"InstanceStorageInfo": {
"VolumeTypes": [
"standard"
],
"VolumeSizeInGiBMin": 1,
"VolumeSizeInGiBMax": 1024
}
},
"GpuInfo": {
"GPUsSupported": false,
"GPUSupported": false,
"GPUSupportedOnDemand": false,
"GPUSupportedSpot": false
},
"FpgaInfo": {
"FPGAsSupported": false,
"FPGASupported": false,
"FPGASupportedOnDemand": false,
"FPGASupportedSpot": false
},
"InferenceAcceleratorInfo": {
"InferenceAcceleratorsSupported": false,
"InferenceAcceleratorsSupportedOnDemand": false,
"InferenceAcceleratorsSupportedSpot": false
},
"EbsInfo": {
"EbsOptimizedSupport": "default",
"EncryptionSupport": "supported",
"EbsOptimizedInfo": {
"BaselineBandwidthInMbps": 2500,
"BaselineThroughputInMBps": 312.5,
"BaselineIops": 12000,
"MaximumBandwidthInMbps": 10000,
"MaximumThroughputInMBps": 1250,
"MaximumIops": 40000
},
"NvmeSupport": "required"
},
"NetworkInfo": {
"NetworkPerformance": "Up to 12.5 Gigabit",
"MaximumNetworkInterfaces": 4,
"MaximumNetworkCards": 1,
"DefaultNetworkCardIndex": 0,
"NetworkCards": [
{
"NetworkCardIndex": 0,
"NetworkPerformance": "Up to 12.5 Gigabit",
"MaximumNetworkInterfaces": 4
}
],
"Ipv4AddressesPerInterface": 15,
"Ipv6AddressesPerInterface": 15,
"Ipv6Supported": true,
"EnaSupport": "required",
"EfaSupported": false,
"EncryptionInTransitSupported": true
},
"PlacementGroupInfo": {
"SupportedStrategies": [
"cluster",
"partition",
"spread"
]
},
"HibernationSupported": false,
"BurstablePerformanceSupported": false,
"DedicatedHostsSupported": true,
"AutoRecoverySupported": true,
"SupportedBootModes": [
"legacy-bios",
"uefi"
]
}
]
}
7 changes: 3 additions & 4 deletions plugins/aws/test/resources/service_quotas_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from fix_plugin_aws.resource.base import AwsResource
from fix_plugin_aws.aws_client import AwsClient
from fix_plugin_aws.resource.base import GraphBuilder, AwsRegion
from fix_plugin_aws.resource.ec2 import AwsEc2InstanceType, AwsEc2Vpc
from fix_plugin_aws.resource.ec2 import AwsEc2InstanceType, AwsEc2Instance, AwsEc2Vpc
from fix_plugin_aws.resource.elbv2 import AwsAlb
from fix_plugin_aws.resource.iam import AwsIamServerCertificate
from fix_plugin_aws.resource.service_quotas import AwsServiceQuota, RegionalQuotas
Expand All @@ -20,11 +20,10 @@ def test_service_quotas() -> None:


def test_instance_type_quotas() -> None:
_, builder = round_trip_for(AwsServiceQuota, "usage", "quota_type")
AwsEc2InstanceType.collect_resources(builder)
_, builder = round_trip_for(AwsServiceQuota, "usage", "quota_type", collect_also=[AwsEc2Instance])
for _, it in builder.global_instance_types.items():
builder.add_node(it, {})
expect_quotas(builder, 3)
expect_quotas(builder, 5)


def test_volume_type_quotas() -> None:
Expand Down

0 comments on commit bc8eae4

Please sign in to comment.