From 4d0a4f32301c719e64cc68472da3923dbb724fd9 Mon Sep 17 00:00:00 2001 From: Dingming Wu Date: Tue, 19 May 2026 00:03:24 -0700 Subject: [PATCH] Fix AWS EC2 Terraform fidelity gaps --- .../aws-ec2/emulator_core/services/ami.py | 50 ++++++- .../emulator_core/services/dhcpoptions.py | 34 +++-- .../emulator_core/services/internetgateway.py | 3 +- .../aws-ec2/emulator_core/services/vpc.py | 139 +++++++++--------- .../emulator_core/services/vpcpeering.py | 5 +- emulators/aws-ec2/emulator_core/utils.py | 14 +- 6 files changed, 154 insertions(+), 91 deletions(-) diff --git a/emulators/aws-ec2/emulator_core/services/ami.py b/emulators/aws-ec2/emulator_core/services/ami.py index 2a68f58..cde59c0 100644 --- a/emulators/aws-ec2/emulator_core/services/ami.py +++ b/emulators/aws-ec2/emulator_core/services/ami.py @@ -151,6 +151,47 @@ class Ami_Backend: def __init__(self): self.state = EC2State.get() self.resources = self.state.amis # alias to shared store + self._ensure_default_images() + + def _ensure_default_images(self) -> None: + if self.resources: + return + + amazon_linux_2 = Ami( + image_id="ami-0f9fc25dd2506cf6d", + owner_id="137112412989", + image_owner_id="137112412989", + image_owner_alias="amazon", + name="amzn2-ami-hvm-2.0.20240109.0-x86_64-gp2", + description="Amazon Linux 2 AMI (HVM), SSD Volume Type", + state="available", + image_state="available", + image_type="machine", + architecture="x86_64", + creation_date="2024-01-09T00:00:00.000Z", + is_public=True, + image_location="amazon/amzn2-ami-hvm-2.0.20240109.0-x86_64-gp2", + platform_details="Linux/UNIX", + root_device_name="/dev/xvda", + root_device_type="ebs", + virtualization_type="hvm", + ena_support=True, + sriov_net_support="simple", + hypervisor="xen", + block_device_mappings=[ + { + "deviceName": "/dev/xvda", + "ebs": { + "deleteOnTermination": True, + "encrypted": False, + "snapshotId": "snap-0f9fc25dd2506cf6d", + "volumeSize": 8, + "volumeType": "gp2", + }, + } + ], + ) + self.resources[amazon_linux_2.image_id] = amazon_linux_2 def _utc_now(self) -> str: return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") @@ -635,7 +676,13 @@ def DescribeImages(self, params: Dict[str, Any]): owners = params.get("Owner.N", []) or [] if owners: - resources = [image for image in resources if image.owner_id in owners or image.image_owner_id in owners] + resources = [ + image + for image in resources + if image.owner_id in owners + or image.image_owner_id in owners + or image.image_owner_alias in owners + ] include_disabled = str2bool(params.get("IncludeDisabled")) include_deprecated = str2bool(params.get("IncludeDeprecated")) @@ -3111,4 +3158,3 @@ def serialize(action: str, data: Dict[str, Any], request_id: str) -> str: if action not in serializers: raise ValueError(f"Unknown action: {action}") return serializers[action](data, request_id) - diff --git a/emulators/aws-ec2/emulator_core/services/dhcpoptions.py b/emulators/aws-ec2/emulator_core/services/dhcpoptions.py index 6552d88..0ebd021 100644 --- a/emulators/aws-ec2/emulator_core/services/dhcpoptions.py +++ b/emulators/aws-ec2/emulator_core/services/dhcpoptions.py @@ -118,7 +118,7 @@ def CreateDhcpOptions(self, params: Dict[str, Any]): "Missing required parameter: DhcpConfiguration.N", ) - dhcp_options_id = self._generate_id("dhcp") + dhcp_options_id = self._generate_id("dopt") tag_set: List[Dict[str, Any]] = [] for spec in params.get("TagSpecification.N", []) or []: tag_set.extend(spec.get("Tags", []) or []) @@ -132,7 +132,7 @@ def CreateDhcpOptions(self, params: Dict[str, Any]): self.resources[dhcp_options_id] = resource return { - 'dhcpOptions': [resource.to_dict()], + 'dhcpOptions': resource.to_dict(), } def DeleteDhcpOptions(self, params: Dict[str, Any]): @@ -211,8 +211,23 @@ def parse_associate_dhcp_options_request(md: Dict[str, Any]) -> Dict[str, Any]: @staticmethod def parse_create_dhcp_options_request(md: Dict[str, Any]) -> Dict[str, Any]: + def parse_dhcp_configurations() -> List[Dict[str, Any]]: + configurations: List[Dict[str, Any]] = [] + index = 1 + while True: + key = get_scalar(md, f"DhcpConfiguration.{index}.Key") + if key is None: + break + values = get_indexed_list(md, f"DhcpConfiguration.{index}.Value") + configurations.append({ + "key": key, + "valueSet": [{"value": value} for value in values], + }) + index += 1 + return configurations + return { - "DhcpConfiguration.N": get_indexed_list(md, "DhcpConfiguration"), + "DhcpConfiguration.N": parse_dhcp_configurations(), "DryRun": str2bool(get_scalar(md, "DryRun")), "TagSpecification.N": parse_tags(md, "TagSpecification"), } @@ -345,15 +360,9 @@ def serialize_create_dhcp_options_response(data: Dict[str, Any], request_id: str if _dhcpOptions_key: param_data = data[_dhcpOptions_key] indent_str = " " * 1 - if param_data: - xml_parts.append(f'{indent_str}') - for item in param_data: - xml_parts.append(f'{indent_str} ') - xml_parts.extend(dhcpoptions_ResponseSerializer._serialize_nested_fields(item, 2)) - xml_parts.append(f'{indent_str} ') - xml_parts.append(f'{indent_str}') - else: - xml_parts.append(f'{indent_str}') + xml_parts.append(f'{indent_str}') + xml_parts.extend(dhcpoptions_ResponseSerializer._serialize_nested_fields(param_data, 2)) + xml_parts.append(f'{indent_str}') xml_parts.append(f'') return "\n".join(xml_parts) @@ -428,4 +437,3 @@ def serialize(action: str, data: Dict[str, Any], request_id: str) -> str: if action not in serializers: raise ValueError(f"Unknown action: {action}") return serializers[action](data, request_id) - diff --git a/emulators/aws-ec2/emulator_core/services/internetgateway.py b/emulators/aws-ec2/emulator_core/services/internetgateway.py index 4aeb81a..51f1ca0 100644 --- a/emulators/aws-ec2/emulator_core/services/internetgateway.py +++ b/emulators/aws-ec2/emulator_core/services/internetgateway.py @@ -395,7 +395,7 @@ def DetachInternetGateway(self, params: Dict[str, Any]): ) resource.vpc_id = None - resource.attachment_set = [{"state": "detached", "vpcId": vpc_id}] + resource.attachment_set = [] if hasattr(vpc, "internet_gateway_ids") and internet_gateway_id in vpc.internet_gateway_ids: vpc.internet_gateway_ids.remove(internet_gateway_id) @@ -778,4 +778,3 @@ def serialize(action: str, data: Dict[str, Any], request_id: str) -> str: if action not in serializers: raise ValueError(f"Unknown action: {action}") return serializers[action](data, request_id) - diff --git a/emulators/aws-ec2/emulator_core/services/vpc.py b/emulators/aws-ec2/emulator_core/services/vpc.py index 61bb534..14cf418 100644 --- a/emulators/aws-ec2/emulator_core/services/vpc.py +++ b/emulators/aws-ec2/emulator_core/services/vpc.py @@ -387,27 +387,21 @@ def DescribeVpcAttribute(self, params: Dict[str, Any]): return create_error_response("InvalidVpcID.NotFound", f"The ID '{vpc_id}' does not exist") supported_attributes = { - "enableDnsSupport": vpc.enable_dns_support, - "enableDnsHostnames": vpc.enable_dns_hostnames, - "enableNetworkAddressUsageMetrics": vpc.enable_network_address_usage_metrics, + "enableDnsSupport": ("enableDnsSupport", vpc.enable_dns_support), + "enableDnsHostnames": ("enableDnsHostnames", vpc.enable_dns_hostnames), + "enableNetworkAddressUsageMetrics": ( + "enableNetworkAddressUsageMetrics", + vpc.enable_network_address_usage_metrics, + ), } if attribute not in supported_attributes: return create_error_response("InvalidParameterValue", f"Invalid attribute '{attribute}'") + response_key, value = supported_attributes[attribute] return { - 'enableDnsHostnames': [ - { - 'Value': vpc.enable_dns_hostnames, - } - ], - 'enableDnsSupport': { - 'Value': vpc.enable_dns_support, + response_key: { + 'value': value, }, - 'enableNetworkAddressUsageMetrics': [ - { - 'Value': vpc.enable_network_address_usage_metrics, - } - ], 'vpcId': vpc.vpc_id, } @@ -429,7 +423,32 @@ def DescribeVpcs(self, params: Dict[str, Any]): else: resources = list(self.resources.values()) - resources = apply_filters(resources, params.get("Filter.N", [])) + filters = params.get("Filter.N", []) or [] + special_filter_names = { + "cidr-block-association.association-id", + "ipv6-cidr-block-association.association-id", + } + for resource_filter in filters: + name = resource_filter.get("Name", "") + values = set(resource_filter.get("Values", []) or []) + if name not in special_filter_names or not values: + continue + association_attr = ( + "ipv6_cidr_block_association_set" + if name.startswith("ipv6-") + else "cidr_block_association_set" + ) + resources = [ + resource + for resource in resources + if any( + str(association.get("associationId", "")) in values + for association in getattr(resource, association_attr, []) or [] + if isinstance(association, dict) + ) + ] + normal_filters = [f for f in filters if f.get("Name", "") not in special_filter_names] + resources = apply_filters(resources, normal_filters) vpc_set = [resource.to_dict() for resource in resources[:max_results]] return { @@ -648,9 +667,12 @@ def parse_disassociate_vpc_cidr_block_request(md: Dict[str, Any]) -> Dict[str, A @staticmethod def parse_modify_vpc_attribute_request(md: Dict[str, Any]) -> Dict[str, Any]: return { - "EnableDnsHostnames": get_scalar(md, "EnableDnsHostnames"), - "EnableDnsSupport": get_scalar(md, "EnableDnsSupport"), - "EnableNetworkAddressUsageMetrics": get_scalar(md, "EnableNetworkAddressUsageMetrics"), + "EnableDnsHostnames": get_scalar(md, "EnableDnsHostnames") + or get_scalar(md, "EnableDnsHostnames.Value"), + "EnableDnsSupport": get_scalar(md, "EnableDnsSupport") + or get_scalar(md, "EnableDnsSupport.Value"), + "EnableNetworkAddressUsageMetrics": get_scalar(md, "EnableNetworkAddressUsageMetrics") + or get_scalar(md, "EnableNetworkAddressUsageMetrics.Value"), "VpcId": get_scalar(md, "VpcId"), } @@ -848,57 +870,39 @@ def serialize_delete_vpc_response(data: Dict[str, Any], request_id: str) -> str: @staticmethod def serialize_describe_vpc_attribute_response(data: Dict[str, Any], request_id: str) -> str: + def append_attribute(xml_parts: List[str], data_key: str, tag_name: str) -> None: + if data_key not in data: + return + param_data = data[data_key] + if isinstance(param_data, list): + param_data = param_data[0] if param_data else {} + if isinstance(param_data, dict) and "Value" in param_data and "value" not in param_data: + param_data = {"value": param_data["Value"]} + indent_str = " " * 1 + xml_parts.append(f'{indent_str}<{tag_name}>') + if isinstance(param_data, dict): + xml_parts.extend(vpc_ResponseSerializer._serialize_nested_fields(param_data, 2)) + elif param_data is not None: + xml_parts.append(f'{indent_str} {str(param_data).lower()}') + xml_parts.append(f'{indent_str}') + xml_parts = [] xml_parts.append(f'') xml_parts.append(f' {esc(request_id)}') - # Serialize enableDnsHostnames - _enableDnsHostnames_key = None - if "enableDnsHostnames" in data: - _enableDnsHostnames_key = "enableDnsHostnames" - elif "EnableDnsHostnames" in data: - _enableDnsHostnames_key = "EnableDnsHostnames" - if _enableDnsHostnames_key: - param_data = data[_enableDnsHostnames_key] - indent_str = " " * 1 - if param_data: - xml_parts.append(f'{indent_str}') - for item in param_data: - xml_parts.append(f'{indent_str} ') - xml_parts.extend(vpc_ResponseSerializer._serialize_nested_fields(item, 2)) - xml_parts.append(f'{indent_str} ') - xml_parts.append(f'{indent_str}') - else: - xml_parts.append(f'{indent_str}') - # Serialize enableDnsSupport - _enableDnsSupport_key = None - if "enableDnsSupport" in data: - _enableDnsSupport_key = "enableDnsSupport" - elif "EnableDnsSupport" in data: - _enableDnsSupport_key = "EnableDnsSupport" - if _enableDnsSupport_key: - param_data = data[_enableDnsSupport_key] - indent_str = " " * 1 - xml_parts.append(f'{indent_str}') - xml_parts.extend(vpc_ResponseSerializer._serialize_nested_fields(param_data, 2)) - xml_parts.append(f'{indent_str}') - # Serialize enableNetworkAddressUsageMetrics - _enableNetworkAddressUsageMetrics_key = None - if "enableNetworkAddressUsageMetrics" in data: - _enableNetworkAddressUsageMetrics_key = "enableNetworkAddressUsageMetrics" - elif "EnableNetworkAddressUsageMetrics" in data: - _enableNetworkAddressUsageMetrics_key = "EnableNetworkAddressUsageMetrics" - if _enableNetworkAddressUsageMetrics_key: - param_data = data[_enableNetworkAddressUsageMetrics_key] - indent_str = " " * 1 - if param_data: - xml_parts.append(f'{indent_str}') - for item in param_data: - xml_parts.append(f'{indent_str} ') - xml_parts.extend(vpc_ResponseSerializer._serialize_nested_fields(item, 2)) - xml_parts.append(f'{indent_str} ') - xml_parts.append(f'{indent_str}') - else: - xml_parts.append(f'{indent_str}') + append_attribute(xml_parts, "enableDnsHostnames", "enableDnsHostnames") + append_attribute(xml_parts, "EnableDnsHostnames", "enableDnsHostnames") + append_attribute(xml_parts, "enableDnsSupport", "enableDnsSupport") + append_attribute(xml_parts, "EnableDnsSupport", "enableDnsSupport") + append_attribute( + xml_parts, + "enableNetworkAddressUsageMetrics", + "enableNetworkAddressUsageMetrics", + ) + append_attribute( + xml_parts, + "EnableNetworkAddressUsageMetrics", + "enableNetworkAddressUsageMetrics", + ) # Serialize vpcId _vpcId_key = None if "vpcId" in data: @@ -1048,4 +1052,3 @@ def serialize(action: str, data: Dict[str, Any], request_id: str) -> str: if action not in serializers: raise ValueError(f"Unknown action: {action}") return serializers[action](data, request_id) - diff --git a/emulators/aws-ec2/emulator_core/services/vpcpeering.py b/emulators/aws-ec2/emulator_core/services/vpcpeering.py index d6ecc6d..c008360 100644 --- a/emulators/aws-ec2/emulator_core/services/vpcpeering.py +++ b/emulators/aws-ec2/emulator_core/services/vpcpeering.py @@ -1,5 +1,5 @@ from typing import Dict, List, Any, Optional -from datetime import datetime, timezone +from datetime import datetime, timezone, timedelta from dataclasses import dataclass, field, asdict from enum import Enum import uuid @@ -189,7 +189,7 @@ def _build_vpc_info(vpc_obj, owner_override: Optional[str], region_override: Opt resource = VpcPeering( accepter_vpc_info=accepter_info, - expiration_time="", + expiration_time=(datetime.now(timezone.utc) + timedelta(days=7)).isoformat().replace("+00:00", "Z"), requester_vpc_info=requester_info, status=status, tag_set=tag_set, @@ -661,4 +661,3 @@ def serialize(action: str, data: Dict[str, Any], request_id: str) -> str: if action not in serializers: raise ValueError(f"Unknown action: {action}") return serializers[action](data, request_id) - diff --git a/emulators/aws-ec2/emulator_core/utils.py b/emulators/aws-ec2/emulator_core/utils.py index 33e6916..6de4ed7 100644 --- a/emulators/aws-ec2/emulator_core/utils.py +++ b/emulators/aws-ec2/emulator_core/utils.py @@ -1,6 +1,7 @@ from typing import Dict, Any, List, Union, Optional from werkzeug.datastructures import MultiDict import html +import fnmatch # ==================== TYPE ALIASES ==================== # These help clarify what types functions return @@ -356,6 +357,13 @@ def serialize_error_response(error_data: ErrorResponse, request_id: str) -> str: """ +def _matches_filter_value(actual: str, expected_values: List[str]) -> bool: + for expected in expected_values: + if fnmatch.fnmatchcase(actual, str(expected)): + return True + return False + + def apply_filters(resources: List[Any], filters: List[Filter]) -> List[Any]: """ Apply AWS-style filters to a list of resource objects or dicts. @@ -405,7 +413,7 @@ def apply_filters(resources: List[Any], filters: List[Filter]) -> List[Any]: if isinstance(tag, dict) and tag.get("Key") == tag_key: tag_value = tag.get("Value", "") break - if tag_value is None or tag_value not in values: + if tag_value is None or not _matches_filter_value(str(tag_value), values): match = False break continue @@ -432,14 +440,14 @@ def apply_filters(resources: List[Any], filters: List[Filter]) -> List[Any]: elif isinstance(obj, list): # List field: pass if any element matches any value list_strs = [str(item) for item in obj] - if not any(v in list_strs for v in values): + if not any(_matches_filter_value(item, values) for item in list_strs): match = False break continue else: val_str = str(obj) - if val_str not in values: + if not _matches_filter_value(val_str, values): match = False break