From 2e684990f391e211bdf42916b77a7e361b0432c2 Mon Sep 17 00:00:00 2001 From: Pushkar Acharya Date: Mon, 12 Dec 2016 14:49:50 -0800 Subject: [PATCH] Update nova driver to Platform9 2.4 release This commit updates the nova driver to latest release of Platform9. Some of the notable changes include: 1. Addition of unit tests 2. Configurable maximum usable resources i.e. memory, disk and cpu 3. Better integration for security groups with latest changes in neutron --- nova/CHANGELOG | 5 + nova/README.md | 21 +- nova/ec2/ec2_group_transformer.py | 20 - nova/ec2/ec2_rule_service.py | 25 - nova/ec2/ec2_rule_transformer.py | 38 -- nova/ec2/ec2driver.py | 737 ++++++++++-------------- nova/ec2/exception_handler.py | 16 +- nova/ec2/group.py | 20 - nova/ec2/group_rule_refresher.py | 55 -- nova/ec2/instance_rule_refresher.py | 27 - nova/ec2/openstack_group_service.py | 23 - nova/ec2/openstack_group_transformer.py | 19 - nova/ec2/openstack_rule_service.py | 25 - nova/ec2/openstack_rule_transformer.py | 33 -- nova/ec2/rule.py | 32 - nova/ec2/rule_comparator.py | 47 -- nova/test-requirements.txt | 1 + nova/tests/ec2/__init__.py | 0 nova/tests/ec2/test_driver.py | 540 +++++++++++++++++ 19 files changed, 866 insertions(+), 818 deletions(-) create mode 100644 nova/CHANGELOG delete mode 100644 nova/ec2/ec2_group_transformer.py delete mode 100644 nova/ec2/ec2_rule_service.py delete mode 100644 nova/ec2/ec2_rule_transformer.py delete mode 100644 nova/ec2/group.py delete mode 100644 nova/ec2/group_rule_refresher.py delete mode 100644 nova/ec2/instance_rule_refresher.py delete mode 100644 nova/ec2/openstack_group_service.py delete mode 100644 nova/ec2/openstack_group_transformer.py delete mode 100644 nova/ec2/openstack_rule_service.py delete mode 100644 nova/ec2/openstack_rule_transformer.py delete mode 100644 nova/ec2/rule.py delete mode 100644 nova/ec2/rule_comparator.py create mode 100644 nova/test-requirements.txt create mode 100644 nova/tests/ec2/__init__.py create mode 100644 nova/tests/ec2/test_driver.py diff --git a/nova/CHANGELOG b/nova/CHANGELOG new file mode 100644 index 0000000..8897ce2 --- /dev/null +++ b/nova/CHANGELOG @@ -0,0 +1,5 @@ +12th December 2016: + a. Adds unit tests + b. Adds additional configurable settings for configuring maximum resources available to nova + c. Better security group integration with latest changes in neutron drivers + diff --git a/nova/README.md b/nova/README.md index 8b2b135..a946704 100644 --- a/nova/README.md +++ b/nova/README.md @@ -1,5 +1,8 @@ ## Setup +Updated: 12th December 2016 + (Updated to be in sync with Platform9 release 2.4) + ### Prerequesites 1. Working green field OpenStack deployment (code currently based out of stable/liberty) 2. The virtualenv used by nova should have Amazon boto package installed @@ -9,15 +12,27 @@ ### Instructions 1. Copy the nova/ec2 directory to /nova/nova/virt/ -2. Update the configuration files - - 1. edit /etc/nova/**nova.conf** +2. Update the configuration files - + 1. edit /etc/nova/**nova.conf** ``` [DEFAULT] compute_driver = ec2.EC2Driver - + [AWS] secret_key = access_key = region_name = + max_cpus = + max_memory_mb = + max_disk_gb = ``` 3. Restart the nova compute services + +### Running unit tests: +1. Copy the nova/tests/ec2 to /nova/tests/unit/virt directory +2. To run the AWS Driver unit tests - + ``` + tox -e nova.tests.unit.virt.ec2 + e.g. to run python 2.7 tests - + tox -e py27 nova.tests.unit.virt.ec2 + ``` diff --git a/nova/ec2/ec2_group_transformer.py b/nova/ec2/ec2_group_transformer.py deleted file mode 100644 index 7b47856..0000000 --- a/nova/ec2/ec2_group_transformer.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (c) 2014 ThoughtWorks -# Copyright (c) 2016 Platform9 Systems Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -class EC2GroupTransformer: - - def to_group(self, ec2_group): - pass diff --git a/nova/ec2/ec2_rule_service.py b/nova/ec2/ec2_rule_service.py deleted file mode 100644 index 6076b5d..0000000 --- a/nova/ec2/ec2_rule_service.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (c) 2014 ThoughtWorks -# Copyright (c) 2016 Platform9 Systems Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -class EC2RuleService: - - def __init__(self, ec2_connection, ec2_rule_transformer): - self.ec2_connection = ec2_connection - self.ec2_rule_transformer = ec2_rule_transformer - - def get_rules_for_group(self, group_name): - group = self.ec2_connection.get_all_security_groups(groupnames=group_name)[0] - return set([self.ec2_rule_transformer.to_rule(rule) for rule in group.rules]) diff --git a/nova/ec2/ec2_rule_transformer.py b/nova/ec2/ec2_rule_transformer.py deleted file mode 100644 index 020e573..0000000 --- a/nova/ec2/ec2_rule_transformer.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) 2014 ThoughtWorks -# Copyright (c) 2016 Platform9 Systems Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from copy import deepcopy -from rule import Rule - - -class EC2RuleTransformer: - - def __init__(self, ec2_connection): - self.ec2_connection = ec2_connection - - def to_rule(self, ec2_rule): - rule_args = {} - rule_args['ip_protocol'] = ec2_rule.ip_protocol - rule_args['from_port'] = ec2_rule.from_port - rule_args['to_port'] = ec2_rule.to_port - - if ec2_rule.grants[0].cidr_ip: - rule_args['ip_range'] = ec2_rule.grants[0].cidr_ip - else: - group_id = ec2_rule.grants[0].group_id - rule_args['group_name'] = self.ec2_connection.get_all_security_groups(group_ids=group_id)[0].name - - return Rule(**rule_args) diff --git a/nova/ec2/ec2driver.py b/nova/ec2/ec2driver.py index 96c6e98..af0c953 100644 --- a/nova/ec2/ec2driver.py +++ b/nova/ec2/ec2driver.py @@ -15,19 +15,22 @@ # under the License. """Connection to the Amazon Web Services - EC2 service""" -from threading import Lock + import base64 -import time -from boto import ec2, vpc import boto.ec2.cloudwatch +import datetime +import hashlib +import json +import sys +import time +import uuid +from threading import Lock +from boto import ec2, vpc from boto import exception as boto_exc from boto.exception import EC2ResponseError from boto.regioninfo import RegionInfo from oslo_config import cfg -from novaclient import client -from ec2_rule_service import EC2RuleService -from ec2_rule_transformer import EC2RuleTransformer -from credentials import Credentials +from nova.i18n import * from nova import block_device from nova.compute import power_state from nova.compute import task_states @@ -40,71 +43,90 @@ from oslo_service import loopingcall from nova.virt import driver from nova.virt import virtapi from nova.virt import hardware -from instance_rule_refresher import InstanceRuleRefresher -from openstack_group_service import OpenstackGroupService -from openstack_rule_service import OpenstackRuleService -from openstack_rule_transformer import OpenstackRuleTransformer -import sys -from group_rule_refresher import GroupRuleRefresher from nova.virt.ec2.exception_handler import Ec2ExceptionHandler -import json LOG = logging.getLogger(__name__) -ec2driver_opts = [ - cfg.StrOpt('snapshot_image_format', - help='Snapshot image format (valid options are : ' - 'raw, qcow2, vmdk, vdi). ' - 'Defaults to same as source image'), - cfg.StrOpt('datastore_regex', - help='Regex to match the name of a datastore.'), - cfg.FloatOpt('task_poll_interval', - default=0.5, - help='The interval used for polling of remote tasks.'), - cfg.IntOpt('api_retry_count', - default=10, - help='The number of times we retry on failures, e.g., ' - 'socket error, etc.'), - cfg.IntOpt('vnc_port', - default=5900, - help='VNC starting port'), - cfg.IntOpt('vnc_port_total', - default=10000, - help='Total number of VNC ports'), - cfg.BoolOpt('use_linked_clone', - default=True, - help='Whether to use linked clone') -] - aws_group = cfg.OptGroup(name='AWS', title='Options to connect to an AWS cloud') aws_opts = [ - cfg.StrOpt('secret_key', help='Secret key of AWS account', required=True, secret=True), - cfg.StrOpt('access_key', help='Access key of AWS account', required=True, secret=True), - cfg.StrOpt('region_name', help='AWS region', required=True) + cfg.StrOpt('secret_key', help='Secret key of AWS account', secret=True), + cfg.StrOpt('access_key', help='Access key of AWS account', secret=True), + cfg.StrOpt('region_name', help='AWS region'), + cfg.IntOpt('vnc_port', + default=5900, + help='VNC starting port'), + # 500 VCPUs + cfg.IntOpt('max_vcpus', + default=500, + help='Max number of vCPUs that can be used'), + # 1000 GB RAM + cfg.IntOpt('max_memory_mb', + default=1024000, + help='Max memory MB that can be used'), + # 1 TB Storage + cfg.IntOpt('max_disk_gb', + default=1024, + help='Max storage in GB that can be used') ] CONF = cfg.CONF -CONF.register_opts(ec2driver_opts, 'ec2driver') CONF.import_opt('my_ip', 'nova.netconf') CONF.register_group(aws_group) CONF.register_opts(aws_opts, group=aws_group) -# TIME_BETWEEN_API_CALL_RETRIES = 1.0 - EC2_STATE_MAP = { "pending": power_state.NOSTATE, "running": power_state.RUNNING, "shutting-down": power_state.NOSTATE, - "terminated": power_state.SHUTDOWN, + "terminated": power_state.CRASHED, "stopping": power_state.NOSTATE, "stopped": power_state.SHUTDOWN } -#Default resources available -VCPUS = 100 -MEMORY_IN_MBS = 88192 -DISK_IN_GB = 1028 +EC2_FLAVOR_MAP = { + 'c3.2xlarge': {'memory_mb': 15360.0, 'vcpus': 8}, + 'c3.4xlarge': {'memory_mb': 30720.0, 'vcpus': 16}, + 'c3.8xlarge': {'memory_mb': 61440.0, 'vcpus': 32}, + 'c3.large': {'memory_mb': 3840.0, 'vcpus': 2}, + 'c3.xlarge': {'memory_mb': 7680.0, 'vcpus': 4}, + 'c4.2xlarge': {'memory_mb': 15360.0, 'vcpus': 8}, + 'c4.4xlarge': {'memory_mb': 30720.0, 'vcpus': 16}, + 'c4.8xlarge': {'memory_mb': 61440.0, 'vcpus': 36}, + 'c4.large': {'memory_mb': 3840.0, 'vcpus': 2}, + 'c4.xlarge': {'memory_mb': 7680.0, 'vcpus': 4}, + 'd2.2xlarge': {'memory_mb': 62464.0, 'vcpus': 8}, + 'd2.4xlarge': {'memory_mb': 124928.0, 'vcpus': 16}, + 'd2.8xlarge': {'memory_mb': 249856.0, 'vcpus': 36}, + 'd2.xlarge': {'memory_mb': 31232.0, 'vcpus': 4}, + 'g2.2xlarge': {'memory_mb': 15360.0, 'vcpus': 8}, + 'g2.8xlarge': {'memory_mb': 61440.0, 'vcpus': 32}, + 'i2.2xlarge': {'memory_mb': 62464.0, 'vcpus': 8}, + 'i2.4xlarge': {'memory_mb': 124928.0, 'vcpus': 16}, + 'i2.8xlarge': {'memory_mb': 249856.0, 'vcpus': 32}, + 'i2.xlarge': {'memory_mb': 31232.0, 'vcpus': 4}, + 'm3.2xlarge': {'memory_mb': 30720.0, 'vcpus': 8}, + 'm3.large': {'memory_mb': 7680.0, 'vcpus': 2}, + 'm3.medium': {'memory_mb': 3840.0, 'vcpus': 1}, + 'm3.xlarge': {'memory_mb': 15360.0, 'vcpus': 4}, + 'm4.10xlarge': {'memory_mb': 163840.0, 'vcpus': 40}, + 'm4.2xlarge': {'memory_mb': 32768.0, 'vcpus': 8}, + 'm4.4xlarge': {'memory_mb': 65536.0, 'vcpus': 16}, + 'm4.large': {'memory_mb': 8192.0, 'vcpus': 2}, + 'm4.xlarge': {'memory_mb': 16384.0, 'vcpus': 4}, + 'r3.2xlarge': {'memory_mb': 62464.0, 'vcpus': 8}, + 'r3.4xlarge': {'memory_mb': 124928.0, 'vcpus': 16}, + 'r3.8xlarge': {'memory_mb': 249856.0, 'vcpus': 32}, + 'r3.large': {'memory_mb': 15616.0, 'vcpus': 2}, + 'r3.xlarge': {'memory_mb': 31232.0, 'vcpus': 4}, + 't2.large': {'memory_mb': 8192.0, 'vcpus': 2}, + 't2.medium': {'memory_mb': 4096.0, 'vcpus': 2}, + 't2.micro': {'memory_mb': 1024.0, 'vcpus': 1}, + 't2.nano': {'memory_mb': 512.0, 'vcpus': 1}, + 't2.small': {'memory_mb': 2048.0, 'vcpus': 1}, + 'x1.32xlarge': {'memory_mb': 1998848.0, 'vcpus': 128} +} + DIAGNOSTIC_KEYS_TO_FILTER = ['group', 'block_device_mapping'] @@ -138,14 +160,12 @@ class EC2Driver(driver.ComputeDriver): "supports_recreate": True, } - """EC2 hypervisor driver. Respurposing for EC2""" - def __init__(self, virtapi, read_only=False): super(EC2Driver, self).__init__(virtapi) self.host_status_base = { - 'vcpus': VCPUS, - 'memory_mb': MEMORY_IN_MBS, - 'local_gb': DISK_IN_GB, + 'vcpus': CONF.AWS.max_vcpus, + 'memory_mb': CONF.AWS.max_memory_mb, + 'local_gb': CONF.AWS.max_disk_gb, 'vcpus_used': 0, 'memory_mb_used': 0, 'local_gb_used': 0, @@ -153,71 +173,59 @@ class EC2Driver(driver.ComputeDriver): 'hypervisor_version': '1.0', 'hypervisor_hostname': CONF.host, 'cpu_info': {}, - 'disk_available_least': DISK_IN_GB, + 'disk_available_least': CONF.AWS.max_disk_gb, } self._mounts = {} self._interfaces = {} - self._pf9_stats = {} - self.nova_creds = Credentials.get_nova_creds() - self.nova = None - if self.nova_creds is None: - LOG.error("Error fetching the Nova Credentials") - else: - VERSION = "2" - self.nova = client.Client(VERSION, self.nova_creds['OS_USERNAME'], self.nova_creds['OS_PASSWORD'], - tenant_id=self.nova_creds['project_id'], auth_url=self.nova_creds['OS_AUTH_URL'], - region_name=self.nova_creds['OS_REGION_NAME'], insecure=True) - + self._uuid_to_ec2_instance = {} + self.ec2_flavor_info = EC2_FLAVOR_MAP aws_region = CONF.AWS.region_name aws_endpoint = "ec2." + aws_region + ".amazonaws.com" region = RegionInfo(name=aws_region, endpoint=aws_endpoint) - LOG.info("******EC2 init with %s region" % aws_region) self.ec2_conn = ec2.EC2Connection(aws_access_key_id=CONF.AWS.access_key, aws_secret_access_key=CONF.AWS.secret_key, region=region) - self.vpc_conn = vpc.VPCConnection(aws_access_key_id=CONF.AWS.access_key, - aws_secret_access_key=CONF.AWS.secret_key, - region=region) - self.cloudwatch_conn = ec2.cloudwatch.connect_to_region( aws_region, aws_access_key_id=CONF.AWS.access_key, aws_secret_access_key=CONF.AWS.secret_key) - self.security_group_lock = Lock() - - self.instance_rule_refresher = InstanceRuleRefresher( - GroupRuleRefresher( - ec2_connection=self.ec2_conn, - openstack_rule_service=OpenstackRuleService( - group_service=OpenstackGroupService(self.nova.security_groups), - openstack_rule_transformer=OpenstackRuleTransformer() - ), - ec2_rule_service=EC2RuleService( - ec2_connection=self.ec2_conn, - ec2_rule_transformer=EC2RuleTransformer(self.ec2_conn) - ) - ) - ) - + LOG.info("EC2 driver init with %s region" % aws_region) if not '_EC2_NODES' in globals(): set_nodes([CONF.host]) def init_host(self, host): - """Initialize anything that is necessary for the driver to function, + """ + Initialize anything that is necessary for the driver to function, including catching up with currently running VM's on the given host. """ return def list_instances(self): - """Return the names of all the instances known to the virtualization + """ + Return the names of all the instances known to the virtualization layer, as a list. """ - all_instances = self.ec2_conn.get_all_instances() + all_instances = self.ec2_conn.get_only_instances() + self._uuid_to_ec2_instance.clear() instance_ids = [] for instance in all_instances: + generate_uuid = False + if instance.state in ['pending', 'shutting-down', 'terminated']: + continue + if len(instance.tags) > 0: + if 'openstack_id' in instance.tags: + self._uuid_to_ec2_instance[instance.tags['openstack_id']] = \ + instance + else: + generate_uuid = True + else: + generate_uuid = True + if generate_uuid: + instance_uuid = self._get_uuid_from_aws_id(instance.id) + self._uuid_to_ec2_instance[instance_uuid] = instance instance_ids.append(instance.id) return instance_ids @@ -229,37 +237,9 @@ class EC2Driver(driver.ComputeDriver): """Unplug VIFs from networks.""" pass - def _configure_default_security_group(self): - """ This function will create and configure Platform9's Default Security Group - """ - LOG.info("Configuring default security groups") - sec_groups = self.ec2_conn.get_all_security_groups() - p9_sec_group_name = "pf9-default" - p9_sec_group_desc = "This is the default Platform9 security group" - ip_protocol = "TCP" - from_port = to_port = 22 - cidr_ip = "0.0.0.0/0" - - try: - for sec_group in sec_groups: - if sec_group.name == p9_sec_group_name: - self.ec2_conn.authorize_security_group(group_name=p9_sec_group_name, ip_protocol=ip_protocol, - from_port=from_port, to_port=to_port, cidr_ip=cidr_ip) - return - - self.ec2_conn.create_security_group(p9_sec_group_name, p9_sec_group_desc) - self.ec2_conn.authorize_security_group(group_name=p9_sec_group_name, ip_protocol=ip_protocol, - from_port=from_port, to_port=to_port, cidr_ip=cidr_ip) - except EC2ResponseError: - exp = sys.exc_value - if exp.error_code == "InvalidPermission.Duplicate": - LOG.info("default security group already exists") - else: - LOG.info("Error in _configure_default_security_group: %s" % exp.message) - def _add_ssh_keys(self, key_name, key_data): """ - Adds SSH Keys into AWS EC2 account + Adds SSH Keys into AWS EC2 account :param key_name: :param key_data: :return: @@ -267,32 +247,21 @@ class EC2Driver(driver.ComputeDriver): # TODO: Need to handle the cases if a key with the same keyname exists and different key content exist_key_pair = self.ec2_conn.get_key_pair(key_name) if not exist_key_pair: - LOG.info("***** Adding SSH key to AWS") + LOG.info("Adding SSH key to AWS") self.ec2_conn.import_key_pair(key_name, key_data) else: - LOG.info("***** SSH key already exists in AWS") - - def _get_subnet_id(self): - """ - Will fetch the Subnet ID of the first Subnet in the VPC - :return: subnet_id - """ - subnets = self.vpc_conn.get_all_subnets() - if len(subnets) > 0: - subnet_id = subnets[0].id - LOG.info("***** Calling SPAWN Subnet found is %s" % subnet_id) - return subnet_id - return None + LOG.info("SSH key already exists in AWS") def _get_image_ami_id_from_meta(self, context, image_lacking_meta): """ - Pulls the Image AMI ID from the location attribute of Image Meta + Pulls the Image AMI ID from the location attribute of Image Meta :param image_meta: :return: ami_id """ image_api = glance.get_default_image_service() - image_meta = image_api._client.call(context, 2, 'get', image_lacking_meta['id']) - LOG.info("***** Calling _get_image_ami_id_from_meta Meta*******: %s", image_meta) + image_meta = image_api._client.call(context, 2, 'get', + image_lacking_meta['id']) + LOG.info("Calling _get_image_ami_id_from_meta Meta: %s", image_meta) try: return image_meta['aws_image_id'] except Exception as e: @@ -301,22 +270,43 @@ class EC2Driver(driver.ComputeDriver): def _process_network_info(self, network_info): """ - Will process network_info object by picking up only one Network out of many + Will process network_info object by picking up only one Network out of many :param network_info: :return: """ - LOG.info("*****Networks ****** %s" % network_info) + LOG.info("Networks to be processed : %s" % network_info) subnet_id = None fixed_ip = None + port_id = None + network_id = None + if len(network_info) > 1: + LOG.warn('AWS does not allow connecting 1 instance to multiple ' + 'VPCs.') for vif in network_info: - LOG.info("*****VIF *****") if 'details' in vif: network_dict = json.loads(vif['details']) subnet_id = network_dict['subnet_id'] LOG.info("Adding subnet ID:" + subnet_id) fixed_ip = network_dict['ip_address'] LOG.info("Fixed IP:" + fixed_ip) - return subnet_id, fixed_ip + port_id = vif['id'] + network_id = vif['network']['id'] + break + return subnet_id, fixed_ip, port_id, network_id + + def _get_instance_sec_grps(self, context, port_id, network_id): + secgrp_ids = [] + from nova import network + network_api = network.API() + port_obj = network_api.show_port(context, port_id) + if port_obj.get('port', {}).get('security_groups', []): + filters = {'tag-value': port_obj['port']['security_groups']} + secgrps = self.ec2_conn.get_all_security_groups(filters=filters) + for secgrp in secgrps: + if network_id and 'openstack_network_id' in secgrp.tags and \ + secgrp.tags['openstack_network_id'] == network_id: + secgrp_ids.append(secgrp.id) + return secgrp_ids def spawn(self, context, instance, image_meta, injected_files, admin_password, network_info=None, block_device_info=None): @@ -343,62 +333,64 @@ class EC2Driver(driver.ComputeDriver): """ image_ami_id = self._get_image_ami_id_from_meta(context, image_meta) - # image_ami_id = "ami-06116566" - subnet_id, fixed_ip = self._process_network_info(network_info) + subnet_id, fixed_ip, port_id, network_id = self._process_network_info( + network_info) if subnet_id is None or fixed_ip is None: raise exception.BuildAbortException("Network configuration failure") - #Flavor + security_groups = self._get_instance_sec_grps(context, port_id, network_id) + + # Flavor flavor_dict = instance['flavor'] - LOG.info("***** Calling SPAWN Flavor Input******: %s " % flavor_dict) - # flavor_type = flavor_map[instance.get_flavor().id] flavor_type = flavor_dict['name'] - LOG.info("***** Calling SPAWN Flavor after mapping ***: %s" % flavor_type) # SSH Keys if instance['key_name'] is not None and instance['key_data'] is not None: self._add_ssh_keys(instance['key_name'], instance['key_data']) - #Security groups - # self._configure_default_security_group() - # security_groups = ["default"] - - #Creating the EC2 instance + # Creating the EC2 instance user_data = None - #passing user_data from the openstack instance which is Base64 encoded after decoding it. + # Passing user_data from the openstack instance which is Base64 encoded + # after decoding it. if 'user_data' in instance and instance['user_data'] is not None: user_data = instance['user_data'] - LOG.info("****** Calling SPAWN user_data.... %s" % user_data) user_data = base64.b64decode(user_data) try: - reservation = self.ec2_conn.run_instances(instance_type=flavor_type, key_name=instance['key_name'], - image_id=image_ami_id, - user_data=user_data, subnet_id=subnet_id, private_ip_address=fixed_ip) + reservation = self.ec2_conn.run_instances( + instance_type=flavor_type, key_name=instance['key_name'], + image_id=image_ami_id, user_data=user_data, + subnet_id=subnet_id, private_ip_address=fixed_ip, + security_group_ids=security_groups) ec2_instance = reservation.instances ec2_instance_obj = ec2_instance[0] ec2_id = ec2_instance[0].id self._wait_for_state(instance, ec2_id, "running", power_state.RUNNING) - LOG.info("****** Instance is UP and Running *********") instance['metadata'].update({'ec2_id': ec2_id}) - LOG.debug("*** ADDing Instance name tag %s to the AWS instance" % instance['display_name']) ec2_instance_obj.add_tag("Name", instance['display_name']) - LOG.debug("*** ADDing Openstack uuid tag %s to the AWS instance" % instance['uuid']) ec2_instance_obj.add_tag("openstack_id", instance['uuid']) + self._uuid_to_ec2_instance[instance.uuid] = ec2_instance_obj # Fetch Public IP of the instance if it has one instances = self.ec2_conn.get_only_instances(instance_ids=[ec2_id]) if len(instances) > 0: public_ip = instances[0].ip_address if public_ip is not None: - LOG.info("****** Updating Public IP address of the device") instance['metadata'].update({'public_ip_address': public_ip}) except EC2ResponseError as ec2_exception: actual_exception = Ec2ExceptionHandler.get_processed_exception(ec2_exception) LOG.info("Error in starting instance %s" % (actual_exception)) raise exception.BuildAbortException(actual_exception.message) + def _get_ec2_id_from_instance(self, instance): + if 'ec2_id' in instance.metadata and instance.metadata['ec2_id']: + return instance.metadata['ec2_id'] + elif instance.uuid in self._uuid_to_ec2_instance: + return self._uuid_to_ec2_instance[instance.uuid].id + # if none of the conditions are met we cannot map OpenStack UUID to + # AWS ID. + raise exception.InstanceNotFound('Instance %s not found' % instance.uuid) def snapshot(self, context, instance, image_id, update_task_state): @@ -408,23 +400,27 @@ class EC2Driver(driver.ComputeDriver): :param instance: nova.objects.instance.Instance :param image_id: Reference to a pre-created image that will hold the snapshot. """ - if instance['metadata']['ec2_id'] is None: - raise exception.InstanceNotRunning(instance_id=instance['uuid']) + if instance.metadata.get('ec2_id', None) is None: + raise exception.InstanceNotFound(instance_id=instance['uuid']) # Adding the below line only alters the state of the instance and not # its image in OpenStack. update_task_state( - task_state=task_states.IMAGE_UPLOADING, expected_state=task_states.IMAGE_SNAPSHOT) - ec2_id = instance['metadata']['ec2_id'] + task_state=task_states.IMAGE_UPLOADING, + expected_state=task_states.IMAGE_SNAPSHOT) + ec2_id = self._get_ec2_id_from_instance(instance) ec_instance_info = self.ec2_conn.get_only_instances( - instance_ids=[ec2_id], filters=None, dry_run=False, max_results=None) + instance_ids=[ec2_id], filters=None, dry_run=False, + max_results=None) ec2_instance = ec_instance_info[0] if ec2_instance.state == 'running': - ec2_image_id = ec2_instance.create_image(name=str( - image_id), description="Image from OpenStack", no_reboot=False, dry_run=False) - LOG.info("Image has been created state to %s." % ec2_image_id) + ec2_image_id = ec2_instance.create_image( + name=str(image_id), description="Image created by OpenStack", + no_reboot=False, dry_run=False) + LOG.info("Image created: %s." % ec2_image_id) - # The instance will be in pending state when it comes up, waiting forit to be in available + # The instance will be in pending state when it comes up, waiting + # for it to be in available self._wait_for_image_state(ec2_image_id, "available") image_api = glance.get_default_image_service() @@ -439,13 +435,14 @@ class EC2Driver(driver.ComputeDriver): 'ramdisk_id': instance['ramdisk_id'], 'ec2_image_id': ec2_image_id } } - # TODO(jhurt): This currently fails, leaving the status of an instance as 'snapshotting' + # TODO(jhurt): This currently fails, leaving the status of an instance + # as 'snapshotting' image_api.update(context, image_id, metadata) def reboot(self, context, instance, network_info, reboot_type, block_device_info=None, bad_volumes_callback=None): - - """Reboot the specified instance. + """ + Reboot the specified instance. After this is called successfully, the instance's state goes back to power_state.RUNNING. The virtualization platform should ensure that the reboot action has completed @@ -460,7 +457,6 @@ class EC2Driver(driver.ComputeDriver): :param bad_volumes_callback: Function to handle any bad volumes encountered """ - if reboot_type == 'SOFT': self._soft_reboot( context, instance, network_info, block_device_info) @@ -469,7 +465,7 @@ class EC2Driver(driver.ComputeDriver): context, instance, network_info, block_device_info) def _soft_reboot(self, context, instance, network_info, block_device_info=None): - ec2_id = instance['metadata']['ec2_id'] + ec2_id = self._get_ec2_id_from_instance(instance) self.ec2_conn.reboot_instances(instance_ids=[ec2_id], dry_run=False) LOG.info("Soft Reboot Complete.") @@ -480,13 +476,12 @@ class EC2Driver(driver.ComputeDriver): @staticmethod def get_host_ip_addr(): - """Retrieves the IP address of the dom0 - """ - LOG.info("***** Calling get_host_ip_addr *******************") + """Retrieves the IP address of the host""" return CONF.my_ip def set_admin_password(self, instance, new_pass): - """Boto doesn't support setting the password at the time of creating an instance. + """ + Boto doesn't support setting the password at the time of creating an instance. hence not implemented. """ pass @@ -524,60 +519,65 @@ class EC2Driver(driver.ComputeDriver): pass def power_off(self, instance, timeout=0, retry_interval=0): - """Power off the specified instance. + """ + Power off the specified instance. :param instance: nova.objects.instance.Instance :param timeout: time to wait for GuestOS to shutdown :param retry_interval: How often to signal guest while waiting for it to shutdown """ # TODO: Need to use timeout and retry_interval - LOG.info("***** Calling POWER OFF *******************") - ec2_id = instance['metadata']['ec2_id'] + ec2_id = self._get_ec2_id_from_instance(instance) self.ec2_conn.stop_instances( instance_ids=[ec2_id], force=False, dry_run=False) self._wait_for_state(instance, ec2_id, "stopped", power_state.SHUTDOWN) def power_on(self, context, instance, network_info, block_device_info): - """Power on the specified instance. - """ - LOG.info("***** Calling POWER ON *******************") - ec2_id = instance['metadata']['ec2_id'] + """Power on the specified instance.""" + ec2_id = self._get_ec2_id_from_instance(instance) self.ec2_conn.start_instances(instance_ids=[ec2_id], dry_run=False) self._wait_for_state(instance, ec2_id, "running", power_state.RUNNING) def soft_delete(self, instance): - """Deleting the specified instance - """ + """Deleting the specified instance""" self.destroy(instance) def restore(self, instance): pass def pause(self, instance): - """Boto doesn't support pause and cannot save system state and hence we've implemented the closest functionality - which is to poweroff the instance. + """ + Boto doesn't support pause and cannot save system state and hence + we've implemented the closest functionality which is to poweroff the + instance. :param instance: nova.objects.instance.Instance """ self.power_off(instance) def unpause(self, instance): - """Since Boto doesn't support pause and cannot save system state, we had implemented the closest functionality - which is to poweroff the instance. and powering on such an instance in this method. + """ + Since Boto doesn't support pause and cannot save system state, we + had implemented the closest functionality which is to poweroff the + instance. and powering on such an instance in this method. :param instance: nova.objects.instance.Instance """ self.power_on( context=None, instance=instance, network_info=None, block_device_info=None) def suspend(self, context, instance): - """Boto doesn't support suspend and cannot save system state and hence we've implemented the closest - functionality which is to poweroff the instance. + """ + Boto doesn't support suspend and cannot save system state and hence + we've implemented the closest functionality which is to poweroff the + instance. :param instance: nova.objects.instance.Instance """ self.power_off(instance) def resume(self, context, instance, network_info, block_device_info=None): - """Since Boto doesn't support suspend and we cannot save system state, we've implemented the closest - functionality which is to power on the instance. + """ + Since Boto doesn't support suspend and we cannot save system state, + we've implemented the closest functionality which is to power on the + instance. :param instance: nova.objects.instance.Instance """ self.power_on(context, instance, network_info, block_device_info) @@ -599,29 +599,34 @@ class EC2Driver(driver.ComputeDriver): :param destroy_disks: Indicates if disks should be destroyed :param migrate_data: implementation specific params """ - LOG.info("***** Calling DESTROY *******************") - if 'ec2_id' not in instance['metadata']: - LOG.warning("Key '%s' not in EC2 instances" % instance['name'], instance=instance) + ec2_id = None + try: + ec2_id = self._get_ec2_id_from_instance(instance) + ec2_instances = self.ec2_conn.get_only_instances( + instance_ids=[ec2_id]) + except exception.InstanceNotFound as ex: + # Exception while fetching instance info from AWS + LOG.exception('Exception in destroy while fetching EC2 id for ' + 'instance %s' % instance.uuid) + return + if len(ec2_instances) == 0: + # Instance already deleted on hypervisor + LOG.warning("EC2 instance with ID %s not found" % ec2_id, + instance=instance) return else: - # Deleting the instance from EC2 - ec2_id = instance['metadata']['ec2_id'] try: - ec2_instances = self.ec2_conn.get_only_instances(instance_ids=[ec2_id]) - except Exception: - return - if ec2_instances.__len__() == 0: - LOG.warning("EC2 instance with ID %s not found" % ec2_id, instance=instance) - return - else: - try: - self.ec2_conn.stop_instances(instance_ids=[ec2_id], force=True) + if ec2_instances[0].state != 'terminated': + if ec2_instances[0].state == 'running': + self.ec2_conn.stop_instances(instance_ids=[ec2_id], + force=True) self.ec2_conn.terminate_instances(instance_ids=[ec2_id]) - self._wait_for_state(instance, ec2_id, "terminated", power_state.SHUTDOWN) - except: - exp = sys.exc_value - LOG.exception("Exception while destroying instance: %s" % exp) - raise exception.NovaException("Exception while destroying instance") + self._wait_for_state(instance, ec2_id, "terminated", + power_state.SHUTDOWN) + except Exception as ex: + LOG.exception("Exception while destroying instance: %s" % + str(ex)) + raise ex def attach_volume(self, context, connection_info, instance, mountpoint, disk_bus=None, device_type=None, encryption=None): @@ -633,8 +638,11 @@ class EC2Driver(driver.ComputeDriver): self._mounts[instance_name][mountpoint] = connection_info volume_id = connection_info['data']['volume_id'] + ec2_id = self._get_ec2_id_from_instance(instance) + # ec2 only attaches volumes at /dev/sdf through /dev/sdp - self.ec2_conn.attach_volume(volume_map[volume_id], instance['metadata']['ec2_id'], "/dev/sdn", dry_run=False) + self.ec2_conn.attach_volume(volume_id, ec2_id, mountpoint, + dry_run=False) def detach_volume(self, connection_info, instance, mountpoint, encryption=None): """Detach the disk attached to the instance. @@ -644,13 +652,16 @@ class EC2Driver(driver.ComputeDriver): except KeyError: pass volume_id = connection_info['data']['volume_id'] - self.ec2_conn.detach_volume(volume_map[volume_id], instance_id=instance['metadata']['ec2_id'], - device="/dev/sdn", force=False, dry_run=False) + ec2_id = self._get_ec2_id_from_instance(instance) + self.ec2_conn.detach_volume(volume_id, instance_id=ec2_id, + device=mountpoint, force=False, + dry_run=False) def swap_volume(self, old_connection_info, new_connection_info, - instance, mountpoint): + instance, mountpoint, resize_to): """Replace the disk attached to the instance. """ + # TODO: Use resize_to parameter instance_name = instance['name'] if instance_name not in self._mounts: self._mounts[instance_name] = {} @@ -662,19 +673,22 @@ class EC2Driver(driver.ComputeDriver): self.detach_volume(old_connection_info, instance, mountpoint) # wait for the old volume to detach successfully to make sure # /dev/sdn is available for the new volume to be attached + # TODO: remove the sleep and poll AWS for the status of volume time.sleep(60) - self.ec2_conn.attach_volume(volume_map[new_volume_id], instance['metadata']['ec2_id'], "/dev/sdn", + ec2_id = self._get_ec2_id_from_instance(instance) + self.ec2_conn.attach_volume(new_volume_id, + ec2_id, mountpoint, dry_run=False) return True def attach_interface(self, instance, image_meta, vif): - LOG.info("******* ATTTACH INTERFACE *******") + LOG.debug("******* ATTTACH INTERFACE *******") if vif['id'] in self._interfaces: raise exception.InterfaceAttachFailed('duplicate') self._interfaces[vif['id']] = vif def detach_interface(self, instance, vif): - LOG.info("******* DETACH INTERFACE *******") + LOG.debug("******* DETACH INTERFACE *******") try: del self._interfaces[vif['id']] except KeyError: @@ -690,32 +704,34 @@ class EC2Driver(driver.ComputeDriver): :num_cpu: (int) the number of virtual CPUs for the domain :cpu_time: (int) the CPU time used in nanoseconds """ - LOG.info("*************** GET INFO ********************") - if 'metadata' not in instance or 'ec2_id' not in instance['metadata']: + + if instance.uuid in self._uuid_to_ec2_instance: + ec2_instance = self._uuid_to_ec2_instance[instance.uuid] + elif 'metadata' in instance and 'ec2_id' in instance['metadata']: + ec2_id = instance['metadata']['ec2_id'] + ec2_instances = self.ec2_conn.get_only_instances( + instance_ids=[ec2_id], filters=None, dry_run=False, + max_results=None) + if len(ec2_instances) == 0: + LOG.warning(_("EC2 instance with ID %s not found") % ec2_id, + instance=instance) + raise exception.InstanceNotFound(instance_id=instance['name']) + ec2_instance = ec2_instances[0] + else: raise exception.InstanceNotFound(instance_id=instance['name']) - ec2_id = instance['metadata']['ec2_id'] - ec2_instances = self.ec2_conn.get_only_instances(instance_ids=[ec2_id], filters=None, dry_run=False, - max_results=None) - if ec2_instances.__len__() == 0: - LOG.warning(_("EC2 instance with ID %s not found") % ec2_id, instance=instance) - raise exception.InstanceNotFound(instance_id=instance['name']) - ec2_instance = ec2_instances[0] - LOG.info(ec2_instance) - LOG.info("state %s max_mem %s mem %s flavor %s" % - (EC2_STATE_MAP.get(ec2_instance.state), ec2_instance.ramdisk, ec2_instance.get_attribute('ramdisk', dry_run=False), ec2_instance.instance_type)) - # return {'state': , - # 'max_mem': ec2_instance.ramdisk, - # 'mem': , - # 'num_cpu': 2, - # 'cpu_time': 0} + power_state = EC2_STATE_MAP.get(ec2_instance.state) + ec2_flavor = self.ec2_flavor_info.get(ec2_instance.instance_type) + memory_mb = ec2_flavor['memory_mb'] + vcpus = ec2_flavor['vcpus'] + return hardware.InstanceInfo( - state=EC2_STATE_MAP.get(ec2_instance.state), - max_mem_kb=ec2_instance.ramdisk, - mem_kb=ec2_instance.get_attribute('ramdisk', dry_run=False), - num_cpu=2, + state=power_state, + max_mem_kb=memory_mb, + mem_kb=memory_mb, + num_cpu=vcpus, cpu_time_ns=0, - id=instance['id']) + id=instance.id) def allow_key(self, key): for key_to_filter in DIAGNOSTIC_KEYS_TO_FILTER: @@ -723,17 +739,17 @@ class EC2Driver(driver.ComputeDriver): return False return True - def get_diagnostics(self, instance_name): - """Return data about VM diagnostics. - """ - LOG.info("******* GET DIAGNOSTICS *********************************************") - instance = self.nova.servers.get(instance_name) + def get_diagnostics(self, instance): + """Return data about VM diagnostics.""" - ec2_id = instance.metadata['ec2_id'] - ec2_instances = self.ec2_conn.get_only_instances(instance_ids=[ec2_id], filters=None, dry_run=False, + ec2_id = self._get_ec2_id_from_instance(instance) + ec2_instances = self.ec2_conn.get_only_instances(instance_ids=[ec2_id], + filters=None, + dry_run=False, max_results=None) - if ec2_instances.__len__() == 0: - LOG.warning(_("EC2 instance with ID %s not found") % ec2_id, instance=instance) + if len(ec2_instances) == 0: + LOG.warning(_("EC2 instance with ID %s not found") % ec2_id, + instance=instance) raise exception.InstanceNotFound(instance_id=instance['name']) ec2_instance = ec2_instances[0] @@ -743,7 +759,6 @@ class EC2Driver(driver.ComputeDriver): diagnostics['instance.' + key] = str(value) metrics = self.cloudwatch_conn.list_metrics(dimensions={'InstanceId': ec2_id}) - import datetime for metric in metrics: end = datetime.datetime.utcnow() @@ -775,7 +790,7 @@ class EC2Driver(driver.ComputeDriver): return [0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L] def get_vnc_console(self, context, instance): - ec2_id = instance['metadata']['ec2_id'] + ec2_id = self._get_ec2_id_from_instance(instance) LOG.info("VNC console connect to %s" % ec2_id) reservations = self.ec2_conn.get_all_instances() @@ -812,108 +827,6 @@ class EC2Driver(driver.ComputeDriver): 'username': 'EC2user', 'password': 'EC2password'} - def _get_ec2_instance_ids_with_security_group(self, ec2_security_group): - return [instance.id for instance in ec2_security_group.instances()] - - def _get_openstack_instances_with_security_group(self, openstack_security_group): - return [instance for instance in (self.nova.servers.list()) - if openstack_security_group.name in [group['name'] for group in instance.security_groups]] - - def _get_id_of_ec2_instance_to_update_security_group(self, ec2_instance_ids_for_security_group, - ec2_ids_for_openstack_instances_for_security_group): - return (set(ec2_ids_for_openstack_instances_for_security_group).symmetric_difference( - set(ec2_instance_ids_for_security_group))).pop() - - def _should_add_security_group_to_instance(self, ec2_instance_ids_for_security_group, - ec2_ids_for_openstack_instances_for_security_group): - return len(ec2_instance_ids_for_security_group) < len(ec2_ids_for_openstack_instances_for_security_group) - - def _add_security_group_to_instance(self, ec2_instance_id, ec2_security_group): - security_group_ids_for_instance = self._get_ec2_security_group_ids_for_instance(ec2_instance_id) - security_group_ids_for_instance.append(ec2_security_group.id) - self.ec2_conn.modify_instance_attribute(ec2_instance_id, "groupSet", security_group_ids_for_instance) - - def _remove_security_group_from_instance(self, ec2_instance_id, ec2_security_group): - security_group_ids_for_instance = self._get_ec2_security_group_ids_for_instance(ec2_instance_id) - security_group_ids_for_instance.remove(ec2_security_group.id) - self.ec2_conn.modify_instance_attribute(ec2_instance_id, "groupSet", security_group_ids_for_instance) - - def _get_ec2_security_group_ids_for_instance(self, ec2_instance_id): - security_groups_for_instance = self.ec2_conn.get_instance_attribute(ec2_instance_id, "groupSet")['groupSet'] - security_group_ids_for_instance = [group.id for group in security_groups_for_instance] - return security_group_ids_for_instance - - def _get_or_create_ec2_security_group(self, openstack_security_group): - try: - return self.ec2_conn.get_all_security_groups(openstack_security_group.name)[0] - except (EC2ResponseError, IndexError) as e: - LOG.warning(e) - return self.ec2_conn.create_security_group(openstack_security_group.name, - openstack_security_group.description) - - def refresh_security_group_rules(self, security_group_id): - """This method is called after a change to security groups. - - All security groups and their associated rules live in the datastore, - and calling this method should apply the updated rules to instances - running the specified security group. - An error should be raised if the operation cannot complete. - """ - LOG.info("************** REFRESH SECURITY GROUP RULES ******************") - - openstack_security_group = self.nova.security_groups.get(security_group_id) - ec2_security_group = self._get_or_create_ec2_security_group(openstack_security_group) - - ec2_ids_for_ec2_instances_with_security_group = self._get_ec2_instance_ids_with_security_group( - ec2_security_group) - - ec2_ids_for_openstack_instances_with_security_group = [ - instance.metadata['ec2_id'] for instance - in self._get_openstack_instances_with_security_group(openstack_security_group) - ] - - self.security_group_lock.acquire() - - try: - ec2_instance_to_update = self._get_id_of_ec2_instance_to_update_security_group( - ec2_ids_for_ec2_instances_with_security_group, - ec2_ids_for_openstack_instances_with_security_group - ) - - should_add_security_group = self._should_add_security_group_to_instance( - ec2_ids_for_ec2_instances_with_security_group, - ec2_ids_for_openstack_instances_with_security_group) - - if should_add_security_group: - self._add_security_group_to_instance(ec2_instance_to_update, ec2_security_group) - else: - self._remove_security_group_from_instance(ec2_instance_to_update, ec2_security_group) - finally: - self.security_group_lock.release() - - return True - - def refresh_security_group_members(self, security_group_id): - LOG.info("************** REFRESH SECURITY GROUP MEMBERS ******************") - LOG.info(security_group_id) - return True - - def _get_allowed_group_name_from_openstack_rule_if_present(self, openstack_rule): - return openstack_rule['group']['name'] if 'name' in openstack_rule['group'] else None - - def _get_allowed_ip_range_from_openstack_rule_if_present(self, openstack_rule): - return openstack_rule['ip_range']['cidr'] if 'cidr' in openstack_rule['ip_range'] else None - - def refresh_instance_security_rules(self, instance): - LOG.info("************** REFRESH INSTANCE SECURITY RULES ******************") - LOG.info(instance) - - # TODO: lock for case when group is associated with multiple instances - - self.instance_rule_refresher.refresh(self.nova.servers.get(instance['id'])) - - return - def refresh_provider_fw_rules(self): pass @@ -927,14 +840,12 @@ class EC2Driver(driver.ComputeDriver): a driver that manages only one node can safely ignore this :returns: Dictionary describing resources """ - LOG.info("************** GET_AVAILABLE_RESOURCE ******************") - if nodename not in _EC2_NODES: return {} - dic = {'vcpus': VCPUS, - 'memory_mb': MEMORY_IN_MBS, - 'local_gb': DISK_IN_GB, + dic = {'vcpus': CONF.AWS.max_vcpus, + 'memory_mb': CONF.AWS.max_memory_mb, + 'local_gb': CONF.AWS.max_disk_gb, 'vcpus_used': 0, 'memory_mb_used': 0, 'local_gb_used': 0, @@ -949,69 +860,6 @@ class EC2Driver(driver.ComputeDriver): dic["supported_instances"] = [supported_tuple] return dic - def get_host_stats_pf9(self, res_types, refresh=False, nodename=None): - """Return currently known physical resource consumption - If 'refresh' is True, run update the stats first. - :param res_types: An array of resources to be queried - """ - LOG.info("*** In get_host_stats_pf9**") - resource_stats = dict() - for resource_type in res_types: - LOG.info("Looking for resource:%s" % resource_type) - resource_dict = self._get_host_stats_pf9(resource_type, - refresh=refresh) - resource_stats.update(resource_dict) - return resource_stats - - def _update_stats_pf9(self, resource_type): - """Retrieve physical resource utilization - """ - if resource_type not in self._pf9_stats.keys(): - self._pf9_stats[resource_type] = {} - - data = 0 - self._pf9_stats[resource_type] = data - - return {resource_type: data} - - def _get_host_stats_pf9(self, res_types, refresh=False): - """Return the current physical resource consumption - """ - if refresh or not self._pf9_stats: - self._update_stats_pf9(res_types) - - return self._pf9_stats - - def get_all_networks_pf9(self, node=None): - ret_list = [] - vpcs = self.vpc_conn.get_all_vpcs() - for vpc in vpcs: - ret_list.append({'bridge': vpc.id}) - return ret_list - - def get_all_ip_mapping_pf9(self, needed_uuids=None): - ip_map = dict() - ec2_instances = self.ec2_conn.get_all_instances() - for reservation in ec2_instances: - if reservation.instances is not None: - for instance in reservation.instances: - if len(instance.tags) > 0: - if 'openstack_id' in instance.tags: - openstack_id = instance.tags['openstack_id'] - intf_list = [] - intf_details = dict() - if len(instance.interfaces) > 0: - for interface in instance.interfaces: - intf_details['bridge'] = interface.vpc_id - intf_details['ip_address'] = interface.private_ip_address - intf_details['mac_address'] = interface.mac_address - LOG.info( - "ID %s VPC %s and IP %s and MAC %s" % ( - openstack_id, interface.vpc_id, instance.private_ip_address, interface.mac_address)) - intf_list.append(intf_details) - ip_map[openstack_id] = intf_list - return ip_map - def ensure_filtering_rules_for_instance(self, instance_ref, network_info): return @@ -1047,8 +895,7 @@ class EC2Driver(driver.ComputeDriver): :param instance: nova.objects.instance.Instance being migrated/resized :param power_on: is True the instance should be powered on """ - LOG.info("***** Calling FINISH MIGRATION *******************") - ec2_id = instance['metadata']['ec2_id'] + ec2_id = self._get_ec2_id_from_instance(instance) ec_instance_info = self.ec2_conn.get_only_instances( instance_ids=[ec2_id], filters=None, dry_run=False, max_results=None) ec2_instance = ec_instance_info[0] @@ -1064,8 +911,7 @@ class EC2Driver(driver.ComputeDriver): """Confirms a resize, destroying the source VM. :param instance: nova.objects.instance.Instance """ - LOG.info("***** Calling CONFIRM MIGRATION *******************") - ec2_id = instance['metadata']['ec2_id'] + ec2_id = self._get_ec2_id_from_instance(instance) ec_instance_info = self.ec2_conn.get_only_instances( instance_ids=[ec2_id], filters=None, dry_run=False, max_results=None) ec2_instance = ec_instance_info[0] @@ -1088,9 +934,9 @@ class EC2Driver(driver.ComputeDriver): host_status['host_hostname'] = nodename host_status['host_name_label'] = nodename host_status['hypervisor_type'] = 'Amazon-EC2' - host_status['vcpus'] = VCPUS - host_status['memory_mb'] = MEMORY_IN_MBS - host_status['local_gb'] = DISK_IN_GB + host_status['vcpus'] = CONF.AWS.max_vcpus + host_status['memory_mb'] = CONF.AWS.max_memory_mb + host_status['local_gb'] = CONF.AWS.max_disk_gb stats.append(host_status) if len(stats) == 0: raise exception.NovaException("EC2Driver has no node") @@ -1135,15 +981,34 @@ class EC2Driver(driver.ComputeDriver): def instance_on_disk(self, instance): return False + def _get_uuid_from_aws_id(self, instance_id): + m = hashlib.md5() + m.update(instance_id) + return str(uuid.UUID(bytes=m.digest(), version=4)) + def list_instance_uuids(self, node=None, template_uuids=None, force=False): - LOG.info("*** list_instance_uuids **") ec2_instances = self.ec2_conn.get_only_instances() - uuid_list = [] + # Clear the cache of UUID->EC2 ID mapping + self._uuid_to_ec2_instance.clear() for instance in ec2_instances: + generate_uuid = False + if instance.state in ['pending', 'shutting-down', 'terminated']: + # Instance is being created or destroyed no need to list it + continue if len(instance.tags) > 0: if 'openstack_id' in instance.tags: - uuid_list.append(instance.tags['openstack_id']) - return uuid_list + self._uuid_to_ec2_instance[instance.tags['openstack_id']] = \ + instance + else: + # Possibly a new discovered instance + generate_uuid = True + else: + generate_uuid = True + + if generate_uuid: + instance_uuid = self._get_uuid_from_aws_id(instance.id) + self._uuid_to_ec2_instance[instance_uuid] = instance + return self._uuid_to_ec2_instance.keys() def _wait_for_state(self, instance, ec2_id, desired_state, desired_power_state): """Wait for the state of the corrosponding ec2 instance to be in completely available state. diff --git a/nova/ec2/exception_handler.py b/nova/ec2/exception_handler.py index de978f4..0ca85c4 100644 --- a/nova/ec2/exception_handler.py +++ b/nova/ec2/exception_handler.py @@ -1,17 +1,3 @@ -# Copyright (c) 2016 Platform9 Systems Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either expressed or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - from nova import exception @@ -28,4 +14,4 @@ class Ec2ExceptionHandler: elif ec2_response_error_exc.error_code == "InvalidAMIID.NotFound": return exception.ImageNotFoundEC2("Invalid Image") else: - return exception.NovaException(ec2_response_error_exc.message) + return exception.NovaException(ec2_response_error_exc.message) \ No newline at end of file diff --git a/nova/ec2/group.py b/nova/ec2/group.py deleted file mode 100644 index e070399..0000000 --- a/nova/ec2/group.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (c) 2014 ThoughtWorks -# Copyright (c) 2016 Platform9 Systems Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -class Group: - - def rule_diff(self, other_group): - pass diff --git a/nova/ec2/group_rule_refresher.py b/nova/ec2/group_rule_refresher.py deleted file mode 100644 index 4bd7a2a..0000000 --- a/nova/ec2/group_rule_refresher.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (c) 2014 ThoughtWorks -# Copyright (c) 2016 Platform9 Systems Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -class GroupRuleRefresher: - - def __init__(self, ec2_connection, openstack_rule_service, ec2_rule_service): - self.ec2_conn = ec2_connection - self.openstack_rule_service = openstack_rule_service - self.ec2_rule_service = ec2_rule_service - - def refresh(self, group_name): - openstack_rules = self.openstack_rule_service.get_rules_for_group(group_name) - ec2_rules = self.ec2_rule_service.get_rules_for_group(group_name) - - self._add_rules_to_ec2(ec2_rules, group_name, openstack_rules) - self._remove_rules_from_ec2(ec2_rules, group_name, openstack_rules) - - def _add_rules_to_ec2(self, ec2_rules, group_name, openstack_rules): - for rule in openstack_rules - ec2_rules: - self._add_rule_on_ec2(group_name, rule) - - def _remove_rules_from_ec2(self, ec2_rules, group_name, openstack_rules): - for rule in ec2_rules - openstack_rules: - self._remove_rule_from_ec2(group_name, rule) - - def _remove_rule_from_ec2(self, group_name, rule): - self.ec2_conn.revoke_security_group( - group_name=group_name, - ip_protocol=rule.ip_protocol, - from_port=rule.from_port, - to_port=rule.to_port, - cidr_ip=rule.ip_range - ) - - def _add_rule_on_ec2(self, group_name, rule): - self.ec2_conn.authorize_security_group( - group_name=group_name, - ip_protocol=rule.ip_protocol, - from_port=rule.from_port, - to_port=rule.to_port, - cidr_ip=rule.ip_range - ) diff --git a/nova/ec2/instance_rule_refresher.py b/nova/ec2/instance_rule_refresher.py deleted file mode 100644 index 1bbc5eb..0000000 --- a/nova/ec2/instance_rule_refresher.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (c) 2014 ThoughtWorks -# Copyright (c) 2016 Platform9 Systems Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -class InstanceRuleRefresher: - - def __init__(self, group_rule_refresher): - self.group_rule_refresher = group_rule_refresher - - def refresh(self, instance): - for group_name in self._get_group_names(instance): - self.group_rule_refresher.refresh(group_name) - - def _get_group_names(self, instance): - return [group['name'] for group in instance.security_groups] diff --git a/nova/ec2/openstack_group_service.py b/nova/ec2/openstack_group_service.py deleted file mode 100644 index 27ddf34..0000000 --- a/nova/ec2/openstack_group_service.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) 2014 ThoughtWorks -# Copyright (c) 2016 Platform9 Systems Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -class OpenstackGroupService(): - - def __init__(self, security_group_manager): - self.security_group_manager = security_group_manager - - def get_group(self, group_name): - return [group for group in self.security_group_manager.list() if group.name == group_name][0] diff --git a/nova/ec2/openstack_group_transformer.py b/nova/ec2/openstack_group_transformer.py deleted file mode 100644 index 4e45f04..0000000 --- a/nova/ec2/openstack_group_transformer.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (c) 2014 ThoughtWorks -# Copyright (c) 2016 Platform9 Systems Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -class OpenstackGroupTransformer: - def to_group(self, openstack_group): - pass diff --git a/nova/ec2/openstack_rule_service.py b/nova/ec2/openstack_rule_service.py deleted file mode 100644 index 3c6f5dc..0000000 --- a/nova/ec2/openstack_rule_service.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (c) 2014 ThoughtWorks -# Copyright (c) 2016 Platform9 Systems Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -class OpenstackRuleService: - def __init__(self, group_service, openstack_rule_transformer): - self.group_service = group_service - self.openstack_rule_transformer = openstack_rule_transformer - - def get_rules_for_group(self, group_name): - openstack_group = self.group_service.get_group(group_name) - return set([self.openstack_rule_transformer.to_rule(rule) for rule in openstack_group.rules]) - # return self.group_service.get_group(group_name).rules diff --git a/nova/ec2/openstack_rule_transformer.py b/nova/ec2/openstack_rule_transformer.py deleted file mode 100644 index 8af8236..0000000 --- a/nova/ec2/openstack_rule_transformer.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright (c) 2014 ThoughtWorks -# Copyright (c) 2016 Platform9 Systems Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from copy import deepcopy -from rule import Rule - - -class OpenstackRuleTransformer: - def to_rule(self, openstack_rule): - rule_args = {} - rule_args['ip_protocol'] = openstack_rule['ip_protocol'] - rule_args['from_port'] = str(openstack_rule['from_port']) - rule_args['to_port'] = str(openstack_rule['to_port']) - - if 'cidr' in openstack_rule['ip_range']: - rule_args['ip_range'] = openstack_rule['ip_range']['cidr'] - else: - rule_args['group_name'] = openstack_rule['group']['name'] - - return Rule(**rule_args) diff --git a/nova/ec2/rule.py b/nova/ec2/rule.py deleted file mode 100644 index 30a8485..0000000 --- a/nova/ec2/rule.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) 2014 ThoughtWorks -# Copyright (c) 2016 Platform9 Systems Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -class Rule: - def __init__(self, ip_protocol, from_port, to_port, ip_range=None, group_name=None): - self.ip_protocol = ip_protocol - self.from_port = from_port - self.to_port = to_port - self.ip_range = ip_range - self.group_name = group_name - - def __key(self): - return self.ip_protocol, self.from_port, self.to_port, self.ip_range, self.group_name - - def __eq__(self, other): - return self.__key() == other.__key() - - def __hash__(self): - return hash(self.__key()) diff --git a/nova/ec2/rule_comparator.py b/nova/ec2/rule_comparator.py deleted file mode 100644 index 9849398..0000000 --- a/nova/ec2/rule_comparator.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (c) 2014 ThoughtWorks -# Copyright (c) 2016 Platform9 Systems Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -class RuleComparator: - def __init__(self, ec2_connection): - self.ec2_connection = ec2_connection - - def rules_are_equal(self, openstack_rule, ec2_rule): - if self._ip_protocols_are_different(ec2_rule, openstack_rule) \ - or self._from_ports_are_different(ec2_rule, openstack_rule) \ - or self._to_ports_are_different(ec2_rule, openstack_rule) \ - or self._ip_ranges_are_present_and_different(ec2_rule, openstack_rule) \ - or self._group_names_are_present_and_different(openstack_rule, ec2_rule): - return False - return True - - def _ip_protocols_are_different(self, ec2_rule, openstack_rule): - return openstack_rule['ip_protocol'] != ec2_rule.ip_protocol - - def _from_ports_are_different(self, ec2_rule, openstack_rule): - return str(openstack_rule['from_port']) != ec2_rule.from_port - - def _to_ports_are_different(self, ec2_rule, openstack_rule): - return str(openstack_rule['to_port']) != ec2_rule.to_port - - def _ip_ranges_are_present_and_different(self, ec2_rule, openstack_rule): - return ('cidr' in openstack_rule['ip_range'] and openstack_rule['ip_range']['cidr'] != ec2_rule.grants[0].cidr_ip) - - def _group_names_are_present_and_different(self, openstack_rule, ec2_rule): - if 'name' not in openstack_rule['group']: - return False - else: - ec2_group_name = self.ec2_connection.get_all_security_groups(group_ids=ec2_rule.grants[0].group_id)[0].name - return openstack_rule['group']['name'] != ec2_group_name diff --git a/nova/test-requirements.txt b/nova/test-requirements.txt new file mode 100644 index 0000000..0a175da --- /dev/null +++ b/nova/test-requirements.txt @@ -0,0 +1 @@ +moto diff --git a/nova/tests/ec2/__init__.py b/nova/tests/ec2/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nova/tests/ec2/test_driver.py b/nova/tests/ec2/test_driver.py new file mode 100644 index 0000000..4731c85 --- /dev/null +++ b/nova/tests/ec2/test_driver.py @@ -0,0 +1,540 @@ +# Copyright 2016 Platform9 Systems Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from moto import mock_ec2 +from moto import mock_cloudwatch +from moto.ec2 import ec2_backends +from nova import context +from nova import exception +from nova import objects +from nova import test +from nova.compute import power_state +from nova.compute import vm_states +from nova.compute import task_states +from nova.image.glance import GlanceImageService +from nova.tests.unit import fake_instance +from nova.tests.unit import matchers +from nova.virt.ec2 import EC2Driver +from oslo_config import cfg +from oslo_utils import uuidutils +import base64 +import boto +import contextlib +import mock + +class EC2DriverTestCase(test.NoDBTestCase): + + @mock_ec2 + @mock_cloudwatch + def setUp(self): + super(EC2DriverTestCase, self).setUp() + self.fake_access_key = 'aws_access_key' + self.fake_secret_key = 'aws_secret_key' + self.region_name = 'us-east-1' + self.region = boto.ec2.get_region(self.region_name) + self.flags(access_key=self.fake_access_key, + secret_key=self.fake_secret_key, + # Region name cannot be fake + region_name=self.region_name, group='AWS') + self.conn = EC2Driver(None, False) + self.type_data = None + self.project_id = 'fake' + self.user_id = 'fake' + self.instance_node = None + self.uuid = None + self.instance = None + self.context = context.RequestContext(self.user_id, self.project_id) + self.fake_vpc_conn = boto.vpc.VPCConnection( + region=self.region, aws_access_key_id=self.fake_access_key, + aws_secret_access_key=self.fake_secret_key) + self.fake_ec2_conn = boto.ec2.EC2Connection( + aws_access_key_id=self.fake_access_key, + aws_secret_access_key=self.fake_secret_key, + region=self.region) + + def tearDown(self): + super(EC2DriverTestCase, self).tearDown() + + def reset(self): + instance_list = self.conn.ec2_conn.get_only_instances() + # terminated instances are considered deleted and hence ignore them + instance_id_list = [x.id for x in instance_list if x.state != 'terminated'] + self.conn.ec2_conn.stop_instances(instance_ids=instance_id_list, + force=True) + self.conn.ec2_conn.terminate_instances(instance_ids=instance_id_list) + self.type_data = None + self.instance = None + self.uuid = None + self.instance_node = None + + @mock_ec2 + def test_list_instances(self): + for x in range(0, 5): + self.conn.ec2_conn.run_instances('ami-1234abc') + fake_list = self.conn.list_instances() + self.assertEqual(5, len(fake_list)) + self.reset() + + @mock_ec2 + def test_add_ssh_keys_key_exists(self): + fake_key = 'fake_key' + fake_key_data = 'abcdefgh' + self.conn.ec2_conn.import_key_pair(fake_key, fake_key_data) + with contextlib.nested( + mock.patch.object(boto.ec2.EC2Connection, 'get_key_pair'), + mock.patch.object(boto.ec2.EC2Connection, 'import_key_pair'), + ) as (fake_get, fake_import): + fake_get.return_value = True + self.conn._add_ssh_keys(fake_key, fake_key_data) + fake_get.assert_called_once_with(fake_key) + fake_import.assert_not_called() + + @mock_ec2 + def test_add_ssh_keys_key_absent(self): + fake_key = 'fake_key' + fake_key_data = 'abcdefgh' + with contextlib.nested( + mock.patch.object(boto.ec2.EC2Connection, 'get_key_pair'), + mock.patch.object(boto.ec2.EC2Connection, 'import_key_pair'), + ) as (fake_get, fake_import): + fake_get.return_value = False + self.conn._add_ssh_keys(fake_key, fake_key_data) + fake_get.assert_called_once_with(fake_key) + fake_import.assert_called_once_with(fake_key, fake_key_data) + + def test_process_network_info(self): + fake_network_info = [ + { + 'profile': {}, + 'ovs_interfaceid': None, + 'preserve_on_delete': False, + 'network': { + 'bridge': None, + 'subnets': [{ + 'ips': [{ + 'meta': {}, + 'version': 4, + 'type': 'fixed', + 'floating_ips': [], + 'address': u'192.168.100.5'}], + 'version': 4, + 'meta': {}, + 'dns': [], + 'routes': [], + 'cidr': u'192.168.100.0/24', + 'gateway': { + 'meta': {}, + 'version': 4, + 'type': 'gateway', + 'address': u'192.168.100.1'}}], + 'meta': { + 'injected': True, + 'tenant_id': '135b1a036a51414ea1f989ab59fefde5'}, + 'id': '4f8ad58d-de60-4b52-94ba-8b988a9b7f33', + 'label': 'test'}, + 'devname': 'tapa9a90cf6-62', + 'vnic_type': 'normal', + 'qbh_params': None, + 'meta': {}, + 'details': '{"subnet_id": "subnet-0107db5a",' + ' "ip_address": "192.168.100.5"}', + 'address': 'fa:16:3e:23:65:2c', + 'active': True, + 'type': 'vip_type_a', + 'id': 'a9a90cf6-627c-46f3-829d-c5a2ae07aaf0', + 'qbg_params': None + } + ] + aws_subnet_id, aws_fixed_ip, port_id, network_id = \ + self.conn._process_network_info(fake_network_info) + self.assertEqual(aws_subnet_id, 'subnet-0107db5a') + self.assertEqual(aws_fixed_ip, '192.168.100.5') + self.assertEqual(port_id, 'a9a90cf6-627c-46f3-829d-c5a2ae07aaf0') + self.assertEqual(network_id, '4f8ad58d-de60-4b52-94ba-8b988a9b7f33') + + def _get_instance_flavor_details(self): + return { + 'memory_mb': 2048.0, 'root_gb': 0, 'deleted_at': None, + 'name': 't2.small', 'deleted': 0, 'created_at': None, + 'ephemeral_gb': 0, 'updated_at': None, 'disabled': False, + 'vcpus': 1, 'extra_specs': {}, 'swap': 0, 'rxtx_factor': 1.0, + 'is_public': True, 'flavorid': '1', 'vcpu_weight': None, 'id': 2 + } + + def _create_instance(self, key_name=None, key_data=None, user_data=None): + uuid = uuidutils.generate_uuid() + self.type_data = self._get_instance_flavor_details() + values = { + 'name': 'fake_instance', + 'id': 1, + 'uuid': uuid, + 'project_id': self.project_id, + 'user_id': self.user_id, + 'kernel_id': 'fake_kernel_id', + 'ramdisk_id': 'fake_ramdisk_id', + 'flavor': objects.flavor.Flavor(**self.type_data), + 'node': 'fake_node', + 'memory_mb': self.type_data['memory_mb'], + 'root_gb': self.type_data['root_gb'], + 'ephemeral_gb': self.type_data['ephemeral_gb'], + 'vpcus': self.type_data['vcpus'], + 'swap': self.type_data['swap'], + 'expected_attrs': ['system_metadata', 'metadata'], + 'display_name': 'fake_instance', + } + if key_name and key_data: + values['key_name'] = key_name + values['key_data'] = key_data + if user_data: + values['user_data'] = user_data + self.instance_node = 'fake_node' + self.uuid = uuid + self.instance = fake_instance.fake_instance_obj(self.context, **values) + + def _create_network(self): + self.vpc = self.fake_vpc_conn.create_vpc('192.168.100.0/24') + self.subnet = self.fake_vpc_conn.create_subnet(self.vpc.id, + '192.168.100.0/24') + self.subnet_id = self.subnet.id + + def _create_nova_vm(self): + self.conn.spawn(self.context, self.instance, None, injected_files=[], + admin_password=None, network_info=None, + block_device_info=None) + + @mock_ec2 + def test_spawn(self): + self._create_instance() + self._create_network() + with contextlib.nested( + mock.patch.object(EC2Driver, '_get_image_ami_id_from_meta'), + mock.patch.object(EC2Driver, '_process_network_info'), + mock.patch.object(EC2Driver, '_get_instance_sec_grps'), + ) as (mock_image, mock_network, mock_secgrp): + mock_image.return_value = 'ami-1234abc' + mock_network.return_value = (self.subnet_id, '192.168.10.5', None, + None) + mock_secgrp.return_value = [] + self._create_nova_vm() + fake_instances = self.fake_ec2_conn.get_only_instances() + self.assertEqual(len(fake_instances), 1) + inst = fake_instances[0] + self.assertEqual(inst.vpc_id, self.vpc.id) + self.assertEqual(self.subnet_id, inst.subnet_id) + self.assertEqual(inst.tags['Name'], 'fake_instance') + self.assertEqual(inst.tags['openstack_id'], self.uuid) + self.assertEqual(inst.image_id, 'ami-1234abc') + self.assertEqual(inst.region.name, self.region_name) + self.assertEqual(inst.key_name, 'None') + self.assertEqual(inst.instance_type, 't2.small') + self.reset() + + @mock_ec2 + def test_spawn_with_key(self): + self._create_instance(key_name='fake_key', key_data='fake_key_data') + self._create_network() + with contextlib.nested( + mock.patch.object(EC2Driver, '_get_image_ami_id_from_meta'), + mock.patch.object(EC2Driver, '_process_network_info'), + mock.patch.object(EC2Driver, '_get_instance_sec_grps'), + ) as (mock_image, mock_network, mock_secgrp): + mock_image.return_value = 'ami-1234abc' + mock_network.return_value = (self.subnet_id, '192.168.10.5', None, + None) + mock_secgrp.return_value = [] + self._create_nova_vm() + fake_instances = self.fake_ec2_conn.get_only_instances() + self.assertEqual(len(fake_instances), 1) + inst = fake_instances[0] + self.assertEqual(inst.key_name, 'fake_key') + self.reset() + + @mock_ec2 + def test_spawn_with_userdata(self): + userdata = """ + #cloud-config + password: password + """ + b64encoded = base64.b64encode(userdata) + self._create_instance(user_data=b64encoded) + self._create_network() + with contextlib.nested( + mock.patch.object(EC2Driver, '_get_image_ami_id_from_meta'), + mock.patch.object(EC2Driver, '_process_network_info'), + mock.patch.object(EC2Driver, '_get_instance_sec_grps'), + ) as (mock_image, mock_network, mock_secgrp): + mock_image.return_value = 'ami-1234abc' + mock_network.return_value = (self.subnet_id, '192.168.10.5', None, + None) + mock_secgrp.return_value = [] + fake_run_instance_op = self.fake_ec2_conn.run_instances( + 'ami-1234abc') + boto.ec2.EC2Connection.run_instances = mock.Mock() + boto.ec2.EC2Connection.run_instances.return_value = \ + fake_run_instance_op + self._create_nova_vm() + fake_instances = self.fake_ec2_conn.get_only_instances() + self.assertEqual(len(fake_instances), 1) + boto.ec2.EC2Connection.run_instances.assert_called_once_with( + instance_type='t2.small', key_name=None, + image_id='ami-1234abc', user_data=userdata, + subnet_id=self.subnet_id, + private_ip_address='192.168.10.5', + security_group_ids=[]) + self.reset() + + @mock_ec2 + def test_spawn_with_network_error(self): + self._create_instance() + with contextlib.nested( + mock.patch.object(EC2Driver, '_get_image_ami_id_from_meta'), + mock.patch.object(EC2Driver, '_process_network_info'), + mock.patch.object(EC2Driver, '_get_instance_sec_grps'), + ) as (mock_image, mock_network, mock_secgrp): + mock_image.return_value = 'ami-1234abc' + mock_network.return_value = (None, None, None, None) + mock_secgrp.return_value = [] + self.assertRaises(exception.BuildAbortException, self._create_nova_vm) + self.reset() + + @mock_ec2 + def test_spawn_with_network_error_from_aws(self): + self._create_instance() + with contextlib.nested( + mock.patch.object(EC2Driver, '_get_image_ami_id_from_meta'), + mock.patch.object(EC2Driver, '_process_network_info'), + mock.patch.object(EC2Driver, '_get_instance_sec_grps'), + ) as (mock_image, mock_network, mock_secgrp): + mock_image.return_value = 'ami-1234abc' + mock_network.return_value = ('subnet-1234abc', '192.168.10.5', + None, None) + mock_secgrp.return_value = [] + self.assertRaises(exception.BuildAbortException, self._create_nova_vm) + self.reset() + + @mock_ec2 + def test_spawn_with_image_error(self): + self._create_instance() + self._create_network() + with contextlib.nested( + mock.patch.object(EC2Driver, '_get_image_ami_id_from_meta'), + mock.patch.object(EC2Driver, '_process_network_info'), + mock.patch.object(EC2Driver, '_get_instance_sec_grps'), + ) as (mock_image, mock_network, mock_secgrp): + mock_image.side_effect = exception.BuildAbortException('fake') + mock_network.return_value = ('subnet-1234abc', '192.168.10.5', + None, None) + mock_secgrp.return_value = [] + self.assertRaises(exception.BuildAbortException, self._create_nova_vm) + self.reset() + + @mock_ec2 + def _create_vm_in_aws_nova(self): + self._create_instance() + self._create_network() + with contextlib.nested( + mock.patch.object(EC2Driver, '_get_image_ami_id_from_meta'), + mock.patch.object(EC2Driver, '_process_network_info'), + mock.patch.object(EC2Driver, '_get_instance_sec_grps'), + ) as (mock_image, mock_network, mock_secgrp): + mock_image.return_value = 'ami-1234abc' + mock_network.return_value = (self.subnet_id, '192.168.10.5', None, + None) + mock_secgrp.return_value = [] + self._create_nova_vm() + + @mock_ec2 + def test_snapshot(self): + self._create_vm_in_aws_nova() + GlanceImageService.update = mock.Mock() + expected_calls = [ + {'args': (), + 'kwargs': + {'task_state': task_states.IMAGE_UPLOADING, + 'expected_state': task_states.IMAGE_SNAPSHOT}}] + func_call_matcher = matchers.FunctionCallMatcher(expected_calls) + self.conn.snapshot(self.context, self.instance, 'test-snapshot', + func_call_matcher.call) + self.assertIsNone(func_call_matcher.match()) + context, snapshot_name, metadata = \ + GlanceImageService.update.call_args[0] + aws_imgs = self.fake_ec2_conn.get_all_images() + self.assertEqual(1, len(aws_imgs)) + aws_img = aws_imgs[0] + self.assertEqual(snapshot_name, 'test-snapshot') + self.assertEqual(aws_img.name, 'test-snapshot') + self.assertEqual(aws_img.id, metadata['properties']['ec2_image_id']) + self.reset() + + @mock_ec2 + def test_snapshot_instance_not_found(self): + boto.ec2.EC2Connection.create_image = mock.Mock() + self._create_instance() + GlanceImageService.update = mock.Mock() + expected_calls = [ + {'args': (), + 'kwargs': + {'task_state': task_states.IMAGE_UPLOADING, + 'expected_state': task_states.IMAGE_SNAPSHOT}}] + func_call_matcher = matchers.FunctionCallMatcher(expected_calls) + self.assertRaises(exception.InstanceNotFound, self.conn.snapshot, + self.context, self.instance, 'test-snapshot', + func_call_matcher.call) + boto.ec2.EC2Connection.create_image.assert_not_called() + self.reset() + + @mock_ec2 + def test_reboot_soft(self): + boto.ec2.EC2Connection.reboot_instances = mock.Mock() + self._create_vm_in_aws_nova() + fake_inst = self.fake_ec2_conn.get_only_instances()[0] + self.conn.reboot(self.context, self.instance, None, 'SOFT', None, None) + boto.ec2.EC2Connection.reboot_instances.assert_called_once_with( + instance_ids=[fake_inst.id], dry_run=False) + self.reset() + + @mock_ec2 + def test_reboot_hard(self): + self._create_vm_in_aws_nova() + fake_inst = self.fake_ec2_conn.get_only_instances()[0] + boto.ec2.EC2Connection.stop_instances = mock.Mock() + boto.ec2.EC2Connection.start_instances = mock.Mock() + EC2Driver._wait_for_state = mock.Mock() + self.conn.reboot(self.context, self.instance, None, 'HARD', None, None) + boto.ec2.EC2Connection.stop_instances.assert_called_once_with( + instance_ids=[fake_inst.id], force=False, dry_run=False) + boto.ec2.EC2Connection.start_instances.assert_called_once_with( + instance_ids=[fake_inst.id], dry_run=False) + wait_state_calls = EC2Driver._wait_for_state.call_args_list + self.assertEqual(2, len(wait_state_calls)) + self.assertEqual('stopped', wait_state_calls[0][0][2]) + self.assertEqual(fake_inst.id, wait_state_calls[0][0][1]) + self.assertEqual('running', wait_state_calls[1][0][2]) + self.assertEqual(fake_inst.id, wait_state_calls[0][0][1]) + self.reset() + + @mock_ec2 + def test_reboot_instance_not_found(self): + self._create_instance() + boto.ec2.EC2Connection.stop_instances = mock.Mock() + self.assertRaises(exception.InstanceNotFound, self.conn.reboot, + self.context, self.instance, None, 'SOFT', None, + None) + boto.ec2.EC2Connection.stop_instances.assert_not_called() + self.reset() + + @mock_ec2 + def test_power_off(self): + self._create_vm_in_aws_nova() + fake_inst = self.fake_ec2_conn.get_only_instances()[0] + self.assertEqual(fake_inst.state, 'running') + self.conn.power_off(self.instance) + fake_inst = self.fake_ec2_conn.get_only_instances()[0] + self.assertEqual(fake_inst.state, 'stopped') + self.reset() + + @mock_ec2 + def test_power_off_instance_not_found(self): + self._create_instance() + self.assertRaises(exception.InstanceNotFound, self.conn.power_off, + self.instance) + self.reset() + + @mock_ec2 + def test_power_on(self): + self._create_vm_in_aws_nova() + fake_inst = self.fake_ec2_conn.get_only_instances()[0] + self.fake_ec2_conn.stop_instances(instance_ids=[fake_inst.id]) + self.conn.power_on(self.context, self.instance, None, None) + fake_inst = self.fake_ec2_conn.get_only_instances()[0] + self.assertEqual(fake_inst.state, 'running') + self.reset() + + @mock_ec2 + def test_power_on_instance_not_found(self): + self._create_instance() + self.assertRaises(exception.InstanceNotFound, self.conn.power_on, + self.context, self.instance, None, None) + self.reset() + + @mock_ec2 + def test_destroy(self): + self._create_vm_in_aws_nova() + self.conn.destroy(self.context, self.instance, None, None) + fake_instance = self.fake_ec2_conn.get_only_instances()[0] + self.assertEqual('terminated', fake_instance.state) + self.reset() + + @mock_ec2 + def test_destroy_instance_not_found(self): + self._create_instance() + with contextlib.nested( + mock.patch.object(boto.ec2.EC2Connection, 'stop_instances'), + mock.patch.object(boto.ec2.EC2Connection, 'terminate_instances'), + mock.patch.object(EC2Driver, '_wait_for_state'), + ) as (fake_stop, fake_terminate, fake_wait): + self.conn.destroy(self.context, self.instance, None, None) + fake_stop.assert_not_called() + fake_terminate.assert_not_called() + fake_wait.assert_not_called() + self.reset() + + @mock_ec2 + def test_destory_instance_terminated_on_aws(self): + self._create_vm_in_aws_nova() + fake_instances = self.fake_ec2_conn.get_only_instances() + self.fake_ec2_conn.stop_instances(instance_ids=[fake_instances[0].id]) + self.fake_ec2_conn.terminate_instances( + instance_ids=[fake_instances[0].id]) + with contextlib.nested( + mock.patch.object(boto.ec2.EC2Connection, 'stop_instances'), + mock.patch.object(boto.ec2.EC2Connection, 'terminate_instances'), + mock.patch.object(EC2Driver, '_wait_for_state'), + ) as (fake_stop, fake_terminate, fake_wait): + self.conn.destroy(self.context, self.instance, None, None) + fake_stop.assert_not_called() + fake_terminate.assert_not_called() + fake_wait.assert_not_called() + self.reset() + + @mock_ec2 + def test_destroy_instance_shut_down_on_aws(self): + self._create_vm_in_aws_nova() + fake_instances = self.fake_ec2_conn.get_only_instances() + self.fake_ec2_conn.stop_instances(instance_ids=[fake_instances[0].id]) + with contextlib.nested( + mock.patch.object(boto.ec2.EC2Connection, 'stop_instances'), + mock.patch.object(boto.ec2.EC2Connection, 'terminate_instances'), + mock.patch.object(EC2Driver, '_wait_for_state'), + ) as (fake_stop, fake_terminate, fake_wait): + self.conn.destroy(self.context, self.instance, None, None) + fake_stop.assert_not_called() + fake_terminate.assert_called_once_with(instance_ids=[fake_instances[0].id]) + self.reset() + + @mock_ec2 + def test_get_info(self): + self._create_vm_in_aws_nova() + vm_info = self.conn.get_info(self.instance) + self.assertEqual(0, vm_info.state) + self.assertEqual(self.instance.id, vm_info.id) + self.reset() + + @mock_ec2 + def test_get_info_instance_not_found(self): + self._create_instance() + self.assertRaises(exception.InstanceNotFound, self.conn.get_info, + self.instance) + self.reset()