From 2ccba6f57ec2eea577a38599b2ef07e2ffc28a41 Mon Sep 17 00:00:00 2001 From: Eric K Date: Wed, 4 Jul 2018 15:48:15 -0700 Subject: [PATCH] explicit data types - part II Additional standard types: UUID, IPNetwork Expanded enum types Custom types defined and registered by data source drivers Type specifications in Nova and Neutron drivers partially-implements: blueprint explicit-data-types Change-Id: Idea95374478e8fffee2b2c1a3589f588846c40c3 --- congress/data_types.py | 90 ++++++++-- congress/datalog/compile.py | 4 +- congress/datasources/datasource_driver.py | 8 +- congress/datasources/datasource_utils.py | 4 + congress/datasources/neutronv2_driver.py | 159 ++++++++++-------- congress/datasources/nova_driver.py | 68 ++++---- congress/tests/datasources/fakes.py | 11 +- .../datasources/test_datasource_driver.py | 13 +- .../datasources/test_neutronv2_driver.py | 6 +- .../tests/datasources/test_nova_driver.py | 4 +- congress/tests/test_data_types.py | 107 +++++++++++- 11 files changed, 335 insertions(+), 139 deletions(-) diff --git a/congress/data_types.py b/congress/data_types.py index 5a4c6fb76..a306c3cfa 100644 --- a/congress/data_types.py +++ b/congress/data_types.py @@ -16,6 +16,8 @@ import abc import collections import ipaddress import json + +from oslo_utils import uuidutils import six @@ -177,16 +179,27 @@ class Float(Scalar): ' or %s' % (value, type(value), int, float)) +class UUID(Str): + + @classmethod + @nullable + def marshal(cls, value): + if uuidutils.is_uuid_like(value): + return value + else: + raise ValueError('Input value (%s) is not an UUID' % value) + + class IPAddress(Str): @classmethod @nullable def marshal(cls, value): try: - return str(ipaddress.IPv4Address(value)) + return str(ipaddress.IPv4Address(six.text_type(value))) except ipaddress.AddressValueError: try: - ipv6 = ipaddress.IPv6Address(value) + ipv6 = ipaddress.IPv6Address(six.text_type(value)) if ipv6.ipv4_mapped: return str(ipv6.ipv4_mapped) else: @@ -196,6 +209,18 @@ class IPAddress(Str): 'as an IP address' % value) +class IPNetwork(Str): + + @classmethod + @nullable + def marshal(cls, value): + try: + return str(ipaddress.ip_network(six.text_type(value))) + except ValueError: + raise ValueError('Input value (%s) is not interprable ' + 'as an IP network' % value) + + @six.add_metaclass(abc.ABCMeta) class CongressTypeFiniteDomain(object): '''Abstract base class for a Congress type of bounded domain. @@ -206,35 +231,68 @@ class CongressTypeFiniteDomain(object): pass -def create_congress_str_enum_type(class_name, enum_items): - '''Return a sub-type of CongressStr +def create_congress_enum_type(class_name, enum_items, base_type, + catch_all_default_value=None): + '''Return a sub-type of base_type - representing a string from a fixed, finite domain. + representing a value of type base_type from a fixed, finite domain. + :param enum_items: collection of items forming the domain + :param catch_all_default_value: value to use for any value outside the + domain. Defaults to None to disallow any avy value outside the domain. ''' + domain = set(enum_items) + if catch_all_default_value is not None: + domain.add(catch_all_default_value) - for item in enum_items: - if not isinstance(item, six.string_types): + for item in domain: + if not base_type.marshal(item) == item: raise ValueError - class NewType(Str, CongressTypeFiniteDomain): - DOMAIN = frozenset(enum_items) + class NewType(base_type, CongressTypeFiniteDomain): + DOMAIN = domain + CATCH_ALL_DEFAULT_VALUE = catch_all_default_value @classmethod @nullable def marshal(cls, value): if value not in cls.DOMAIN: - raise ValueError( - 'Input value (%s) is not in the expected domain of values ' - '%s' % (value, cls.DOMAIN)) + if cls.CATCH_ALL_DEFAULT_VALUE is None: + raise ValueError( + 'Input value (%s) is not in the expected domain of ' + 'values %s' % (value, cls.DOMAIN)) + else: + return cls.CATCH_ALL_DEFAULT_VALUE return value NewType.__name__ = class_name return NewType -NetworkDirection = create_congress_str_enum_type( - 'NetworkDirection', ('ingress', 'egress')) +class TypesRegistry(object): + _type_name_to_type_class = {} -TYPES = [Scalar, Str, Bool, Int, Float, IPAddress] + @classmethod + def register(cls, type_class): + # skip if type already registered + if not issubclass(type_class, Scalar): + raise TypeError('Attempted to register a type which is not a ' + 'subclass of the top type %s.' % Scalar) + elif str(type_class) in cls._type_name_to_type_class: + if type_class == cls._type_name_to_type_class[str(type_class)]: + pass # type already registered + else: # conflicting types with same name + raise Exception('Attempted to register new type with the same ' + 'name \'%s\' as previously registered type.' % + type_class) + else: # register new type + cls._type_name_to_type_class[str(type_class)] = type_class -TYPE_NAME_TO_TYPE_CLASS = {str(type_obj): type_obj for type_obj in TYPES} + @classmethod + def type_class(cls, type_name): + return cls._type_name_to_type_class[type_name] + + +TYPES = [Scalar, Str, Bool, Int, Float, IPAddress, IPNetwork] + +for type_class in TYPES: + TypesRegistry.register((type_class)) diff --git a/congress/datalog/compile.py b/congress/datalog/compile.py index b55d2c533..d60b744e3 100644 --- a/congress/datalog/compile.py +++ b/congress/datalog/compile.py @@ -99,8 +99,8 @@ class Schema(object): # 'type': 'typename', 'nullable': True/False} if len(cols) and isinstance(cols[0], dict): return [data_types.TypeNullabilityTuple( - data_types.TYPE_NAME_TO_TYPE_CLASS.get( - x.get('type', str(data_types.Scalar))), + data_types.TypesRegistry.type_class( + x.get('type', 'Scalar')), x.get('nullable', True)) for x in cols] else: return [data_types.TypeNullabilityTuple(data_types.Scalar, True) diff --git a/congress/datasources/datasource_driver.py b/congress/datasources/datasource_driver.py index c0468143d..e834022f2 100644 --- a/congress/datasources/datasource_driver.py +++ b/congress/datasources/datasource_driver.py @@ -804,7 +804,13 @@ class DataSourceDriver(data_service.DataService): # check that data type matches if specified in translator if data_type is not None and value is not None: - value = data_type.marshal(value) + try: + value = data_type.marshal(value) + except ValueError: + # Note(types): Log but tolerate type error for now so that + # an unintentionally over-specified type does not interfere + # with the operation of untyped policy engines + LOG.exception('Type error.') return value diff --git a/congress/datasources/datasource_utils.py b/congress/datasources/datasource_utils.py index 18dcfa118..6cdb43ca1 100644 --- a/congress/datasources/datasource_utils.py +++ b/congress/datasources/datasource_utils.py @@ -25,6 +25,10 @@ from keystoneauth1 import loading as kaloading from congress.datasources import constants +def typed_value_trans(type): + return {'translation-type': 'VALUE', 'data-type': type} + + def get_openstack_required_config(): return {'auth_url': constants.REQUIRED, 'endpoint': constants.OPTIONAL, diff --git a/congress/datasources/neutronv2_driver.py b/congress/datasources/neutronv2_driver.py index 1cd0d9e28..dad1589c3 100644 --- a/congress/datasources/neutronv2_driver.py +++ b/congress/datasources/neutronv2_driver.py @@ -20,6 +20,7 @@ from __future__ import absolute_import import neutronclient.v2_0.client from oslo_log import log as logging +from congress import data_types from congress.datasources import constants from congress.datasources import datasource_driver from congress.datasources import datasource_utils as ds_utils @@ -27,6 +28,25 @@ from congress.datasources import datasource_utils as ds_utils LOG = logging.getLogger(__name__) +IngressEgress = data_types.create_congress_enum_type( + 'IngressEgress', ('ingress', 'egress'), data_types.Str) +data_types.TypesRegistry.register(IngressEgress) + +FloatingIPStatus = data_types.create_congress_enum_type( + 'FloatingIPStatus', ('ACTIVE', 'DOWN', 'ERROR'), data_types.Str, + catch_all_default_value='OTHER') +data_types.TypesRegistry.register(FloatingIPStatus) + +NeutronStatus = data_types.create_congress_enum_type( + 'NeutronStatus', ('ACTIVE', 'DOWN', 'BUILD', 'ERROR'), data_types.Str, + catch_all_default_value='OTHER') +data_types.TypesRegistry.register(NeutronStatus) + +IPVersion = data_types.create_congress_enum_type( + 'IPv4IPv6', (4, 6), data_types.Int) +data_types.TypesRegistry.register(IPVersion) + + class NeutronV2Driver(datasource_driver.PollingDataSourceDriver, datasource_driver.ExecutionDriver): @@ -45,8 +65,9 @@ class NeutronV2Driver(datasource_driver.PollingDataSourceDriver, SECURITY_GROUPS = 'security_groups' FLOATING_IPS = 'floating_ips' - # This is the most common per-value translator, so define it once here. - value_trans = {'translation-type': 'VALUE'} + value_trans_str = ds_utils.typed_value_trans(data_types.Str) + value_trans_bool = ds_utils.typed_value_trans(data_types.Bool) + value_trans_int = ds_utils.typed_value_trans(data_types.Int) floating_ips_translator = { 'translation-type': 'HDICT', @@ -54,23 +75,24 @@ class NeutronV2Driver(datasource_driver.PollingDataSourceDriver, 'selector-type': 'DICT_SELECTOR', 'field-translators': ({'fieldname': 'id', 'desc': 'The UUID of the floating IP address', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'router_id', 'desc': 'UUID of router', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'tenant_id', 'desc': 'Tenant ID', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'floating_network_id', 'desc': 'The UUID of the network associated with floating IP', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'fixed_ip_address', 'desc': 'Fixed IP address associated with floating IP address', - 'translator': value_trans}, + 'translator': ds_utils.typed_value_trans(data_types.IPAddress)}, {'fieldname': 'floating_ip_address', - 'desc': 'The floating IP address', 'translator': value_trans}, + 'desc': 'The floating IP address', + 'translator': ds_utils.typed_value_trans(data_types.IPAddress)}, {'fieldname': 'port_id', 'desc': 'UUID of port', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'status', 'desc': 'The floating IP status', - 'translator': value_trans})} + 'translator': ds_utils.typed_value_trans(FloatingIPStatus)})} networks_translator = { 'translation-type': 'HDICT', @@ -78,19 +100,19 @@ class NeutronV2Driver(datasource_driver.PollingDataSourceDriver, 'selector-type': 'DICT_SELECTOR', 'field-translators': ({'fieldname': 'id', 'desc': 'Network ID', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'tenant_id', 'desc': 'Tenant ID', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'name', 'desc': 'Network name', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'status', 'desc': 'Network status', - 'translator': value_trans}, + 'translator': ds_utils.typed_value_trans(NeutronStatus)}, {'fieldname': 'admin_state_up', 'desc': 'Administrative state of the network (true/false)', - 'translator': value_trans}, + 'translator': value_trans_bool}, {'fieldname': 'shared', 'desc': 'Indicates if network is shared across all tenants', - 'translator': value_trans})} + 'translator': value_trans_bool})} ports_fixed_ips_translator = { 'translation-type': 'HDICT', @@ -103,10 +125,10 @@ class NeutronV2Driver(datasource_driver.PollingDataSourceDriver, 'field-translators': ({'fieldname': 'ip_address', 'desc': 'The IP addresses for the port', - 'translator': value_trans}, + 'translator': ds_utils.typed_value_trans(data_types.IPAddress)}, {'fieldname': 'subnet_id', 'desc': 'The UUID of the subnet to which the port is attached', - 'translator': value_trans})} + 'translator': value_trans_str})} ports_security_groups_translator = { 'translation-type': 'LIST', @@ -116,7 +138,7 @@ class NeutronV2Driver(datasource_driver.PollingDataSourceDriver, 'parent-key-desc': 'UUID of port', 'val-col': 'security_group_id', 'val-col-desc': 'UUID of security group', - 'translator': value_trans} + 'translator': value_trans_str} ports_translator = { 'translation-type': 'HDICT', @@ -124,26 +146,27 @@ class NeutronV2Driver(datasource_driver.PollingDataSourceDriver, 'selector-type': 'DICT_SELECTOR', 'field-translators': ({'fieldname': 'id', 'desc': 'UUID of port', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'tenant_id', 'desc': 'tenant ID', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'name', 'desc': 'port name', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'network_id', 'desc': 'UUID of attached network', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'mac_address', 'desc': 'MAC address of the port', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'admin_state_up', 'desc': 'Administrative state of the port', - 'translator': value_trans}, + 'translator': value_trans_bool}, {'fieldname': 'status', 'desc': 'Port status', - 'translator': value_trans}, + 'translator': ds_utils.typed_value_trans(NeutronStatus)}, {'fieldname': 'device_id', - 'desc': 'The UUID of the device that uses this port', - 'translator': value_trans}, + 'desc': 'The ID of the device that uses this port', + 'translator': value_trans_str}, {'fieldname': 'device_owner', - 'desc': 'The UUID of the entity that uses this port', - 'translator': value_trans}, + 'desc': 'The entity type that uses this port.' + 'E.g., compute:nova, network:router_interface', + 'translator': value_trans_str}, {'fieldname': 'fixed_ips', 'desc': 'The IP addresses for the port', 'translator': ports_fixed_ips_translator}, @@ -161,10 +184,10 @@ class NeutronV2Driver(datasource_driver.PollingDataSourceDriver, 'field-translators': ({'fieldname': 'start', 'desc': 'The start address for the allocation pools', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'end', 'desc': 'The end address for the allocation pools', - 'translator': value_trans})} + 'translator': value_trans_str})} subnets_dns_nameservers_translator = { 'translation-type': 'LIST', @@ -174,7 +197,7 @@ class NeutronV2Driver(datasource_driver.PollingDataSourceDriver, 'parent-key-desc': 'UUID of subnet', 'val-col': 'dns_nameserver', 'val-col-desc': 'The DNS server', - 'translator': value_trans} + 'translator': value_trans_str} subnets_routes_translator = { 'translation-type': 'HDICT', @@ -187,10 +210,10 @@ class NeutronV2Driver(datasource_driver.PollingDataSourceDriver, 'field-translators': ({'fieldname': 'destination', 'desc': 'The destination for static route', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'nexthop', 'desc': 'The next hop for the destination', - 'translator': value_trans})} + 'translator': value_trans_str})} subnets_translator = { 'translation-type': 'HDICT', @@ -198,26 +221,26 @@ class NeutronV2Driver(datasource_driver.PollingDataSourceDriver, 'selector-type': 'DICT_SELECTOR', 'field-translators': ({'fieldname': 'id', 'desc': 'UUID of subnet', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'tenant_id', 'desc': 'tenant ID', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'name', 'desc': 'subnet name', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'network_id', 'desc': 'UUID of attached network', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'ip_version', 'desc': 'The IP version, which is 4 or 6', - 'translator': value_trans}, + 'translator': ds_utils.typed_value_trans(IPVersion)}, {'fieldname': 'cidr', 'desc': 'The CIDR', - 'translator': value_trans}, + 'translator': ds_utils.typed_value_trans(data_types.IPNetwork)}, {'fieldname': 'gateway_ip', 'desc': 'The gateway IP address', - 'translator': value_trans}, + 'translator': ds_utils.typed_value_trans(data_types.IPAddress)}, {'fieldname': 'enable_dhcp', 'desc': 'Is DHCP is enabled or not', - 'translator': value_trans}, + 'translator': value_trans_bool}, {'fieldname': 'ipv6_ra_mode', 'desc': 'The IPv6 RA mode', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'ipv6_address_mode', - 'desc': 'The IPv6 address mode', 'translator': value_trans}, + 'desc': 'The IPv6 address mode', 'translator': value_trans_str}, {'fieldname': 'allocation_pools', 'translator': subnets_allocation_pools_translator}, {'fieldname': 'dns_nameservers', @@ -235,9 +258,9 @@ class NeutronV2Driver(datasource_driver.PollingDataSourceDriver, 'in-list': True, 'field-translators': ({'fieldname': 'subnet_id', 'desc': 'UUID of the subnet', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'ip_address', 'desc': 'IP Address', - 'translator': value_trans})} + 'translator': ds_utils.typed_value_trans(data_types.IPAddress)})} routers_external_gateway_infos_translator = { 'translation-type': 'HDICT', @@ -248,10 +271,10 @@ class NeutronV2Driver(datasource_driver.PollingDataSourceDriver, 'selector-type': 'DICT_SELECTOR', 'field-translators': ({'fieldname': 'network_id', 'desc': 'Network ID', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'enable_snat', 'desc': 'current Source NAT status for router', - 'translator': value_trans}, + 'translator': value_trans_bool}, {'fieldname': 'external_fixed_ips', 'translator': external_fixed_ips_translator})} @@ -261,19 +284,19 @@ class NeutronV2Driver(datasource_driver.PollingDataSourceDriver, 'selector-type': 'DICT_SELECTOR', 'field-translators': ({'fieldname': 'id', 'desc': 'uuid of the router', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'tenant_id', 'desc': 'tenant ID', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'status', 'desc': 'router status', - 'translator': value_trans}, + 'translator': ds_utils.typed_value_trans(NeutronStatus)}, {'fieldname': 'admin_state_up', 'desc': 'administrative state of router', - 'translator': value_trans}, + 'translator': value_trans_bool}, {'fieldname': 'name', 'desc': 'router name', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'distributed', 'desc': "indicates if it's distributed router ", - 'translator': value_trans}, + 'translator': value_trans_bool}, {'fieldname': 'external_gateway_info', 'translator': routers_external_gateway_infos_translator})} @@ -287,29 +310,29 @@ class NeutronV2Driver(datasource_driver.PollingDataSourceDriver, 'in-list': True, 'field-translators': ({'fieldname': 'id', 'desc': 'The UUID of the security group rule', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'tenant_id', 'desc': 'tenant ID', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'remote_group_id', 'desc': 'remote group id to associate with security group rule', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'direction', 'desc': 'Direction in which the security group rule is applied', - 'translator': value_trans}, + 'translator': ds_utils.typed_value_trans(IngressEgress)}, {'fieldname': 'ethertype', 'desc': 'IPv4 or IPv6', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'protocol', 'desc': 'protocol that is matched by the security group rule.', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'port_range_min', 'desc': 'Min port number in the range', - 'translator': value_trans}, + 'translator': value_trans_int}, {'fieldname': 'port_range_max', 'desc': 'Max port number in the range', - 'translator': value_trans}, + 'translator': value_trans_int}, {'fieldname': 'remote_ip_prefix', 'desc': 'Remote IP prefix to be associated', - 'translator': value_trans})} + 'translator': value_trans_str})} security_group_translator = { 'translation-type': 'HDICT', @@ -317,13 +340,13 @@ class NeutronV2Driver(datasource_driver.PollingDataSourceDriver, 'selector-type': 'DICT_SELECTOR', 'field-translators': ({'fieldname': 'id', 'desc': 'The UUID for the security group', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'tenant_id', 'desc': 'Tenant ID', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'name', 'desc': 'The security group name', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'description', 'desc': 'security group description', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'security_group_rules', 'translator': security_group_rules_translator})} diff --git a/congress/datasources/nova_driver.py b/congress/datasources/nova_driver.py index acc9d8a8c..98ccaed2a 100644 --- a/congress/datasources/nova_driver.py +++ b/congress/datasources/nova_driver.py @@ -37,10 +37,12 @@ import novaclient.client from oslo_log import log as logging import six +from congress import data_types from congress.datasources import constants from congress.datasources import datasource_driver from congress.datasources import datasource_utils as ds_utils + LOG = logging.getLogger(__name__) @@ -53,8 +55,9 @@ class NovaDriver(datasource_driver.PollingDataSourceDriver, AVAILABILITY_ZONES = "availability_zones" TAGS = "tags" - # This is the most common per-value translator, so define it once here. - value_trans = {'translation-type': 'VALUE'} + value_trans_str = ds_utils.typed_value_trans(data_types.Str) + value_trans_bool = ds_utils.typed_value_trans(data_types.Bool) + value_trans_int = ds_utils.typed_value_trans(data_types.Int) def safe_id(x): if isinstance(x, six.string_types): @@ -70,18 +73,18 @@ class NovaDriver(datasource_driver.PollingDataSourceDriver, 'selector-type': 'DOT_SELECTOR', 'field-translators': ({'fieldname': 'id', 'desc': 'The UUID for the server', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'name', 'desc': 'Name of the server', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'hostId', 'col': 'host_id', - 'desc': 'The UUID for the host', 'translator': value_trans}, + 'desc': 'The UUID for the host', 'translator': value_trans_str}, {'fieldname': 'status', 'desc': 'The server status', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'tenant_id', 'desc': 'The tenant ID', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'user_id', 'desc': 'The user ID of the user who owns the server', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'image', 'col': 'image_id', 'desc': 'Name or ID of image', 'translator': {'translation-type': 'VALUE', @@ -92,11 +95,11 @@ class NovaDriver(datasource_driver.PollingDataSourceDriver, 'extract-fn': safe_id}}, {'fieldname': 'OS-EXT-AZ:availability_zone', 'col': 'zone', 'desc': 'The availability zone of host', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'OS-EXT-SRV-ATTR:hypervisor_hostname', 'desc': ('The hostname of hypervisor where the server is ' 'running'), - 'col': 'host_name', 'translator': value_trans}, + 'col': 'host_name', 'translator': value_trans_str}, {'fieldname': 'tags', 'translator': {'translation-type': 'LIST', 'table-name': TAGS, @@ -105,7 +108,7 @@ class NovaDriver(datasource_driver.PollingDataSourceDriver, 'parent-key-desc': 'UUID of server', 'val-col': 'tag', 'val-col-desc': 'server tag string', - 'translator': value_trans}})} + 'translator': value_trans_str}})} flavors_translator = { 'translation-type': 'HDICT', @@ -113,19 +116,19 @@ class NovaDriver(datasource_driver.PollingDataSourceDriver, 'selector-type': 'DOT_SELECTOR', 'field-translators': ({'fieldname': 'id', 'desc': 'ID of the flavor', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'name', 'desc': 'Name of the flavor', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'vcpus', 'desc': 'Number of vcpus', - 'translator': value_trans}, + 'translator': value_trans_int}, {'fieldname': 'ram', 'desc': 'Memory size in MB', - 'translator': value_trans}, + 'translator': value_trans_int}, {'fieldname': 'disk', 'desc': 'Disk size in GB', - 'translator': value_trans}, + 'translator': value_trans_int}, {'fieldname': 'ephemeral', 'desc': 'Ephemeral space size in GB', - 'translator': value_trans}, + 'translator': value_trans_int}, {'fieldname': 'rxtx_factor', 'desc': 'RX/TX factor', - 'translator': value_trans})} + 'translator': ds_utils.typed_value_trans(data_types.Float)})} hypervisors_translator = { 'translation-type': 'HDICT', @@ -133,13 +136,14 @@ class NovaDriver(datasource_driver.PollingDataSourceDriver, 'selector-type': 'DOT_SELECTOR', 'field-translators': ({'fieldname': 'hypervisor_hostname', 'desc': 'Hypervisor host', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'id', 'desc': 'hypervisori id', - 'translator': value_trans}, + # untyped: depends on api microversion + 'translator': {'translation-type': 'VALUE'}}, {'fieldname': 'state', 'desc': 'State of the hypervisor', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'status', 'desc': 'Status of the hypervisor', - 'translator': value_trans})} + 'translator': value_trans_str})} services_translator = { 'translation-type': 'HDICT', @@ -147,21 +151,21 @@ class NovaDriver(datasource_driver.PollingDataSourceDriver, 'selector-type': 'DOT_SELECTOR', 'field-translators': ({'fieldname': 'id', 'col': 'service_id', 'desc': 'Service ID', - 'translator': value_trans}, + 'translator': value_trans_int}, {'fieldname': 'binary', 'desc': 'Service binary', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'host', 'desc': 'Host Name', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'zone', 'desc': 'Availability Zone', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'status', 'desc': 'Status of service', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'state', 'desc': 'State of service', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'updated_at', 'desc': 'Last updated time', - 'translator': value_trans}, + 'translator': value_trans_str}, {'fieldname': 'disabled_reason', 'desc': 'Disabled reason', - 'translator': value_trans})} + 'translator': value_trans_str})} availability_zones_translator = { 'translation-type': 'HDICT', @@ -169,10 +173,10 @@ class NovaDriver(datasource_driver.PollingDataSourceDriver, 'selector-type': 'DOT_SELECTOR', 'field-translators': ({'fieldname': 'zoneName', 'col': 'zone', - 'desc': 'Availability zone name', 'translator': value_trans}, + 'desc': 'Availability zone name', 'translator': value_trans_str}, {'fieldname': 'zoneState', 'col': 'state', 'desc': 'Availability zone state', - 'translator': value_trans})} + 'translator': value_trans_str})} TRANSLATORS = [servers_translator, flavors_translator, services_translator, hypervisors_translator, availability_zones_translator] diff --git a/congress/tests/datasources/fakes.py b/congress/tests/datasources/fakes.py index e00107640..6a57ba45c 100644 --- a/congress/tests/datasources/fakes.py +++ b/congress/tests/datasources/fakes.py @@ -204,9 +204,14 @@ class NovaFakeClient(mock.MagicMock): h.status = status return h - def get_hypervisor_list(self): - h_one = self.get_hypervisor('host1', '2', 'up', 'enabled') - h_two = self.get_hypervisor('host2', '3', 'down', 'enabled') + def get_hypervisor_list(self, nova_api_version='2.26'): + from distutils.version import StrictVersion + if StrictVersion(nova_api_version) <= StrictVersion('2.52'): + h_one = self.get_hypervisor('host1', 2, 'up', 'enabled') + h_two = self.get_hypervisor('host2', 3, 'down', 'enabled') + else: + h_one = self.get_hypervisor('host1', '2', 'up', 'enabled') + h_two = self.get_hypervisor('host2', '3', 'down', 'enabled') return [h_one, h_two] diff --git a/congress/tests/datasources/test_datasource_driver.py b/congress/tests/datasources/test_datasource_driver.py index 6ebe83753..22c81c515 100644 --- a/congress/tests/datasources/test_datasource_driver.py +++ b/congress/tests/datasources/test_datasource_driver.py @@ -41,9 +41,6 @@ class TestDatasourceDriver(base.TestCase): super(TestDatasourceDriver, self).setUp() self.val_trans = {'translation-type': 'VALUE'} - def typed_value_trans(self, type): - return {'translation-type': 'VALUE', 'data-type': type} - def compute_hash(self, obj): s = json.dumps(sorted(obj, key=(lambda x: str(type(x)) + repr(x))), sort_keys=True) @@ -94,7 +91,7 @@ class TestDatasourceDriver(base.TestCase): 'in-list': True, 'field-translators': ({'fieldname': 'ip_address', - 'translator': self.typed_value_trans(Type2)}, + 'translator': datasource_utils.typed_value_trans(Type2)}, {'fieldname': 'subnet_id', 'translator': self.val_trans})} ports_translator = { @@ -103,7 +100,7 @@ class TestDatasourceDriver(base.TestCase): 'selector-type': 'DICT_SELECTOR', 'field-translators': ({'fieldname': 'id', - 'translator': self.typed_value_trans(Type2)}, + 'translator': datasource_utils.typed_value_trans(Type2)}, {'fieldname': 'fixed_ips', 'translator': ports_fixed_ips_translator})} @@ -1110,7 +1107,8 @@ class TestDatasourceDriver(base.TestCase): {'fieldname': 'a', 'col': 'a1', 'translator': - self.typed_value_trans( + datasource_utils. + typed_value_trans( data_types.Bool)}, {'fieldname': 'b', 'col': 'b1', @@ -1128,7 +1126,8 @@ class TestDatasourceDriver(base.TestCase): 'col': 'd1', 'translator': self.val_trans})}}, {'fieldname': 'ztestfield3', 'col': 'zparent_col3', - 'translator': self.typed_value_trans(data_types.Str)}, + 'translator': datasource_utils.typed_value_trans( + data_types.Str)}, {'fieldname': 'testfield4', 'col': 'parent_col4', 'translator': {'translation-type': 'VALUE', 'extract-fn': lambda x: x.id}}, diff --git a/congress/tests/datasources/test_neutronv2_driver.py b/congress/tests/datasources/test_neutronv2_driver.py index e4ca4e8e8..42c5f39c5 100644 --- a/congress/tests/datasources/test_neutronv2_driver.py +++ b/congress/tests/datasources/test_neutronv2_driver.py @@ -219,8 +219,8 @@ class TestNeutronV2Driver(base.TestCase): {u'direction': u'egress', u'ethertype': u'IPv4', u'id': u'1d943e83-e4e6-472a-9655-f74eb22f3668', - u'port_range_max': None, - u'port_range_min': None, + u'port_range_max': 22, + u'port_range_min': 11, u'protocol': None, u'remote_group_id': None, u'remote_ip_prefix': None, @@ -353,7 +353,7 @@ class TestNeutronV2Driver(base.TestCase): 'egress', 'IPv6', None, None, None, None), ('a268fc32-1a59-4154-9a7c-f453ef92560c', '1d943e83-e4e6-472a-9655-f74eb22f3668', '', None, - 'egress', 'IPv4', None, None, None, None), + 'egress', 'IPv4', None, 11, 22, None), ('a268fc32-1a59-4154-9a7c-f453ef92560c', '30be5ee1-5b0a-4929-aca5-0c25f1c6b733', '', 'a268fc32-1a59-4154-9a7c-f453ef92560c', 'ingress', diff --git a/congress/tests/datasources/test_nova_driver.py b/congress/tests/datasources/test_nova_driver.py index a814eb582..cb55a15de 100644 --- a/congress/tests/datasources/test_nova_driver.py +++ b/congress/tests/datasources/test_nova_driver.py @@ -208,11 +208,11 @@ class TestNovaDriver(base.TestCase): status = host[3] if host_name == 'host1': - self.assertEqual('2', id) + self.assertEqual(2, id) self.assertEqual('up', state) self.assertEqual('enabled', status) elif host_name == 'host2': - self.assertEqual('3', id) + self.assertEqual(3, id) self.assertEqual('down', state) self.assertEqual('enabled', status) diff --git a/congress/tests/test_data_types.py b/congress/tests/test_data_types.py index dc402f4a8..8c44db8cd 100644 --- a/congress/tests/test_data_types.py +++ b/congress/tests/test_data_types.py @@ -24,8 +24,105 @@ from congress import data_types class TestDataTypes(testtools.TestCase): - def test_congress_str_nullable(self): - self.assertEqual(data_types.Str.marshal('test-str-value'), - 'test-str-value') - self.assertIsNone(data_types.Str.marshal(None)) - self.assertRaises(ValueError, data_types.Str.marshal, True) + def test_nullable(self): + for type_class in data_types.TYPES: + self.assertIsNone(type_class.marshal(None)) + + def test_Scalar(self): + valid_values = [1, 1.0, 'str', u'str', True] + invalid_values = [{}, []] + for val in valid_values: + self.assertEqual(val, data_types.Scalar.marshal(val)) + + for val in invalid_values: + self.assertRaises(ValueError, data_types.Scalar.marshal, val) + + def test_Str(self): + valid_values = ['str', u'str', '1'] + invalid_values = [{}, [], True, 1] + for val in valid_values: + self.assertEqual(val, data_types.Str.marshal(val)) + + for val in invalid_values: + self.assertRaises(ValueError, data_types.Str.marshal, val) + + def test_Bool(self): + valid_values = [True, False] + invalid_values = [{}, [], 'True', 0, 1] + for val in valid_values: + self.assertEqual(val, data_types.Bool.marshal(val)) + + for val in invalid_values: + self.assertRaises(ValueError, data_types.Bool.marshal, val) + + def test_Int(self): + valid_values = [1, 1.0, -1, True, False] + invalid_values = [{}, [], 1.1, '1'] + for val in valid_values: + self.assertEqual(val, data_types.Int.marshal(val)) + + for val in invalid_values: + self.assertRaises(ValueError, data_types.Int.marshal, val) + + def test_Float(self): + valid_values = [1, 1.0, -1, True, False] + invalid_values = [{}, [], '1'] + for val in valid_values: + self.assertEqual(val, data_types.Int.marshal(val)) + + for val in invalid_values: + self.assertRaises(ValueError, data_types.Int.marshal, val) + + def test_UUID(self): + valid_values = ['026f66d3-d6f1-44d1-8451-0a95ee984ffa', + '026f66d3d6f144d184510a95ee984ffa', + '-0-2-6f66d3d6f144d184510a95ee984ffa'] + invalid_values = [{}, [], '1', True, 1, + 'z26f66d3d6f144d184510a95ee984ffa'] + for val in valid_values: + self.assertEqual(val, data_types.UUID.marshal(val)) + + for val in invalid_values: + self.assertRaises(ValueError, data_types.UUID.marshal, val) + + def test_IPAddress(self): + valid_values = [('10.0.0.1', '10.0.0.1'), + ('::ffff:0a00:0001', '10.0.0.1'), + ('0000:0000:0000:0000:0000:ffff:0a00:0001', + '10.0.0.1'), + ('2001:db8::ff00:42:8329', '2001:db8::ff00:42:8329'), + ('2001:0db8:0000:0000:0000:ff00:0042:8329', + '2001:db8::ff00:42:8329')] + invalid_values = [{}, [], '1', True, 1, + '256.0.0.1'] + for val in valid_values: + self.assertEqual(val[1], data_types.IPAddress.marshal(val[0])) + + for val in invalid_values: + self.assertRaises(ValueError, data_types.IPAddress.marshal, val) + + def test_IPNetwork(self): + valid_values = [('10.0.0.0/16', '10.0.0.0/16'), + ('2001:db00::0/24', '2001:db00::/24'), + ('::ffff:0a00:0000/128', '::ffff:a00:0/128')] + invalid_values = [{}, [], '1', True, 1, + '10.0.0.0/4' + '10.0.0.1/16'] + for val in valid_values: + self.assertEqual(val[1], data_types.IPNetwork.marshal(val[0])) + + for val in invalid_values: + self.assertRaises(ValueError, data_types.IPNetwork.marshal, val) + + def test_enum_types(self): + Test1 = data_types.create_congress_enum_type( + 'Test', [1, 2], data_types.Int) + self.assertEqual(1, Test1.marshal(1)) + self.assertIsNone(Test1.marshal(None)) + self.assertRaises(ValueError, Test1.marshal, 0) + + Test2 = data_types.create_congress_enum_type( + 'Test', [1, 2], data_types.Int, catch_all_default_value=0) + self.assertEqual(1, Test2.marshal(1)) + self.assertEqual(0, Test2.marshal(-1)) + self.assertEqual(0, Test2.marshal('blah'))