From 8d3b5e3de96b4582fcfe8830f3efa876c5a5a14e Mon Sep 17 00:00:00 2001 From: Artem Grechanichenko Date: Fri, 19 Aug 2016 01:05:58 +0300 Subject: [PATCH] Add devops_template for each test in fuel_library and puppet-openstack mapping Closes-Bug: #1614546 Change-Id: Ifaacf0fab80de26a55d2ba1cfe5afeb6867766c1 --- fuelweb_test/tests/base_test_case.py | 10 +- gates_tests/devops_templates/default.yaml | 235 ++++++++++++++++++ .../helpers/fuel_library_modules_mapping.yaml | 218 +++++++++------- .../openstack_puppet_projects_mapping.yaml | 12 + gates_tests/helpers/utils.py | 40 ++- 5 files changed, 415 insertions(+), 100 deletions(-) create mode 100644 gates_tests/devops_templates/default.yaml diff --git a/fuelweb_test/tests/base_test_case.py b/fuelweb_test/tests/base_test_case.py index 6aaf183f2..b57412467 100644 --- a/fuelweb_test/tests/base_test_case.py +++ b/fuelweb_test/tests/base_test_case.py @@ -12,9 +12,11 @@ # License for the specific language governing permissions and limitations # under the License. +import os import time from devops.helpers.helpers import wait +from devops.helpers.templates import yaml_template_load from proboscis import TestProgram from proboscis import SkipTest @@ -83,7 +85,13 @@ class TestBasic(object): @property def env(self): if self.__env is None: - self.__env = EnvironmentModel(self._devops_config) + # hack before we find proper solution for loading devops templates + external_config = os.environ.get("DEVOPS_SETTINGS_TEMPLATE", None) + if external_config: + config = yaml_template_load(external_config) + else: + config = self._devops_config + self.__env = EnvironmentModel(config) return self.__env @property diff --git a/gates_tests/devops_templates/default.yaml b/gates_tests/devops_templates/default.yaml new file mode 100644 index 000000000..aac456611 --- /dev/null +++ b/gates_tests/devops_templates/default.yaml @@ -0,0 +1,235 @@ +--- +aliases: + + dynamic_address_pool: + - &pool_default !os_env POOL_DEFAULT, 10.109.0.0/16:24 + + default_interface_model: + - &interface_model !os_env INTERFACE_MODEL, e1000 + +template: + devops_settings: + env_name: !os_env ENV_NAME + + address_pools: + # Network pools used by the environment + + fuelweb_admin-pool01: + net: *pool_default + params: + vlan_start: 0 + + # Reserved IP addresses. + # - 'l2_network_device' will be used as an IP address for creating + # libvirt network with assigned IP on it, if specified. + # - 'gateway' should be used in fuel-qa framework to get the gateway + # of the address pool. Can be different from libvirt IP address + # in case if a network already have another gateway. + # - all other IP addresses specified in the template can be used + # for different purposes in different test cases. + ip_reserved: + + # Address of the gateway in the address pool fuelweb_admin-pool01. + # If string - than is used 'as is', as a real IP address + # If numeric - than assumed as a relative address + # from the start of the address pool. + gateway: +1 + + # Libvirt bridge will get this IP address + l2_network_device: +1 + + # IP ranges, reserved in the address pool. + # - 'default' IP range stands for 'default' nodegroup in Fuel. + # - any other IP range names such as 'floating' can be specified to + # provide a special IP ranges to configure cluster settings. + # For special test cases, several IP ranges can be configured here + # for different networks. Names of these ranges should be + # agreed for template and test cases. + ip_ranges: + + # Relative or real IP addresses + default: [+2, -2] # admin IP range for 'default' nodegroup name + + public-pool01: + net: *pool_default + params: + vlan_start: 0 + ip_reserved: + gateway: +1 + l2_network_device: +1 # l2_network_device will get this IP address + ip_ranges: + default: [+2, +127] # public IP range for 'default' nodegroup name + floating: [+128, -2] + + storage-pool01: + net: *pool_default + params: + vlan_start: 101 + + management-pool01: + net: *pool_default + params: + vlan_start: 102 + + private-pool01: + net: *pool_default + params: + vlan_start: 960 + vlan_end: 1000 + + groups: + - name: rack-01 + driver: + name: devops.driver.libvirt + params: + connection_string: !os_env CONNECTION_STRING, qemu:///system + storage_pool_name: !os_env STORAGE_POOL_NAME, default + stp: True + hpet: False + use_host_cpu: !os_env DRIVER_USE_HOST_CPU, true + + network_pools: # Address pools for OpenStack networks. + # Actual names should be used for keys + # (the same as in Nailgun, for example) + + fuelweb_admin: fuelweb_admin-pool01 + public: public-pool01 + storage: storage-pool01 + management: management-pool01 + private: private-pool01 + + l2_network_devices: # Libvirt bridges. It is *NOT* Nailgun networks + admin: + address_pool: fuelweb_admin-pool01 + dhcp: false + forward: + mode: nat + + public: + address_pool: public-pool01 + dhcp: false + forward: + mode: nat + + storage: + address_pool: storage-pool01 + dhcp: false + + management: + address_pool: management-pool01 + dhcp: false + + private: + address_pool: private-pool01 + dhcp: false + + nodes: + - name: admin # Custom name of VM for Fuel admin node + role: fuel_master # Fixed role for Fuel master node properties + params: + vcpu: !os_env ADMIN_NODE_CPU, 2 + memory: !os_env ADMIN_NODE_MEMORY, 3072 + boot: + - hd + - cdrom # for boot from usb - without 'cdrom' + volumes: + - name: system + capacity: !os_env ADMIN_NODE_VOLUME_SIZE, 75 + format: qcow2 + - name: iso + source_image: !os_env ISO_PATH # if 'source_image' set, then volume capacity is calculated from it's size + format: raw + device: cdrom # for boot from usb - 'disk' + bus: ide # for boot from usb - 'usb' + interfaces: + - label: eth0 + l2_network_device: admin # Libvirt bridge name. It is *NOT* a Nailgun network + interface_model: *interface_model + network_config: + eth0: + networks: + - fuelweb_admin + + # Slave nodes + + - name: slave-01 + role: fuel_slave + params: &rack-01-slave-node-params + vcpu: !os_env SLAVE_NODE_CPU, 2 + memory: !os_env SLAVE_NODE_MEMORY, 3072 + boot: + - network + - hd + volumes: + - name: system + capacity: !os_env NODE_VOLUME_SIZE, 50 + format: qcow2 + - name: cinder + capacity: !os_env NODE_VOLUME_SIZE, 50 + format: qcow2 + - name: swift + capacity: !os_env NODE_VOLUME_SIZE, 50 + format: qcow2 + + # List of node interfaces + interfaces: + - label: eth0 + l2_network_device: admin # Libvirt bridge name. It is *NOT* Nailgun networks + interface_model: *interface_model + - label: eth1 + l2_network_device: public + interface_model: *interface_model + - label: eth2 + l2_network_device: storage + interface_model: *interface_model + - label: eth3 + l2_network_device: management + interface_model: *interface_model + - label: eth4 + l2_network_device: private + interface_model: *interface_model + + # How Nailgun/OpenStack networks should assigned for interfaces + network_config: + eth0: + networks: + - fuelweb_admin # Nailgun/OpenStack network name + eth1: + networks: + - public + eth2: + networks: + - storage + eth3: + networks: + - management + eth4: + networks: + - private + + + - name: slave-02 + role: fuel_slave + params: *rack-01-slave-node-params + - name: slave-03 + role: fuel_slave + params: *rack-01-slave-node-params + - name: slave-04 + role: fuel_slave + params: *rack-01-slave-node-params + - name: slave-05 + role: fuel_slave + params: *rack-01-slave-node-params + - name: slave-06 + role: fuel_slave + params: *rack-01-slave-node-params + - name: slave-07 + role: fuel_slave + params: *rack-01-slave-node-params + - name: slave-08 + role: fuel_slave + params: *rack-01-slave-node-params + - name: slave-09 + role: fuel_slave + params: *rack-01-slave-node-params + diff --git a/gates_tests/helpers/fuel_library_modules_mapping.yaml b/gates_tests/helpers/fuel_library_modules_mapping.yaml index 3929beab6..ad866b4ac 100644 --- a/gates_tests/helpers/fuel_library_modules_mapping.yaml +++ b/gates_tests/helpers/fuel_library_modules_mapping.yaml @@ -1,103 +1,109 @@ bvt_2: - - apache - - apache_api_proxy - - apache_mpm - - api-proxy - - api_proxy.conf - - apt - - astute - - atop - - atop_retention - - auth_file - - credentials_file - - ceph - - ceph_nova_compute - - cluster - - cluster_haproxy - - cluster-haproxy - - cluster_vrouter - - cluster-vrouter - - cobbler - - compute - - concat - - controller - - corosync - - database - - datacat - - deployment_groups - - dnsmasq - - dnsmasq.conf - - docker - - fuel - - fuel_pkgs - - glance - - globals - - globals_yaml - - haproxy - - hiera - - horizon - - hosts - - inifile - - install_ssh_keys - - keystone - - l23network - - logging - - mon - - mcollective - - memcached - - monit - - nailgun - - neutron - - netconfig - - nova - - openrc.erb - - openssl - - openstack - - openstack-controller - - openstack_controller - - openstack-haproxy - - openstack_haproxy - - openstack-network - - openstack_network - - openstacklib - - pacemaker - - pacemaker_wrappers - - package_pins - - postgresql - - puppet_pull - - puppet-pull.sh - - rabbitmq - - roles/controller - - roles/compute - - roles/enable_compute - - roles/tasks - - rsync - - rsyslog - - spec - - ssh - - ssl - - staging - - stdlib - - sysctl - - sysfs - - test_compute - - test_controller - - tftp - - tools - - tweaks - - vcsrepo - - virtual_ips - - xinetd - - vmware - - wait_for_backend - - wait_for_keystone_backends - - wait_for_glance_backends + devops_settings_template: devops_templates/default.yaml + modules: + - apache + - apache_api_proxy + - apache_mpm + - api-proxy + - api_proxy.conf + - apt + - astute + - atop + - atop_retention + - auth_file + - credentials_file + - ceph + - ceph_nova_compute + - cluster + - cluster_haproxy + - cluster-haproxy + - cluster_vrouter + - cluster-vrouter + - cobbler + - compute + - concat + - controller + - corosync + - database + - datacat + - deployment_groups + - dnsmasq + - dnsmasq.conf + - docker + - fuel + - fuel_pkgs + - glance + - globals + - globals_yaml + - haproxy + - hiera + - horizon + - hosts + - inifile + - install_ssh_keys + - keystone + - l23network + - logging + - mon + - mcollective + - memcached + - monit + - nailgun + - neutron + - netconfig + - nova + - openrc.erb + - openssl + - openstack + - openstack-controller + - openstack_controller + - openstack-haproxy + - openstack_haproxy + - openstack-network + - openstack_network + - openstacklib + - pacemaker + - pacemaker_wrappers + - package_pins + - postgresql + - puppet_pull + - puppet-pull.sh + - rabbitmq + - roles/controller + - roles/compute + - roles/enable_compute + - roles/tasks + - rsync + - rsyslog + - spec + - ssh + - ssl + - staging + - stdlib + - sysctl + - sysfs + - test_compute + - test_controller + - tftp + - tools + - tweaks + - vcsrepo + - virtual_ips + - xinetd + - vmware + - wait_for_backend + - wait_for_keystone_backends + - wait_for_glance_backends deploy_ha_controller_neutron_example: + devops_settings_template: devops_templates/default.yaml + modules: - plugins deploy_heat_ha: + devops_settings_template: devops_templates/default.yaml + modules: - aodh - ceilometer - ceilometer_ha @@ -108,6 +114,8 @@ deploy_heat_ha: - roles/mongo ha_neutron: + devops_settings_template: devops_templates/default.yaml + modules: - cinder - openstack-cinder - roles/cinder @@ -115,13 +123,19 @@ ha_neutron: - openstack_cinder reduced_footprint: + devops_settings_template: devops_templates/default.yaml + modules: - generate_vms - vm_libvirt ha_neutron_firewall: + devops_settings_template: devops_templates/default.yaml + modules: - firewall ha_neutron_mysql_termination: + devops_settings_template: devops_templates/default.yaml + modules: - galera - mysql - mysql.access.cnf @@ -130,6 +144,8 @@ ha_neutron_mysql_termination: - mysql_user_access ironic_deploy_ceph: + devops_settings_template: devops_templates/ironic_template.yaml + modules: - ceph - ironic - ironic-conductor @@ -137,9 +153,13 @@ ironic_deploy_ceph: - roles/ironic_conductor deploy_murano_ha_with_tun: + devops_settings_template: devops_templates/default.yaml + modules: - murano deploy_ha_dns_ntp: + devops_settings_template: devops_templates/default.yaml + modules: - dns - ntp - resolvconf @@ -147,22 +167,34 @@ deploy_ha_dns_ntp: - resolv.dnsmasq.conf deploy_sahara_ha_tun: + devops_settings_template: devops_templates/default.yaml + modules: - sahara - sahara_templates neutron_vlan_ha: + devops_settings_template: devops_templates/default.yaml + modules: - swift - proxy positive_cic_maintenance_mode: + devops_settings_template: devops_templates/default.yaml + modules: - umm deploy_ha_cgroup: + devops_settings_template: devops_templates/default.yaml + modules: - cgroups basic_env_for_hugepages: + devops_settings_template: devops_templates/default.yaml + modules: - allocated_hugepages test_logrotate: + devops_settings_template: devops_templates/default.yaml + modules: - apache2.logrotate - apache2.prerotate diff --git a/gates_tests/helpers/openstack_puppet_projects_mapping.yaml b/gates_tests/helpers/openstack_puppet_projects_mapping.yaml index 98e6cbc28..2acab529d 100644 --- a/gates_tests/helpers/openstack_puppet_projects_mapping.yaml +++ b/gates_tests/helpers/openstack_puppet_projects_mapping.yaml @@ -1,4 +1,6 @@ bvt_2: + devops_settings_template: devops_templates/default.yaml + projects: - openstack/puppet-ceph - openstack/puppet-glance - openstack/puppet-horizon @@ -10,19 +12,29 @@ bvt_2: - openstack/puppet-oslo neutron_vlan_ha: + devops_settings_template: devops_templates/default.yaml + projects: - openstack/puppet-cinder - openstack/puppet-swift deploy_sahara_ha_tun: + devops_settings_template: devops_templates/default.yaml + projects: - openstack/puppet-sahara deploy_murano_ha_with_tun: + devops_settings_template: devops_templates/default.yaml + projects: - openstack/puppet-murano ironic_deploy_ceph: + devops_settings_template: devops_templates/ironic_template.yaml + projects: - openstack/puppet-ironic deploy_heat_ha: + devops_settings_template: devops_templates/default.yaml + projects: - openstack/puppet-aodh - openstack/puppet-ceilometer - openstack/puppet-heat diff --git a/gates_tests/helpers/utils.py b/gates_tests/helpers/utils.py index 4ccf642ae..9af91400a 100644 --- a/gates_tests/helpers/utils.py +++ b/gates_tests/helpers/utils.py @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. +import itertools import os import yaml @@ -19,7 +20,6 @@ from proboscis import register from proboscis.asserts import assert_equal from devops.helpers import helpers - from fuelweb_test.helpers.fuel_actions import BaseActions from fuelweb_test.helpers.gerrit.gerrit_info_provider import \ FuelLibraryModulesProvider @@ -303,7 +303,9 @@ def fuel_library_modules_mapping(modules): mapping = yaml.load(f) if modules and isinstance(modules, dict): - all_modules = set([j for i in mapping.values() for j in i]) + all_modules = set(list(itertools.chain.from_iterable( + [mapping[test_group]['modules'] for test_group in mapping]))) + logger.debug( "List of puppet modules covered by system_tests {}".format( all_modules)) @@ -312,7 +314,7 @@ def fuel_library_modules_mapping(modules): # checking that module from review covered by system_test for module in modules.keys(): - if module.split('.')[0] not in all_modules: + if module not in all_modules: logger.warning( "{}:{} module not exist or not covered by system_test" .format(module, modules[module])) @@ -325,10 +327,23 @@ def fuel_library_modules_mapping(modules): set(modules)): for test in mapping: test_intersection = len( - set(mapping[test]).intersection(set(modules))) + set(mapping[test]['modules']).intersection(set(modules))) if test_intersection > max_intersection: max_intersection = test_intersection system_test = test + + devops_template = mapping[system_test]['devops_settings_template'] + + import gates_tests + + path_to_template = os.path.join( + os.path.dirname(os.path.abspath(gates_tests.__file__)), + devops_template) + + logger.debug("devops template is {}".format(path_to_template)) + + os.environ['DEVOPS_SETTINGS_TEMPLATE'] = path_to_template + # To completely check ceph module we can't mix ceph and cinder togeher else: logger.warning( @@ -364,7 +379,8 @@ def openstack_puppet_project_mapping(project): "r") as f: mapping = yaml.load(f) - all_projects = set([j for i in mapping.values() for j in i]) + all_projects = set(list(itertools.chain.from_iterable( + [mapping[test_group]['modules'] for test_group in mapping]))) logger.debug( "List of openstack/puppet-projects " "covered by system_tests {}".format( @@ -381,9 +397,21 @@ def openstack_puppet_project_mapping(project): # find test group which cover project edited in review system_test = "bvt_2" for test in mapping: - if project in mapping[test]: + if project in mapping[test]['projects']: system_test = test break + + devops_template = mapping[system_test]['devops_settings_template'] + + import gates_tests + + path_to_template = os.path.join( + os.path.dirname(os.path.abspath(gates_tests.__file__)), + devops_template) + + logger.debug("devops template is {}".format(path_to_template)) + + os.environ['DEVOPS_SETTINGS_TEMPLATE'] = path_to_template logger.info( "Edited project in review - '{}'" " will be checked by next system test: {}".format(