From cb358e96344513e832fb1dfd87ef0bfd336a179a Mon Sep 17 00:00:00 2001 From: Wes Hayutin Date: Thu, 21 Mar 2019 14:32:08 -0600 Subject: [PATCH] add tempest skip list for stein Need a skip list for stein in place Change-Id: Ib38e4e07be2c3429b801c5a9d57e3834f0d969e1 --- .../vars/tempest_skip_stein.yml | 254 ++++++++++++++++++ 1 file changed, 254 insertions(+) create mode 100644 roles/validate-tempest/vars/tempest_skip_stein.yml diff --git a/roles/validate-tempest/vars/tempest_skip_stein.yml b/roles/validate-tempest/vars/tempest_skip_stein.yml new file mode 100644 index 000000000..adf7f91a1 --- /dev/null +++ b/roles/validate-tempest/vars/tempest_skip_stein.yml @@ -0,0 +1,254 @@ +--- +## PLEASE READ ## +## Do not add tests to the skip list that are in ## +## featureset016 - featureset019 w/o permission from ## +## the TripleO PTL, Thanks ## + +known_failures: + - test: 'tempest.api.network.test_ports.PortsIpV6TestJSON.test_update_port_with_two_security_groups_and_extra_attributes' + reason: 'Test is failing with SecurityGroupNotFound' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1818860' + - test: 'tempest.api.compute.admin.test_quotas_negative.QuotasAdminNegativeTest.test_create_server_when_memory_quota_is_full' + reason: 'Test is failing on unknown API error: nova.exception.GlanceConnectionFailed' + lp: 'https://bugs.launchpad.net/tempest/+bug/1818886' + - test: 'tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_port_security_macspoofing_port' + reason: 'It can be removed when bug for OVN is repaired' + lp: 'https://bugs.launchpad.net/tempest/+bug/1728886' + - test: 'tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_hotplug_nic' + reason: 'workaround skip for not to not block job pending proper fix' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1806075' + - test: 'tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_implied_roles_create_check_show_delete' + reason: 'Difference in Implied Roles check API return code' + lp: 'https://bugs.launchpad.net/keystone/+bug/1660603' + - test: 'neutron_tempest_plugin.api.admin.test_shared_network_extension.RBACSharedNetworksTest' + reason: 'RBAC policy resource conflict' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1753209' + - test: 'tempest.api.volume.test_volumes_snapshots.VolumesSnapshotTestJSON' + reason: 'network resource conflict' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1753211' + - test: 'mistral_tempest_tests.tests.api.v2.test_actions.ActionTestsV2.test_get_list_actions_not_in_list_filter' + reason: 'mismatch error, mistral team is investigating' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1736950' + - test: 'patrole_tempest_plugin' + reason: 'Not supported in TripleO/tempestconf' + - test: 'zaqar_tempest_plugin' + reason: 'Not supported in TripleO/tempestconf' + - test: 'watcher_tempest_plugin' + reason: 'Not supported in TripleO' + - test: 'vitrage_tempest_plugin' + reason: 'Not supported in TripleO' + - test: 'trove_tempest_plugin' + reason: 'Not supported in TripleO' + - test: 'tripleo_common_tempest_plugin' + reason: 'Not supported in TripleO' + - test: 'tempest_horizon' + reason: 'Not supported in TripleO' + - test: 'octavia_tempest_plugin' + reason: 'Not supported in TripleO' + - test: 'murano_tempest_tests' + reason: 'Not supported in TripleO' + - test: 'mistral_tempest_tests' + reason: 'Not supported in TripleO' + - test: 'magnum_tempest_plugin' + reason: 'Not supported in TripleO' + - test: 'kuryr_tempest_plugin' + reason: 'Not supported in TripleO' + - test: 'heat_tempest_plugin' + reason: 'Not supported in TripleO' + - test: 'designate_tempest_plugin' + reason: 'Not supported in TripleO' + - test: 'congress_tempest_plugin' + reason: 'Not supported in TripleO' + - test: 'ec2api_tempest_plugin' + reason: 'Not supported in TripleO/tempestconf' + - test: 'telemetry_tempest_plugin.scenario.test_telemetry_integration.TestTelemetryIntegration' + reason: 'telemetry tests failed due to One or more ports have an IP allocation from this subnet' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1792862' + - test: 'neutron_tempest_plugin.api.test_trunk_negative' + reason: 'neutron tempest tests are failing by giving 503 Service Unavailable' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1793482' + - test: 'neutron_tempest_plugin.api.test_timestamp' + reason: 'neutron tempest tests are failing by giving 503 Service Unavailable' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1793482' + - test: 'neutron_tempest_plugin.api.test_subnetpools_negative' + reason: 'neutron tempest tests are failing by giving 503 Service Unavailable' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1793482' + - test: 'neutron_tempest_plugin.api.test_service_type_management' + reason: 'neutron tempest tests are failing by giving 503 Service Unavailable' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1793482' + - test: 'neutron_tempest_plugin.api.test_security_groups' + reason: 'neutron tempest tests are failing by giving 503 Service Unavailable' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1793482' + - test: 'neutron_tempest_plugin.api.test_routers_negative' + reason: 'neutron tempest tests are failing by giving 503 Service Unavailable' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1793482' + - test: 'neutron_tempest_plugin.api.test_routers' + reason: 'neutron tempest tests are failing by giving 503 Service Unavailable' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1793482' + - test: 'neutron_tempest_plugin.api.test_qos_negative' + reason: 'neutron tempest tests are failing by giving 503 Service Unavailable' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1793482' + - test: 'neutron_tempest_plugin.api.test_qos' + reason: 'neutron tempest tests are failing by giving 503 Service Unavailable' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1793482' + - test: 'neutron_tempest_plugin.api.test_ports' + reason: 'neutron tempest tests are failing by giving 503 Service Unavailable' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1793482' + - test: 'neutron_tempest_plugin.api.test_networks_negative' + reason: 'neutron tempest tests are failing by giving 503 Service Unavailable' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1793482' + - test: 'neutron_tempest_plugin.api.test_networks' + reason: 'neutron tempest tests are failing by giving 503 Service Unavailable' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1793482' + - test: 'neutron_tempest_plugin.api.test_floating_ips_negative' + reason: 'neutron tempest tests are failing by giving 503 Service Unavailable' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1793482' + - test: 'neutron_tempest_plugin.api.test_floating_ips' + reason: 'neutron tempest tests are failing by giving 503 Service Unavailable' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1793482' + - test: 'neutron_tempest_plugin.api.test_flavors_extensions' + reason: 'neutron tempest tests are failing by giving 503 Service Unavailable' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1793482' + - test: 'neutron_tempest_plugin.api.test_extension_driver_port_security' + reason: 'neutron tempest tests are failing by giving 503 Service Unavailable' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1793482' + - test: 'neutron_tempest_plugin.api.test_allowed_address_pair' + reason: 'neutron tempest tests are failing by giving 503 Service Unavailable' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1793482' + - test: 'neutron_tempest_plugin.api.test_address_scopes_negative' + reason: 'neutron tempest tests are failing by giving 503 Service Unavailable' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1793482' + - test: 'neutron_tempest_plugin.api.test_address_scopes' + reason: 'neutron tempest tests are failing by giving 503 Service Unavailable' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1793482' + - test: 'neutron_tempest_plugin.api.admin.test_shared_network_extension' + reason: 'neutron tempest tests are failing by giving 503 Service Unavailable' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1793482' + - test: 'neutron_tempest_plugin.api.admin.test_routers_ha' + reason: 'neutron tempest tests are failing by giving 503 Service Unavailable' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1793482' + - test: 'neutron_tempest_plugin.api.admin.test_networks.NetworksTestAdmin' + reason: 'neutron tempest tests are failing by giving 503 Service Unavailable' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1793482' + - test: 'neutron_tempest_plugin.api.admin.test_l3_agent_scheduler' + reason: 'neutron tempest tests are failing by giving 503 Service Unavailable' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1793482' + - test: 'tempest.scenario.test_volume_boot_pattern.TestVolumeBootPattern' + reason: 'neutron ' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1802971' + - test: 'tempest.api.compute.servers.test_attach_interfaces.AttachInterfacesUnderV243Test.test_add_remove_fixed_ip' + reason: 'Got error: Class PyRoute2 cannot be found ' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1810325' + - test: 'tempest.api.network.admin.test_dhcp_agent_scheduler.DHCPAgentSchedulersTestJSON.test_add_remove_network_from_dhcp_agent' + reason: 'DHCP Agent is disabled by default in OVN' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1817705' + - test: 'tempest.api.network.admin.test_dhcp_agent_scheduler.DHCPAgentSchedulersTestJSON.test_list_networks_hosted_by_one_dhcp' + reason: 'DHCP Agent is disabled by default in OVN' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1817705' + - test: 'neutron_tempest_plugin.api.admin.test_dhcp_agent_scheduler.DHCPAgentSchedulersTestJSON.test_add_remove_network_from_dhcp_agent' + reason: 'DHCP Agent is disabled by default in OVN' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1817705' + - test: 'neutron_tempest_plugin.api.admin.test_dhcp_agent_scheduler.DHCPAgentSchedulersTestJSON.test_dhcp_port_status_active' + reason: 'DHCP Agent is disabled by default in OVN' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1817705' + - test: 'neutron_tempest_plugin.api.admin.test_dhcp_agent_scheduler.DHCPAgentSchedulersTestJSON.test_list_networks_hosted_by_one_dhcp' + reason: 'DHCP Agent is disabled by default in OVN' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1817705' + - test: 'neutron_tempest_plugin.scenario.test_qos.QoSTest.test_qos_basic_and_update' + reason: 'Possible bug for QoS + FIP in OVN' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1817705' + - test: 'barbican_tempest_plugin.tests.scenario.test_image_signing.ImageSigningSnapshotTest' + reason: 'barbican and python-tempestconf not playing well' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1820668' + - test: 'barbican_tempest_plugin.tests.scenario.test_image_signing.ImageSigningTest' + reason: 'barbican and python-tempestconf not playing well' + lp: 'https://bugs.launchpad.net/tripleo/+bug/1820671' + + # skip list for undercloud + - test: 'tempest.api.network.test_networks.NetworksTest.test_external_network_visibility' + reason: 'Running on undercloud' + undercloud: true + - test: 'tempest.api.network.test_floating_ips.FloatingIPTestJSON' + reason: 'Running on undercloud' + undercloud: true + - test: 'tempest.api.compute.servers' + reason: 'Running on undercloud' + undercloud: true + - test: 'tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_add_existent_host' + reason: 'Running on undercloud' + undercloud: true + - test: 'tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_add_host_as_user' + reason: 'Running on undercloud' + undercloud: true + - test: 'tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_remove_host_as_user' + reason: 'Running on undercloud' + undercloud: true + - test: 'tempest.api.compute.admin.test_create_server.ServersWithSpecificFlavorTestJSON' + reason: 'Running on undercloud' + undercloud: true + - test: 'tempest.api.compute.admin.test_aggregates.AggregatesAdminTestJSON' + reason: 'Running on undercloud' + undercloud: true + - test: 'tempest.api.compute.admin.test_create_server.ServersWithSpecificFlavorTestJSON' + reason: 'Running on undercloud' + undercloud: true + - test: 'tempest.api.compute.admin.test_hypervisor' + reason: 'Running on undercloud' + undercloud: true + - test: 'tempest.api.compute.admin.test_hypervisor_negative' + reason: 'Running on undercloud' + undercloud: true + - test: 'tempest.api.compute.admin.test_delete_server' + reason: 'Running on undercloud' + undercloud: true + - test: 'tempest.api.compute.admin.test_hosts.HostsAdminTestJSON.test_show_host_detail' + reason: 'Running on undercloud' + undercloud: true + - test: 'tempest.api.compute.admin.test_servers' + reason: 'Running on undercloud' + undercloud: true + - test: 'tempest.api.compute.admin.test_simple_tenant_usage' + reason: 'Running on undercloud' + undercloud: true + - test: 'tempest.api.compute.admin.test_servers_negative' + reason: 'Running on undercloud' + undercloud: true + - test: 'tempest.api.compute.floating_ips' + reason: 'Running on undercloud' + undercloud: true + - test: 'tempest.api.compute.admin.test_server_diagnostics' + reason: 'Running on undercloud' + undercloud: true + - test: 'tempest.api.compute.admin.test_server_diagnostics_negative' + reason: 'Running on undercloud' + undercloud: true + - test: 'tempest.api.compute.images' + reason: 'Running on undercloud' + undercloud: true + - test: 'tempest.api.network.admin.test_external_networks_negative' + reason: 'Running on undercloud' + undercloud: true + - test: 'tempest.api.network.admin.test_floating_ips_admin_actions' + reason: 'Running on undercloud' + undercloud: true + - test: 'tempest.api.network.admin.test_routers' + reason: 'Running on undercloud' + undercloud: true + - test: 'tempest.api.network.admin.test_routers_negative' + reason: 'Running on undercloud' + undercloud: true + - test: 'tempest.api.network.test_floating_ips_negative' + reason: 'Running on undercloud' + undercloud: true + - test: 'tempest.api.network.test_networks.NetworksIpV6Test.test_external_network_visibility' + reason: 'Running on undercloud' + undercloud: true + - test: 'tempest.api.network.admin.test_ports' + reason: 'Running on undercloud' + undercloud: true + - test: 'tempest.api.compute.security_groups.test_security_groups.SecurityGroupsTestJSON' + reason: 'Running on undercloud' + undercloud: true + - test: 'tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON' + reason: 'Running on undercloud' + undercloud: true