Fixes on tests and new lists for openstack-operators

This patch changes the following:

* Add a separated skip and allow list for openstack operators.
* Fix tox replacing the whitelist_externals with allowlist_externals
  since the first is now no longer supported.
* Fix validate tests to work on tox python 3.8 and 3.9 jobs

Change-Id: Ifdfa31105ab4231ae3bc48c480265f461d928df7

openstack-operators/tempest_allow.yml

@@ -0,0 +1,6 @@
+groups:
+  - name: openstack-operator
+    tests:
+      - tempest.api.identity.*.v3
+    releases:
+      - all

openstack-operators/tempest_skip.yml

@@ -0,0 +1,19 @@
+known_failures:
+  - test: 'tempest.api.network.admin.test_ports.PortsAdminExtendedAttrsIpV6TestJSON'
+    deployment:
+      - 'overcloud'
+    releases:
+      - name: 'master'
+        reason: 'No controller'
+        lp: 'http://no.bug'
+    jobs:
+      - neutron-operator
+  - test: 'tempest.api.network.admin.test_ports.PortsAdminExtendedAttrsTestJSON'
+    deployment:
+      - 'overcloud'
+    releases:
+      - name: 'master'
+        reason: 'No controller'
+        lp: 'http://no.bug'
+    jobs:
+      - neutron-operator

roles/validate_yaml/tasks/main.yaml

@@ -5,6 +5,7 @@
       set -ex
       export PATH=$PATH:/usr/local/sbin:/usr/sbin:$HOME/.local/bin
       tempest-skip validate --skipped --file roles/validate-tempest/vars/tempest_skip.yml
+      tempest-skip validate --skipped --file openstack-operators/tempest_skip.yml
     args:
       chdir: "{{ tempest_skip_path }}"
       executable: /bin/bash
@@ -14,6 +15,7 @@
       set -ex
       export PATH=$PATH:/usr/local/sbin:/usr/sbin:$HOME/.local/bin
       tempest-skip validate --allowed --file roles/validate-tempest/vars/tempest_allow.yml
+      tempest-skip validate --allowed --file openstack-operators/tempest_allow.yml
     args:
       chdir: "{{ tempest_skip_path }}"
       executable: /bin/bash

setup.cfg

@@ -27,6 +27,7 @@ data_files =
     usr/local/share/ansible/roles/list_skipped = roles/list_skipped/*
     usr/local/share/ansible/roles/list_allowed = roles/list_allowed*
     usr/local/share/ansible/roles/validate-tempest = roles/validate-tempest/*
+    usr/local/share/openstack-operator = openstack-operator/*
 
 [entry_points]
 console_scripts =

tempest_skip/tests/test_validate.py

@@ -14,24 +14,20 @@
 # under the License.
 
 import os
-import subprocess
 import tempfile
 
 from tempest_skip.tests import base
+from tempest_skip.validate import Validate
+from voluptuous.error import MultipleInvalid
 
 
 class TestValidate(base.TestCase):
     def setUp(self):
         super(TestValidate, self).setUp()
-
-    def assertRunExit(self, cmd, expected):
-        p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
-                             stderr=subprocess.PIPE)
-        out, err = p.communicate()
-        msg = ("Running %s got an unexpected returncode\n"
-               "Stdout: %s\nStderr: %s" % (' '.join(cmd), out, err))
-        self.assertEqual(p.returncode, expected, msg)
-        return out, err
+        self.cmd = Validate(__name__, [])
+        self.parser = self.cmd.get_parser(__name__)
+        self.parser.skipped = False
+        self.parser.allowed = False
 
 
 class TestValidateAllowed(TestValidate):
@@ -61,8 +57,11 @@ class TestValidateAllowed(TestValidate):
         yaml_file.write(valid_yaml)
         yaml_file.close()
 
-        self.assertRunExit(['tempest-skip', 'validate',
-                            '--allowed', '--file', path], 0)
+        self.parser.allowed = True
+        self.parser.filename = path
+        cmd_result = self.cmd.take_action(self.parser)
+
+        self.assertEqual(cmd_result, None)
 
     def test_validate_fails(self):
         fd, path = tempfile.mkstemp()
@@ -83,8 +82,9 @@ class TestValidateAllowed(TestValidate):
         yaml_file.write(valid_yaml)
         yaml_file.close()
 
-        self.assertRunExit(['tempest-skip', 'validate',
-                            '--allowed', '--file', path], 1)
+        self.parser.allowed = True
+        self.parser.filename = path
+        self.assertRaises(MultipleInvalid, self.cmd.take_action, self.parser)
 
 
 class TestValidateSkipped(TestValidate):
@@ -124,8 +124,11 @@ class TestValidateSkipped(TestValidate):
         yaml_file.write(valid_yaml)
         yaml_file.close()
 
-        self.assertRunExit(['tempest-skip', 'validate',
-                            '--skipped', '--file', path], 0)
+        self.parser.skipped = True
+        self.parser.filename = path
+        cmd_result = self.cmd.take_action(self.parser)
+
+        self.assertEqual(cmd_result, None)
 
     def test_validate_fails(self):
         fd, path = tempfile.mkstemp()
@@ -157,5 +160,6 @@ class TestValidateSkipped(TestValidate):
         yaml_file.write(valid_yaml)
         yaml_file.close()
 
-        self.assertRunExit(['tempest-skip', 'validate',
-                            '--skipped', '--file', path], 1)
+        self.parser.skipped = True
+        self.parser.filename = path
+        self.assertRaises(MultipleInvalid, self.cmd.take_action, self.parser)

tox.ini

@@ -13,7 +13,7 @@ install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://releases.ope
 setenv =
    VIRTUAL_ENV={envdir}
    PYTHONWARNINGS=default::DeprecationWarning
-whitelist_externals = *
+allowlist_externals = *
 deps =
     -r{toxinidir}/requirements.txt
     -r{toxinidir}/test-requirements.txt
@@ -46,7 +46,7 @@ commands = sphinx-build -b html doc/source doc/build/html
 
 [testenv:pdf-docs]
 deps = {[testenv:docs]deps}
-whitelist_externals =
+allowlist_externals =
    make
 commands =
    sphinx-build -W -b latex doc/source doc/build/pdf