From af57dd6719a9f99ff91e93949a7d14f2951cbe9a Mon Sep 17 00:00:00 2001 From: Martin Kopec Date: Mon, 3 Dec 2018 08:57:51 +0000 Subject: [PATCH] Add profile argument The patch implements --profile argument, which defines a path to a profile.yaml file - a file which contains python-tempestconf arguments and their values. Also --generate-profile argument is introduced. A user can generate a sample profile.yaml file using this argument. The sample contains definitions of all python-tempestconf arguments set to their default values. Story: 2004503 Task: 28225 Change-Id: I80848d8ef9868ed3975b938d61880753eadc5ae9 --- config_tempest/main.py | 26 +++ config_tempest/profile.py | 103 ++++++++++++ config_tempest/tests/test_profile.py | 32 ++++ doc/source/user/import.rst | 4 + doc/source/user/index.rst | 1 + doc/source/user/profile.rst | 159 ++++++++++++++++++ ...Add-profile-argument-70bfa9606826ff81.yaml | 15 ++ .../tasks/main.yaml | 47 ++++++ 8 files changed, 387 insertions(+) create mode 100644 config_tempest/profile.py create mode 100644 config_tempest/tests/test_profile.py create mode 100644 doc/source/user/profile.rst create mode 100644 releasenotes/notes/Add-profile-argument-70bfa9606826ff81.yaml diff --git a/config_tempest/main.py b/config_tempest/main.py index 56974ecc..d699fc86 100755 --- a/config_tempest/main.py +++ b/config_tempest/main.py @@ -49,6 +49,7 @@ from config_tempest import constants as C from config_tempest.constants import LOG from config_tempest.credentials import Credentials from config_tempest.flavors import Flavors +from config_tempest import profile from config_tempest.services.services import Services from config_tempest.tempest_conf import TempestConf from config_tempest.users import Users @@ -275,6 +276,21 @@ def get_arg_parser(): For example: --create-accounts-file $HOME/accounts.yaml """) + parser.add_argument('--profile', default=None, metavar='PATH', + help="""python-tempestconf's profile.yaml file + A file which contains definition of + python-tempestconf's arguments. + NOTE: If this argument is used, other + arguments cannot be defined!""") + parser.add_argument('--generate-profile', default=None, + metavar='PATH', + help="""Generate a sample profile.yaml file. + A sample profile.yaml will be generated in the + specified path. After that python-tempestconf + ends. + For example: + --generate-profile $HOME/profile.yaml + """) parser.add_argument('--image-disk-format', default=C.DEFAULT_IMAGE_FORMAT, help="""A format of an image to be uploaded to glance. Default is '%s'""" % C.DEFAULT_IMAGE_FORMAT) @@ -526,6 +542,16 @@ def config_tempest(**kwargs): def main(): args = parse_arguments() + if args.generate_profile: + profile.generate_profile(args, args.generate_profile) + sys.exit(0) + if args.profile: + profile_args = profile.read_profile_file(args.profile) + # update default args by values gained from the profile + # Namespace can't be updated, so translate it to a dict first + args_dict = vars(args) + args_dict.update(profile_args) + args = argparse.Namespace(**args_dict) cloud_creds = get_cloud_creds(args) config_tempest( append=args.append, diff --git a/config_tempest/profile.py b/config_tempest/profile.py new file mode 100644 index 00000000..e9ee22e0 --- /dev/null +++ b/config_tempest/profile.py @@ -0,0 +1,103 @@ +# Copyright 2018 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import yaml + + +def _convert_remove_append(options): + """Convert append and remove dicts read from .yaml file. + + :param options: {'section.key': 'value1,value2', ...} + :type options: dict + :return: ['section.key=value1,value2', ...] + :rtype: list + """ + converted = [] + for key in options: + v = options[key] + if isinstance(v, list): + v = ','.join(v) + converted.append(key + "=" + v) + return converted + + +def read_profile_file(path): + """Read python-tempestconf arguments from a .yaml file. + + :param path: path to the profile.yaml file + :type path: string + :return: profile arguments + :rtype: dict + """ + with open(path, 'r') as stream: + profile_args = yaml.load(stream) + # convert overrides, to a list of tuples (s, k, v) + overrides = [] + if 'overrides' in profile_args: + for key in profile_args['overrides']: + s, k = key.split('.') + v = profile_args['overrides'][key] + if isinstance(v, list): + v = ','.join(v) + overrides.append((s, k, v)) + profile_args['overrides'] = overrides + # convert remove + remove = [] + if 'remove' in profile_args: + remove = _convert_remove_append(profile_args['remove']) + profile_args['remove'] = remove + # convert append values + append = [] + if 'append' in profile_args: + append = _convert_remove_append(profile_args['append']) + profile_args['append'] = append + return profile_args + + +def generate_profile(args, path): + """Generates a sample profile.yaml file. + + :type args: argparse.Namespace + :param path: Specifies where the sample file will be generated in. + :type path: string + """ + iterable_args = vars(args) + # pop following arguments as they are written + # more detailed to the file below + iterable_args.pop('append') + iterable_args.pop('overrides') + iterable_args.pop('remove') + # pop profile as that shouldn't be in a profile.yaml + iterable_args.pop('profile') + with open(path, 'w') as outfile: + yaml.safe_dump(iterable_args, outfile, default_flow_style=False) + outfile.write("""append: {} + #identity.username: username + #compute-feature-enabled.api_extensions: + # - dvr + # - extension +overrides: {} + #identity.username: username + #identity.password: + # - my_password + #compute-feature-enabled.api_extensions: + # - dvr + # - extension +remove: {} + #identity.username: username + #compute-feature-enabled.api_extensions: + # - dvr + # - extension +""") diff --git a/config_tempest/tests/test_profile.py b/config_tempest/tests/test_profile.py new file mode 100644 index 00000000..3b22b64d --- /dev/null +++ b/config_tempest/tests/test_profile.py @@ -0,0 +1,32 @@ +# Copyright 2018 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from config_tempest import profile +from config_tempest.tests.base import BaseConfigTempestTest + + +class TestProfile(BaseConfigTempestTest): + + def test_convert_remove_append(self): + in_data = { + 'section.key1': 'value1', + 'section.key2': 'value1,value2' + } + expected = [ + 'section.key1=value1', + 'section.key2=value1,value2' + ] + out_data = profile._convert_remove_append(in_data) + self.assertItemsEqual(expected, out_data) diff --git a/doc/source/user/import.rst b/doc/source/user/import.rst index ebb8de05..4a3293b6 100644 --- a/doc/source/user/import.rst +++ b/doc/source/user/import.rst @@ -79,6 +79,10 @@ List of arguments which may be passed to `config_tempest` * test_accounts * verbose + OR + + * profile, see why **or** in `CLI documentation`_ + .. note:: For detailed description of the options see our `CLI documentation`_ diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst index b34fbce5..40816f51 100644 --- a/doc/source/user/index.rst +++ b/doc/source/user/index.rst @@ -7,4 +7,5 @@ User Guide usage import + profile default diff --git a/doc/source/user/profile.rst b/doc/source/user/profile.rst new file mode 100644 index 00000000..5eea94da --- /dev/null +++ b/doc/source/user/profile.rst @@ -0,0 +1,159 @@ +=============================================== +Use python-tempestconf with a profile.yaml file +=============================================== + +A ``profile.yaml`` is helpful mainly in jobs running on different versions of +OpenStack, because arguments for ``python-tempestconf`` may differ in each +OpenStack version. Using the ``--profile`` argument those jobs can use +a ``profile.yaml`` file for each OpenStack version which makes the code of +those jobs much clearer and more readable, as for example they contain less if +else statements, ... + +.. note:: + + Apart from ``--deployer-input`` config file which specifies content of + a tempest.conf, ``profile.yaml`` file defines arguments which are passed + to ``python-tempestconf``, see `CLI documentation`_. + + .. _CLI documentation: ../cli/cli_options.html + +.. warning:: + + If this argument is used, other arguments cannot be defined, it means, + user uses either CLI arguments or profile.yaml file. + +Generating a sample profile.yaml file ++++++++++++++++++++++++++++++++++++++ + +.. code-block:: Bash + + $ discover-tempest-config --generate-profile ./etc/profile.yaml + +.. code-block:: yaml + + $ cat ./etc/profile.yaml + collect_timing: false + create: false + create_accounts_file: null + debug: false + deployer_input: null + endpoint_type: null + generate_profile: ./etc/profile.yaml + http_timeout: null + image: http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img + image_disk_format: qcow2 + insecure: false + network_id: null + no_default_deployer: false + non_admin: false + os_api_version: null + os_auth_type: password + os_auth_url: null + os_cacert: null + os_cert: null + os_cloud: null + os_default_domain_id: null + os_default_domain_name: null + os_domain_id: null + os_domain_name: null + os_endpoint_override: null + os_endpoint_type: null + os_interface: public + os_key: null + os_password: null + os_project_domain_id: null + os_project_domain_name: null + os_project_id: null + os_project_name: null + os_region_name: null + os_service_name: null + os_service_type: null + os_system_scope: null + os_trust_id: null + os_user_domain_id: null + os_user_domain_name: null + os_user_id: null + os_username: null + out: etc/tempest.conf + test_accounts: null + timeout: 600 + verbose: false + append: {} + #identity.username: username + #compute-feature-enabled.api_extensions: + # - dvr + # - extension + overrides: {} + #identity.username: username + #identity.password: + # - my_password + #compute-feature-enabled.api_extensions: + # - dvr + # - extension + remove: {} + #identity.username: username + #compute-feature-enabled.api_extensions: + # - dvr + # - extension + +.. note:: + + The generated sample of a ``profile.yaml`` file contains all + ``python-tempestconf`` arguments set to their default values. That means, + that you can **remove arguments you didn't modify** to keep the file simple + and more readable. + + +``python-tempestconf`` accepts both of the following inputs, so you can use +what suits you better, either strings or lists: + +.. code-block:: yaml + + create: True + out: ./etc/tempest.conf + deployer-input: ./deploy.txt + no-default-deployer: False + overrides: + identity.username: my_override + identity.password: my_password + network-feature-enabled.api_extensions: all + compute-feature-enabled.api_extensions: dvr,mine + remove: + auth.identity: username + network-feature-enabled.api_extensions: '' + compute-feature-enabled.api_extensions: dvr,mine + + +.. code-block:: yaml + + create: True + out: ./etc/tempest.conf + deployer-input: ./deploy.txt + no-default-deployer: False + overrides: + identity.username: my_override + identity.password: + - my_password + network-feature-enabled.api_extensions: + - all + compute-feature-enabled.api_extensions: + - dvr + - mine + remove: + auth.identity: username + network-feature-enabled.api_extensions: + - '' + compute-feature-enabled.api_extensions: + - dvr + - mine + + +Using profile.yaml file ++++++++++++++++++++++++ + +After you've created your customized ``profile.yaml`` file, let's say in +``./etc/profile.yaml``, use it as follows: + +.. code-block:: Bash + + $ discover-tempest-config --profile ./etc/profile.yaml diff --git a/releasenotes/notes/Add-profile-argument-70bfa9606826ff81.yaml b/releasenotes/notes/Add-profile-argument-70bfa9606826ff81.yaml new file mode 100644 index 00000000..71c75e75 --- /dev/null +++ b/releasenotes/notes/Add-profile-argument-70bfa9606826ff81.yaml @@ -0,0 +1,15 @@ +--- +features: + - | + --profile argument specifies a path to a profile.yaml file which contains + definitions of python-tempestconf arguments. + + NOTE: If this argument is used, other arguments cannot be defined, it means + a user uses either CLI arguments or profile.yaml file. + + A user can generate a sample profile.yaml file using --generate-profile + argument. The sample contains definitions of all python-tempestconf + arguments set to their default values. + + NOTE: If this argument is used, python-tempestconf ends right after the + a sample profile.yaml file is generated. diff --git a/roles/generate-tempestconf-file-cloud/tasks/main.yaml b/roles/generate-tempestconf-file-cloud/tasks/main.yaml index 8e57ea4d..f64e24e0 100644 --- a/roles/generate-tempestconf-file-cloud/tasks/main.yaml +++ b/roles/generate-tempestconf-file-cloud/tasks/main.yaml @@ -44,6 +44,53 @@ chdir: "{{ tempestconf_src_relative_path }}" executable: /bin/bash + - set_fact: + profile: + debug: true + out: etc/tempest_profile.conf + verbose: true + create: "{{ cloud_user == 'devstack-admin' }}" + non_admin: "{{ cloud_user == 'devstack' }}" + os_cloud: "{{ cloud_user }}" + overrides: + auth.tempest_roles: Member + + - name: Generate sample profile.yaml file + shell: | + set -ex + export PATH=$PATH:/usr/local/sbin:/usr/sbin + source {{ virtualenvs.tempestconf }}/bin/activate + printenv + discover-tempest-config \ + --generate-profile ./etc/profile.yaml + args: + chdir: "{{ tempestconf_src_relative_path }}" + executable: /bin/bash + + - name: Edit the profile.yaml file + copy: + content: "{{ profile | to_nice_yaml }}" + dest: "{{ tempestconf_src_relative_path }}/etc/profile.yaml" + + - name: Generate tempest configuration file with profile.yaml + shell: | + set -ex + export PATH=$PATH:/usr/local/sbin:/usr/sbin + source {{ virtualenvs.tempestconf }}/bin/activate + printenv + cat ./etc/profile.yaml + discover-tempest-config \ + --profile ./etc/profile.yaml + args: + chdir: "{{ tempestconf_src_relative_path }}" + executable: /bin/bash + + - name: Diff tempest.conf and tempest_profile.conf + shell: | + diff ./etc/cloud_tempest.conf ./etc/tempest_profile.conf + args: + chdir: "{{ tempestconf_src_relative_path }}" + - name: Print generated tempest.conf shell: | set -ex