tripleo-keystone-resources: allow to manage keystone services

Management of Keystone resources for each OpenStack service:
- services
- projects
- endpoints
- roles
- users and their assignment to roles

It's using batch + async so it runs faster than with Puppet where it
calls openstack client for each resource. Here it's using the python
openstacksdk with concurrency.

Change-Id: Ib9615c55d0fb4ea71208d74c5ee22594db52f46a

tripleo_ansible/ansible_plugins/filter/helpers.py

@@ -27,7 +27,8 @@ class FilterModule(object):
             'subsort': self.subsort,
             'needs_delete': self.needs_delete,
             'haskey': self.haskey,
-            'list_of_keys': self.list_of_keys
+            'list_of_keys': self.list_of_keys,
+            'get_key_from_dict': self.get_key_from_dict
         }
 
     def subsort(self, dict_to_sort, attribute, null_value=0):
@@ -182,3 +183,32 @@ class FilterModule(object):
             for k, v in i.items():
                 list_of_keys.append(k)
         return list_of_keys
+
+    def get_key_from_dict(self, data, key, strict=False, default=None):
+        """Return a list of unique values from a specific key from a dict.
+
+        This filter takes in input a list of dictionaries and for each of them
+        it will add the value of a specific key into returned_list and
+        returns it sorted. If the key has to be part of the dict, set strict to
+        True. A default can be set if the key doesn't exist but strict has to
+        be set to False.
+        """
+        returned_list = []
+        for i in data.items():
+            value = i[1].get(key)
+            if value is None and not strict and default is not None:
+                value = default
+            if value is None:
+                if strict:
+                    raise TypeError('Missing %s key in '
+                                    '%s' % (key, i[0]))
+                else:
+                    continue
+            if isinstance(value, list):
+                for v in value:
+                    if v not in returned_list:
+                        returned_list.append(v)
+            else:
+                if value not in returned_list:
+                    returned_list.append(value)
+        return sorted(returned_list)

tripleo_ansible/roles/tripleo-keystone-resources/defaults/main.yml

@@ -24,3 +24,4 @@ tripleo_keystone_resources_clouds_file_path: /etc/openstack/clouds.yaml
 tripleo_keystone_resources_clouds_file_owner: root
 tripleo_keystone_resources_clouds_file_group: root
 tripleo_keystone_resources_clouds_file_mode: 0644
+tripleo_keystone_resources_catalog_config: {}

tripleo_ansible/roles/tripleo-keystone-resources/tasks/endpoints.yml

@@ -0,0 +1,40 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: "Async creation of Keystone {{ keystone_endpoint_type }} endpoint"
+  os_keystone_endpoint:
+    cloud: "{{ tripleo_keystone_resources_cloud_name }}"
+    service: "{{ tripleo_keystone_resources_data.key }}"
+    url: "{{ tripleo_keystone_resources_data['value']['urls'][keystone_endpoint_type] }}"
+    endpoint_interface: "{{ keystone_endpoint_type }}"
+    region: "{{ tripleo_keystone_resources_data.value.region }}"
+    state: present
+  async: 60
+  poll: 0
+  register: tripleo_keystone_resources_endpoint_results
+  loop: "{{ batched_tripleo_keystone_resources_data }}"
+  loop_control:
+    loop_var: tripleo_keystone_resources_data
+
+- name: "Check Keystone {{ keystone_endpoint_type }} endpoint status"
+  async_status:
+    jid: "{{ tripleo_keystone_resources_endpoint_async_result_item.ansible_job_id }}"
+  loop: "{{ tripleo_keystone_resources_endpoint_results.results }}"
+  loop_control:
+    loop_var: "tripleo_keystone_resources_endpoint_async_result_item"
+  register: tripleo_keystone_resources_endpoint_async_poll_results
+  until: tripleo_keystone_resources_endpoint_async_poll_results.finished
+  retries: 30

tripleo_ansible/roles/tripleo-keystone-resources/tasks/loop-endpoints.yml

@@ -0,0 +1,21 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: "Create Keystone {{ keystone_endpoint_type }} endpoints"
+  include_tasks: endpoints.yml
+  loop: "{{ tripleo_keystone_resources_catalog_config | dict2items | batch(5) | list }}"
+  loop_control:
+    loop_var: batched_tripleo_keystone_resources_data

tripleo_ansible/roles/tripleo-keystone-resources/tasks/main.yml

@@ -31,3 +31,42 @@
         - "{{ ansible_os_family | lower }}.yml"
   tags:
     - always
+
+- name: Create Keystone Projects
+  include_tasks: projects.yml
+  loop: "{{ tripleo_keystone_resources_catalog_config | get_key_from_dict(key='project', default='service') | batch(5) | list }}"
+  loop_control:
+    loop_var: batched_tripleo_keystone_resources_projects
+
+- name: Create Keystone Services
+  include_tasks: services.yml
+  loop: "{{ tripleo_keystone_resources_catalog_config | dict2items | batch(5) | list }}"
+  loop_control:
+    loop_var: batched_tripleo_keystone_resources_data
+
+- name: Create Keystone Endpoints
+  include_tasks: loop-endpoints.yml
+  loop:
+    - public
+    - admin
+    - internal
+  loop_control:
+    loop_var: keystone_endpoint_type
+
+- name: Create Keystone Roles
+  include_tasks: roles.yml
+  loop: "{{ tripleo_keystone_resources_catalog_config | get_key_from_dict(key='roles', default='service') | batch(5) | list }}"
+  loop_control:
+    loop_var: batched_tripleo_keystone_resources_roles
+
+- name: "Create Keystone Users"
+  include_tasks: users.yml
+  loop: "{{ tripleo_keystone_resources_catalog_config | dict2items | batch(5) | list }}"
+  loop_control:
+    loop_var: batched_tripleo_keystone_resources_data
+
+- name: "Assign Keystone Users to Roles"
+  include_tasks: user_roles.yml
+  loop: "{{ tripleo_keystone_resources_catalog_config | dict2items | batch(5) | list }}"
+  loop_control:
+    loop_var: batched_tripleo_keystone_resources_data

tripleo_ansible/roles/tripleo-keystone-resources/tasks/projects.yml

@@ -0,0 +1,38 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Async creation of Keystone project
+  os_project:
+    cloud: "{{ tripleo_keystone_resources_cloud_name }}"
+    name: "{{ tripleo_keystone_resources_project }}"
+    domain_id: default
+    state: present
+  async: 60
+  poll: 0
+  register: tripleo_keystone_resources_project_results
+  loop: "{{ batched_tripleo_keystone_resources_projects }}"
+  loop_control:
+    loop_var: tripleo_keystone_resources_project
+
+- name: Check Keystone project status
+  async_status:
+    jid: "{{ tripleo_keystone_resources_project_async_result_item.ansible_job_id }}"
+  loop: "{{ tripleo_keystone_resources_project_results.results }}"
+  loop_control:
+    loop_var: "tripleo_keystone_resources_project_async_result_item"
+  register: tripleo_keystone_resources_project_async_poll_results
+  until: tripleo_keystone_resources_project_async_poll_results.finished
+  retries: 30

tripleo_ansible/roles/tripleo-keystone-resources/tasks/roles.yml

@@ -0,0 +1,38 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Async creation of Keystone role
+  os_keystone_role:
+    cloud: "{{ tripleo_keystone_resources_cloud_name }}"
+    name: "{{ tripleo_keystone_resources_role }}"
+    domain_id: default
+    state: present
+  async: 60
+  poll: 0
+  register: tripleo_keystone_resources_role_results
+  loop: "{{ batched_tripleo_keystone_resources_roles }}"
+  loop_control:
+    loop_var: tripleo_keystone_resources_role
+
+- name: Check Keystone role status
+  async_status:
+    jid: "{{ tripleo_keystone_resources_role_async_result_item.ansible_job_id }}"
+  loop: "{{ tripleo_keystone_resources_role_results.results }}"
+  loop_control:
+    loop_var: "tripleo_keystone_resources_role_async_result_item"
+  register: tripleo_keystone_resources_role_async_poll_results
+  until: tripleo_keystone_resources_role_async_poll_results.finished
+  retries: 30

tripleo_ansible/roles/tripleo-keystone-resources/tasks/services.yml

@@ -0,0 +1,39 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Async creation of Keystone service
+  os_keystone_service:
+    cloud: "{{ tripleo_keystone_resources_cloud_name }}"
+    name: "{{ tripleo_keystone_resources_data.key }}"
+    service_type: "{{ tripleo_keystone_resources_data.value.service }}"
+    description: "OpenStack {{ tripleo_keystone_resources_data.value.service | title() }} Service"
+    state: present
+  async: 60
+  poll: 0
+  register: tripleo_keystone_resources_service_results
+  loop: "{{ batched_tripleo_keystone_resources_data }}"
+  loop_control:
+    loop_var: tripleo_keystone_resources_data
+
+- name: Check Keystone service status
+  async_status:
+    jid: "{{ tripleo_keystone_resources_service_async_result_item.ansible_job_id }}"
+  loop: "{{ tripleo_keystone_resources_service_results.results }}"
+  loop_control:
+    loop_var: "tripleo_keystone_resources_service_async_result_item"
+  register: tripleo_keystone_resources_service_async_poll_results
+  until: tripleo_keystone_resources_service_async_poll_results.finished
+  retries: 30

tripleo_ansible/roles/tripleo-keystone-resources/tasks/user_roles.yml

@@ -0,0 +1,39 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: "Async assignment of Keystone user to roles"
+  os_user:
+    cloud: "{{ tripleo_keystone_resources_cloud_name }}"
+    user: "{{ tripleo_keystone_resources_data.key }}"
+    role: "{{ tripleo_keystone_resources_data.value.roles }}"
+    project: "{{ tripleo_keystone_resources_data.value.projects }}"
+    state: present
+  async: 60
+  poll: 0
+  register: tripleo_keystone_resources_user_role_results
+  loop: "{{ batched_tripleo_keystone_resources_data }}"
+  loop_control:
+    loop_var: tripleo_keystone_resources_data
+
+- name: "Check Keystone user assignment to roles status"
+  async_status:
+    jid: "{{ tripleo_keystone_resources_user_role_async_result_item.ansible_job_id }}"
+  loop: "{{ tripleo_keystone_resources_user_role_results.results }}"
+  loop_control:
+    loop_var: "tripleo_keystone_resources_user_role_async_result_item"
+  register: tripleo_keystone_resources_user_role_async_poll_results
+  until: tripleo_keystone_resources_user_role_async_poll_results.finished
+  retries: 30

tripleo_ansible/roles/tripleo-keystone-resources/tasks/users.yml

@@ -0,0 +1,40 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: "Async creation of Keystone user"
+  os_user:
+    cloud: "{{ tripleo_keystone_resources_cloud_name }}"
+    name: "{{ tripleo_keystone_resources_data.key }}"
+    password: "{{ tripleo_keystone_resources_data.value.password }}"
+    email: "{{ tripleo_keystone_resources_data.key }}@localhost"
+    domain: default
+    state: present
+  async: 60
+  poll: 0
+  register: tripleo_keystone_resources_user_results
+  loop: "{{ batched_tripleo_keystone_resources_data }}"
+  loop_control:
+    loop_var: tripleo_keystone_resources_data
+
+- name: "Check Keystone user status"
+  async_status:
+    jid: "{{ tripleo_keystone_resources_user_async_result_item.ansible_job_id }}"
+  loop: "{{ tripleo_keystone_resources_user_results.results }}"
+  loop_control:
+    loop_var: "tripleo_keystone_resources_user_async_result_item"
+  register: tripleo_keystone_resources_user_async_poll_results
+  until: tripleo_keystone_resources_user_async_poll_results.finished
+  retries: 30

tripleo_ansible/tests/plugins/filter/test_helpers.py

@@ -256,3 +256,43 @@ class TestHelperFilters(tests_base.TestCase):
                                      reverse=True,
                                      any=True)
         self.assertEqual(result, expected_list)
+
+    def test_get_key_from_dict(self):
+        data = {
+           'nova_api': {
+             'project': 'service1'
+           },
+           'glance_api': {
+             'project': 'service1'
+           },
+           'heat_api': {
+             'user': 'heat'
+           },
+           'cinder_api': {
+             'project': 'service2'
+           }
+        }
+        expected_list = ['service1', 'service2', 'service3']
+        result = self.filters.get_key_from_dict(data, key='project',
+                                                default='service3')
+        self.assertEqual(result, expected_list)
+
+    def test_get_key_from_dict_with_list_input(self):
+        data = {
+           'nova_api': {
+             'roles': ['service', 'admin']
+           },
+           'glance_api': {
+             'roles': 'service1'
+           },
+           'heat_api': {
+             'user': 'heat'
+           },
+           'cinder_api': {
+             'project': 'service2'
+           }
+        }
+        expected_list = ['admin', 'service', 'service1']
+        result = self.filters.get_key_from_dict(data, key='roles',
+                                                default='service')
+        self.assertEqual(result, expected_list)