Gather script for containers

Currently our gather playbook assumes all configuration files will be in
/etc. This assumption isn't true with container deployments. Currently
the configuration files are located in
/var/lib/config-data/<service>/etc/<service>/.

The Gather script needs to support both container and non-container
deployments. This patchset updates the config parser python script
to check if a service is in the running containers list and then
determine it's appropriate path, grab all of multiple config files
in that path, then parse and drop them off for ansible to use.

This method automagically supports all possible mixes of containerized
uncontainerized services and will always grab the correct config even
if that changes build to build or run to run.

It's also easily extensible for many possible config locations or
different container types by adding another condition or additional
search paths.

Change-Id: I95a3059c7fc263733ac64aa894c6dbf11e2a909f
Closes-bug: #1701264

ansible/gather/roles/ceilometer/tasks/main.yml

@@ -9,7 +9,7 @@
 
 - name: Parse Ceilometer config
   become: true
-  shell: python /tmp/openstack-config-parser.py ceilometer /etc/ceilometer/ceilometer.conf /tmp/out.yml
+  shell: python /tmp/openstack-config-parser.py ceilometer /tmp/out.yml
   when: ceilometer_config.stat.exists
 
 - name: Fetch output

ansible/gather/roles/cinder/tasks/main.yml

@@ -9,7 +9,7 @@
 
 - name: Parse Cinder config
   become: true
-  shell: python /tmp/openstack-config-parser.py cinder /etc/cinder/cinder.conf /tmp/out.yml
+  shell: python /tmp/openstack-config-parser.py cinder /tmp/out.yml
   when: cinder_config.stat.exists
 
 - name: Fetch output

ansible/gather/roles/common/files/openstack-config-parser.py

@@ -12,31 +12,44 @@
 #   limitations under the License.
 
 import sys
-# usage: openstack-config-parser.py [service] [config file] [output file]
+import os
+import subprocess
+# usage: openstack-config-parser.py [service] [output file]
 
+def run_cmd(cmd):
+    process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE)
+    stdout, stderr = process.communicate()
+    output_dict = {}
+    output_dict['stdout'] = stdout.strip()
+    output_dict['stderr'] = stderr.strip()
+    output_dict['rc'] = process.returncode
+    return output_dict
+
+def strip_chars(line):
+    forbidden_chars = ['#', '\n', '"', '\\', ' ', '<', '>']
+    for char in forbidden_chars:
+        line = line.replace(char, '')
+    return line
 
 def parse_config(serviceName, fileName):
-    # a dict containing key/value
-    # pairs, last value is what is
-    # stored.
+    # a dict containing key/value pairs, last value is what is stored.
     values = {}
     with open(fileName) as config:
         section = None
         for line in config:
-            pair = line.replace('#', '')
-            pair = pair.replace('\n', '')
-            pair = pair.replace('"', '')
-            pair = pair.replace('\\', '')
-            pair = pair.replace(' ', '')
-            pair = pair.replace('<', '')
-            pair = pair.replace('>', '')
+            pair = strip_chars(line)
             pair = pair.split('=')
             # excludes any line without a key/val pair
-            valid_line = not line.startswith(
-                "# ") and '[' not in line and line != '\n' and line != '#\n' and "password" not in line.lower()
+            valid_line = not line.startswith("# ") and \
+                         '[' not in line and line != '\n' \
+                         and line != '#\n' and "password" \
+                         not in line.lower()
             if line.startswith('['):
                 section = line.replace('[','').replace(']','').replace('\n','')
-            if '#' not in line and valid_line and not section == None:
+            if '#' not in line and valid_line and not section == None and len(pair) == 2:
+                pair[0] = strip_chars(pair[0])
+                pair[1] = strip_chars(pair[1])
                 values["openstack_S_" + serviceName + "_S_" + section + "_S_" + pair[0]] = pair[1]
     return values
 
@@ -45,26 +58,97 @@ def try_type(val):
     try:
         int(val)
         return val
-    except ValueError:
+    except (ValueError, TypeError):
         try:
             float(val)
             return val
-        except ValueError:
-            if val.lower() in ("true", "false"):
+        except (ValueError, TypeError):
+            if type(val) is list:
+                return "\"" + str(val) + "\""
+            elif val.lower() in ("true", "false"):
                 return val
             else:
                 return "\"" + val + "\""
 
 
+def add_conf_location(serviceName, fileName, values): 
+    # Stores the exact location we gathered this config from.
+    index = "openstack_S_" + serviceName + "_S_" + "Browbeat" + "_S_" + "gather_conf_path"
+    if index in values:
+        values[index].append(fileName)
+    else:
+        values[index] = [fileName]
+
 def print_vars_file(values, fileName):
     with open(fileName, 'w') as output:
         for key in values:
             output.write(key + ": " + try_type(values[key]) + "\n")
 
+def is_containerized(service_name):
+    out = run_cmd("docker ps")
+    if service_name in out['stdout']:
+        return True
+    else:
+        return False
+
+def get_configs_list(path, extension='.conf'):
+    configs = []
+    for item in os.listdir(path):
+        if item.endswith(extension):
+            configs.extend([item])
+    return configs
+
+def get_neutron_plugin(output, cfg_path):
+    plugin = output['openstack_S_neutron_S_DEFAULT_S_core_plugin']
+    plugin_path = "{}/plugins/{}/".format(cfg_path, plugin)
+    for item in get_configs_list(plugin_path, extension='.ini'):
+        full_path = "{}/{}".format(plugin_path,
+                                   item)
+        output.update(parse_config("neutron-plugin", full_path))
+        add_conf_location("neutron", full_path, output)
+    return output
 
 def main():
-    output = parse_config(sys.argv[1], sys.argv[2])
-    print_vars_file(output, sys.argv[3])
+    if len(sys.argv) < 3:
+        print("usage: openstack-config-parser.py [service] [output file]")
+        exit(1)
+
+    service_name = sys.argv[1]
+    outfile = sys.argv[2]
+
+    # This is a list of services that require exceptions from the usual
+    # pattern when gathering their config files
+    pattern_exceptions = ['glance']
+    in_container = is_containerized(service_name)
+
+
+    if 'undercloud' in service_name:
+        cfg_path = "/home/stack"
+    elif in_container and service_name not in pattern_exceptions:
+        cfg_path = "/var/lib/config-data/{}/etc/{}".format(service_name,
+                                                           service_name)
+    # Glance has all configs in a folder named glance_api, ps shows no
+    # processes outside of the container, so I assume those are the right
+    # configs, even though the container is also named glance-api
+    # jkilpatr 7/13/17
+    elif in_container and 'glance' in service_name:
+        cfg_path = "/var/lib/config-data/glance_api/etc/glance"
+    else:
+        cfg_path = "/etc/{}".format(service_name)
+
+    print("Parsing all .conf files in {}".format(cfg_path))
+    output = {}
+    for item in get_configs_list(cfg_path):
+        full_path = "{}/{}".format(cfg_path,
+                                   item)
+        output.update(parse_config(service_name, full_path))
+        add_conf_location(service_name, full_path, output)
+    # Required to find and load the active neutron plugin file.
+    if 'neutron' in service_name:
+        output.update(get_neutron_plugin(output, cfg_path))
+
+    print_vars_file(output, outfile)
 
 if __name__ == '__main__':
     sys.exit(main())
+

ansible/gather/roles/common/tasks/main.yml

@@ -1,3 +1,18 @@
 ---
 - name: Copy config parser script to remote
   copy: src=openstack-config-parser.py dest=/tmp/openstack-config-parser.py
+
+- name: Determine if docker is running
+  shell: docker ps | wc -l
+  register: docker_ps
+
+- name: Set var for container deployment
+  set_fact:
+    containers: True
+    config_path: /var/lib/config-data/
+  when: docker_ps.stdout|int > 1
+
+- name: Set fact for non-container deployment
+  set_fact:
+    config_path: /etc
+  when: docker_ps.stdout|int < 2

ansible/gather/roles/gather/tasks/main.yml

@@ -0,0 +1,65 @@
+---
+#
+# Tasks to get facts
+#
+
+- name: Check for the config - container
+  become: true
+  stat: path="container_config_paths[item].config"
+  register: config_containers
+  when: hostvars[inventory_hostname]['containers'] is defined
+  with_items: "{{ container_config_paths }}"
+
+- name: Check for the config
+  become: true
+  stat: path="{{hostvars[inventory_hostname]['config_path']}}nova/nova.conf"
+  register: config
+  when: hostvars[inventory_hostname]['containers'] is not defined
+
+- name: Create tmp dir
+  become: true
+  shell: mktemp -d -p /tmp -t XXX-metadata
+  register: tmp
+
+- name: Parse config - containers
+  become: true
+  shell: "python /tmp/openstack-config-parser.py {{item}} {{container_config_paths[item].config}} {{tmp.stdout}}/{{item}}.yml"
+  when: hostvars[inventory_hostname]['containers'] is defined
+  ignore_errors: true
+  with_items: "{{ container_config_paths }}"
+
+- name: Parse config
+  become: true
+  shell: python /tmp/openstack-config-parser.py {{config_paths[item]}} {{hostvars[inventory_hostname]['config_path']}}/nova/nova.conf /tmp/out.yml
+  when: config.stat.exists and hostvars[inventory_hostname]['containers'] is not defined
+  ignore_errors: true
+
+- name: Create local tmp dir
+  become: false
+  local_action: shell mktemp -d -p /tmp -t XXX-metadata
+  register: localtmp
+
+- name: Fetch output - containers
+  fetch: src={{tmp.stdout}}/{{item}}.yml dest={{localtmp.stdout}}/{{item}}.yml flat=yes
+  when: hostvars[inventory_hostname]['containers'] is defined
+  ignore_errors: true
+  with_items: "{{ container_config_paths }}"
+
+- name: Assemble metadata - containers
+  local_action: assemble src="{{localtmp.stdout}}" dest="{{localtmp.stdout}}/out.yml"
+
+- name: Load configuration variables - containers
+  include_vars: "{{localtmp.stdout}}/out.yml"
+  when: hostvars[inventory_hostname]['containers'] is defined
+  ignore_errors: true
+
+- name: Fetch output
+  fetch: src=/tmp/out.yml dest=/tmp/out.yml flat=yes
+  when: hostvars[inventory_hostname]['containers'] is not defined
+  ignore_errors: true
+
+- name: Load configuration variables
+  include_vars: /tmp/out.yml
+  when: hostvars[inventory_hostname]['containers'] is not defined
+  ignore_errors: true
+

ansible/gather/roles/glance/tasks/main.yml

@@ -3,19 +3,12 @@
 # Tasks to get Glance facts
 #
 
-- name: Get config files for Glance
-  shell: "ls /etc/glance/*.conf"
-  register: glance_config
-
 - name: Parse Glance config files
   become: true
-  shell: "python /tmp/openstack-config-parser.py glance {{ item }} /tmp/{{ item | basename }}.yml"
-  with_items: "{{ glance_config.stdout_lines }}"
+  shell: "python /tmp/openstack-config-parser.py glance /tmp/out.yml"
 
 - name: Fetch output
-  fetch: "src=/tmp/{{ item | basename }}.yml dest=/tmp/{{ item | basename  }}.yml flat=yes"
-  with_items: "{{ glance_config.stdout_lines }}"
+  fetch: "src=/tmp/out.yml dest=/tmp/out.yml flat=yes"
 
 - name: Load configuration variables
-  include_vars: "/tmp/{{ item | basename }}.yml"
-  with_items: "{{ glance_config.stdout_lines }}"
+  include_vars: "/tmp/out.yml"

ansible/gather/roles/gnocchi/tasks/main.yml

@@ -10,7 +10,7 @@
 
 - name: Parse Gnocchi config
   become: true
-  shell: python /tmp/openstack-config-parser.py gnocchi /etc/gnocchi/gnocchi.conf /tmp/out.yml
+  shell: python /tmp/openstack-config-parser.py gnocchi /tmp/out.yml
   when: gnocchi_config.stat.exists
 
 - name: Fetch output

ansible/gather/roles/heat/tasks/main.yml

@@ -10,7 +10,7 @@
 
 - name: Parse Heat config
   become: true
-  shell: python /tmp/openstack-config-parser.py heat /etc/heat/heat.conf /tmp/out.yml
+  shell: python /tmp/openstack-config-parser.py heat /tmp/out.yml
   when: heat_config.stat.exists
 
 - name: Fetch output

ansible/gather/roles/keystone/tasks/main.yml

@@ -10,7 +10,7 @@
 
 - name: Parse Keystone config
   become: true
-  shell: python /tmp/openstack-config-parser.py keystone /etc/keystone/keystone.conf /tmp/out.yml
+  shell: python /tmp/openstack-config-parser.py keystone /tmp/out.yml
   when: keystone_config.stat.exists
 
 - name: Fetch output

ansible/gather/roles/mysql/tasks/main.yml

@@ -1,3 +1,5 @@
+---
+
 #
 # Get mysql facts
 #
@@ -5,10 +7,23 @@
     shell: mysql -e "show variables like 'max_connections';" | grep max_connections | awk '{print $2}'
     register: max_conn
     ignore_errors: true
+    when: hostvars[inventory_hostname]['containers'] is not defined
+
+  - name: Get max_connections on the database
+    shell: docker exec mysql cat /etc/my.cnf.d/galera.cnf| grep max_connections | awk -F ' = ' '{print $2}'
+    register: max_conn_container
+    ignore_errors: true
+    when: hostvars[inventory_hostname]['containers'] is defined
 
   - name: Set max database connections
     set_fact:
       openstack_mysql_max_connections: "{{ max_conn.stdout }}"
+    when: hostvars[inventory_hostname]['containers'] is not defined
+
+  - name: Set max database connections
+    set_fact:
+      openstack_mysql_max_connections: "{{ max_conn_container.stdout }}"
+    when: hostvars[inventory_hostname]['containers'] is defined
 
   - name : Get file descriptors for the mysql process
     shell: cat /proc/$(pgrep mysqld_safe)/limits | grep "open files" | awk '{print $4}'
@@ -17,5 +32,3 @@
   - name: Set file descriptors fact for mysql
     set_fact:
       openstack_mysql_file_descriptors: "{{ mysql_desc.stdout }}"
-
-

ansible/gather/roles/neutron/tasks/main.yml

@@ -10,7 +10,7 @@
 
 - name: Parse Neutron config
   become: true
-  shell: python /tmp/openstack-config-parser.py neutron /etc/neutron/neutron.conf /tmp/out.yml
+  shell: python /tmp/openstack-config-parser.py neutron /tmp/out.yml
   when: neutron_config.stat.exists
 
 - name: Fetch output
@@ -21,12 +21,3 @@
   include_vars: /tmp/out.yml
   when: neutron_config.stat.exists
 
-- name: Parse Neutron plugin.ini
-  become: true
-  shell: python /tmp/openstack-config-parser.py neutron-plugin /etc/neutron/plugin.ini /tmp/out.yml
-
-- name: Fetch output
-  fetch: src=/tmp/out.yml dest=/tmp/out.yml flat=yes
-
-- name: Load configuration variables
-  include_vars: /tmp/out.yml

ansible/gather/roles/nova/tasks/main.yml

@@ -10,7 +10,7 @@
 
 - name: Parse Nova config
   become: true
-  shell: python /tmp/openstack-config-parser.py nova /etc/nova/nova.conf /tmp/out.yml
+  shell: python /tmp/openstack-config-parser.py nova /tmp/out.yml
   when: nova_config.stat.exists
 
 - name: Fetch output

ansible/gather/roles/rabbitmq/tasks/main.yml

@@ -6,9 +6,20 @@
     shell: rabbitmqctl status | grep file_descriptors | awk -F',' '{print $3}' | sed 's/.$//'
     register: rabbitmq_desc
     ignore_errors: true
+    when: hostvars[inventory_hostname]['containers'] is not defined
+
+  - name : Get rabbitmq file descriptors - containers
+    shell: docker exec rabbitmq rabbitmqctl status | grep total_limit | awk -F',' '{print $2}'| sed 's/.$//'
+    register: rabbitmq_desc_container
+    ignore_errors: true
+    when: hostvars[inventory_hostname]['containers'] is defined
 
   - name: Set rabbitmq file descriptors
     set_fact:
       openstack_rabbitmq_file_descriptors: "{{ rabbitmq_desc.stdout }}"
+    when: hostvars[inventory_hostname]['containers'] is not defined
 
-
+  - name: Set rabbitmq file descriptors - containers
+    set_fact:
+      openstack_rabbitmq_file_descriptors: "{{ rabbitmq_desc_container.stdout }}"
+    when: hostvars[inventory_hostname]['containers'] is defined

ansible/gather/roles/undercloud/tasks/main.yml

@@ -11,7 +11,7 @@
 
   - name: Undercloud.conf
     become: true
-    shell: python /tmp/openstack-config-parser.py undercloud /home/stack/undercloud.conf /tmp/out.yml
+    shell: python /tmp/openstack-config-parser.py undercloud /tmp/out.yml
     when: undercloud_conf.stat.exists
 
   - name: Fetch output

ansible/gather/site.yml

@@ -3,6 +3,7 @@
   remote_user: "{{ host_remote_user }}"
   become: true
   roles:
+    - common
     - compute
 
 - hosts: controller