Merge "Add host info collection to collect-logs"

roles/collect-logs/files/heat-deploy-times.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+# Copyright 2016 Red Hat Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# Usage: openstack stack event list -f json overcloud | \
+#        heat-deploy-times.py [list of resource names]
+# If no resource names are provided, all of the resources will be output.
+
+import json
+import sys
+import time
+
+
+def process_events(all_events, events):
+    times = {}
+    for event in all_events:
+        name = event['resource_name']
+        status = event['resource_status']
+        # Older clients return timestamps in the first format, newer ones
+        # append a Z.  This way we can handle both formats.
+        try:
+            strptime = time.strptime(event['event_time'],
+                                     '%Y-%m-%dT%H:%M:%S')
+        except ValueError:
+            strptime = time.strptime(event['event_time'],
+                                     '%Y-%m-%dT%H:%M:%SZ')
+        etime = time.mktime(strptime)
+        if name in events:
+            if status == 'CREATE_IN_PROGRESS':
+                times[name] = {'start': etime, 'elapsed': None}
+            elif status == 'CREATE_COMPLETE':
+                times[name]['elapsed'] = etime - times[name]['start']
+    for name, data in sorted(times.items(),
+                             key=lambda x: x[1]['elapsed'],
+                             reverse=True):
+        elapsed = 'Still in progress'
+        if times[name]['elapsed'] is not None:
+            elapsed = times[name]['elapsed']
+        print('%s %s') % (name, elapsed)
+
+if __name__ == '__main__':
+    stdin = sys.stdin.read()
+    all_events = json.loads(stdin)
+    events = sys.argv[1:]
+    if not events:
+        events = set()
+        for event in all_events:
+            events.add(event['resource_name'])
+    process_events(all_events, events)

roles/collect-logs/tasks/collect.yml

@@ -164,6 +164,23 @@
             done;
         fi
 
+    # Collect host info as done presently in TripleO-CI.
+    # Will be deprecated.
+    - name: Create get_host_info script
+      template:
+        src: "get_host_info.sh.j2"
+        dest: "/tmp/get_host_info.sh"
+        mode: 0755
+
+    - name: Copy heat-deploy-times.py to host
+      copy:
+        src: heat-deploy-times.py
+        dest: "/tmp/heat-deploy-times.py"
+        mode: 0755
+
+    - name: Call get_host_info script
+      shell: /tmp/get_host_info.sh &>/var/log/host_info.txt
+
     - name: Erase temporary log directory if exists
       file:
         path: "/tmp/{{ inventory_hostname }}"

roles/collect-logs/templates/get_host_info.sh.j2

@@ -0,0 +1,46 @@
+#!/bin/bash
+echo "This script will be deprecated in favor of collect-logs files"
+set -x
+export PATH=$PATH:/sbin
+ps -eaufxZ
+ls -Z /var/run/
+df -h
+uptime
+sudo netstat -lpn
+sudo iptables-save
+sudo ovs-vsctl show
+ip addr
+ip route
+ip -6 route
+free -h
+top -n 1 -b -o RES
+rpm -qa
+yum repolist -v
+sudo os-collect-config --print
+which pcs &> /dev/null && sudo pcs status --full
+which pcs &> /dev/null && sudo pcs constraint show --full
+which pcs &> /dev/null && sudo pcs stonith show --full
+which crm_verify &> /dev/null && sudo crm_verify -L -VVVVVV
+which ceph &> /dev/null && sudo ceph status
+sudo facter
+find ~jenkins -iname tripleo-overcloud-passwords -execdir cat '{}' ';'
+sudo systemctl list-units --full --all
+
+
+if [ -e /home/{{ undercloud_user }}/stackrc ] ; then
+bash <<-EOF &>/var/log/postci.txt
+    source /home/{{ undercloud_user }}/stackrc
+    nova list
+    # If there's no overcloud then there's no point in continuing
+    heat stack-show overcloud || (echo 'No active overcloud found' && exit 0)
+    heat resource-list -n5 overcloud
+    heat event-list overcloud
+    # --nested-depth 2 seems to get us a reasonable list of resources without
+    # taking an excessive amount of time
+    openstack stack event list --nested-depth 2 -f json overcloud | /tmp/heat-deploy-times.py | tee /var/log/heat-deploy-times.log || echo 'Failed to process resource deployment times. This is expected for stable/liberty.'
+    # useful to see what failed when puppet fails
+    # NOTE(bnemec): openstack stack failures list only exists in Newton and above.
+    # On older releases we still need to manually query the deployments.
+    openstack stack failures list --long overcloud || for failed_deployment in \$(heat resource-list --nested-depth 5 overcloud | grep FAILED | grep 'StructuredDeployment ' | cut -d '|' -f3); do heat deployment-show \$failed_deployment; done;
+EOF
+fi