openstack
/
tripleo-quickstart-extras
Code Issues Proposed changes
tripleo-quickstart-extras/roles/collect-logs/tasks/collect.yml

245 lines
8.6 KiB
YAML
Raw Blame History

---
- become: true
ignore_errors: true
block:
- name: Ensure required rpms for logging are installed
yum: name={{ item }} state=present
with_flattened:
- gzip
- tar
- name: Prepare directory with extra logs
file: dest=/var/log/extra state=directory
- name: rpm -qa
shell: rpm -qa | sort -f >/var/log/extra/rpm-list.txt
- name: yum list installed
shell: yum list installed >/var/log/extra/yum-list-installed.txt
- name: Collecting cpu information
shell: cat /proc/cpuinfo &> /var/log/extra/cpuinfo.txt
# used by OSP Release Engineering to import into internal builds
- name: package import delorean
shell: |
repoquery --disablerepo='*' --enablerepo='delorean'\
-a --qf '%{sourcerpm}'|sort -u|sed 's/.src.rpm//g' >> /var/log/extra/import-delorean.txt
# used by OSP Release Engineering to import into internal builds
- name: package import delorean-testing
shell: |
repoquery --disablerepo='*' --enablerepo='delorean-*-testing'\
-a --qf '%{sourcerpm}'|sort -u|sed 's/.src.rpm//g' >> /var/log/extra/import-delorean-testing.txt
- name: Collect logs from all failed systemd services
shell: >
systemctl -t service --failed --no-legend | awk '{print $1}'
| xargs -r -n1 journalctl -u > /var/log/extra/services.txt 2>&1
- name: Collect network status info
shell: >
(ip a; ip r; iptables-save; iptables -nL) &> /var/log/extra/network.txt;
(for NS in $(ip netns list); do
echo "==== $NS ====";
ip netns exec $NS ip a;
ip netns exec $NS ip r;
ip netns exec $NS ip iptables-save;
PIDS="$(ip netns pids $NS)";
[[ ! -z "$PIDS" ]] && ps --no-headers -f --pids "$PIDS";
echo "";
done) &> /var/log/extra/network-netns;
(for NB in $(ovs-vsctl show | grep Bridge |awk '{print $2}'); do
echo "==== Bridge name - $NB ====";
ovs-ofctl show $NB;
ovs-ofctl dump-flows $NB;
echo "";
done;
ovsdb-client dump) &> /var/log/extra/network-bridges;
- name: lsof -P
shell: "lsof -P &> /var/log/extra/lsof.txt"
- name: pstree -p
shell: "pstree -p &> /var/log/extra/pstree.txt"
- name: sysctl -a
shell: "sysctl -a &> /var/log/extra/sysctl.txt"
- name: netstat -lnp
shell: "netstat -lnp &> /var/log/extra/netstat.txt"
- name: openstack-status
shell: "which openstack-status &> /dev/null && (. ~/keystonerc_admin; openstack-status &> /var/log/extra/openstack-status.txt)"
when: "'controller' in inventory_hostname"
- name: List nova servers on undercloud
shell: >
if [[ -e {{ working_dir }}/stackrc ]]; then
source {{ working_dir }}/stackrc;
nova list &> /var/log/extra/nova_list.txt;
fi
- name: lsmod
shell: "lsmod &> /var/log/extra/lsmod.txt"
- name: lspci
shell: "lspci &> /var/log/extra/lspci.txt"
- name: pip list
shell: "pip list &> /var/log/extra/pip.txt"
- name: lvm debug
shell: "(vgs; pvs; lvs) &> /var/log/extra/lvm.txt"
- name: Generate human-readable SAR logs
shell: "[[ -f /usr/lib64/sa/sa2 ]] && /usr/lib64/sa/sa2 -A"
- name: check for dstat log file
stat: path=/var/log/extra/dstat-csv.log
register: dstat_logfile
- name: Get dstat_graph tool
git: repo="https://github.com/Dabz/dstat_graph.git" dest="/tmp/dstat_graph"
when: dstat_logfile.stat.exists
- name: Generate HTML dstat graphs if it exists
shell: "/tmp/dstat_graph/generate_page.sh /var/log/extra/dstat-csv.log > /var/log/extra/dstat.html"
when: dstat_logfile.stat.exists
args:
chdir: "/tmp/dstat_graph"
- name: Search for AVC denied
shell: >
grep -i denied /var/log/audit/audit* &&
grep -i denied /var/log/audit/audit* > /var/log/extra/denials.txt
- name: Search for segfaults in logs
shell: >
grep -v ansible-command /var/log/messages | grep segfault &&
grep -v ansible-command /var/log/messages | grep segfault > /var/log/extra/segfaults.txt
- name: Search for oom-killer instances in logs
shell: >
grep -v ansible-command /var/log/messages | grep oom-killer &&
grep -v ansible-command /var/log/messages | grep oom-killer > /var/log/extra/oom-killers.txt
- name: Ensure sos package is installed when collect sosreport(s)
yum: name=sos state=present
when: artcl_collect_sosreport|bool
- name: Collect sosreport
command: >
sosreport --batch
when: artcl_collect_sosreport|bool
- name: Collect delorean logs
shell: >
if [[ -e /home/{{ undercloud_user }}/DLRN/data/repos ]]; then
rm -rf /tmp/delorean_logs && mkdir /tmp/delorean_logs;
find /home/{{ undercloud_user }}/DLRN/data/repos/ -name '*.log' -exec cp --parents \{\} /tmp/delorean_logs/ \; ;
find /tmp/delorean_logs -name '*.log' -exec gzip \{\} \; ;
find /tmp/delorean_logs -name '*.log.gz' -exec sh -c 'x="{}"; mv "$x" "${x%.log.gz}.log.txt.gz"' \; ;
rm -rf {{ artcl_collect_dir }}/delorean_logs && mkdir {{ artcl_collect_dir }}/delorean_logs;
mv /tmp/delorean_logs/home/{{ undercloud_user }}/DLRN/data/repos/* {{ artcl_collect_dir }}/delorean_logs/;
fi
- name: Collect docker info and logs
shell: >
if command -v docker && systemctl is-active docker; then
All_FILE=/var/log/extra/docker_allinfo.log;
docker ps --all --size &> $All_FILE;
docker images &>> $All_FILE;
docker stats --all --no-stream &>> $All_FILE;
docker info &>> $All_FILE;
for cont in $(docker ps | awk {'print $NF'} | grep -v NAMES); do
INFO_FILE=/var/log/extra/docker_${cont}_info.log;
echo "+ docker top $cont auxw" > $INFO_FILE;
docker top $cont auxw &>> $INFO_FILE;
echo "+ docker exec $cont top -bwn1" >> $INFO_FILE;
docker exec $cont top -bwn1 &>> $INFO_FILE;
echo "+ docker inspect $cont" >> $INFO_FILE;
docker inspect $cont &>> $INFO_FILE;
docker logs $cont &> /var/log/extra/docker_${cont}.log;
done;
fi
- name: Erase temporary log directory if exists
file:
path: "/tmp/{{ inventory_hostname }}"
state: absent
- name: Set default collect list
set_fact:
collect_list: "{{ artcl_collect_list | join(' ') }}"
- name: Override collect list
set_fact:
collect_list: "{{ artcl_collect_override[inventory_hostname] | join(' ') }}"
when:
- artcl_collect_override is defined
- artcl_collect_override[inventory_hostname] is defined
- name: Gather the logs to /tmp
become: yes
shell: >
mkdir -p /tmp/{{ inventory_hostname }};
for F in $(ls -d1 /var/log/rpm.list /var/log/extra {{ collect_list }}); do
cp -rL --parents $F /tmp/{{ inventory_hostname }};
done;
find /tmp/{{ inventory_hostname }} -not -type f -not -type d -delete;
for D in {{ artcl_exclude_list | default([]) | join(' ') }}; do
rm -rf "/tmp/{{ inventory_hostname }}/$D";
done;
find /tmp/{{ inventory_hostname }} -type d -print0 | xargs -0 chmod 755;
find /tmp/{{ inventory_hostname }} -type f -print0 | xargs -0 chmod 644;
chown -R {{ ansible_user }}: /tmp/{{ inventory_hostname }};
- name: Compress logs to tar.gz
shell: >
chdir=/tmp
tar czf {{ inventory_hostname }}.tar.gz {{ inventory_hostname }};
when: artcl_tar_gz|bool
- name: gzip logs individually and tar them
shell: >
chdir=/tmp
gzip -r ./{{ inventory_hostname }};
tar cf {{ inventory_hostname }}.tar {{ inventory_hostname }};
when: artcl_gzip_only|bool
- name: Fetch log archive (tar.gz)
fetch:
src: "/tmp/{{ inventory_hostname }}.tar.gz"
dest: "{{ artcl_collect_dir }}/{{ inventory_hostname }}.tar.gz"
flat: yes
validate_checksum: no
when: artcl_tar_gz|bool
- name: Fetch log archive (tar)
fetch:
src: "/tmp/{{ inventory_hostname }}.tar"
dest: "{{ artcl_collect_dir }}/{{ inventory_hostname }}.tar"
flat: yes
validate_checksum: no
when: artcl_gzip_only|bool
- name: Delete temporary log directory after collection
file:
path: "/tmp/{{ inventory_hostname }}"
state: absent
ignore_errors: true
- delegate_to: localhost
when: artcl_gzip_only|bool
block:
- name: Extract the logs
shell: >
chdir={{ artcl_collect_dir }}
tar xf {{ inventory_hostname }}.tar;
- name: delete the tar file after extraction
file:
path: "{{ artcl_collect_dir }}/{{ inventory_hostname }}.tar"
state: absent
View Git Blame Copy Permalink