diff --git a/ansible/generate_tripleo_hostfile.sh b/ansible/generate_tripleo_hostfile.sh index 1bb14be9d..dcbbaa946 100755 --- a/ansible/generate_tripleo_hostfile.sh +++ b/ansible/generate_tripleo_hostfile.sh @@ -78,6 +78,16 @@ if [ ${#clouds} -gt 0 ]; then exit 1 fi + networker_id=$(ssh -tt -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" stack@${tripleo_ip_address} ". ~/stackrc; openstack stack resource show $overcloud_name Networker > >(grep physical_resource_id) 2>/dev/null" | awk '{print $4}') + if [ ${#networker_id} -lt 3 ]; then + echo "Info: No Networker resources." + else + networker_ids=$(ssh -tt -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" stack@${tripleo_ip_address} ". ~/stackrc; openstack stack resource list ${networker_id} > >(grep -i networker) 2>/dev/null" | awk '{print $2}') + if [ ${#networker_ids} -lt 1 ]; then + echo "Info: No Networker resources." + fi + fi + blockstorage_id=$(ssh -tt -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" stack@${tripleo_ip_address} ". ~/stackrc; openstack stack resource show $overcloud_name BlockStorage > >(grep physical_resource_id) 2>/dev/null" | awk '{print $4}') if [ ${#blockstorage_id} -lt 3 ]; then echo "Info: No BlockStorage resources." @@ -125,6 +135,15 @@ if [ ${#clouds} -gt 0 ]; then controller_uuids+=$(ssh -tt -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" stack@${tripleo_ip_address} ". ~/stackrc; openstack stack resource show ${controller_id} ${controller} > >(grep -oP \"'nova_server_resource': u'([a-z0-9]+-[a-z0-9]+-[a-z0-9]+-[a-z0-9]+-[a-z0-9]+)'\") 2>/dev/null" | awk '{print $2}' | grep -oP [a-z0-9]+-[a-z0-9]+-[a-z0-9]+-[a-z0-9]+-[a-z0-9]+) fi done + networker_uuids=() + for networker in ${networker_ids} + do + if [[ ${version_tripleo} -lt 2 ]] ; then + networker_uuids+=$(ssh -tt -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" stack@${tripleo_ip_address} ". ~/stackrc; heat resource-show ${networker_id} ${networker} | grep -i nova_server_resource" | awk '{print $4}') + else + networker_uuids+=$(ssh -tt -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" stack@${tripleo_ip_address} ". ~/stackrc; openstack stack resource show ${networker_id} ${networker} > >(grep -oP \"'nova_server_resource': u'([a-z0-9]+-[a-z0-9]+-[a-z0-9]+-[a-z0-9]+-[a-z0-9]+)'\") 2>/dev/null" | awk '{print $2}' | grep -oP [a-z0-9]+-[a-z0-9]+-[a-z0-9]+-[a-z0-9]+-[a-z0-9]+) + fi + done blockstorage_uuids=() for blockstorage in ${blockstorage_ids} do @@ -191,11 +210,12 @@ echo " IdentityFile ~/.ssh/id_rsa" | tee -a ${ssh_config_file} echo " StrictHostKeyChecking no" | tee -a ${ssh_config_file} echo " UserKnownHostsFile=/dev/null" | tee -a ${ssh_config_file} +blockstorage_hn=() +cephstorage_hn=() compute_hn=() controller_hn=() -blockstorage_hn=() +networker_hn=() objectstorage_hn=() -cephstorage_hn=() IFS=$'\n' for line in $nodes; do uuid=$(echo $line | awk '{print $2}') @@ -203,6 +223,8 @@ for line in $nodes; do IP=$(echo $line | awk '{print $8}' | cut -d "=" -f2) if grep -q $uuid <<< {$controller_uuids}; then controller_hn+=("$host") +elif grep -q $uuid <<< {$networker_uuids}; then + networker_hn+=("$host") elif grep -q $uuid <<< {$blockstorage_uuids}; then blockstorage_hn+=("$host") elif grep -q $uuid <<< {$objectstorage_uuids}; then @@ -231,6 +253,11 @@ controller_hn=( $( do echo "$item" done | sort) ) +networker_hn=( $( + for item in "${networker_hn[@]}" + do + echo "$item" + done | sort) ) blockstorage_hn=( $( for item in "${blockstorage_hn[@]}" do @@ -286,6 +313,22 @@ if [[ ${#controller_hn} -gt 0 ]]; then done fi echo "" | tee -a ${ansible_inventory_file} +echo "[networker]" | tee -a ${ansible_inventory_file} +if [[ ${#networker_hn} -gt 0 ]]; then + for networker in ${networker_hn[@]}; do + ironic_uuid='' + for line in ${ironic_uuids}; do + uuid=$(echo $line | awk '{print $2}') + host=$(echo $line | awk '{print $6}') + if [ "$host" == "$networker" ]; then + ironic_uuid=$uuid + break + fi + done + echo "${networker} ironic_uuid=${ironic_uuid}" | tee -a ${ansible_inventory_file} + done +fi +echo "" | tee -a ${ansible_inventory_file} echo "[blockstorage]" | tee -a ${ansible_inventory_file} if [[ ${#blockstorage_hn} -gt 0 ]]; then for blockstorage in ${blockstorage_hn[@]}; do @@ -355,6 +398,9 @@ if [[ ${#controller_hn} -gt 0 ]] || [[ ${#blockstorage_hn} -gt 0 ]] || [[ ${#obj if [[ ${#controller_hn} -gt 0 ]]; then echo "controller" | tee -a ${ansible_inventory_file} fi + if [[ ${#networker_hn} -gt 0 ]]; then + echo "networker" | tee -a ${ansible_inventory_file} + fi if [[ ${#blockstorage_hn} -gt 0 ]]; then echo "blockstorage" | tee -a ${ansible_inventory_file} fi diff --git a/ansible/install/collectd-openstack.yml b/ansible/install/collectd-openstack.yml index 92d9d1d9c..388d0ec86 100644 --- a/ansible/install/collectd-openstack.yml +++ b/ansible/install/collectd-openstack.yml @@ -8,6 +8,7 @@ # Or use tags: # ansible-playbook -i hosts install/collectd-openstack.yml --tag "undercloud" # ansible-playbook -i hosts install/collectd-openstack.yml --tag "controller" +# ansible-playbook -i hosts install/collectd-openstack.yml --tag "networker" # ansible-playbook -i hosts install/collectd-openstack.yml --tag "blockstorage" # ansible-playbook -i hosts install/collectd-openstack.yml --tag "objectstorage" # ansible-playbook -i hosts install/collectd-openstack.yml --tag "cephstorage" @@ -52,6 +53,24 @@ tags: controller environment: "{{proxy_env}}" +- hosts: networker + remote_user: "{{ host_remote_user }}" + vars: + config_type: networker + roles: + - { role: common, when: collectd_networker } + - { role: epel, when: collectd_networker } + - { role: repo } + - { role: collectd-openstack, when: collectd_networker } + tasks: + - name: Collectd off if not collectd_networker + service: name=collectd state=stopped enabled=false + become: true + when: not collectd_networker + ignore_errors: true + tags: networker + environment: "{{proxy_env}}" + - hosts: blockstorage remote_user: "{{ host_remote_user }}" vars: diff --git a/ansible/install/group_vars/all.yml b/ansible/install/group_vars/all.yml index 640e73d3d..c16fdc828 100644 --- a/ansible/install/group_vars/all.yml +++ b/ansible/install/group_vars/all.yml @@ -140,6 +140,7 @@ collectd_write_graphite_port: 2003 # Run collectd on specific openstack nodes: collectd_undercloud: true collectd_controller: true +collectd_networker: true collectd_blockstorage: true collectd_objectstorage: true collectd_cephstorage: true @@ -197,6 +198,7 @@ gnocchi_status_controller_collectd_interval: 10 # Disk plugin metrics are opt-out, IOStat metrics are opt-in disk_undercloud_collectd_plugin: true disk_controller_collectd_plugin: true +disk_networker_collectd_plugin: true disk_cephstorage_collectd_plugin: true disk_compute_collectd_plugin: true disk_blockstorage_collectd_plugin: true @@ -206,6 +208,8 @@ iostat_undercloud_collectd_plugin: false iostat_undercloud_collectd_interval: 10 iostat_controller_collectd_plugin: false iostat_controller_collectd_interval: 10 +iostat_networker_collectd_plugin: false +iostat_networker_collectd_interval: 10 iostat_cephstorage_collectd_plugin: false iostat_cephstorage_collectd_interval: 10 iostat_compute_collectd_plugin: false @@ -257,10 +261,14 @@ controller_monitored_queues: ######################## ovsagent_compute_monitor: false ovsagent_controller_monitor: false +ovsagent_networker_monitor: false controller_monitored_ints: - "tap" +networker_monitored_ints: + - "tap" + compute_monitored_ints: - "qvo" @@ -268,6 +276,10 @@ controller_monitored_ns: - "qrouter" - "qdhcp" +networker_monitored_ns: + - "qrouter" + - "qdhcp" + ######################## # Swift stat plugin ######################## diff --git a/ansible/install/roles/collectd-openstack/tasks/main.yml b/ansible/install/roles/collectd-openstack/tasks/main.yml index 82a73d043..1ceb37d5d 100644 --- a/ansible/install/roles/collectd-openstack/tasks/main.yml +++ b/ansible/install/roles/collectd-openstack/tasks/main.yml @@ -239,7 +239,7 @@ with_items: - src: collectd_ovsagent.py dest: /usr/local/bin/collectd_ovsagent.py - when: "('controller' in group_names ) or ('compute' in group_names)" + when: "('controller' in group_names ) or ('compute' in group_names) or ('networker' in group_names)" # Rabbitmq monitoring - name: Install pyrabbit diff --git a/ansible/install/roles/collectd-openstack/templates/networker.collectd.conf.j2 b/ansible/install/roles/collectd-openstack/templates/networker.collectd.conf.j2 new file mode 100644 index 000000000..e8d944af1 --- /dev/null +++ b/ansible/install/roles/collectd-openstack/templates/networker.collectd.conf.j2 @@ -0,0 +1,294 @@ +# Installed by Browbeat Ansible Installer +# Config type: {{config_type}} + +# Interval default is 10s +Interval {{collectd_interval}} + +# Hostname for this machine, if not defined, use gethostname(2) system call +Hostname "{{inventory_hostname}}" + +# Loaded Plugins: +LoadPlugin "logfile" + + File "/var/log/collectd.log" + LogLevel "info" + PrintSeverity true + Timestamp true + + +LoadPlugin write_graphite +LoadPlugin cpu +LoadPlugin conntrack +LoadPlugin df +LoadPlugin disk +LoadPlugin exec +LoadPlugin interface +LoadPlugin irq +LoadPlugin load +LoadPlugin match_regex +LoadPlugin memory +LoadPlugin numa +LoadPlugin processes + + Globals true + +LoadPlugin swap +LoadPlugin tail +LoadPlugin turbostat +LoadPlugin unixsock +LoadPlugin uptime + +# Open unix domain socket for collectdctl + + SocketFile "/var/run/collectd-unixsock" + SocketGroup "collectd" + SocketPerms "0770" + DeleteSocket true + + +PreCacheChain "PreCache" + + + + Plugin "^interface$" + PluginInstance "^tap*" + + Target "stop" + + + + Plugin "^interface$" + PluginInstance "^q.*" + + Target "stop" + + Target "return" + + +# Graphite Host Configuration + + + Host "{{graphite_host}}" + Port "{{collectd_write_graphite_port}}" + Prefix "{{graphite_prefix}}." + Protocol "tcp" + LogSendErrors true + StoreRates true + AlwaysAppendDS false + EscapeCharacter "_" + + + + + FSType anon_inodefs + FSType bdev + FSType cgroup + FSType cpuset + FSType debugfs + FSType devpts + FSType devtmpfs + FSType ecryptfs + FSType fuse + FSType fusectl + FSType hugetlbfs + FSType mqueue + FSType nfs + FSType nfs4 + FSType nfsd + FSType overlay + FSType pipefs + FSType proc + FSType pstore + FSType ramfs + #FSType rootfs + FSType rpc_pipefs + FSType securityfs + FSType sockfs + FSType sysfs + FSType tmpfs + FSType vboxsf + IgnoreSelected true + ValuesPercentage true + ReportInodes true + + +{%if disk_networker_collectd_plugin %} + + Disk "/^[hsv]d[a-z]+[0-9]?$/" + Disk "/^nvm/" + IgnoreSelected false + + +{% endif %} +{%if iostat_networker_collectd_plugin %} + + ModulePath "/usr/local/bin/" + Import "collectd_iostat_python" + + + Path "/usr/bin/iostat" + Interval {{iostat_networker_collectd_interval}} + IostatInterval 2 + Count 2 + Verbose false + NiceNames false + PluginName collectd_iostat_python + + + +{% endif %} + +# (akrzos) Including the version of OpenStack that the process was verified as running after +# OpenStack Installation with a comment at the end of each Process/ProcessMatch statement. +# A Minus before the version means the process was not found in that version. (Ex -10) + + # Collectd (Browbeat Installed) + ProcessMatch "collectd" "/usr/sbin/collectd" + + # Everything Else (OpenStack Installed) + # (Processes displayed under "Everything Else" on Grafana Dashboards) + ProcessMatch "dnsmasq" "^dnsmasq.+" # 12 + ProcessMatch "haproxy" "haproxy.+-f.+/var/lib/neutron/ns-metadata-proxy.*" # 12 + Process "keepalived" # 12 + + # Neutron (OpenStack Installed) + ProcessMatch "neutron-dhcp-agent" "python.+neutron-dhcp-agent" # 12 + ProcessMatch "neutron-l3-agent" "python.+neutron-l3-agent" # 12 + ProcessMatch "neutron-metadata-agent" "python.+neutron-metadata-agent" # 12 + ProcessMatch "neutron-ns-metadata-proxy" "python.+neutron-ns-metadata-proxy" # + ProcessMatch "neutron-openvswitch-agent" "python.+neutron-openvswitch-agent" # 12 + + # OVS (OpenStack Installed) + ProcessMatch "ovs-vswitchd" "ovs-vswitchd.+openvswitch" # 10,11,12 + ProcessMatch "ovsdb-client" "ovsdb-client" # 10,11,12 + ProcessMatch "ovsdb-server" "ovsdb-server.+openvswitch" # 10,11,12 + + + + ReportBytes true + ValuesPercentage true + + +# Tail plugin configuration + + + Instance "neutron-dhcp-agent" + + Regex " ERROR " + DSType "CounterInc" + Type "counter" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} + + + Instance "neutron-l3-agent" + + Regex " ERROR " + DSType "CounterInc" + Type "counter" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} + + + Instance "neutron-metadata-agent" + + Regex " ERROR " + DSType "CounterInc" + Type "counter" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} + + + Instance "neutron-openvswitch-agent" + + Regex " ERROR " + DSType "CounterInc" + Type "counter" + Instance "error" + +{%if regex_warn %} + + Regex " WARNING " + DSType "CounterInc" + Type "counter" + Instance "warn" + +{% endif %} +{%if regex_info %} + + Regex " INFO " + DSType "CounterInc" + Type "counter" + Instance "info" + +{% endif %} + + + +{% if ovsagent_networker_monitor %} + + ModulePath "/usr/local/bin/" + LogTraces true + Interactive false + Import "collectd_ovsagent" + + interval 10 + prefix ovsagent + interfaces {% for int in networker_monitored_ints %} {{int}} {% endfor %} + + namespaces {% for ns in networker_monitored_ns %} {{ns}} {% endfor %} + + + +{% endif %} + +# Include other collectd configuration files +Include "/etc/collectd.d"