More modifications for telemetry testing

* Adding some OSP10 (Newton) Compat
  * Adjust pipeline.yaml (instead of polling.yaml)
  * Ability to Sync polling agents on Newton
* Ability to reduce the archive-policy to a reduced workload
* Clean up (delete) existing resources for consistent archive-policy
* Add repos to all overcloud nodes for upgrade

Change-Id: Ia0b80b9747617c56c2939d68a799b37dbc2e19c9

ansible/browbeat/adjustment-gnocchi-reduce-archive-policy.yml

@@ -0,0 +1,28 @@
+---
+#
+# Reduce the archive-policy to a single retention period to reduce the telemetry load.
+#
+
+- hosts: undercloud
+  remote_user: "{{ local_remote_user }}"
+  gather_facts: false
+  vars_files:
+    - ../install/group_vars/all.yml
+  tasks:
+    - name: Create low-workload archive-policy
+      shell: ". {{ overcloudrc }}; gnocchi archive-policy create -d points:8640,granularity:0:05:00 -b 0 -m std -m count -m min -m max -m sum -m mean low-workload"
+
+- hosts: controller
+  remote_user: "{{ host_remote_user }}"
+  gather_facts: false
+  vars_files:
+    - ../install/group_vars/all.yml
+  vars:
+    restart_notification: false
+    restart_collector: true
+    ceilometer_configuration:
+      - section: dispatcher_gnocchi
+        option: archive_policy
+        value: low-workload
+  roles:
+    - ceilometer-config

ansible/browbeat/adjustment-gnocchi.yml

@@ -28,12 +28,19 @@
         value: "{{item.value}}"
         backup: yes
       with_items:
+        # Newton, Ocata, Pike
         - section: metricd
           option: workers
           value: "{{metricd_workers}}"
+        # Newton
+        - section: storage
+          option: metric_processing_delay
+          value: "{{metric_processing_delay}}"
+        # Ocata, Pike
         - section: metricd
           option: metric_processing_delay
           value: "{{metric_processing_delay}}"
+        # Pike
         - section: metricd
           option: processing_replicas
           value: "{{processing_replicas}}"

ansible/browbeat/clean-gnocchi-resources.yml

@@ -0,0 +1,16 @@
+---
+#
+# Clean Gnocchi resources
+#
+# Used incase of adding a new default archive-policy and thus all resources should
+# use the same new archive-policy.
+#
+
+- hosts: undercloud
+  remote_user: "{{ local_remote_user }}"
+  gather_facts: false
+  vars_files:
+    - ../install/group_vars/all.yml
+  tasks:
+    - name: Clean Gnocchi Resources
+      shell: ". {{ overcloudrc }}; gnocchi resource list -c type -c id -c revision_end  | grep None | awk '{print $2}' | xargs -I % gnocchi resource delete %"

ansible/browbeat/ntp-sync.yml

@@ -9,6 +9,7 @@
 
 - hosts: overcloud
   remote_user: "{{ host_remote_user }}"
+  gather_facts: false
   vars_files:
     - ../install/group_vars/all.yml
   tasks:

ansible/browbeat/roles/ceilometer-polling/tasks/main.yml

@@ -3,7 +3,29 @@
 # Deploy the Ceilometer polling.yaml file
 #
 
-- name: Deploy polling.yaml file
+- name: (Newton) Deploy pipeline.yaml file
+  become: true
+  template:
+    src: pipeline.yaml.j2
+    dest: /etc/ceilometer/pipeline.yaml
+    owner: root
+    group: ceilometer
+    mode: 0640
+    backup: true
+  when: "{{reduced_metrics}} == false and ('Newton' in osp_version['content'] | b64decode)"
+
+- name: (Newton) Deploy the reduced metrics pipeline.yaml file
+  become: true
+  template:
+    src: reduced_pipeline.yaml.j2
+    dest: /etc/ceilometer/pipeline.yaml
+    owner: root
+    group: ceilometer
+    mode: 0640
+    backup: true
+  when: "{{reduced_metrics}} == true and ('Newton' in osp_version['content'] | b64decode)"
+
+- name: (Ocata, Pike) Deploy polling.yaml file
   become: true
   template:
     src: polling.yaml.j2
@@ -12,9 +34,9 @@
     group: ceilometer
     mode: 0640
     backup: true
-  when: "{{reduced_metrics}} == false"
+  when: "{{reduced_metrics}} == false and ('Ocata' in osp_version['content'] | b64decode or 'Pike' in osp_version['content'] | b64decode)"
 
-- name: Deploy the reduced metrics polling.yaml file
+- name: (Ocata, Pike) Deploy the reduced metrics polling.yaml file
   become: true
   template:
     src: reduced_polling.yaml.j2
@@ -23,4 +45,4 @@
     group: ceilometer
     mode: 0640
     backup: true
-  when: reduced_metrics
+  when: "{{reduced_metrics}} == true and ('Ocata' in osp_version['content'] | b64decode or 'Pike' in osp_version['content'] | b64decode)"

ansible/browbeat/roles/ceilometer-polling/templates/pipeline.yaml.j2

@@ -0,0 +1,92 @@
+---
+sources:
+    - name: meter_source
+      interval: {{polling_interval}}
+      meters:
+          - "*"
+      sinks:
+          - meter_sink
+    - name: cpu_source
+      interval: {{polling_interval}}
+      meters:
+          - "cpu"
+      sinks:
+          - cpu_sink
+          - cpu_delta_sink
+    - name: disk_source
+      interval: {{polling_interval}}
+      meters:
+          - "disk.read.bytes"
+          - "disk.read.requests"
+          - "disk.write.bytes"
+          - "disk.write.requests"
+          - "disk.device.read.bytes"
+          - "disk.device.read.requests"
+          - "disk.device.write.bytes"
+          - "disk.device.write.requests"
+      sinks:
+          - disk_sink
+    - name: network_source
+      interval: {{polling_interval}}
+      meters:
+          - "network.incoming.bytes"
+          - "network.incoming.packets"
+          - "network.outgoing.bytes"
+          - "network.outgoing.packets"
+      sinks:
+          - network_sink
+sinks:
+    - name: meter_sink
+      transformers:
+      publishers:
+          - notifier://
+    - name: cpu_sink
+      transformers:
+          - name: "rate_of_change"
+            parameters:
+                target:
+                    name: "cpu_util"
+                    unit: "%"
+                    type: "gauge"
+                    scale: "100.0 / (10**9 * (resource_metadata.cpu_number or 1))"
+      publishers:
+          - notifier://
+    - name: cpu_delta_sink
+      transformers:
+          - name: "delta"
+            parameters:
+                target:
+                    name: "cpu.delta"
+                growth_only: True
+      publishers:
+          - notifier://
+    - name: disk_sink
+      transformers:
+          - name: "rate_of_change"
+            parameters:
+                source:
+                    map_from:
+                        name: "(disk\\.device|disk)\\.(read|write)\\.(bytes|requests)"
+                        unit: "(B|request)"
+                target:
+                    map_to:
+                        name: "\\1.\\2.\\3.rate"
+                        unit: "\\1/s"
+                    type: "gauge"
+      publishers:
+          - notifier://
+    - name: network_sink
+      transformers:
+          - name: "rate_of_change"
+            parameters:
+                source:
+                   map_from:
+                       name: "network\\.(incoming|outgoing)\\.(bytes|packets)"
+                       unit: "(B|packet)"
+                target:
+                    map_to:
+                        name: "network.\\1.\\2.rate"
+                        unit: "\\1/s"
+                    type: "gauge"
+      publishers:
+          - notifier://

ansible/browbeat/roles/ceilometer-polling/templates/reduced_pipeline.yaml.j2

@@ -0,0 +1,112 @@
+---
+sources:
+    - name: some_pollsters
+      interval: {{polling_interval}}
+      meters:
+        - cpu
+        - memory.usage
+        - network.incoming.bytes
+        - network.incoming.packets
+        - network.outgoing.bytes
+        - network.outgoing.packets
+        - disk.read.bytes
+        - disk.read.requests
+        - disk.write.bytes
+        - disk.write.requests
+        - hardware.cpu.util
+        - hardware.memory.used
+        - hardware.memory.total
+        - hardware.memory.buffer
+        - hardware.memory.cached
+        - hardware.memory.swap.avail
+        - hardware.memory.swap.total
+        - hardware.system_stats.io.outgoing.blocks
+        - hardware.system_stats.io.incoming.blocks
+        - hardware.network.ip.incoming.datagrams
+        - hardware.network.ip.outgoing.datagrams
+      sinks:
+          - meter_sink
+    - name: cpu_source
+      interval: {{polling_interval}}
+      meters:
+          - "cpu"
+      sinks:
+          - cpu_sink
+          - cpu_delta_sink
+    - name: disk_source
+      interval: {{polling_interval}}
+      meters:
+          - "disk.read.bytes"
+          - "disk.read.requests"
+          - "disk.write.bytes"
+          - "disk.write.requests"
+          - "disk.device.read.bytes"
+          - "disk.device.read.requests"
+          - "disk.device.write.bytes"
+          - "disk.device.write.requests"
+      sinks:
+          - disk_sink
+    - name: network_source
+      interval: {{polling_interval}}
+      meters:
+          - "network.incoming.bytes"
+          - "network.incoming.packets"
+          - "network.outgoing.bytes"
+          - "network.outgoing.packets"
+      sinks:
+          - network_sink
+sinks:
+    - name: meter_sink
+      transformers:
+      publishers:
+          - notifier://
+    - name: cpu_sink
+      transformers:
+          - name: "rate_of_change"
+            parameters:
+                target:
+                    name: "cpu_util"
+                    unit: "%"
+                    type: "gauge"
+                    scale: "100.0 / (10**9 * (resource_metadata.cpu_number or 1))"
+      publishers:
+          - notifier://
+    - name: cpu_delta_sink
+      transformers:
+          - name: "delta"
+            parameters:
+                target:
+                    name: "cpu.delta"
+                growth_only: True
+      publishers:
+          - notifier://
+    - name: disk_sink
+      transformers:
+          - name: "rate_of_change"
+            parameters:
+                source:
+                    map_from:
+                        name: "(disk\\.device|disk)\\.(read|write)\\.(bytes|requests)"
+                        unit: "(B|request)"
+                target:
+                    map_to:
+                        name: "\\1.\\2.\\3.rate"
+                        unit: "\\1/s"
+                    type: "gauge"
+      publishers:
+          - notifier://
+    - name: network_sink
+      transformers:
+          - name: "rate_of_change"
+            parameters:
+                source:
+                   map_from:
+                       name: "network\\.(incoming|outgoing)\\.(bytes|packets)"
+                       unit: "(B|packet)"
+                target:
+                    map_to:
+                        name: "network.\\1.\\2.rate"
+                        unit: "\\1/s"
+                    type: "gauge"
+      publishers:
+          - notifier://

ansible/browbeat/sync-ceilometer-polling.yml

@@ -20,15 +20,15 @@
       register: osp_version
       become: true
 
-    - name: (Ocata) Set Controller Task
+    - name: (Newton, Ocata) Set Controller Task
       set_fact:
         the_task: "systemctl restart openstack-ceilometer-central.service"
-      when: "('Ocata' in osp_version['content'] | b64decode) and ('controller' in group_names)"
+      when: "('Newton' in osp_version['content'] | b64decode or 'Ocata' in osp_version['content'] | b64decode) and ('controller' in group_names)"
 
-    - name: (Ocata) Set Compute Task
+    - name: (Newton, Ocata) Set Compute Task
       set_fact:
         the_task: "systemctl restart openstack-ceilometer-compute.service"
-      when: "('Ocata' in osp_version['content'] | b64decode) and ('compute' in group_names)"
+      when: "('Newton' in osp_version['content'] | b64decode or 'Ocata' in osp_version['content'] | b64decode) and ('compute' in group_names)"
 
     - name: (Pike) Set Controller Task
       set_fact:

ansible/browbeat/telemetry-polling.yml

@@ -21,5 +21,11 @@
   vars:
     polling_interval: 300
     reduced_metrics: false
+  pre_tasks:
+    - name: Get OSP Version
+      slurp:
+        src: "/etc/rhosp-release"
+      register: osp_version
+      become: true
   roles:
     - ceilometer-polling

ansible/install/repos.yml

@@ -0,0 +1,12 @@
+---
+#
+# Add repos
+#
+# Can be used in conjunction with doing an upgrade
+#
+
+- hosts: overcloud
+  remote_user: "{{ host_remote_user }}"
+  gather_facts: false
+  roles:
+    - { role: repo }