Make Heka collect Nova logs

Partially implements: blueprint heka
Change-Id: I67dfc5d762d74454e4a852dd61c57fe9cd4b78c8

ansible/roles/common/tasks/config.yml

@@ -31,3 +31,4 @@
     - "global"
     - "haproxy"
     - "rabbitmq"
+    - "openstack"

ansible/roles/common/templates/heka-openstack.toml.j2

@@ -0,0 +1,10 @@
+[openstack_log_decoder]
+type = "SandboxDecoder"
+filename = "lua_decoders/os_openstack_log.lua"
+
+[openstack_logstreamer_input]
+type = "LogstreamerInput"
+decoder = "openstack_log_decoder"
+log_directory = "/var/log/kolla"
+file_match = '(?P<Service>nova)/(?P<Program>.*)\.log'
+differentiator = ["Service", "_", "Program"]

ansible/roles/common/templates/heka.json.j2

@@ -18,6 +18,12 @@
             "dest": "/etc/heka/heka-rabbitmq.toml",
             "owner": "heka",
             "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/heka-openstack.toml",
+            "dest": "/etc/heka/heka-openstack.toml",
+            "owner": "heka",
+            "perm": "0600"
         }
     ]
 }

ansible/roles/common/templates/rsyslog.conf.j2

@@ -16,24 +16,6 @@ $template GlanceApiFile,"/var/log/glance/glance_api.log"
 $template GlanceRegistryFile,"/var/log/glance/glance_registry.log"
 :syslogtag,contains,"glance-registry" ?GlanceRegistryFile
 
-$template NovaApiFile,"/var/log/nova/nova-api.log"
-:syslogtag,contains,"nova-api" ?NovaApiFile
-
-$template NovaConductorFile,"/var/log/nova/nova-conductor.log"
-:syslogtag,contains,"nova-conductor" ?NovaConductorFile
-
-$template NovaConsoleauthFile,"/var/log/nova/nova-consoleauth.log"
-:syslogtag,contains,"nova-consoleauth" ?NovaConsoleauthFile
-
-$template NovaNovncFile,"/var/log/nova/nova-novncproxy.log"
-:syslogtag,contains,"nova-novncproxy" ?NovaNovncFile
-
-$template NovaSchedulerFile,"/var/log/nova/nova-scheduler.log"
-:syslogtag,contains,"nova-scheduler" ?NovaSchedulerFile
-
-$template NovaComputeFile,"/var/log/nova/nova-compute.log"
-:syslogtag,contains,"nova-compute" ?NovaComputeFile
-
 $template HeatAPIFile,"/var/log/heat/heat-api.log"
 :syslogtag,contains,"heat-api" ?HeatAPIFile
 

ansible/roles/nova/tasks/bootstrap_service.yml

@@ -12,7 +12,9 @@
       BOOTSTRAP:
     name: "bootstrap_nova"
     restart_policy: "never"
-    volumes: "{{ node_config_directory }}/nova-api/:{{ container_config_directory }}/:ro"
+    volumes:
+      - "{{ node_config_directory }}/nova-api/:{{ container_config_directory }}/:ro"
+      - "kolla_logs:/var/log/kolla/"
   run_once: True
   delegate_to: "{{ groups['nova-api'][0] }}"
 
@@ -41,6 +43,7 @@
     restart_policy: "never"
     volumes:
       - "{{ node_config_directory }}/nova-compute/:{{ container_config_directory }}/:ro"
+      - "kolla_logs:/var/log/kolla/"
       - "nova_compute:/var/lib/nova/"
   when:
     - inventory_hostname in groups['compute']

ansible/roles/nova/tasks/start_compute.yml

@@ -27,9 +27,9 @@
       - "{{ node_config_directory }}/nova-compute/:{{ container_config_directory }}/:ro"
       - "/lib/modules:/lib/modules:ro"
       - "/run:/run"
+      - "kolla_logs:/var/log/kolla/"
       - "nova_compute:/var/lib/nova/"
       - "nova_libvirt:/var/lib/libvirt"
-      - "rsyslog_socket:/var/lib/kolla/rsyslog/"
   when:
     - inventory_hostname in groups['compute']
     - not enable_nova_fake | bool
@@ -42,7 +42,7 @@
     name: "nova_compute_ironic"
     volumes:
       - "{{ node_config_directory }}/nova-compute-ironic/:{{ container_config_directory }}/:ro"
-      - "rsyslog_socket:/var/lib/kolla/rsyslog/"
+      - "kolla_logs:/var/log/kolla/"
   when:
     - inventory_hostname in groups['nova-compute-ironic']
     - enable_ironic | bool
@@ -58,7 +58,7 @@
       - "{{ node_config_directory }}/nova-compute-fake-{{ item }}/:{{ container_config_directory }}/:ro"
       - "/lib/modules:/lib/modules:ro"
       - "/run:/run"
-      - "rsyslog_socket:/var/lib/kolla/rsyslog/"
+      - "kolla_logs:/var/log/kolla/"
   with_sequence: start=1 end={{ num_nova_fake_per_node }}
   when:
     - inventory_hostname in groups['compute']

ansible/roles/nova/tasks/start_conductors.yml

@@ -7,5 +7,5 @@
     image: "{{ nova_conductor_image_full }}"
     volumes:
       - "{{ node_config_directory }}/nova-conductor/:{{ container_config_directory }}/:ro"
-      - "rsyslog_socket:/var/lib/kolla/rsyslog/"
+      - "kolla_logs:/var/log/kolla/"
   when: inventory_hostname in groups['nova-conductor']

ansible/roles/nova/tasks/start_controllers.yml

@@ -9,7 +9,7 @@
     volumes:
       - "{{ node_config_directory }}/nova-api/:{{ container_config_directory }}/:ro"
       - "/lib/modules:/lib/modules:ro"
-      - "rsyslog_socket:/var/lib/kolla/rsyslog/"
+      - "kolla_logs:/var/log/kolla/"
   when: inventory_hostname in groups['nova-api']
 
 - name: Starting nova-consoleauth container
@@ -20,7 +20,7 @@
     name: "nova_consoleauth"
     volumes:
       - "{{ node_config_directory }}/nova-consoleauth/:{{ container_config_directory }}/:ro"
-      - "rsyslog_socket:/var/lib/kolla/rsyslog/"
+      - "kolla_logs:/var/log/kolla/"
   when: inventory_hostname in groups['nova-consoleauth']
 
 - name: Starting nova-novncproxy container
@@ -31,7 +31,7 @@
     name: "nova_novncproxy"
     volumes:
       - "{{ node_config_directory }}/nova-novncproxy/:{{ container_config_directory }}/:ro"
-      - "rsyslog_socket:/var/lib/kolla/rsyslog/"
+      - "kolla_logs:/var/log/kolla/"
   when:
     - inventory_hostname in groups['nova-novncproxy']
     - nova_console == 'novnc'
@@ -44,7 +44,7 @@
     name: "nova_scheduler"
     volumes:
       - "{{ node_config_directory }}/nova-scheduler/:{{ container_config_directory }}/:ro"
-      - "rsyslog_socket:/var/lib/kolla/rsyslog/"
+      - "kolla_logs:/var/log/kolla/"
   when: inventory_hostname in groups['nova-scheduler']
 
 - name: Starting nova-spicehtml5proxy container
@@ -55,7 +55,7 @@
     name: "nova_spicehtml5proxy"
     volumes:
       - "{{ node_config_directory }}/nova-spicehtml5proxy/:{{ container_config_directory }}/:ro"
-      - "rsyslog_socket:/var/lib/kolla/rsyslog/"
+      - "kolla_logs:/var/log/kolla/"
   when:
     - inventory_hostname in groups['nova-spicehtml5proxy']
     - nova_console == 'spice'

ansible/roles/nova/templates/nova.conf.j2

@@ -2,8 +2,7 @@
 [DEFAULT]
 debug = {{ nova_logging_debug }}
 
-syslog_log_facility=LOG_LOCAL0
-use_syslog=yes
+log_dir = /var/log/kolla/nova
 
 api_paste_config = /etc/nova/api-paste.ini
 state_path = /var/lib/nova

docker/nova/nova-api/extend_start.sh

@@ -1,5 +1,12 @@
 #!/bin/bash
 
+if [[ ! -d "/var/log/kolla/nova" ]]; then
+    mkdir -p /var/log/kolla/nova
+fi
+if [[ $(stat -c %a /var/log/kolla/nova) != "755" ]]; then
+    chmod 755 /var/log/kolla/nova
+fi
+
 # Bootstrap and exit if KOLLA_BOOTSTRAP variable is set. This catches all cases
 # of the KOLLA_BOOTSTRAP variable being set, including empty.
 if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then

docker/nova/nova-base/Dockerfile.j2

@@ -44,9 +44,9 @@ ADD nova-base-archive /nova-base-source
 RUN ln -s nova-base-source/* nova \
     && useradd --user-group nova \
     && /var/lib/kolla/venv/bin/pip --no-cache-dir install --upgrade -c requirements/upper-constraints.txt /nova \
-    && mkdir -p /etc/nova /var/log/nova /home/nova /var/lib/nova \
+    && mkdir -p /etc/nova /home/nova /var/lib/nova \
     && cp -r /nova/etc/nova/* /etc/nova/ \
-    && chown -R nova: /etc/nova /var/log/nova /home/nova /var/lib/nova \
+    && chown -R nova: /etc/nova /home/nova /var/lib/nova \
     && sed -i 's|^exec_dirs.*|exec_dirs=/var/lib/kolla/venv/bin,/sbin,/usr/sbin,/bin,/usr/bin,/usr/local/bin,/usr/local/sbin|g' /etc/nova/rootwrap.conf
 
 COPY nova_sudoers /etc/sudoers.d/nova_sudoers
@@ -55,4 +55,7 @@ RUN chmod 750 /etc/sudoers.d \
 
 {% endif %}
 
+COPY extend_start.sh /usr/local/bin/kolla_extend_start
+RUN chmod 755 /usr/local/bin/kolla_extend_start
+
 RUN usermod -a -G kolla nova

docker/nova/nova-base/extend_start.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+if [[ ! -d "/var/log/kolla/nova" ]]; then
+    mkdir -p /var/log/kolla/nova
+fi
+if [[ $(stat -c %a /var/log/kolla/nova) != "755" ]]; then
+    chmod 755 /var/log/kolla/nova
+fi

docker/nova/nova-compute/extend_start.sh

@@ -1,5 +1,13 @@
 #!/bin/bash
 
+
+if [[ ! -d "/var/log/kolla/nova" ]]; then
+    mkdir -p /var/log/kolla/nova
+fi
+if [[ $(stat -c %a /var/log/kolla/nova) != "755" ]]; then
+    chmod 755 /var/log/kolla/nova
+fi
+
 # Bootstrap and exit if KOLLA_BOOTSTRAP variable is set. This catches all cases
 # of the KOLLA_BOOTSTRAP variable being set, including empty.
 if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then

docker/nova/nova-libvirt/extend_start.sh

@@ -7,3 +7,10 @@ if [[ -c /dev/kvm ]]; then
     chmod 660 /dev/kvm
     chown root:kvm /dev/kvm
 fi
+
+if [[ ! -d "/var/log/kolla/nova" ]]; then
+    mkdir -p /var/log/kolla/nova
+fi
+if [[ $(stat -c %a /var/log/kolla/nova) != "755" ]]; then
+    chmod 755 /var/log/kolla/nova
+fi