openstack/octavia
Code Issues Proposed changes

Amphora logging

Browse Source 
Configure rsyslog to forward logs to a target host

Co-Authored-By: Michael Johnson <johnsomor@gmail.com>
Story: 1665069
Task: 33646

Change-Id: I00703f86555cbb574b943794b14a36fbc644f1b2
This commit is contained in:
German Eichberger 2018-12-12 15:55:35 -08:00 committed by Michael Johnson
parent 80ddbaeef4
commit 686303e79d
28 changed files with 668 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
devstack/etc/rsyslog/10-octavia-log-offloading.conf Normal file
View File

@ -0,0 +1,11 @@
# provides UDP syslog reception
module(load="imudp")
input(type="imudp" port=["%ADMIN_PORT%", "%TENANT_PORT%"])
if ($inputname == "imudp" and $syslogfacility-text == "local0" and $syslogseverity-text == "info") then {
action(type="omfile" FileCreateMode="0644" File="/var/log/octavia-tenant-traffic.log")&stop
}
if ($inputname == "imudp" and $syslogfacility-text != "local0") then {
action(type="omfile" FileCreateMode="0644" File="/var/log/octavia-amphora.log")&stop
}

1
devstack/files/debs/octavia
View File

@ -1,2 +1,3 @@
golang
debootstrap
rsyslog

1
devstack/files/rpms-suse/octavia
View File

@ -1,3 +1,4 @@
debootstrap
dpkg
go
rsyslog

1
devstack/files/rpms/octavia
View File

@ -1,2 +1,3 @@
golang
debootstrap
rsyslog

38
devstack/plugin.sh
View File

@ -386,7 +386,15 @@ function create_mgmt_network_interface {
die "Unknown network controller. Please define octavia_create_network_interface_device"
fi
sudo ip link set dev o-hm0 address $MGMT_PORT_MAC
sudo iptables -I INPUT -i o-hm0 -p udp --dport 5555 -j ACCEPT
sudo iptables -I INPUT -i o-hm0 -p udp --dport $OCTAVIA_HM_LISTEN_PORT -j ACCEPT
sudo iptables -I INPUT -i o-hm0 -p udp --dport $OCTAVIA_AMP_LOG_ADMIN_PORT -j ACCEPT
sudo iptables -I INPUT -i o-hm0 -p udp --dport $OCTAVIA_AMP_LOG_TENANT_PORT -j ACCEPT
if [ $IPV6_ENABLED == 'true' ] ; then
sudo ip6tables -I INPUT -i o-hm0 -p udp --dport $OCTAVIA_HM_LISTEN_PORT -j ACCEPT
sudo ip6tables -I INPUT -i o-hm0 -p udp --dport $OCTAVIA_AMP_LOG_ADMIN_PORT -j ACCEPT
sudo ip6tables -I INPUT -i o-hm0 -p udp --dport $OCTAVIA_AMP_LOG_TENANT_PORT -j ACCEPT
fi
if [ $OCTAVIA_CONTROLLER_IP_PORT_LIST == 'auto' ] ; then
iniset $OCTAVIA_CONF health_manager controller_ip_port_list $MGMT_PORT_IP:$OCTAVIA_HM_LISTEN_PORT
@ -397,6 +405,13 @@ function create_mgmt_network_interface {
iniset $OCTAVIA_CONF health_manager bind_ip $MGMT_PORT_IP
iniset $OCTAVIA_CONF health_manager bind_port $OCTAVIA_HM_LISTEN_PORT
iniset $OCTAVIA_CONF amphora_agent admin_log_targets "${MGMT_PORT_IP}:${OCTAVIA_AMP_LOG_ADMIN_PORT}"
iniset $OCTAVIA_CONF amphora_agent tenant_log_targets "${MGMT_PORT_IP}:${OCTAVIA_AMP_LOG_TENANT_PORT}"
# Setting these here as the devstack rsyslog configuration expects
# these values.
iniset $OCTAVIA_CONF amphora_agent user_log_facility 0
iniset $OCTAVIA_CONF amphora_agent administrative_log_facility 1
}
function build_mgmt_network {
@ -418,8 +433,13 @@ function build_mgmt_network {
# Create security group and rules
openstack security group create lb-health-mgr-sec-grp
openstack security group rule create --protocol udp --dst-port $OCTAVIA_HM_LISTEN_PORT lb-health-mgr-sec-grp
openstack security group rule create --protocol udp --dst-port $OCTAVIA_AMP_LOG_ADMIN_PORT lb-health-mgr-sec-grp
openstack security group rule create --protocol udp --dst-port $OCTAVIA_AMP_LOG_TENANT_PORT lb-health-mgr-sec-grp
if [ $IPV6_ENABLED == 'true' ] ; then
openstack security group rule create --protocol udp --dst-port $OCTAVIA_HM_LISTEN_PORT --ethertype IPv6 --remote-ip ::/0 lb-health-mgr-sec-grp
openstack security group rule create --protocol udp --dst-port $OCTAVIA_AMP_LOG_ADMIN_PORT --ethertype IPv6 --remote-ip ::/0 lb-health-mgr-sec-grp
openstack security group rule create --protocol udp --dst-port $OCTAVIA_AMP_LOG_TENANT_PORT --ethertype IPv6 --remote-ip ::/0 lb-health-mgr-sec-grp
fi
}
@ -454,6 +474,14 @@ function configure_octavia_api_haproxy {
}
function configure_rsyslog {
sudo cp ${OCTAVIA_DIR}/devstack/etc/rsyslog/10-octavia-log-offloading.conf /etc/rsyslog.d/
sudo sed -e "
s|%ADMIN_PORT%|${OCTAVIA_AMP_LOG_ADMIN_PORT}|g;
s|%TENANT_PORT%|${OCTAVIA_AMP_LOG_TENANT_PORT}|g;
" -i /etc/rsyslog.d/10-octavia-log-offloading.conf
}
function octavia_start {
if ! ps aux | grep -q [o]-hm0 && [ $OCTAVIA_NODE != 'api' ] ; then
@ -474,6 +502,8 @@ function octavia_start {
run_process $OCTAVIA_CONSUMER "$OCTAVIA_CONSUMER_BINARY $OCTAVIA_CONSUMER_ARGS"
run_process $OCTAVIA_HOUSEKEEPER "$OCTAVIA_HOUSEKEEPER_BINARY $OCTAVIA_HOUSEKEEPER_ARGS"
run_process $OCTAVIA_HEALTHMANAGER "$OCTAVIA_HEALTHMANAGER_BINARY $OCTAVIA_HEALTHMANAGER_ARGS"
restart_service rsyslog
}
function octavia_stop {
@ -534,6 +564,10 @@ function octavia_cleanup {
fi
sudo rm -rf $NOVA_STATE_PATH $NOVA_AUTH_CACHE_DIR
sudo rm -f /etc/rsyslog.d/10-octavia-log-offloading.conf
restart_service rsyslog
}
function add_load-balancer_roles {
@ -589,6 +623,8 @@ function octavia_init {
create_octavia_accounts
add_load-balancer_roles
configure_rsyslog
elif [ $OCTAVIA_NODE == 'api' ] ; then
create_octavia_accounts

3
devstack/settings
View File

@ -49,6 +49,9 @@ OCTAVIA_AMP_IMAGE_TAG="amphora"
OCTAVIA_AMP_CONN_TIMEOUT=${OCTAVIA_AMP_CONN_TIMEOUT:-"10"}
OCTAVIA_AMP_READ_TIMEOUT=${OCTAVIA_AMP_READ_TIMEOUT:-"120"}
OCTAVIA_AMP_LOG_ADMIN_PORT=${OCTAVIA_AMP_LOG_ADMIN_PORT:="10514"}
OCTAVIA_AMP_LOG_TENANT_PORT=${OCTAVIA_AMP_LOG_TENANT_PORT:="20514"}
OCTAVIA_HEALTH_KEY=${OCTAVIA_HEALTH_KEY:-"insecure"}
OCTAVIA_LB_TOPOLOGY=${OCTAVIA_LB_TOPOLOGY:-"SINGLE"}

1
doc/source/admin/index.rst
View File

@ -29,6 +29,7 @@ Operator Reference
:maxdepth: 1
../contributor/guides/dev-quick-start.rst
log-offloading.rst
api-audit.rst
guides/certificates.rst
../configuration/configref.rst

277
doc/source/admin/log-offloading.rst Normal file
View File

@ -0,0 +1,277 @@
..
Copyright 2019 Red Hat, Inc. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
==============================
Octavia Amphora Log Offloading
==============================
The default logging configuration will store the logs locally, on the amphora
filesystem with file rotation.
Octavia Amphorae can offload their log files via the syslog protocol to syslog
receivers via the load balancer management network (lb-mgmt-net). This allows
log aggregation of both administrative logs and also tenant traffic flow logs.
The syslog receivers can either be local to the load balancer management
network or routable via the load balancer management network.
By default any syslog receiver that supports UDP or TCP syslog protocol can
be used, however the operator also has the option to create an override
rsyslog configuration template to enable other features or protocols their
Amphora image may support.
This guide will discuss the features of :term:`Amphora` log offloading and how
to configure them.
Administrative Logs
===================
The administrative log offloading feature of the :term:`Amphora` covers all of
the system logging inside the :term:`Amphora` except for the tenant flow logs.
Tenant flow logs can be sent to and processed by the same syslog receiver used
by the administrative logs, but they are configured seperately.
All administrative log messages will be sent using the native log format
for the application sending the message.
Enabling Administrative Log Offloading
--------------------------------------
One or more syslog receiver endpoints must be configured in the Octavia
configuration file to enable administrative log offloading. The first endpoint
will be the primary endpoint to receive the syslog packets. Should the first
endpoint become unavailable, the additional endpoints listed will be tried
one at a time.
.. note::
Secondary syslog endpoints will only be used if the log_protocol is
configured for TCP. With the UDP syslog protocol, rsyslog is unable
to detect if the primary endpoint has failed.
To configure administrative log offloading, set the following setting in your
Octavia configuration file for all of the controllers and restart them:
.. code-block:: ini
[amphora_agent]
admin_log_targets = 192.0.2.1:10514, 2001:db8:1::10:10514
In this example, the primary syslog receiver will be 192.0.2.1 on port 10514.
The backup syslog receiver will be 2001:db8:1::10 on port 10514.
.. note::
Make sure your syslog receiver endpoints are accessible from the load
balancer management network and you have configured the required
security group or firewall rules to allow the traffic. These endpoints
can be routable addresses from the load balancer management network.
The load balancer related administrative logs will be sent using a
LOG_LOCAL[0-7] facility. The facility number defaults to 1, but is configurable
using the administrative_log_facility setting in the Octavia configuration
file.
To configure administrative log facility, set the following setting in your
Octavia configuration file for all of the controllers and restart them:
.. code-block:: ini
[amphora_agent]
administrative_log_facility = 1
Forwarding All Administrative Logs
----------------------------------
By default, the Amphorae will only forward load balancer related administrative
logs, such as the haproxy admin logs, keepalived, and :term:`Amphora` agent
logs.
You can optionally configure the Amphorae to send all of the administrative
logs from the :term:`Amphora`, such as the kernel, system, and security logs.
Even with this setting the tenant flow logs will not be included. You can
configure tenant flow log forwarding in the `Tenant Flow Logs`_ section.
The load balancer related administrative logs will be sent using the
LOG_LOCAL[0-7] configured using the administrative_log_facility setting. All
other administrative log messages will use their native syslog facilities.
To configure the Amphorae to forward all administrative logs, set the following
setting in your Octavia configuration file for all of the controllers and
restart them:
.. code-block:: ini
[amphora_agent]
forward_all_logs = True
Tenant Flow Logs
================
Enabling Tenant Flow Log Offloading
-----------------------------------
One or more syslog receiver endpoints must be configured in the Octavia
configuration file to enable tenant flow log offloading. The first endpoint
will be the primary endpoint to receive the syslog packets. Should the first
endpoint become unavailable, the additional endpoints listed will be tried
one at a time. The endpoints configured for tenant flow log offloading may be
the same endpoints as the administrative log offloading configuration.
.. warning::
Tenant flow logging can produce a large number of syslog messages
depending on how many connections the load balancers are receiving.
Tenant flow logging produces one log entry per connection to the
load balancer. We recommend you monitor, size, and configure your syslog
receivers appropriately based on the expected number of connections your
load balancers will be handling.
.. note::
Secondary syslog endpoints will only be used if the log_protocol is
configured for TCP. With the UDP syslog protocol, rsyslog is unable
to detect if the primary endpoint has failed.
To configure tenant flow log offloading, set the following setting in your
Octavia configuration file for all of the controllers and restart them:
.. code-block:: ini
[amphora_agent]
tenant_log_targets = 192.0.2.1:10514, 2001:db8:1::10:10514
In this example, the primary syslog receiver will be 192.0.2.1 on port 10514.
The backup syslog receiver will be 2001:db8:1::10 on port 10514.
.. note::
Make sure your syslog receiver endpoints are accessible from the load
balancer management network and you have configured the required
security group or firewall rules to allow the traffic. These endpoints
can be routable addresses from the load balancer management network.
The load balancer related tenant flow logs will be sent using a
LOG_LOCAL[0-7] facility. The facility number defaults to 0, but is configurable
using the user_log_facility setting in the Octavia configuration file.
To configure the tenant flow log facility, set the following setting in your
Octavia configuration file for all of the controllers and restart them:
.. code-block:: ini
[amphora_agent]
user_log_facility = 0
Tenant Flow Log Format
----------------------
The default tenant flow log format is:
.. code-block::
project_id loadbalancer_id listener_id client_ip client_port data_time
request_string http_status bytes_read bytes_uploaded
client_certificate_verify(0 or 1) client_certificate_distinguised_name
pool_id member_id processing_time(ms) termination_state
Any field that is unknown or not applicable to the connection will have a '-'
character in its place.
An example log entry when using rsyslog as the syslog receiver is:
.. note::
The prefix[1] in this example comes from the rsyslog receiver and is not
part of the syslog message from the amphora.
[1] "Jun 12 00:44:13 amphora-3e0239c3-5496-4215-b76c-6abbe18de573 haproxy[1644]:"
.. code-block::
Jun 12 00:44:13 amphora-3e0239c3-5496-4215-b76c-6abbe18de573 haproxy[1644]: 5408b89aa45b48c69a53dca1aaec58db fd8f23df-960b-4b12-ba62-2b1dff661ee7 261ecfc2-9e8e-4bba-9ec2-3c903459a895 172.24.4.1 41152 12/Jun/2019:00:44:13.030 "GET / HTTP/1.1" 200 76 73 - "" e37e0e04-68a3-435b-876c-cffe4f2138a4 6f2720b3-27dc-4496-9039-1aafe2fee105 4 --
Custom Tenant Flow Log Format
-----------------------------
You can optionally specify a custom log format for the tenant flow logs.
This string follows the HAProxy log format variables with the exception of
the "{{ project_id }}" and "{{ lb_id }}" variables that will be replaced
by the Octavia :term:`Amphora` driver. These custom variables are optional.
See the HAProxy documentation for `Custom log format <http://cbonte.github.io/haproxy-dconv/1.9/configuration.html#8.2.4>`_ variable definitions.
To configure a custom log format, set the following setting in your
Octavia configuration file for all of the controllers and restart them:
.. code-block:: ini
[haproxy_amphora]
user_log_format = '{{ project_id }} {{ lb_id }} %f %ci %cp %t %{+Q}r %ST %B %U %[ssl_c_verify] %{+Q}[ssl_c_s_dn] %b %s %Tt %tsc'
Disabling Logging
=================
There may be cases where you need to disable logging inside the
:term:`Amphora`, such as complying with regulatory standards.
Octavia provides multiple options for disabling :term:`Amphora` logging.
Disable Local Log Storage
-------------------------
This setting stops log entries from being written to the disk inside the
:term:`Amphora`. Logs can still be sent via :term:`Amphora` log offloading if
log offloading is configured for the Amphorae. Enabling this setting may
provide a performance benefit to the load balancer.
.. warning::
This feature disables ALL log storage in the :term:`Amphora`, including
kernel, system, and security logging.
.. note::
If you enable this setting and are not using :term:`Amphora` log
offloading, we recommend you also `Disable Tenant Flow Logging`_ to
improve load balancing performance.
To disable local log storage in the :term:`Amphora`, set the following setting
in your Octavia configuration file for all of the controllers and restart them:
.. code-block:: ini
[amphora_agent]
disable_local_log_storage = True
Disable Tenant Flow Logging
---------------------------
This setting allows you to disable tenant flow logging irrespective of the
other logging configuration settings. It will take precedent over the other
settings. When this setting is enabled, no tenant flow (connection) logs will
be written to the disk inside the :term:`Amphora` or be sent via the
:term:`Amphora` log offloading.
.. note::
Disabling tenant flow logging can also improve the load balancing
performance of the amphora. Due to the potential performance improvement,
we recommend you enable this setting when using the
`Disable Local Log Storage`_ setting.
To disable tenant flow logging, set the following setting in your Octavia
configuration file for all of the controllers and restart them:
.. code-block:: ini
[haproxy_amphora]
connection_logging = False

62
etc/octavia.conf
View File

@ -195,15 +195,7 @@
# facilitate "fail fast" scenarios like failovers
# active_connection_max_retries = 15
# active_connection_rety_interval = 2
#
# Sets the syslog LOG_LOCAL[0-7] facility number for amphora log offloading.
# user_log_facility will receive the traffic flow logs.
# administrative_log_facility will receive the amphora processes logs.
# Note: Some processes only support LOG_LOCAL, so we are restricted to the
# LOG_LOCAL facilities.
# user_log_facility = 0
# administrative_log_facility = 1
#
# The user flow log format for HAProxy.
# {{ project_id }} and {{ lb_id }} will be automatically substituted by the
# controller when configuring HAProxy if they are present in the string.
@ -310,6 +302,58 @@
#
# amphora_udp_driver = keepalived_lvs
##### Log offloading
#
# Note: The admin and tenant logs can point to the same endpoints.
#
# List of log server ip and port pairs for Administrative logs.
# Additional hosts are backup to the primary server. If none are
# specified, remote logging is disabled.
# Example 192.0.2.1:10514, 2001:db8:1::10:10514'
#
# admin_log_targets =
#
# List of log server ip and port pairs for tenant traffic logs.
# Additional hosts are backup to the primary server. If none are
# specified, remote logging is disabled.
# Example 192.0.2.1:10514, 2001:db8:2::15:10514'
#
# tenant_log_targets =
# Sets the syslog LOG_LOCAL[0-7] facility number for amphora log offloading.
# user_log_facility will receive the traffic flow logs.
# administrative_log_facility will receive the amphora processes logs.
# Note: Some processes only support LOG_LOCAL, so we are restricted to the
# LOG_LOCAL facilities.
#
# user_log_facility = 0
# administrative_log_facility = 1
# The log forwarding protocol to use. One of TCP or UDP.
# log_protocol = UDP
# The maximum attempts to retry connecting to the logging host.
# log_retry_count = 5
# The time, in seconds, to wait between retries connecting to the logging host.
# log_retry_interval = 2
# The queue size (messages) to buffer log messages.
# log_queue_size = 10000
# Controller local path to a custom logging configuration template.
# Currently this is an rsyslog configuration file template.
# logging_template_override =
# When True, the amphora will forward all of the system logs (except tenant
# traffice logs) to the admin log target(s). When False, only amphora specific
# admin logs will be forwarded.
# forward_all_logs = False
# When True, no logs will be written to the amphora filesystem. When False,
# log files will be written to the local filesystem.
# disable_local_log_storage = False
[keepalived_vrrp]
# Amphora Role/Priority advertisement interval in seconds
# vrrp_advert_int = 1

4
octavia/amphorae/backends/agent/agent_jinja_cfg.py
View File

@ -61,5 +61,5 @@ class AgentJinjaTemplater(object):
'agent_tls_protocol': CONF.amphora_agent.agent_tls_protocol,
'topology': topology,
'administrative_log_facility':
CONF.haproxy_amphora.administrative_log_facility,
'user_log_facility': CONF.haproxy_amphora.user_log_facility})
CONF.amphora_agent.administrative_log_facility,
'user_log_facility': CONF.amphora_agent.user_log_facility})

2
octavia/amphorae/backends/agent/api_server/keepalived.py
View File

@ -96,7 +96,7 @@ class Keepalived(object):
amphora_nsname=consts.AMPHORA_NAMESPACE,
amphora_netns=consts.AMP_NETNS_SVC_PREFIX,
administrative_log_facility=(
CONF.haproxy_amphora.administrative_log_facility),
CONF.amphora_agent.administrative_log_facility),
)
text_file.write(text)

2
octavia/amphorae/backends/agent/api_server/keepalivedlvs.py
View File

@ -131,7 +131,7 @@ class KeepalivedLvs(udp_listener_base.UdpListenerApiServerBase):
amphora_nsname=consts.AMPHORA_NAMESPACE,
amphora_netns=consts.AMP_NETNS_SVC_PREFIX,
administrative_log_facility=(
CONF.haproxy_amphora.administrative_log_facility),
CONF.amphora_agent.administrative_log_facility),
)
text_file.write(text)

2
octavia/cmd/agent.py
View File

@ -82,7 +82,7 @@ def main():
'loglevel': 'debug',
'syslog': True,
'syslog_facility': 'local{}'.format(
CONF.haproxy_amphora.administrative_log_facility),
CONF.amphora_agent.administrative_log_facility),
'syslog_addr': 'unix://dev/log',
}

51
octavia/common/config.py
View File

@ -22,6 +22,7 @@ import ssl
import sys
from keystoneauth1 import loading as ks_loading
from octavia_lib.common import constants as lib_consts
from oslo_config import cfg
from oslo_db import options as db_options
from oslo_log import log as logging
@ -118,6 +119,50 @@ amphora_agent_opts = [
help=_("Minimum TLS protocol for communication with the "
"amphora agent."),
choices=TLS_PROTOCOL_CHOICES),
# Logging setup
cfg.ListOpt('admin_log_targets',
help=_('List of log server ip and port pairs for '
'Administrative logs. Additional hosts are backup to '
'the primary server. If none is '
'specified remote logging is disabled. Example '
'127.0.0.1:10514, 192.168.0.1:10514')),
cfg.ListOpt('tenant_log_targets',
help=_('List of log server ip and port pairs for '
'tenant traffic logs. Additional hosts are backup to '
'the primary server. If none is '
'specified remote logging is disabled. Example '
'127.0.0.1:10514, 192.168.0.1:10514')),
cfg.IntOpt('user_log_facility', default=0, min=0, max=7,
help=_('LOG_LOCAL facility number to use for user traffic '
'logs.')),
cfg.IntOpt('administrative_log_facility', default=1, min=0, max=7,
help=_('LOG_LOCAL facility number to use for amphora processes '
'logs.')),
cfg.StrOpt('log_protocol', default=lib_consts.PROTOCOL_UDP,
choices=[lib_consts.PROTOCOL_TCP, lib_consts.PROTOCOL_UDP],
help=_("The log forwarding transport protocol. One of UDP or "
"TCP.")),
cfg.IntOpt('log_retry_count', default=5,
help=_('The maximum attempts to retry connecting to the '
'logging host.')),
cfg.IntOpt('log_retry_interval', default=2,
help=_('The time, in seconds, to wait between retries '
'connecting to the logging host.')),
cfg.IntOpt('log_queue_size', default=10000,
help=_('The queue size (messages) to buffer log messages.')),
cfg.StrOpt('logging_template_override',
help=_('Custom logging configuration template.')),
cfg.BoolOpt('forward_all_logs', default=False,
help=_('When True, the amphora will forward all of the '
'system logs (except tenant traffic logs) to the '
'admin log target(s). When False, '
'only amphora specific admin logs will be forwarded.')),
cfg.BoolOpt('disable_local_log_storage', default=False,
help=_('When True, no logs will be written to the amphora '
'filesystem. When False, log files will be written to '
'the local filesystem.')),
# Do not specify in octavia.conf, loaded at runtime
cfg.StrOpt('amphora_id', help=_("The amphora ID.")),
cfg.StrOpt('amphora_udp_driver',
@ -253,12 +298,6 @@ haproxy_amphora_opts = [
cfg.StrOpt('haproxy_stick_size', default='10k',
help=_('Size of the HAProxy stick table. Accepts k, m, g '
'suffixes. Example: 10k')),
cfg.IntOpt('user_log_facility', default=0, min=0, max=7,
help=_('LOG_LOCAL facility number to use for user traffic '
'logs.')),
cfg.IntOpt('administrative_log_facility', default=1, min=0, max=7,
help=_('LOG_LOCAL facility number to use for amphora processes '
'logs.')),
cfg.StrOpt('user_log_format',
default='{project_id} {lb_id} %f %ci %cp %t %{+Q}r %ST %B %U '
'%[ssl_c_verify] %{+Q}[ssl_c_s_dn] %b %s %Tt %tsc',

2
octavia/common/constants.py
View File

@ -483,8 +483,10 @@ AUTH_HEADER_PROTOCOL_NUMBER = 51
TEMPLATES = '/templates'
AGENT_API_TEMPLATES = '/templates'
LOGGING_TEMPLATES = '/templates'
AGENT_CONF_TEMPLATE = 'amphora_agent_conf.template'
LOGGING_CONF_TEMPLATE = '10-rsyslog.conf.template'
USER_DATA_CONFIG_DRIVE_TEMPLATE = 'user_data_config_drive.template'
OPEN = 'OPEN'

4
octavia/common/jinja/haproxy/jinja_cfg.py
View File

@ -162,8 +162,8 @@ class JinjaTemplater(object):
'log_http': self.log_http,
'log_server': self.log_server,
'administrative_log_facility':
CONF.haproxy_amphora.administrative_log_facility,
'user_log_facility': CONF.haproxy_amphora.user_log_facility,
CONF.amphora_agent.administrative_log_facility,
'user_log_facility': CONF.amphora_agent.user_log_facility,
'connection_logging': self.connection_logging},
constants=constants)

0
octavia/common/jinja/logging/__init__.py Normal file
View File

66
octavia/common/jinja/logging/logging_jinja_cfg.py Normal file
View File

@ -0,0 +1,66 @@
# Copyright 2018 Rackspace, US Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import jinja2
from octavia.common.config import cfg
from octavia.common import constants
CONF = cfg.CONF
TEMPLATES_DIR = (os.path.dirname(os.path.realpath(__file__)) +
constants.LOGGING_TEMPLATES + '/')
class LoggingJinjaTemplater(object):
def __init__(self, logging_templates=None):
self.logging_templates = logging_templates or TEMPLATES_DIR
template_loader = jinja2.FileSystemLoader(searchpath=os.path.dirname(
self.logging_templates))
jinja_env = jinja2.Environment(loader=template_loader, autoescape=True)
self.logging_template = jinja_env.get_template(
constants.LOGGING_CONF_TEMPLATE)
def build_logging_config(self):
admin_log_hosts = []
for server in CONF.amphora_agent.admin_log_targets or []:
(host, port) = server.rsplit(':', 1)
admin_log_hosts.append({
'host': host,
'port': port,
})
tenant_log_hosts = []
for server in CONF.amphora_agent.tenant_log_targets or []:
(host, port) = server.rsplit(':', 1)
tenant_log_hosts.append({
'host': host,
'port': port,
})
return self.logging_template.render(
{'admin_log_hosts': admin_log_hosts,
'tenant_log_hosts': tenant_log_hosts,
'protocol': CONF.amphora_agent.log_protocol,
'retry_count': CONF.amphora_agent.log_retry_count,
'retry_interval': CONF.amphora_agent.log_retry_interval,
'queue_size': CONF.amphora_agent.log_queue_size,
'forward_all_logs': CONF.amphora_agent.forward_all_logs,
'disable_local_log_storage':
CONF.amphora_agent.disable_local_log_storage,
'admin_log_facility':
CONF.amphora_agent.administrative_log_facility,
'user_log_facility': CONF.amphora_agent.user_log_facility,
})

27
octavia/common/jinja/logging/templates/10-rsyslog.conf.template Normal file
View File

@ -0,0 +1,27 @@
{# Copyright 2018 Rackspace, US Inc.
# Copyright 2019 Red Hat, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#}
{#- Tenant traffic logs -#}
local{{ user_log_facility }}.=info {% for host in tenant_log_hosts %}{% if not loop.first %}&{% endif %}action(type="omfwd" target="{{ host['host'] }}" port="{{ host['port'] }}" protocol="{{ protocol }}" action.resumeRetryCount="{{ retry_count }}" action.resumeInterval="{{ retry_interval }}" queue.type="linkedList" queue.size="{{ queue_size }}" {% if not loop.first %}action.execOnlyWhenPreviousIsSuspended="on"{% endif %}){% endfor %}
{#- Administrative logs -#}
{%- if forward_all_logs %}
*.*;local{{ user_log_facility }}.none {% for host in admin_log_hosts %}{% if not loop.first %}&{% endif %}action(type="omfwd" target="{{ host['host'] }}" port="{{ host['port'] }}" protocol="{{ protocol }}" action.resumeRetryCount="{{ retry_count }}" action.resumeInterval="{{ retry_interval }}" queue.type="linkedList" queue.size="{{ queue_size }}" {% if not loop.first %}action.execOnlyWhenPreviousIsSuspended="on"{% endif %}){% endfor %}
{% else %}
local{{ admin_log_facility }}.* {% for host in admin_log_hosts %}{% if not loop.first %}&{% endif %}action(type="omfwd" target="{{ host['host'] }}" port="{{ host['port'] }}" protocol="{{ protocol }}" action.resumeRetryCount="{{ retry_count }}" action.resumeInterval="{{ retry_interval }}" queue.type="linkedList" queue.size="{{ queue_size }}" {% if not loop.first %}action.execOnlyWhenPreviousIsSuspended="on"{% endif %}){% endfor -%}
{%- endif -%}
{%- if disable_local_log_storage -%}
*.* stop
{%- endif -%}

7
octavia/controller/worker/v1/tasks/compute_tasks.py
View File

@ -25,6 +25,7 @@ from taskflow.types import failure
from octavia.amphorae.backends.agent import agent_jinja_cfg
from octavia.common import constants
from octavia.common import exceptions
from octavia.common.jinja.logging import logging_jinja_cfg
from octavia.common.jinja import user_data_jinja_cfg
from octavia.common import utils
from octavia.controller.worker import amphora_rate_limit
@ -88,6 +89,12 @@ class ComputeCreate(BaseComputeTask):
agent_cfg = agent_jinja_cfg.AgentJinjaTemplater()
config_drive_files['/etc/octavia/amphora-agent.conf'] = (
agent_cfg.build_agent_config(amphora_id, topology))
logging_cfg = logging_jinja_cfg.LoggingJinjaTemplater(
CONF.amphora_agent.logging_template_override)
config_drive_files['/etc/rsyslog.d/10-rsyslog.conf'] = (
logging_cfg.build_logging_config())
if user_data_config_drive:
udtemplater = user_data_jinja_cfg.UserDataJinjaCfg()
user_data = udtemplater.build_user_data_config(

7
octavia/controller/worker/v2/tasks/compute_tasks.py
View File

@ -25,6 +25,7 @@ from taskflow.types import failure
from octavia.amphorae.backends.agent import agent_jinja_cfg
from octavia.common import constants
from octavia.common import exceptions
from octavia.common.jinja.logging import logging_jinja_cfg
from octavia.common.jinja import user_data_jinja_cfg
from octavia.common import utils
from octavia.controller.worker import amphora_rate_limit
@ -88,6 +89,12 @@ class ComputeCreate(BaseComputeTask):
agent_cfg = agent_jinja_cfg.AgentJinjaTemplater()
config_drive_files['/etc/octavia/amphora-agent.conf'] = (
agent_cfg.build_agent_config(amphora_id, topology))
logging_cfg = logging_jinja_cfg.LoggingJinjaTemplater(
CONF.amphora_agent.logging_template_override)
config_drive_files['/etc/rsyslog.d/10-rsyslog.conf'] = (
logging_cfg.build_logging_config())
if user_data_config_drive:
udtemplater = user_data_jinja_cfg.UserDataJinjaCfg()
user_data = udtemplater.build_user_data_config(

12
octavia/tests/unit/amphorae/backends/agent/test_agent_jinja_cfg.py
View File

@ -57,9 +57,9 @@ class AgentJinjaTestCase(base.TestCase):
# Test execution order could influence this with the test below
self.conf.config(group='amphora_agent',
agent_server_network_file=None)
self.conf.config(group="haproxy_amphora",
self.conf.config(group="amphora_agent",
administrative_log_facility=1)
self.conf.config(group="haproxy_amphora", user_log_facility=0)
self.conf.config(group="amphora_agent", user_log_facility=0)
expected_config = ('\n[DEFAULT]\n'
'debug = False\n'
'use_syslog = True\n'
@ -101,9 +101,9 @@ class AgentJinjaTestCase(base.TestCase):
self.conf.config(group="amphora_agent",
agent_server_network_file='/etc/network/interfaces')
self.conf.config(group="haproxy_amphora", use_upstart='False')
self.conf.config(group="haproxy_amphora",
self.conf.config(group="amphora_agent",
administrative_log_facility=1)
self.conf.config(group="haproxy_amphora", user_log_facility=0)
self.conf.config(group="amphora_agent", user_log_facility=0)
expected_config = ('\n[DEFAULT]\n'
'debug = False\n'
'use_syslog = True\n'
@ -149,9 +149,9 @@ class AgentJinjaTestCase(base.TestCase):
agent_server_network_file=None)
self.conf.config(group="amphora_agent",
amphora_udp_driver='new_udp_driver')
self.conf.config(group="haproxy_amphora",
self.conf.config(group="amphora_agent",
administrative_log_facility=1)
self.conf.config(group="haproxy_amphora", user_log_facility=0)
self.conf.config(group="amphora_agent", user_log_facility=0)
expected_config = ('\n[DEFAULT]\n'
'debug = False\n'
'use_syslog = True\n'

0
octavia/tests/unit/common/jinja/logging/__init__.py Normal file
View File

61
octavia/tests/unit/common/jinja/logging/test_logging_jinja_cfg.py Normal file
View File

@ -0,0 +1,61 @@
# Copyright 2018 Rackspace, US Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from octavia_lib.common import constants as lib_consts
from oslo_config import cfg
from oslo_config import fixture as oslo_fixture
from octavia.common.jinja.logging import logging_jinja_cfg
import octavia.tests.unit.base as base
class LoggingJinjaTestCase(base.TestCase):
def setUp(self):
super(LoggingJinjaTestCase, self).setUp()
self.conf = oslo_fixture.Config(cfg.CONF)
self.conf.config(debug=False)
self.conf.config(
group="amphora_agent",
admin_log_targets='192.0.2.17:10514,192.51.100.4:10514')
self.conf.config(
group="amphora_agent",
tenant_log_targets='192.0.2.7:20514,192.51.100.9:20514')
self.conf.config(group="amphora_agent",
log_protocol=lib_consts.PROTOCOL_UDP)
self.conf.config(group="amphora_agent", log_retry_count=5)
self.conf.config(group="amphora_agent", log_retry_interval=2)
self.conf.config(group="amphora_agent", log_queue_size=10000)
def test_build_agent_config(self):
lj = logging_jinja_cfg.LoggingJinjaTemplater()
expected_config = (
u'local0.=info action(type="omfwd" target="192.0.2.7" '
'port="20514" protocol="UDP" action.resumeRetryCount="5" '
'action.resumeInterval="2" queue.type="linkedList" '
'queue.size="10000" )&action(type="omfwd" target="192.51.100.9" '
'port="20514" protocol="UDP" action.resumeRetryCount="5" '
'action.resumeInterval="2" queue.type="linkedList" '
'queue.size="10000" action.execOnlyWhenPreviousIsSuspended="on")\n'
'local1.* action(type="omfwd" target="192.0.2.17" port="10514" '
'protocol="UDP" action.resumeRetryCount="5" '
'action.resumeInterval="2" queue.type="linkedList" '
'queue.size="10000" )&action(type="omfwd" target="192.51.100.4" '
'port="10514" protocol="UDP" action.resumeRetryCount="5" '
'action.resumeInterval="2" queue.type="linkedList" '
'queue.size="10000" action.execOnlyWhenPreviousIsSuspended="on")')
logging_cfg = lj.build_logging_config()
self.assertEqual(expected_config, logging_cfg)

26
octavia/tests/unit/controller/worker/v1/tasks/test_compute_tasks.py
View File

@ -86,16 +86,20 @@ class TestComputeTasks(base.TestCase):
super(TestComputeTasks, self).setUp()
@mock.patch('octavia.common.jinja.logging.logging_jinja_cfg.'
'LoggingJinjaTemplater.build_logging_config')
@mock.patch('jinja2.Environment.get_template')
@mock.patch('octavia.amphorae.backends.agent.'
'agent_jinja_cfg.AgentJinjaTemplater.'
'build_agent_config', return_value='test_conf')
@mock.patch('stevedore.driver.DriverManager.driver')
def test_compute_create(self, mock_driver, mock_conf, mock_jinja):
def test_compute_create(self, mock_driver, mock_conf, mock_jinja,
mock_log_cfg):
image_owner_id = uuidutils.generate_uuid()
self.conf.config(
group="controller_worker", amp_image_owner_id=image_owner_id)
mock_log_cfg.return_value = 'FAKE CFG'
createcompute = compute_tasks.ComputeCreate()
@ -116,7 +120,8 @@ class TestComputeTasks(base.TestCase):
network_ids=AMP_NET,
port_ids=[PORT_ID],
config_drive_files={'/etc/octavia/'
'amphora-agent.conf': 'test_conf'},
'amphora-agent.conf': 'test_conf',
'/etc/rsyslog.d/10-rsyslog.conf': 'FAKE CFG'},
user_data=None,
server_group_id=SERVER_GRPOUP_ID)
@ -205,13 +210,15 @@ class TestComputeTasks(base.TestCase):
createcompute.revert(COMPUTE_ID, _amphora_mock.id)
@mock.patch('octavia.common.jinja.logging.logging_jinja_cfg.'
'LoggingJinjaTemplater.build_logging_config')
@mock.patch('jinja2.Environment.get_template')
@mock.patch('octavia.amphorae.backends.agent.'
'agent_jinja_cfg.AgentJinjaTemplater.'
'build_agent_config', return_value='test_conf')
@mock.patch('stevedore.driver.DriverManager.driver')
def test_compute_create_without_ssh_access(self, mock_driver,
mock_conf, mock_jinja):
def test_compute_create_without_ssh_access(
self, mock_driver, mock_conf, mock_jinja, mock_log_cfg):
createcompute = compute_tasks.ComputeCreate()
@ -220,6 +227,7 @@ class TestComputeTasks(base.TestCase):
group="controller_worker", amp_ssh_access_allowed=False)
self.conf.config(
group="controller_worker", user_data_config_drive=False)
mock_log_cfg.return_value = 'FAKE CFG'
# Test execute()
compute_id = createcompute.execute(_amphora_mock.id, ports=[_port],
@ -237,7 +245,8 @@ class TestComputeTasks(base.TestCase):
network_ids=AMP_NET,
port_ids=[PORT_ID],
config_drive_files={'/etc/octavia/'
'amphora-agent.conf': 'test_conf'},
'amphora-agent.conf': 'test_conf',
'/etc/rsyslog.d/10-rsyslog.conf': 'FAKE CFG'},
user_data=None,
server_group_id=SERVER_GRPOUP_ID)
@ -265,15 +274,19 @@ class TestComputeTasks(base.TestCase):
createcompute.revert(COMPUTE_ID, _amphora_mock.id)
@mock.patch('octavia.common.jinja.logging.logging_jinja_cfg.'
'LoggingJinjaTemplater.build_logging_config')
@mock.patch('jinja2.Environment.get_template')
@mock.patch('octavia.amphorae.backends.agent.'
'agent_jinja_cfg.AgentJinjaTemplater.'
'build_agent_config', return_value='test_conf')
@mock.patch('stevedore.driver.DriverManager.driver')
def test_compute_create_cert(self, mock_driver, mock_conf, mock_jinja):
def test_compute_create_cert(self, mock_driver, mock_conf, mock_jinja,
mock_log_cfg):
createcompute = compute_tasks.CertComputeCreate()
key = utils.get_six_compatible_server_certs_key_passphrase()
fer = fernet.Fernet(key)
mock_log_cfg.return_value = 'FAKE CFG'
mock_driver.build.return_value = COMPUTE_ID
path = '/etc/octavia/certs/ca_01.pem'
@ -299,6 +312,7 @@ class TestComputeTasks(base.TestCase):
port_ids=[],
user_data=None,
config_drive_files={
'/etc/rsyslog.d/10-rsyslog.conf': 'FAKE CFG',
'/etc/octavia/certs/server.pem': fer.decrypt(test_cert),
'/etc/octavia/certs/client_ca.pem': 'test',
'/etc/octavia/amphora-agent.conf': 'test_conf'},

26
octavia/tests/unit/controller/worker/v2/tasks/test_compute_tasks.py
View File

@ -86,16 +86,20 @@ class TestComputeTasks(base.TestCase):
super(TestComputeTasks, self).setUp()
@mock.patch('octavia.common.jinja.logging.logging_jinja_cfg.'
'LoggingJinjaTemplater.build_logging_config')
@mock.patch('jinja2.Environment.get_template')
@mock.patch('octavia.amphorae.backends.agent.'
'agent_jinja_cfg.AgentJinjaTemplater.'
'build_agent_config', return_value='test_conf')
@mock.patch('stevedore.driver.DriverManager.driver')
def test_compute_create(self, mock_driver, mock_conf, mock_jinja):
def test_compute_create(self, mock_driver, mock_conf, mock_jinja,
mock_log_cfg):
image_owner_id = uuidutils.generate_uuid()
self.conf.config(
group="controller_worker", amp_image_owner_id=image_owner_id)
mock_log_cfg.return_value = 'FAKE CFG'
createcompute = compute_tasks.ComputeCreate()
@ -116,7 +120,8 @@ class TestComputeTasks(base.TestCase):
network_ids=AMP_NET,
port_ids=[PORT_ID],
config_drive_files={'/etc/octavia/'
'amphora-agent.conf': 'test_conf'},
'amphora-agent.conf': 'test_conf',
'/etc/rsyslog.d/10-rsyslog.conf': 'FAKE CFG'},
user_data=None,
server_group_id=SERVER_GRPOUP_ID)
@ -205,13 +210,15 @@ class TestComputeTasks(base.TestCase):
createcompute.revert(COMPUTE_ID, _amphora_mock.id)
@mock.patch('octavia.common.jinja.logging.logging_jinja_cfg.'
'LoggingJinjaTemplater.build_logging_config')
@mock.patch('jinja2.Environment.get_template')
@mock.patch('octavia.amphorae.backends.agent.'
'agent_jinja_cfg.AgentJinjaTemplater.'
'build_agent_config', return_value='test_conf')
@mock.patch('stevedore.driver.DriverManager.driver')
def test_compute_create_without_ssh_access(self, mock_driver,
mock_conf, mock_jinja):
def test_compute_create_without_ssh_access(
self, mock_driver, mock_conf, mock_jinja, mock_log_cfg):
createcompute = compute_tasks.ComputeCreate()
@ -220,6 +227,7 @@ class TestComputeTasks(base.TestCase):
group="controller_worker", amp_ssh_access_allowed=False)
self.conf.config(
group="controller_worker", user_data_config_drive=False)
mock_log_cfg.return_value = 'FAKE CFG'
# Test execute()
compute_id = createcompute.execute(_amphora_mock.id, ports=[_port],
@ -237,7 +245,8 @@ class TestComputeTasks(base.TestCase):
network_ids=AMP_NET,
port_ids=[PORT_ID],
config_drive_files={'/etc/octavia/'
'amphora-agent.conf': 'test_conf'},
'amphora-agent.conf': 'test_conf',
'/etc/rsyslog.d/10-rsyslog.conf': 'FAKE CFG'},
user_data=None,
server_group_id=SERVER_GRPOUP_ID)
@ -265,15 +274,19 @@ class TestComputeTasks(base.TestCase):
createcompute.revert(COMPUTE_ID, _amphora_mock.id)
@mock.patch('octavia.common.jinja.logging.logging_jinja_cfg.'
'LoggingJinjaTemplater.build_logging_config')
@mock.patch('jinja2.Environment.get_template')
@mock.patch('octavia.amphorae.backends.agent.'
'agent_jinja_cfg.AgentJinjaTemplater.'
'build_agent_config', return_value='test_conf')
@mock.patch('stevedore.driver.DriverManager.driver')
def test_compute_create_cert(self, mock_driver, mock_conf, mock_jinja):
def test_compute_create_cert(self, mock_driver, mock_conf, mock_jinja,
mock_log_cfg):
createcompute = compute_tasks.CertComputeCreate()
key = utils.get_six_compatible_server_certs_key_passphrase()
fer = fernet.Fernet(key)
mock_log_cfg.return_value = 'FAKE CFG'
mock_driver.build.return_value = COMPUTE_ID
path = '/etc/octavia/certs/ca_01.pem'
@ -299,6 +312,7 @@ class TestComputeTasks(base.TestCase):
port_ids=[],
user_data=None,
config_drive_files={
'/etc/rsyslog.d/10-rsyslog.conf': 'FAKE CFG',
'/etc/octavia/certs/server.pem': fer.decrypt(test_cert),
'/etc/octavia/certs/client_ca.pem': 'test',
'/etc/octavia/amphora-agent.conf': 'test_conf'},

12
releasenotes/notes/Add-log-offloading-233cd8612c0dd2b5.yaml Normal file
View File

@ -0,0 +1,12 @@
---
features:
- |
Octavia now supports Amphora log offloading. Operators can define syslog
targets for the Amphora administrative logs and for the tenant load
balancer flow logs.
issues:
- |
Amphorae are unable to provide tenant flow logs for UDP listeners.
upgrade:
- |
To enable log offloading, the amphora image needs to be updated.

4
zuul.d/jobs.yaml
View File

@ -19,7 +19,9 @@
- ^releasenotes/.*$
vars:
zuul_copy_output:
'/var/log/dib-build': 'logs'
'/var/log/dib-build/': 'logs'
'/var/log/octavia-tenant-traffic.log': 'logs'
'/var/log/octavia-amphora.log': 'logs'
- job:
name: publish-openstack-octavia-amphora-image