Merge "Add support for FWaaS v2 logging"

This commit is contained in:
Zuul 2019-06-25 18:37:42 +00:00 committed by Gerrit Code Review
commit bc3e150f88
19 changed files with 385 additions and 16 deletions

View File

@ -312,3 +312,29 @@ options:
description: |
YAML-formatted associative array of sysctl key/value pairs to be set
persistently e.g. '{ kernel.pid_max : 4194303 }'.
firewall-group-log-output-base:
type: string
default:
description: |
This option allows setting a path for Firewall Group logs.
A valid file system path must be provided. If this option is not
provided Neutron will use syslog as a destination.
(Available from Stein)
firewall-group-log-rate-limit:
type: int
default:
description: |
Log entries are queued for writing to a log file when a packet rate
exceeds the limit set by this option.
Possible values: null (no rate limitation), integer values greater than 100.
WARNING: Should be NOT LESS than 100, if set (if null logging will not be
rate limited).
(Available from Stein)
firewall-group-log-burst-limit:
type: int
default: 25
description: |
This option sets the maximum queue size for log entries.
Can be used to avoid excessive memory consumption.
WARNING: Should be NOT LESS than 25.
(Available from Stein)

View File

@ -294,8 +294,10 @@ class OpenStackAmuletDeployment(AmuletDeployment):
('bionic', None): self.bionic_queens,
('bionic', 'cloud:bionic-rocky'): self.bionic_rocky,
('bionic', 'cloud:bionic-stein'): self.bionic_stein,
('bionic', 'cloud:bionic-train'): self.bionic_train,
('cosmic', None): self.cosmic_rocky,
('disco', None): self.disco_stein,
('eoan', None): self.eoan_train,
}
return releases[(self.series, self.openstack)]
@ -313,6 +315,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
('bionic', 'queens'),
('cosmic', 'rocky'),
('disco', 'stein'),
('eoan', 'train'),
])
if self.openstack:
os_origin = self.openstack.split(':')[1]

View File

@ -54,11 +54,15 @@ NOVA_CLIENT_VERSION = "2"
OPENSTACK_RELEASES_PAIRS = [
'trusty_icehouse', 'trusty_kilo', 'trusty_liberty',
'trusty_mitaka', 'xenial_mitaka', 'xenial_newton',
'yakkety_newton', 'xenial_ocata', 'zesty_ocata',
'xenial_pike', 'artful_pike', 'xenial_queens',
'bionic_queens', 'bionic_rocky', 'cosmic_rocky',
'bionic_stein', 'disco_stein']
'trusty_mitaka', 'xenial_mitaka',
'xenial_newton', 'yakkety_newton',
'xenial_ocata', 'zesty_ocata',
'xenial_pike', 'artful_pike',
'xenial_queens', 'bionic_queens',
'bionic_rocky', 'cosmic_rocky',
'bionic_stein', 'disco_stein',
'bionic_train', 'eoan_train',
]
class OpenStackAmuletUtils(AmuletUtils):

View File

@ -521,6 +521,86 @@ class IdentityCredentialsContext(IdentityServiceContext):
return {}
class NovaVendorMetadataContext(OSContextGenerator):
"""Context used for configuring nova vendor metadata on nova.conf file."""
def __init__(self, os_release_pkg, interfaces=None):
"""Initialize the NovaVendorMetadataContext object.
:param os_release_pkg: the package name to extract the OpenStack
release codename from.
:type os_release_pkg: str
:param interfaces: list of string values to be used as the Context's
relation interfaces.
:type interfaces: List[str]
"""
self.os_release_pkg = os_release_pkg
if interfaces is not None:
self.interfaces = interfaces
def __call__(self):
cmp_os_release = CompareOpenStackReleases(
os_release(self.os_release_pkg))
ctxt = {'vendor_data': False}
vdata_providers = []
vdata = config('vendor-data')
vdata_url = config('vendor-data-url')
if vdata:
try:
# validate the JSON. If invalid, we do not set anything here
json.loads(vdata)
except (TypeError, ValueError) as e:
log('Error decoding vendor-data. {}'.format(e), level=ERROR)
else:
ctxt['vendor_data'] = True
# Mitaka does not support DynamicJSON
# so vendordata_providers is not needed
if cmp_os_release > 'mitaka':
vdata_providers.append('StaticJSON')
if vdata_url:
if cmp_os_release > 'mitaka':
ctxt['vendor_data_url'] = vdata_url
vdata_providers.append('DynamicJSON')
else:
log('Dynamic vendor data unsupported'
' for {}.'.format(cmp_os_release), level=ERROR)
if vdata_providers:
ctxt['vendordata_providers'] = ','.join(vdata_providers)
return ctxt
class NovaVendorMetadataJSONContext(OSContextGenerator):
"""Context used for writing nova vendor metadata json file."""
def __init__(self, os_release_pkg):
"""Initialize the NovaVendorMetadataJSONContext object.
:param os_release_pkg: the package name to extract the OpenStack
release codename from.
:type os_release_pkg: str
"""
self.os_release_pkg = os_release_pkg
def __call__(self):
ctxt = {'vendor_data_json': '{}'}
vdata = config('vendor-data')
if vdata:
try:
# validate the JSON. If invalid, we return empty.
json.loads(vdata)
except (TypeError, ValueError) as e:
log('Error decoding vendor-data. {}'.format(e), level=ERROR)
else:
ctxt['vendor_data_json'] = vdata
return ctxt
class AMQPContext(OSContextGenerator):
def __init__(self, ssl_dir=None, rel_name='amqp', relation_prefix=None,
@ -647,6 +727,10 @@ class AMQPContext(OSContextGenerator):
if notification_format:
ctxt['notification_format'] = notification_format
send_notifications_to_logs = conf.get('send-notifications-to-logs', None)
if send_notifications_to_logs:
ctxt['send_notifications_to_logs'] = send_notifications_to_logs
if not self.complete:
return {}
@ -698,6 +782,25 @@ class CephContext(OSContextGenerator):
ensure_packages(['ceph-common'])
return ctxt
def context_complete(self, ctxt):
"""Overridden here to ensure the context is actually complete.
We set `key` and `auth` to None here, by default, to ensure
that the context will always evaluate to incomplete until the
Ceph relation has actually sent these details; otherwise,
there is a potential race condition between the relation
appearing and the first unit actually setting this data on the
relation.
:param ctxt: The current context members
:type ctxt: Dict[str, ANY]
:returns: True if the context is complete
:rtype: bool
"""
if 'auth' not in ctxt or 'key' not in ctxt:
return False
return super(CephContext, self).context_complete(ctxt)
class HAProxyContext(OSContextGenerator):
"""Provides half a context for the haproxy template, which describes
@ -1607,6 +1710,10 @@ class NeutronAPIContext(OSContextGenerator):
'rel_key': 'enable-nsg-logging',
'default': False,
},
'enable_nfg_logging': {
'rel_key': 'enable-nfg-logging',
'default': False,
},
'global_physnet_mtu': {
'rel_key': 'global-physnet-mtu',
'default': 1500,

View File

@ -217,6 +217,11 @@ def neutron_plugins():
plugins['nsx']['config'] = '/etc/neutron/nsx.ini'
plugins['vsp']['driver'] = (
'nuage_neutron.plugins.nuage.plugin.NuagePlugin')
if CompareOpenStackReleases(release) >= 'newton':
plugins['vsp']['config'] = '/etc/neutron/plugins/ml2/ml2_conf.ini'
plugins['vsp']['driver'] = 'neutron.plugins.ml2.plugin.Ml2Plugin'
plugins['vsp']['server_packages'] = ['neutron-server',
'neutron-plugin-ml2']
return plugins

View File

@ -2,6 +2,9 @@
[oslo_messaging_notifications]
driver = {{ oslo_messaging_driver }}
transport_url = {{ transport_url }}
{% if send_notifications_to_logs %}
driver = log
{% endif %}
{% if notification_topics -%}
topics = {{ notification_topics }}
{% endif -%}

View File

@ -0,0 +1 @@
{{ vendor_data_json }}

View File

@ -120,6 +120,7 @@ OPENSTACK_RELEASES = (
'queens',
'rocky',
'stein',
'train',
)
UBUNTU_OPENSTACK_RELEASE = OrderedDict([
@ -139,6 +140,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
('bionic', 'queens'),
('cosmic', 'rocky'),
('disco', 'stein'),
('eoan', 'train'),
])
@ -159,6 +161,7 @@ OPENSTACK_CODENAMES = OrderedDict([
('2018.1', 'queens'),
('2018.2', 'rocky'),
('2019.1', 'stein'),
('2019.2', 'train'),
])
# The ugly duckling - must list releases oldest to newest
@ -195,6 +198,8 @@ SWIFT_CODENAMES = OrderedDict([
['2.18.0', '2.19.0']),
('stein',
['2.20.0', '2.21.0']),
('train',
['2.22.0']),
])
# >= Liberty version->codename mapping
@ -208,6 +213,7 @@ PACKAGE_CODENAMES = {
('17', 'queens'),
('18', 'rocky'),
('19', 'stein'),
('20', 'train'),
]),
'neutron-common': OrderedDict([
('7', 'liberty'),
@ -218,6 +224,7 @@ PACKAGE_CODENAMES = {
('12', 'queens'),
('13', 'rocky'),
('14', 'stein'),
('15', 'train'),
]),
'cinder-common': OrderedDict([
('7', 'liberty'),
@ -228,6 +235,7 @@ PACKAGE_CODENAMES = {
('12', 'queens'),
('13', 'rocky'),
('14', 'stein'),
('15', 'train'),
]),
'keystone': OrderedDict([
('8', 'liberty'),
@ -238,6 +246,7 @@ PACKAGE_CODENAMES = {
('13', 'queens'),
('14', 'rocky'),
('15', 'stein'),
('16', 'train'),
]),
'horizon-common': OrderedDict([
('8', 'liberty'),
@ -248,6 +257,7 @@ PACKAGE_CODENAMES = {
('13', 'queens'),
('14', 'rocky'),
('15', 'stein'),
('16', 'train'),
]),
'ceilometer-common': OrderedDict([
('5', 'liberty'),
@ -258,6 +268,7 @@ PACKAGE_CODENAMES = {
('10', 'queens'),
('11', 'rocky'),
('12', 'stein'),
('13', 'train'),
]),
'heat-common': OrderedDict([
('5', 'liberty'),
@ -268,6 +279,7 @@ PACKAGE_CODENAMES = {
('10', 'queens'),
('11', 'rocky'),
('12', 'stein'),
('13', 'train'),
]),
'glance-common': OrderedDict([
('11', 'liberty'),
@ -278,6 +290,7 @@ PACKAGE_CODENAMES = {
('16', 'queens'),
('17', 'rocky'),
('18', 'stein'),
('19', 'train'),
]),
'openstack-dashboard': OrderedDict([
('8', 'liberty'),
@ -288,6 +301,7 @@ PACKAGE_CODENAMES = {
('13', 'queens'),
('14', 'rocky'),
('15', 'stein'),
('16', 'train'),
]),
}

View File

@ -1488,7 +1488,7 @@ def is_broker_action_done(action, rid=None, unit=None):
@param action: name of action to be performed
@returns True if action complete otherwise False
"""
rdata = relation_get(rid, unit) or {}
rdata = relation_get(rid=rid, unit=unit) or {}
broker_rsp = rdata.get(get_broker_rsp_key())
if not broker_rsp:
return False
@ -1510,7 +1510,7 @@ def mark_broker_action_done(action, rid=None, unit=None):
@param action: name of action to be performed
@returns None
"""
rdata = relation_get(rid, unit) or {}
rdata = relation_get(rid=rid, unit=unit) or {}
broker_rsp = rdata.get(get_broker_rsp_key())
if not broker_rsp:
return

View File

@ -173,6 +173,14 @@ CLOUD_ARCHIVE_POCKETS = {
'stein/proposed': 'bionic-proposed/stein',
'bionic-stein/proposed': 'bionic-proposed/stein',
'bionic-proposed/stein': 'bionic-proposed/stein',
# Train
'train': 'bionic-updates/train',
'bionic-train': 'bionic-updates/train',
'bionic-train/updates': 'bionic-updates/train',
'bionic-updates/train': 'bionic-updates/train',
'train/proposed': 'bionic-proposed/train',
'bionic-train/proposed': 'bionic-proposed/train',
'bionic-proposed/train': 'bionic-proposed/train',
}
@ -522,14 +530,16 @@ def add_source(source, key=None, fail_invalid=False):
for r, fn in six.iteritems(_mapping):
m = re.match(r, source)
if m:
# call the assoicated function with the captured groups
# raises SourceConfigError on error.
fn(*m.groups())
if key:
# Import key before adding the source which depends on it,
# as refreshing packages could fail otherwise.
try:
import_key(key)
except GPGKeyError as e:
raise SourceConfigError(str(e))
# call the associated function with the captured groups
# raises SourceConfigError on error.
fn(*m.groups())
break
else:
# nothing matched. log an error and maybe sys.exit

View File

@ -47,6 +47,9 @@ IPTABLES_HYBRID = 'iptables_hybrid'
OPENVSWITCH = 'openvswitch'
VALID_FIREWALL_DRIVERS = (IPTABLES_HYBRID, OPENVSWITCH)
NFG_LOG_RATE_LIMIT_MIN = 100
NFG_LOG_BURST_LIMIT_MIN = 25
def _get_firewall_driver(ovs_ctxt):
'''
@ -103,6 +106,33 @@ def get_nsg_log_path(desired_nsg_log_path):
return desired_nsg_log_path
def validate_nfg_log_path(desired_nfg_log_path):
if not desired_nfg_log_path:
# None means "we need to use syslog" - no need
# to check anything on filesystem
return None
dst_dir, _ = os.path.split(desired_nfg_log_path)
path_exists = os.path.exists(dst_dir)
if not path_exists:
log(
"Desired NFG log directory {} not exists! "
"falling back to syslog".format(dst_dir),
"WARN"
)
return None
if path_exists and os.path.isdir(desired_nfg_log_path):
log(
"Desired NFG log path {} should be file, not directory! "
"falling back to syslog".format(desired_nfg_log_path),
"WARN"
)
return None
return desired_nfg_log_path
class OVSPluginContext(context.NeutronContext):
interfaces = []
@ -278,6 +308,30 @@ class L3AgentContext(OSContextGenerator):
else:
ctxt['agent_mode'] = 'legacy'
ctxt['enable_nfg_logging'] = (
neutron_api_settings['enable_nfg_logging']
)
ctxt['nfg_log_output_base'] = validate_nfg_log_path(
config('firewall-group-log-output-base')
)
ctxt['nfg_log_rate_limit'] = config(
'firewall-group-log-rate-limit'
)
if ctxt['nfg_log_rate_limit'] is not None:
ctxt['nfg_log_rate_limit'] = max(
ctxt['nfg_log_rate_limit'],
NFG_LOG_RATE_LIMIT_MIN
)
ctxt['nfg_log_burst_limit'] = config(
'firewall-group-log-burst-limit'
)
if ctxt['nfg_log_burst_limit'] is not None:
ctxt['nfg_log_burst_limit'] = max(
ctxt['nfg_log_burst_limit'],
NFG_LOG_BURST_LIMIT_MIN
)
return ctxt

View File

@ -106,7 +106,10 @@ ML2_CONF = '%s/plugins/ml2/ml2_conf.ini' % NEUTRON_CONF_DIR
OVS_CONF = '%s/plugins/ml2/openvswitch_agent.ini' % NEUTRON_CONF_DIR
EXT_PORT_CONF = '/etc/init/ext-port.conf'
NEUTRON_METADATA_AGENT_CONF = "/etc/neutron/metadata_agent.ini"
DVR_PACKAGES = ['neutron-l3-agent']
DVR_PACKAGES = [
'neutron-l3-agent',
'libnetfilter-log1',
]
DHCP_PACKAGES = ['neutron-dhcp-agent']
# haproxy is required for isolated provider networks
# ns-metadata-proxy LP#1831935
@ -116,6 +119,7 @@ L3HA_PACKAGES = ['keepalived']
PY3_PACKAGES = [
'python3-neutron',
'python3-zmq', # fwaas_v2_log
]
PURGE_PACKAGES = [

View File

@ -0,0 +1,9 @@
# newton
###############################################################################
# [ WARNING ]
# Configuration file maintained by Juju. Local changes may be overwritten.
###############################################################################
[fwaas]
agent_version = v1
driver = iptables
enabled = True

View File

@ -0,0 +1,9 @@
# stein
###############################################################################
# [ WARNING ]
# Configuration file maintained by Juju. Local changes may be overwritten.
###############################################################################
[fwaas]
agent_version = v2
driver = iptables_v2
enabled = True

View File

@ -0,0 +1,29 @@
# stein
###############################################################################
# [ WARNING ]
# Configuration file maintained by Juju. Local changes may be overwritten.
# {{ restart_trigger_l3agent }}
###############################################################################
[DEFAULT]
interface_driver = openvswitch
agent_mode = {{ agent_mode }}
{% if external_configuration_new -%}
gateway_external_network_id =
external_network_bridge =
{% endif %}
[AGENT]
{% if enable_nfg_logging -%}
extensions = fwaas_v2,fwaas_v2_log
[network_log]
{% if nfg_log_rate_limit -%}
rate_limit = {{ nfg_log_rate_limit }}
{% endif -%}
burst_limit = {{ nfg_log_burst_limit }}
{% if nfg_log_output_base -%}
local_output_log_base = {{ nfg_log_output_base }}
{% endif -%}
{% else %}
extensions = fwaas_v2
{% endif -%}

View File

@ -435,8 +435,80 @@ class L3AgentContextTest(CharmTestCase):
}
_rget.side_effect = lambda *args, **kwargs: rdata
self.assertEqual(
context.L3AgentContext()(), {'agent_mode': 'dvr',
'external_configuration_new': True}
context.L3AgentContext()(), {
'agent_mode': 'dvr',
'external_configuration_new': True,
'enable_nfg_logging': False,
'nfg_log_burst_limit': 25,
'nfg_log_output_base': None,
'nfg_log_rate_limit': None,
}
)
@patch.object(context, 'validate_nfg_log_path')
@patch.object(charmhelpers.contrib.openstack.context, 'relation_get')
@patch.object(charmhelpers.contrib.openstack.context, 'relation_ids')
@patch.object(charmhelpers.contrib.openstack.context, 'related_units')
def test_dvr_nfg_enabled(self, _runits, _rids, _rget,
_validate_nfg_log_path):
_runits.return_value = ['unit1']
_rids.return_value = ['rid2']
rdata = {
'neutron-security-groups': 'True',
'enable-dvr': 'True',
'l2-population': 'True',
'overlay-network-type': 'vxlan',
'network-device-mtu': 1500,
'enable-nfg-logging': 'True',
}
_rget.side_effect = lambda *args, **kwargs: rdata
_validate_nfg_log_path.side_effect = lambda x: x
self.test_config.set('firewall-group-log-output-base',
'/var/log/neutron/firewall.log')
self.test_config.set('firewall-group-log-rate-limit', 200)
self.test_config.set('firewall-group-log-burst-limit', 30)
self.assertEqual(
context.L3AgentContext()(), {
'agent_mode': 'dvr',
'external_configuration_new': True,
'enable_nfg_logging': True,
'nfg_log_burst_limit': 30,
'nfg_log_output_base': '/var/log/neutron/firewall.log',
'nfg_log_rate_limit': 200,
}
)
@patch.object(context, 'validate_nfg_log_path')
@patch.object(charmhelpers.contrib.openstack.context, 'relation_get')
@patch.object(charmhelpers.contrib.openstack.context, 'relation_ids')
@patch.object(charmhelpers.contrib.openstack.context, 'related_units')
def test_dvr_nfg_enabled_mins(self, _runits, _rids, _rget,
_validate_nfg_log_path):
_runits.return_value = ['unit1']
_rids.return_value = ['rid2']
rdata = {
'neutron-security-groups': 'True',
'enable-dvr': 'True',
'l2-population': 'True',
'overlay-network-type': 'vxlan',
'network-device-mtu': 1500,
'enable-nfg-logging': 'True',
}
_rget.side_effect = lambda *args, **kwargs: rdata
_validate_nfg_log_path.side_effect = lambda x: x
self.test_config.set('firewall-group-log-output-base',
'/var/log/neutron/firewall.log')
self.test_config.set('firewall-group-log-rate-limit', 90)
self.test_config.set('firewall-group-log-burst-limit', 20)
self.assertEqual(
context.L3AgentContext()(), {
'agent_mode': 'dvr',
'external_configuration_new': True,
'enable_nfg_logging': True,
'nfg_log_burst_limit': 25,
'nfg_log_output_base': '/var/log/neutron/firewall.log',
'nfg_log_rate_limit': 100,
}
)
@patch.object(charmhelpers.contrib.openstack.context, 'relation_get')
@ -455,8 +527,14 @@ class L3AgentContextTest(CharmTestCase):
}
_rget.side_effect = lambda *args, **kwargs: rdata
self.assertEqual(
context.L3AgentContext()(), {'agent_mode': 'dvr_snat',
'external_configuration_new': True}
context.L3AgentContext()(), {
'agent_mode': 'dvr_snat',
'external_configuration_new': True,
'enable_nfg_logging': False,
'nfg_log_burst_limit': 25,
'nfg_log_output_base': None,
'nfg_log_rate_limit': None,
}
)
@patch.object(charmhelpers.contrib.openstack.context, 'relation_get')
@ -473,7 +551,13 @@ class L3AgentContextTest(CharmTestCase):
'network-device-mtu': 1500,
}
_rget.side_effect = lambda *args, **kwargs: rdata
self.assertEqual(context.L3AgentContext()(), {'agent_mode': 'legacy'})
self.assertEqual(context.L3AgentContext()(), {
'agent_mode': 'legacy',
'enable_nfg_logging': False,
'nfg_log_burst_limit': 25,
'nfg_log_output_base': None,
'nfg_log_rate_limit': None,
})
class SharedSecretContext(CharmTestCase):

View File

@ -169,6 +169,7 @@ class NeutronOVSHooksTests(CharmTestCase):
self.assertTrue(self.CONFIGS.write_all.called)
_plugin_joined.assert_called_with(relation_id='rid')
self.purge_packages.assert_called_with(['neutron-l3-agent',
'libnetfilter-log1',
'keepalived'])
def test_neutron_plugin_joined_dvr_dhcp(self):

View File

@ -211,6 +211,7 @@ class TestNeutronOVSUtils(CharmTestCase):
head_pkg,
'neutron-plugin-openvswitch-agent',
'neutron-l3-agent',
'libnetfilter-log1',
]
self.assertEqual(pkg_list, expect)
@ -229,8 +230,10 @@ class TestNeutronOVSUtils(CharmTestCase):
expect = [
head_pkg,
'neutron-l3-agent',
'libnetfilter-log1',
'neutron-openvswitch-agent',
'python3-neutron',
'python3-zmq',
'python3-neutron-fwaas',
]
self.assertEqual(pkg_list, expect)
@ -251,6 +254,7 @@ class TestNeutronOVSUtils(CharmTestCase):
expect = [
head_pkg,
'neutron-l3-agent',
'libnetfilter-log1',
'keepalived',
'neutron-openvswitch-agent',
]
@ -272,6 +276,7 @@ class TestNeutronOVSUtils(CharmTestCase):
expect = [
head_pkg,
'neutron-l3-agent',
'libnetfilter-log1',
'neutron-openvswitch-agent',
]
self.assertEqual(pkg_list, expect)
@ -292,6 +297,7 @@ class TestNeutronOVSUtils(CharmTestCase):
expect = [
head_pkg,
'neutron-l3-agent',
'libnetfilter-log1',
'neutron-openvswitch-agent',
]
self.assertEqual(pkg_list, expect)