Fuel 10 uplift
- generate override actions - clean up outdated manifests - align function calls Change-Id: Ie1bf8f2360f72f9de4d3f63c0a6b397f3e94909d Signed-off-by: Michal Skalski <mskalski@mirantis.com>
This commit is contained in:
parent
f5c52caac2
commit
0574e57f39
@ -17,7 +17,7 @@ Requirements
|
||||
|
||||
| Requirement | Version/Comment |
|
||||
|----------------------------------|-----------------|
|
||||
| Mirantis OpenStack compatibility | 9.0 |
|
||||
| Mirantis OpenStack compatibility | 10.0 |
|
||||
|
||||
Limitations
|
||||
-----------
|
||||
|
@ -6,181 +6,179 @@ $odl = $opendaylight::odl_settings
|
||||
$management_vip = hiera('management_vip')
|
||||
$ovsdb_managers = odl_ovsdb_managers($opendaylight::odl_mgmt_ips)
|
||||
|
||||
if $use_neutron {
|
||||
|
||||
package {'python-networking-odl': ensure => installed }
|
||||
package {'python-networking-odl':
|
||||
ensure => installed,
|
||||
}
|
||||
|
||||
exec { 'ovs-set-manager':
|
||||
command => "ovs-vsctl set-manager ${ovsdb_managers}",
|
||||
path => '/usr/bin'
|
||||
}
|
||||
exec { 'ovs-set-manager':
|
||||
command => "ovs-vsctl set-manager ${ovsdb_managers}",
|
||||
path => '/usr/bin'
|
||||
}
|
||||
|
||||
if $odl['enable_l3_odl'] or roles_include(['primary-controller', 'controller']) {
|
||||
$patch_jacks_names = get_pair_of_jack_names(['br-ex', 'br-ex-lnx'])
|
||||
$ext_interface = $patch_jacks_names[0]
|
||||
}
|
||||
if $odl['enable_l3_odl'] or roles_include(['primary-controller', 'controller']) {
|
||||
$patch_jacks_names = get_pair_of_jack_names(['br-ex', 'br-ex-lnx'])
|
||||
$ext_interface = $patch_jacks_names[0]
|
||||
}
|
||||
|
||||
$openstack_network_hash = hiera_hash('openstack_network', { })
|
||||
$neutron_config = hiera_hash('neutron_config')
|
||||
$neutron_advanced_config = hiera_hash('neutron_advanced_configuration', { })
|
||||
$enable_qos = pick($neutron_advanced_config['neutron_qos'], false)
|
||||
$openstack_network_hash = hiera_hash('openstack_network', { })
|
||||
$neutron_config = hiera_hash('neutron_config')
|
||||
$neutron_advanced_config = hiera_hash('neutron_advanced_configuration', { })
|
||||
$enable_qos = pick($neutron_advanced_config['neutron_qos'], false)
|
||||
|
||||
$core_plugin = 'neutron.plugins.ml2.plugin.Ml2Plugin'
|
||||
$core_plugin = 'neutron.plugins.ml2.plugin.Ml2Plugin'
|
||||
|
||||
if $odl['odl_v2'] {
|
||||
$odl_plugin = 'odl-router_v2'
|
||||
} else {
|
||||
$odl_plugin = 'odl-router'
|
||||
}
|
||||
if $odl['odl_v2'] {
|
||||
$odl_plugin = 'odl-router_v2'
|
||||
} else {
|
||||
$odl_plugin = 'odl-router'
|
||||
}
|
||||
|
||||
if $odl['enable_l3_odl'] {
|
||||
$default_service_plugins = [
|
||||
$odl_plugin,
|
||||
'neutron.services.metering.metering_plugin.MeteringPlugin',
|
||||
]
|
||||
} else {
|
||||
$default_service_plugins = [
|
||||
'neutron.services.l3_router.l3_router_plugin.L3RouterPlugin',
|
||||
'neutron.services.metering.metering_plugin.MeteringPlugin',
|
||||
]
|
||||
}
|
||||
if $odl['enable_l3_odl'] {
|
||||
$default_service_plugins = [
|
||||
$odl_plugin,
|
||||
'neutron.services.metering.metering_plugin.MeteringPlugin',
|
||||
]
|
||||
} else {
|
||||
$default_service_plugins = [
|
||||
'neutron.services.l3_router.l3_router_plugin.L3RouterPlugin',
|
||||
'neutron.services.metering.metering_plugin.MeteringPlugin',
|
||||
]
|
||||
}
|
||||
|
||||
if $enable_qos {
|
||||
$service_plugins = concat($default_service_plugins, ['qos'])
|
||||
} else {
|
||||
$service_plugins = $default_service_plugins
|
||||
}
|
||||
if $enable_qos {
|
||||
$service_plugins = concat($default_service_plugins, ['qos'])
|
||||
} else {
|
||||
$service_plugins = $default_service_plugins
|
||||
}
|
||||
|
||||
$neutron_config_l3 = pick($neutron_config['l3'], {})
|
||||
$dhcp_lease_duration = pick($neutron_config_l3['dhcp_lease_duration'], '600')
|
||||
$neutron_config_l3 = pick($neutron_config['l3'], {})
|
||||
$dhcp_lease_duration = pick($neutron_config_l3['dhcp_lease_duration'], '600')
|
||||
|
||||
$rabbit_hash = hiera_hash('rabbit', {})
|
||||
$ceilometer_hash = hiera_hash('ceilometer', {})
|
||||
$network_scheme = hiera_hash('network_scheme', {})
|
||||
$rabbit_hash = hiera_hash('rabbit', {})
|
||||
$ceilometer_hash = hiera_hash('ceilometer', {})
|
||||
$network_scheme = hiera_hash('network_scheme', {})
|
||||
|
||||
$verbose = pick($openstack_network_hash['verbose'], hiera('verbose', true))
|
||||
$debug = pick($openstack_network_hash['debug'], hiera('debug', true))
|
||||
# TODO(aschultz): LP#1499620 - neutron in UCA liberty fails to start with
|
||||
# syslog enabled.
|
||||
$use_syslog = $::os_package_type ? {
|
||||
'ubuntu' => false,
|
||||
default => hiera('use_syslog', true)
|
||||
}
|
||||
$use_stderr = hiera('use_stderr', false)
|
||||
$log_facility = hiera('syslog_log_facility_neutron', 'LOG_LOCAL4')
|
||||
$verbose = pick($openstack_network_hash['verbose'], hiera('verbose', true))
|
||||
$debug = pick($openstack_network_hash['debug'], hiera('debug', true))
|
||||
# TODO(aschultz): LP#1499620 - neutron in UCA liberty fails to start with
|
||||
# syslog enabled.
|
||||
$use_syslog = $::os_package_type ? {
|
||||
'ubuntu' => false,
|
||||
default => hiera('use_syslog', true)
|
||||
}
|
||||
$use_stderr = hiera('use_stderr', false)
|
||||
$log_facility = hiera('syslog_log_facility_neutron', 'LOG_LOCAL4')
|
||||
|
||||
prepare_network_config($network_scheme)
|
||||
$bind_host = get_network_role_property('neutron/api', 'ipaddr')
|
||||
prepare_network_config($network_scheme)
|
||||
$bind_host = get_network_role_property('neutron/api', 'ipaddr')
|
||||
|
||||
$base_mac = $neutron_config['L2']['base_mac']
|
||||
$amqp_hosts = split(hiera('amqp_hosts', ''), ',')
|
||||
$amqp_user = $rabbit_hash['user']
|
||||
$amqp_password = $rabbit_hash['password']
|
||||
$base_mac = $neutron_config['L2']['base_mac']
|
||||
$amqp_hosts = split(hiera('amqp_hosts', ''), ',')
|
||||
$amqp_user = $rabbit_hash['user']
|
||||
$amqp_password = $rabbit_hash['password']
|
||||
|
||||
$kombu_compression = hiera('kombu_compression', '')
|
||||
$kombu_compression = hiera('kombu_compression', $::os_service_default)
|
||||
|
||||
$segmentation_type = try_get_value($neutron_config, 'L2/segmentation_type')
|
||||
$segmentation_type = dig44($neutron_config, ['L2', 'segmentation_type'])
|
||||
|
||||
$nets = $neutron_config['predefined_networks']
|
||||
$nets = $neutron_config['predefined_networks']
|
||||
|
||||
if $segmentation_type == 'vlan' {
|
||||
$net_role_property = 'neutron/private'
|
||||
if $segmentation_type == 'vlan' {
|
||||
$net_role_property = 'neutron/private'
|
||||
|
||||
if $ext_interface {
|
||||
exec { 'ovs-set-provider-mapping':
|
||||
command => "ovs-vsctl set Open_vSwitch $(ovs-vsctl show | head -n 1) other_config:provider_mappings=br-ex:${ext_interface},physnet2:br-aux",
|
||||
path => '/usr/bin',
|
||||
require => Exec['ovs-set-manager'],
|
||||
}
|
||||
} else {
|
||||
exec { 'ovs-set-provider-mapping':
|
||||
command => "ovs-vsctl set Open_vSwitch $(ovs-vsctl show | head -n 1) other_config:provider_mappings=physnet2:br-aux",
|
||||
path => '/usr/bin',
|
||||
require => Exec['ovs-set-manager'],
|
||||
}
|
||||
if $ext_interface {
|
||||
exec { 'ovs-set-provider-mapping':
|
||||
command => "ovs-vsctl set Open_vSwitch $(ovs-vsctl show | head -n 1) other_config:provider_mappings=br-ex:${ext_interface},physnet2:br-aux",
|
||||
path => '/usr/bin',
|
||||
require => Exec['ovs-set-manager'],
|
||||
}
|
||||
} else {
|
||||
$net_role_property = 'neutron/mesh'
|
||||
$tunneling_ip = get_network_role_property($net_role_property, 'ipaddr')
|
||||
|
||||
if $odl['enable_bgpvpn'] {
|
||||
exec { 'ovs-set-provider-mapping':
|
||||
command => "ovs-vsctl set Open_vSwitch $(ovs-vsctl show | head -n 1) other_config:provider_mappings=physnet1:br-floating",
|
||||
path => '/usr/bin',
|
||||
require => Exec['ovs-set-manager'],
|
||||
}
|
||||
} else {
|
||||
if $ext_interface {
|
||||
exec { 'ovs-set-provider-mapping':
|
||||
command => "ovs-vsctl set Open_vSwitch $(ovs-vsctl show | head -n 1) other_config:provider_mappings=br-ex:${ext_interface}",
|
||||
path => '/usr/bin',
|
||||
require => Exec['ovs-set-manager'],
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
exec { 'ovs-set-tunnel-endpoint':
|
||||
command => "ovs-vsctl set Open_vSwitch $(ovs-vsctl show | head -n 1) other_config:local_ip=${tunneling_ip}",
|
||||
exec { 'ovs-set-provider-mapping':
|
||||
command => "ovs-vsctl set Open_vSwitch $(ovs-vsctl show | head -n 1) other_config:provider_mappings=physnet2:br-aux",
|
||||
path => '/usr/bin',
|
||||
require => Exec['ovs-set-manager'],
|
||||
}
|
||||
}
|
||||
$iface = get_network_role_property($net_role_property, 'phys_dev')
|
||||
$physical_net_mtu = pick(get_transformation_property('mtu', $iface[0]), '1500')
|
||||
} else {
|
||||
$net_role_property = 'neutron/mesh'
|
||||
$tunneling_ip = get_network_role_property($net_role_property, 'ipaddr')
|
||||
|
||||
$default_log_levels = hiera_hash('default_log_levels')
|
||||
|
||||
class { '::neutron' :
|
||||
verbose => $verbose,
|
||||
debug => $debug,
|
||||
use_syslog => $use_syslog,
|
||||
use_stderr => $use_stderr,
|
||||
lock_path => '/var/lib/neutron/lock',
|
||||
log_facility => $log_facility,
|
||||
bind_host => $bind_host,
|
||||
base_mac => $base_mac,
|
||||
core_plugin => $core_plugin,
|
||||
service_plugins => $service_plugins,
|
||||
allow_overlapping_ips => true,
|
||||
mac_generation_retries => '32',
|
||||
dhcp_lease_duration => $dhcp_lease_duration,
|
||||
dhcp_agents_per_network => '2',
|
||||
report_interval => $neutron_config['neutron_report_interval'],
|
||||
rabbit_user => $amqp_user,
|
||||
rabbit_hosts => $amqp_hosts,
|
||||
rabbit_password => $amqp_password,
|
||||
network_device_mtu => $physical_net_mtu,
|
||||
advertise_mtu => true,
|
||||
}
|
||||
|
||||
# TODO (iberezovskiy): remove this workaround in N when neutron module
|
||||
# will be switched to puppet-oslo usage for rabbit configuration
|
||||
if $kombu_compression in ['gzip','bz2'] {
|
||||
if !defined(Oslo::Messaging_rabbit['neutron_config']) and !defined(Neutron_config['oslo_messaging_rabbit/kombu_compression']) {
|
||||
neutron_config { 'oslo_messaging_rabbit/kombu_compression': value => $kombu_compression; }
|
||||
} else {
|
||||
Neutron_config<| title == 'oslo_messaging_rabbit/kombu_compression' |> { value => $kombu_compression }
|
||||
}
|
||||
}
|
||||
|
||||
if $default_log_levels {
|
||||
neutron_config {
|
||||
'DEFAULT/default_log_levels' :
|
||||
value => join(sort(join_keys_to_values($default_log_levels, '=')), ',');
|
||||
if $odl['enable_bgpvpn'] {
|
||||
exec { 'ovs-set-provider-mapping':
|
||||
command => "ovs-vsctl set Open_vSwitch $(ovs-vsctl show | head -n 1) other_config:provider_mappings=physnet1:br-floating",
|
||||
path => '/usr/bin',
|
||||
require => Exec['ovs-set-manager'],
|
||||
}
|
||||
} else {
|
||||
neutron_config { 'DEFAULT/default_log_levels' : ensure => absent; }
|
||||
}
|
||||
if $ext_interface {
|
||||
exec { 'ovs-set-provider-mapping':
|
||||
command => "ovs-vsctl set Open_vSwitch $(ovs-vsctl show | head -n 1) other_config:provider_mappings=br-ex:${ext_interface}",
|
||||
path => '/usr/bin',
|
||||
require => Exec['ovs-set-manager'],
|
||||
}
|
||||
}
|
||||
|
||||
if $use_syslog {
|
||||
neutron_config { 'DEFAULT/use_syslog_rfc_format': value => true; }
|
||||
}
|
||||
|
||||
neutron_config {
|
||||
'DEFAULT/notification_driver': value => $ceilometer_hash['notification_driver'];
|
||||
exec { 'ovs-set-tunnel-endpoint':
|
||||
command => "ovs-vsctl set Open_vSwitch $(ovs-vsctl show | head -n 1) other_config:local_ip=${tunneling_ip}",
|
||||
path => '/usr/bin',
|
||||
require => Exec['ovs-set-manager'],
|
||||
}
|
||||
|
||||
}
|
||||
$iface = get_network_role_property($net_role_property, 'phys_dev')
|
||||
|
||||
if $iface {
|
||||
$physical_net_mtu = pick(get_transformation_property('mtu', $iface[0]), '1500')
|
||||
} else {
|
||||
$physical_net_mtu = '1500'
|
||||
}
|
||||
|
||||
|
||||
$default_log_levels = hiera_hash('default_log_levels')
|
||||
|
||||
# manually add line to neutron_sudoers in case of UCA packages
|
||||
# because UCA doesn't have such line
|
||||
if $::os_package_type == 'ubuntu' {
|
||||
file_line { 'root_helper_daemon':
|
||||
line => 'neutron ALL = (root) NOPASSWD: /usr/bin/neutron-rootwrap-daemon /etc/neutron/rootwrap.conf',
|
||||
path => '/etc/sudoers.d/neutron_sudoers',
|
||||
match => '^neutron ALL = (root) NOPASSWD: /usr/bin/neutron-rootwrap-daemon',
|
||||
}
|
||||
Package['neutron'] -> File_line[ 'root_helper_daemon'] -> Neutron_config<||>
|
||||
}
|
||||
|
||||
class { '::neutron' :
|
||||
lock_path => '/var/lib/neutron/lock',
|
||||
bind_host => $bind_host,
|
||||
base_mac => $base_mac,
|
||||
core_plugin => $core_plugin,
|
||||
service_plugins => $service_plugins,
|
||||
allow_overlapping_ips => true,
|
||||
mac_generation_retries => '32',
|
||||
dhcp_lease_duration => $dhcp_lease_duration,
|
||||
dhcp_agents_per_network => '2',
|
||||
report_interval => $neutron_config['neutron_report_interval'],
|
||||
rabbit_user => $amqp_user,
|
||||
rabbit_hosts => $amqp_hosts,
|
||||
rabbit_password => $amqp_password,
|
||||
rabbit_heartbeat_timeout_threshold => 0,
|
||||
kombu_compression => $kombu_compression,
|
||||
global_physnet_mtu => $physical_net_mtu,
|
||||
advertise_mtu => true,
|
||||
notification_driver => $ceilometer_hash['notification_driver'],
|
||||
manage_logging => false,
|
||||
root_helper_daemon => 'sudo neutron-rootwrap-daemon /etc/neutron/rootwrap.conf'
|
||||
}
|
||||
|
||||
class { '::neutron::logging':
|
||||
debug => $debug,
|
||||
use_syslog => $use_syslog,
|
||||
use_stderr => $use_stderr,
|
||||
syslog_log_facility => $log_facility,
|
||||
default_log_levels => $default_log_levels,
|
||||
}
|
||||
|
||||
|
||||
### SYSCTL ###
|
||||
|
||||
|
@ -1,83 +1,82 @@
|
||||
notice('MODULAR: odl-ml2.pp')
|
||||
|
||||
$use_neutron = hiera('use_neutron', false)
|
||||
|
||||
class neutron {}
|
||||
class { 'neutron' :}
|
||||
|
||||
if $use_neutron {
|
||||
include ::neutron::params
|
||||
include ::neutron::params
|
||||
|
||||
$node_name = hiera('node_name')
|
||||
$neutron_primary_controller_roles = hiera('neutron_primary_controller_roles', ['primary-controller'])
|
||||
$neutron_compute_roles = hiera('neutron_compute_nodes', ['compute'])
|
||||
$primary_controller = roles_include($neutron_primary_controller_roles)
|
||||
$compute = roles_include($neutron_compute_roles)
|
||||
$node_name = hiera('node_name')
|
||||
$neutron_primary_controller_roles = hiera('neutron_primary_controller_roles', ['primary-controller'])
|
||||
$neutron_compute_roles = hiera('neutron_compute_nodes', ['compute'])
|
||||
$primary_controller = roles_include($neutron_primary_controller_roles)
|
||||
$compute = roles_include($neutron_compute_roles)
|
||||
|
||||
$neutron_config = hiera_hash('neutron_config')
|
||||
$neutron_server_enable = pick($neutron_config['neutron_server_enable'], true)
|
||||
$neutron_nodes = hiera_hash('neutron_nodes')
|
||||
$neutron_config = hiera_hash('neutron_config')
|
||||
$neutron_server_enable = pick($neutron_config['neutron_server_enable'], true)
|
||||
$neutron_nodes = hiera_hash('neutron_nodes')
|
||||
|
||||
$dpdk_config = hiera_hash('dpdk', {})
|
||||
$enable_dpdk = pick($dpdk_config['enabled'], false)
|
||||
$dpdk_config = hiera_hash('dpdk', {})
|
||||
$enable_dpdk = pick($dpdk_config['enabled'], false)
|
||||
|
||||
$management_vip = hiera('management_vip')
|
||||
$service_endpoint = hiera('service_endpoint', $management_vip)
|
||||
$ssl_hash = hiera_hash('use_ssl', {})
|
||||
$internal_auth_protocol = get_ssl_property($ssl_hash, {}, 'keystone', 'internal', 'protocol', 'http')
|
||||
$internal_auth_address = get_ssl_property($ssl_hash, {}, 'keystone', 'internal', 'hostname', [$service_endpoint])
|
||||
$management_vip = hiera('management_vip')
|
||||
$service_endpoint = hiera('service_endpoint', $management_vip)
|
||||
$ssl_hash = hiera_hash('use_ssl', {})
|
||||
$internal_auth_protocol = get_ssl_property($ssl_hash, {}, 'keystone', 'internal', 'protocol', 'http')
|
||||
$internal_auth_address = get_ssl_property($ssl_hash, {}, 'keystone', 'internal', 'hostname', [$service_endpoint])
|
||||
|
||||
$auth_api_version = 'v2.0'
|
||||
$identity_uri = "${internal_auth_protocol}://${internal_auth_address}:5000"
|
||||
$auth_url = "${identity_uri}/${auth_api_version}"
|
||||
$auth_password = $neutron_config['keystone']['admin_password']
|
||||
$auth_user = pick($neutron_config['keystone']['admin_user'], 'neutron')
|
||||
$auth_tenant = pick($neutron_config['keystone']['admin_tenant'], 'services')
|
||||
$auth_region = hiera('region', 'RegionOne')
|
||||
$auth_endpoint_type = 'internalURL'
|
||||
$auth_api_version = 'v2.0'
|
||||
$identity_uri = "${internal_auth_protocol}://${internal_auth_address}:5000"
|
||||
$auth_url = "${identity_uri}/${auth_api_version}"
|
||||
$auth_password = $neutron_config['keystone']['admin_password']
|
||||
$auth_user = pick($neutron_config['keystone']['admin_user'], 'neutron')
|
||||
$auth_tenant = pick($neutron_config['keystone']['admin_tenant'], 'services')
|
||||
$auth_region = hiera('region', 'RegionOne')
|
||||
$auth_endpoint_type = 'internalURL'
|
||||
|
||||
# Synchronize database after plugin was configured
|
||||
if $primary_controller {
|
||||
include ::neutron::db::sync
|
||||
notify{"Trigger neutron-db-sync": } ~> Exec['neutron-db-sync']
|
||||
# Synchronize database after plugin was configured
|
||||
if $primary_controller {
|
||||
include ::neutron::db::sync
|
||||
notify{"Trigger neutron-db-sync": } ~> Exec['neutron-db-sync']
|
||||
}
|
||||
|
||||
if $node_name in keys($neutron_nodes) {
|
||||
if $neutron_server_enable {
|
||||
$service_ensure = 'running'
|
||||
} else {
|
||||
$service_ensure = 'stopped'
|
||||
}
|
||||
|
||||
if $node_name in keys($neutron_nodes) {
|
||||
if $neutron_server_enable {
|
||||
$service_ensure = 'running'
|
||||
} else {
|
||||
$service_ensure = 'stopped'
|
||||
}
|
||||
service { 'neutron-server':
|
||||
name => $::neutron::params::server_service,
|
||||
enable => $neutron_server_enable,
|
||||
ensure => $service_ensure,
|
||||
hasstatus => true,
|
||||
hasrestart => true,
|
||||
tag => 'neutron-service',
|
||||
} ->
|
||||
exec { 'waiting-for-neutron-api':
|
||||
environment => [
|
||||
"OS_TENANT_NAME=${auth_tenant}",
|
||||
"OS_USERNAME=${auth_user}",
|
||||
"OS_PASSWORD=${auth_password}",
|
||||
"OS_AUTH_URL=${auth_url}",
|
||||
"OS_REGION_NAME=${auth_region}",
|
||||
"OS_ENDPOINT_TYPE=${auth_endpoint_type}",
|
||||
],
|
||||
path => '/usr/sbin:/usr/bin:/sbin:/bin',
|
||||
tries => '30',
|
||||
try_sleep => '4',
|
||||
command => 'neutron net-list --http-timeout=4 2>&1 > /dev/null',
|
||||
provider => 'shell'
|
||||
}
|
||||
|
||||
service { 'neutron-server':
|
||||
name => $::neutron::params::server_service,
|
||||
enable => $neutron_server_enable,
|
||||
ensure => $service_ensure,
|
||||
hasstatus => true,
|
||||
hasrestart => true,
|
||||
tag => 'neutron-service',
|
||||
}
|
||||
|
||||
# Stub for upstream neutron manifests
|
||||
package { 'neutron':
|
||||
name => 'binutils',
|
||||
ensure => 'installed',
|
||||
exec { 'waiting-for-neutron-api':
|
||||
environment => [
|
||||
"OS_TENANT_NAME=${auth_tenant}",
|
||||
"OS_USERNAME=${auth_user}",
|
||||
"OS_PASSWORD=${auth_password}",
|
||||
"OS_AUTH_URL=${auth_url}",
|
||||
"OS_REGION_NAME=${auth_region}",
|
||||
"OS_ENDPOINT_TYPE=${auth_endpoint_type}",
|
||||
],
|
||||
path => '/usr/sbin:/usr/bin:/sbin:/bin',
|
||||
tries => '30',
|
||||
try_sleep => '4',
|
||||
command => 'neutron net-list --http-timeout=4 2>&1 > /dev/null',
|
||||
provider => 'shell',
|
||||
subscribe => Service['neutron-server'],
|
||||
refreshonly => true,
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
# Stub for upstream neutron manifests
|
||||
package { 'neutron':
|
||||
name => 'binutils',
|
||||
ensure => 'installed',
|
||||
}
|
||||
|
@ -1,127 +0,0 @@
|
||||
|
||||
notice('MODULAR: odl-netconfig.pp')
|
||||
|
||||
$roles = hiera('roles')
|
||||
#ODL custome network schema
|
||||
$network_scheme = odl_network_scheme($roles)
|
||||
prepare_network_config($network_scheme)
|
||||
|
||||
if ( $::l23_os =~ /(?i:centos6)/ and $::kernelmajversion == '3.10' ) {
|
||||
$ovs_datapath_package_name = 'kmod-openvswitch-lt'
|
||||
}
|
||||
|
||||
$dpdk_options = hiera_hash('dpdk', {})
|
||||
|
||||
class { '::l23network' :
|
||||
use_ovs => hiera('use_ovs', false),
|
||||
use_ovs_dkms_datapath_module => $::l23_os ? {
|
||||
/(?i:redhat7|centos7)/ => false,
|
||||
default => true
|
||||
},
|
||||
ovs_datapath_package_name => $ovs_datapath_package_name,
|
||||
use_dpdk => pick($dpdk_options['enabled'], false),
|
||||
dpdk_options => $dpdk_options,
|
||||
}
|
||||
$sdn = generate_network_config()
|
||||
notify {'SDN': message => $sdn }
|
||||
|
||||
#Set arp_accept to 1 by default #lp1456272
|
||||
sysctl::value { 'net.ipv4.conf.all.arp_accept': value => '1' }
|
||||
sysctl::value { 'net.ipv4.conf.default.arp_accept': value => '1' }
|
||||
|
||||
### TCP connections keepalives and failover related parameters ###
|
||||
# configure TCP keepalive for host OS.
|
||||
# Send 3 probes each 8 seconds, if the connection was idle
|
||||
# for a 30 seconds. Consider it dead, if there was no responces
|
||||
# during the check time frame, i.e. 30+3*8=54 seconds overall.
|
||||
# (note: overall check time frame should be lower then
|
||||
# nova_report_interval).
|
||||
class { '::openstack::keepalive' :
|
||||
tcpka_time => '30',
|
||||
tcpka_probes => '8',
|
||||
tcpka_intvl => '3',
|
||||
tcp_retries2 => '5',
|
||||
}
|
||||
|
||||
# increase network backlog for performance on fast networks
|
||||
sysctl::value { 'net.core.netdev_max_backlog': value => '261144' }
|
||||
|
||||
L2_port<||> -> Sysfs_config_value<||>
|
||||
L3_ifconfig<||> -> Sysfs_config_value<||>
|
||||
L3_route<||> -> Sysfs_config_value<||>
|
||||
|
||||
class { '::sysfs' :}
|
||||
|
||||
if hiera('set_rps', true) {
|
||||
sysfs_config_value { 'rps_cpus' :
|
||||
ensure => 'present',
|
||||
name => '/etc/sysfs.d/rps_cpus.conf',
|
||||
value => cpu_affinity_hex($::processorcount),
|
||||
sysfs => '/sys/class/net/*/queues/rx-*/rps_cpus',
|
||||
exclude => '/sys/class/net/lo/*',
|
||||
}
|
||||
} else {
|
||||
sysfs_config_value { 'rps_cpus' :
|
||||
ensure => 'absent',
|
||||
name => '/etc/sysfs.d/rps_cpus.conf',
|
||||
}
|
||||
}
|
||||
|
||||
if hiera('set_xps', true) {
|
||||
sysfs_config_value { 'xps_cpus' :
|
||||
ensure => 'present',
|
||||
name => '/etc/sysfs.d/xps_cpus.conf',
|
||||
value => cpu_affinity_hex($::processorcount),
|
||||
sysfs => '/sys/class/net/*/queues/tx-*/xps_cpus',
|
||||
exclude => '/sys/class/net/lo/*',
|
||||
}
|
||||
} else {
|
||||
sysfs_config_value { 'xps_cpus' :
|
||||
ensure => 'absent',
|
||||
name => '/etc/sysfs.d/xps_cpus.conf',
|
||||
}
|
||||
}
|
||||
|
||||
if !defined(Package['irqbalance']) {
|
||||
package { 'irqbalance':
|
||||
ensure => installed,
|
||||
}
|
||||
}
|
||||
|
||||
if !defined(Service['irqbalance']) {
|
||||
service { 'irqbalance':
|
||||
ensure => running,
|
||||
require => Package['irqbalance'],
|
||||
}
|
||||
}
|
||||
|
||||
# We need to wait at least 30 seconds for the bridges and other interfaces to
|
||||
# come up after being created. This should allow for all interfaces to be up
|
||||
# and ready for traffic before proceeding with further deploy steps. LP#1458954
|
||||
exec { 'wait-for-interfaces':
|
||||
path => '/usr/bin:/bin',
|
||||
command => 'sleep 32',
|
||||
}
|
||||
|
||||
$run_ping_checker = hiera('run_ping_checker', true)
|
||||
|
||||
if $run_ping_checker {
|
||||
# check that network was configured successfully
|
||||
# and the default gateway is online
|
||||
if hiera('default_gateway', false) {
|
||||
$default_gateway = hiera('default_gateway')
|
||||
} else {
|
||||
$default_gateway = get_default_gateways()
|
||||
}
|
||||
|
||||
ping_host { $default_gateway :
|
||||
ensure => 'up',
|
||||
}
|
||||
L2_port<||> -> Ping_host[$default_gateway]
|
||||
L2_bond<||> -> Ping_host[$default_gateway]
|
||||
L3_ifconfig<||> -> Ping_host[$default_gateway]
|
||||
L3_route<||> -> Ping_host[$default_gateway]
|
||||
}
|
||||
|
||||
Class['::l23network'] ->
|
||||
Exec['wait-for-interfaces']
|
@ -1,30 +0,0 @@
|
||||
notice('MODULAR: configure_default_route.pp')
|
||||
|
||||
|
||||
$roles = hiera('roles')
|
||||
$network_scheme = odl_network_scheme($roles)
|
||||
$management_vrouter_vip = hiera('management_vrouter_vip')
|
||||
$management_role = 'management'
|
||||
$fw_admin_role = 'fw-admin'
|
||||
|
||||
if ( $::l23_os =~ /(?i:centos6)/ and $::kernelmajversion == '3.10' ) {
|
||||
$ovs_datapath_package_name = 'kmod-openvswitch-lt'
|
||||
}
|
||||
|
||||
class { 'l23network' :
|
||||
use_ovs => hiera('use_ovs', false),
|
||||
use_ovs_dkms_datapath_module => $::l23_os ? {
|
||||
/(?i:redhat7|centos7)/ => false,
|
||||
default => true
|
||||
},
|
||||
ovs_datapath_package_name => $ovs_datapath_package_name,
|
||||
}
|
||||
|
||||
$new_network_scheme = configure_default_route($network_scheme, $management_vrouter_vip, $fw_admin_role, $management_role )
|
||||
notice ($new_network_scheme)
|
||||
|
||||
if !empty($new_network_scheme) {
|
||||
prepare_network_config($new_network_scheme)
|
||||
$sdn = generate_network_config()
|
||||
notify {'SDN': message => $sdn }
|
||||
}
|
@ -1,27 +1,23 @@
|
||||
require 'yaml'
|
||||
|
||||
module Puppet::Parser::Functions
|
||||
newfunction(:odl_hiera_overrides) do |args|
|
||||
filename = args[0]
|
||||
node_roles = args[1]
|
||||
odl = function_hiera(['opendaylight'])
|
||||
management_vip = function_hiera(['management_vip'])
|
||||
newfunction(:odl_hiera_overrides, :arity => 6) do |args|
|
||||
|
||||
filename, odl, neutron_config, neutron_advanced_configuration, network_scheme, mgmt_vip = args
|
||||
|
||||
hiera_overrides = {}
|
||||
configuration = {}
|
||||
|
||||
if odl['odl_v2']
|
||||
mechanism_driver = 'opendaylight_v2'
|
||||
else
|
||||
mechanism_driver = 'opendaylight'
|
||||
end
|
||||
mechanism_driver = odl['odl_v2'] ? 'opendaylight_v2' : 'opendaylight'
|
||||
|
||||
ml2_plugin = {'neutron_plugin_ml2' =>
|
||||
{'ml2/mechanism_drivers' => {'value' => mechanism_driver},
|
||||
'ml2_odl/password' => {'value' => 'admin'},
|
||||
'ml2_odl/username' => {'value' => 'admin'},
|
||||
'ml2_odl/url' => {'value' => "http://#{management_vip}:#{odl['rest_api_port']}/controller/nb/v2/neutron"}
|
||||
}
|
||||
}
|
||||
ml2_plugin = {
|
||||
'neutron_plugin_ml2' => {
|
||||
'ml2/mechanism_drivers' => {'value' => mechanism_driver},
|
||||
'ml2_odl/password' => {'value' => 'admin'},
|
||||
'ml2_odl/username' => {'value' => 'admin'},
|
||||
'ml2_odl/url' => {'value' => "http://#{mgmt_vip}:#{odl['rest_api_port']}/controller/nb/v2/neutron"}
|
||||
}
|
||||
}
|
||||
|
||||
# When L3 forward is disabled in ODL set external_network_bridge option
|
||||
# to use neutron L3 agent to create qg port on selected bridge
|
||||
@ -55,26 +51,18 @@ module Puppet::Parser::Functions
|
||||
hiera_overrides['configuration'] = configuration
|
||||
|
||||
# override neutron_config/quantum_settings
|
||||
neutron_config = function_hiera(['neutron_config'])
|
||||
neutron_config['L2']['mechanism_drivers'] = 'opendaylight'
|
||||
if odl['enable_bgpvpn']
|
||||
neutron_config['L2']['phys_nets']['physnet1']['bridge'] = 'br-ex'
|
||||
else
|
||||
neutron_config['L2']['phys_nets']['physnet1']['bridge'] = 'br-int'
|
||||
end
|
||||
neutron_config['L2']['phys_nets']['physnet1']['bridge'] = odl['enable_bgpvpn'] ? 'br-ex' : 'br-int'
|
||||
|
||||
hiera_overrides['neutron_config'] = neutron_config
|
||||
hiera_overrides['quantum_settings'] = neutron_config
|
||||
|
||||
# override neutron_advanced_configuration
|
||||
neutron_advanced_configuration = function_hiera(['neutron_advanced_configuration'])
|
||||
neutron_advanced_configuration['neutron_dvr'] = false
|
||||
neutron_advanced_configuration['neutron_l2_pop'] = false
|
||||
neutron_advanced_configuration['l2_agent_ha'] = false
|
||||
hiera_overrides['neutron_advanced_configuration'] = neutron_advanced_configuration
|
||||
|
||||
# override network scheme
|
||||
network_scheme = function_odl_network_scheme( [node_roles] )
|
||||
|
||||
hiera_overrides['network_scheme'] = network_scheme
|
||||
# write to hiera override yaml file
|
||||
File.open(filename, 'w') { |file| file.write(hiera_overrides.to_yaml) }
|
||||
|
@ -1,62 +1,108 @@
|
||||
#
|
||||
# Modify default network schema.
|
||||
# ODL use only br-int and br-ex bridges and both of them
|
||||
# should be created on OVS.
|
||||
# Override hiera hierarchy is not enough because of default hash deep merge policy.
|
||||
#
|
||||
module Puppet::Parser::Functions
|
||||
newfunction(:odl_network_scheme, :type => :rvalue) do |args|
|
||||
# override network_scheme
|
||||
odl = function_hiera(['opendaylight'])
|
||||
network_scheme = function_hiera(['network_scheme'])
|
||||
newfunction(:odl_network_scheme, :arity => 2, :type => :rvalue, :doc => <<-ENDHEREDOC
|
||||
Modify/override default network scheme.
|
||||
In case of odl-ovsdb-openstack feature is enabled only br-int and br-ex
|
||||
bridges are used and both of them should be created on OVS.
|
||||
|
||||
input:
|
||||
<boolean> bgpvpn enabled or not
|
||||
<hash> original network scheme
|
||||
|
||||
output: overridden network scheme
|
||||
|
||||
ENDHEREDOC
|
||||
) do |args|
|
||||
|
||||
odl_bgpvpn_enabled, network_scheme = args
|
||||
|
||||
# get original network scheme
|
||||
orig_endpoints = network_scheme['endpoints']
|
||||
orig_roles = network_scheme['roles']
|
||||
orig_transformations = network_scheme['transformations']
|
||||
|
||||
# init overridden network scheme
|
||||
endpoints = {}
|
||||
roles = {}
|
||||
transformations = []
|
||||
|
||||
# If bgpvpn extensions are enabled br-floating is not needed
|
||||
delete_bridges = ['br-prv']
|
||||
|
||||
debug "ODL network before transformation: #{network_scheme}"
|
||||
delete_bridges.each do |bridge|
|
||||
transformations << {
|
||||
'override' => bridge,
|
||||
'override-action' => 'noop',
|
||||
}
|
||||
end
|
||||
|
||||
endpoints = network_scheme['endpoints']
|
||||
transformations = network_scheme['transformations']
|
||||
transformations.delete_if { |action| action['action'] == 'add-br' and delete_bridges.include?(action['name']) }
|
||||
transformations.delete_if { |action| action['action'] == 'add-patch' and not (action['bridges'] & delete_bridges).empty? }
|
||||
transformations.delete_if { |action| action['action'] == 'add-port' and delete_bridges.include?(action['bridge']) }
|
||||
|
||||
if not odl['enable_bgpvpn']
|
||||
debug "Changing network_scheme for the non bgpvpn case."
|
||||
# Modify only once
|
||||
if not endpoints.has_key? 'br-ex-lnx'
|
||||
transformations.each { |action| action['name'] = 'br-ex-lnx' if (action['action'] == 'add-br' and action['name'] == 'br-ex') }
|
||||
transformations.each { |action| action['bridge'] = 'br-ex-lnx' if (action['action'] == 'add-port' and action['bridge'] == 'br-ex') }
|
||||
transformations.each { |action| action['bridge'] = 'br-ex-lnx' if (action['action'] == 'add-bond' and action['bridge'] == 'br-ex') }
|
||||
end
|
||||
|
||||
transformations.each { |action| action['name'] = 'br-ex' if (action['action'] == 'add-br' and action['name'] == 'br-floating') }
|
||||
transformations.each { |action| action['bridge'] = 'br-ex' if (action['action'] == 'add-port' and action['bridge'] == 'br-floating') }
|
||||
|
||||
transformations.each { |action| action['bridges'] = ['br-ex', 'br-ex-lnx'] if (action['action'] == 'add-patch' and action['bridges'] == ['br-floating', 'br-ex']) }
|
||||
|
||||
roles = network_scheme['roles']
|
||||
roles.each { |role,bridge| roles[role] = 'br-ex-lnx' if bridge == 'br-ex' }
|
||||
roles['neutron/private'] = 'br-aux' if roles.has_key?('neutron/private')
|
||||
roles['neutron/floating'] = 'br-ex' if roles.has_key?('neutron/floating')
|
||||
|
||||
if endpoints.has_key? 'br-ex' and not endpoints.has_key? 'br-ex-lnx'
|
||||
endpoints['br-ex-lnx'] = endpoints.delete 'br-ex'
|
||||
end
|
||||
|
||||
if endpoints.has_key? 'br-floating'
|
||||
endpoints['br-ex'] = endpoints.delete 'br-floating'
|
||||
end
|
||||
|
||||
if endpoints.has_key? 'br-prv'
|
||||
endpoints['br-aux'] = endpoints.delete 'br-prv'
|
||||
end
|
||||
else
|
||||
debug "Changing network_scheme for the bgpvpn case"
|
||||
if endpoints.has_key? 'br-prv'
|
||||
endpoints.delete 'br-prv'
|
||||
orig_transformations.each do |tf|
|
||||
case tf['action']
|
||||
when 'add-port'
|
||||
transformations << {
|
||||
'override' => tf['name'],
|
||||
'override-action' => 'noop',
|
||||
} if delete_bridges.include?(tf['bridge'])
|
||||
when 'add-patch'
|
||||
transformations << {
|
||||
'override' => 'patch-%s:%s' % tf['bridges'],
|
||||
'name' => 'patch__%s--%s' % tf['bridges'],
|
||||
'override-action' => 'noop',
|
||||
} if delete_bridges.any? { |br| tf['bridges'].include? br }
|
||||
end
|
||||
end
|
||||
debug "ODL network after transformation: #{network_scheme}"
|
||||
network_scheme
|
||||
|
||||
if not odl_bgpvpn_enabled
|
||||
Puppet.debug 'Changing network_scheme for the non bgpvpn case'
|
||||
|
||||
BRIDGE_MAPPING = {
|
||||
'br-ex' => 'br-ex-lnx',
|
||||
'br-floating' => 'br-ex'
|
||||
}
|
||||
|
||||
BRIDGE_MAPPING.each do |orig_br, br|
|
||||
transformations << {
|
||||
'override' => orig_br,
|
||||
'name' => br,
|
||||
}
|
||||
end
|
||||
|
||||
orig_transformations.each do |tf|
|
||||
transformations << {
|
||||
'override' => tf['name'],
|
||||
'bridge' => BRIDGE_MAPPING[tf['bridge']],
|
||||
} if BRIDGE_MAPPING.keys.any? { |br| br == tf['bridge'] }
|
||||
end
|
||||
|
||||
transformations << {
|
||||
'override' => 'patch-br-ex:br-floating',
|
||||
'bridges' => ['br-ex', 'br-ex-lnx'],
|
||||
}
|
||||
|
||||
orig_roles.each { |role, br| roles[role] = 'br-ex-lnx' if br == 'br-ex' }
|
||||
|
||||
roles['neutron/private'] = 'br-aux' if orig_roles.key?('neutron/private')
|
||||
roles['neutron/floating'] = 'br-ex' if orig_roles.key?('neutron/floating')
|
||||
|
||||
endpoints['br-floating'] = '' if orig_endpoints.key? 'br-floating'
|
||||
|
||||
if orig_endpoints.key? 'br-ex'
|
||||
endpoints['br-ex-lnx'] = orig_endpoints['br-ex']
|
||||
endpoints['br-ex'] = orig_endpoints['br-floating'] || ''
|
||||
endpoints['br-ex']['gateway'] ||= ''
|
||||
endpoints['br-ex']['vendor_specific'] ||= {}
|
||||
end
|
||||
|
||||
if orig_endpoints.key? 'br-prv'
|
||||
endpoints['br-aux'] = orig_endpoints['br-prv']
|
||||
endpoints['br-prv'] = ''
|
||||
end
|
||||
end
|
||||
|
||||
# return overridden network scheme
|
||||
{
|
||||
'endpoints' => endpoints,
|
||||
'roles' => roles,
|
||||
'transformations' => transformations.map { |tf| tf.merge({'action' => 'override'}) },
|
||||
}
|
||||
end
|
||||
end
|
||||
|
@ -1,5 +1,17 @@
|
||||
class opendaylight::hiera_override {
|
||||
include opendaylight
|
||||
$override_file = '/etc/hiera/plugins/opendaylight.yaml'
|
||||
$roles = hiera('roles')
|
||||
odl_hiera_overrides($override_file, $roles)
|
||||
|
||||
# override network scheme
|
||||
$orig_network_scheme = hiera_hash('network_scheme')
|
||||
$network_scheme = odl_network_scheme($opendaylight::odl_settings['enable_bgpvpn'], $orig_network_scheme)
|
||||
|
||||
odl_hiera_overrides(
|
||||
$override_file,
|
||||
$opendaylight::odl_settings,
|
||||
hiera('neutron_config'),
|
||||
hiera('neutron_advanced_configuration'),
|
||||
$network_scheme,
|
||||
hiera('management_vip')
|
||||
)
|
||||
}
|
||||
|
@ -22,45 +22,6 @@
|
||||
puppet_modules: puppet/modules:/etc/puppet/modules
|
||||
timeout: 720
|
||||
|
||||
- id: netconfig
|
||||
type: puppet
|
||||
version: 2.1.0
|
||||
groups: ['/.*/']
|
||||
required_for: [deploy_end]
|
||||
requires: [tools]
|
||||
condition:
|
||||
yaql_exp: >
|
||||
changedAny($.network_scheme, $.dpdk, $.get('use_ovs'), $.get('set_rps'),
|
||||
$.get('set_rps'), $.get('run_ping_checker'),
|
||||
$.network_scheme.endpoints.values().where(
|
||||
$.get('gateway') != null).gateway)
|
||||
parameters:
|
||||
puppet_manifest: puppet/manifests/odl-netconfig.pp
|
||||
puppet_modules: puppet/modules:/etc/puppet/modules
|
||||
timeout: 300
|
||||
test_pre:
|
||||
cmd: ruby /etc/puppet/modules/osnailyfacter/modular/netconfig/netconfig_pre.rb
|
||||
test_post:
|
||||
cmd: ruby /etc/puppet/modules/osnailyfacter/modular/netconfig/netconfig_post.rb
|
||||
# Run netconfig task on all nodes (except mongo and vrouter VIP roles) after virtual_ips.
|
||||
# Thus we make sure that our default gateway (vrouter VIP) is up before configuring
|
||||
# routing on nodes.
|
||||
cross-depends:
|
||||
yaql_exp: >
|
||||
switch(
|
||||
(
|
||||
$.roles.any($.matches('^(primary-)?(mongo)$'))
|
||||
or ($.network_metadata.get('vips',{}).get('vrouter',{}).get('ipaddr') = null)
|
||||
or (
|
||||
len($.roles.toSet().intersect($.network_metadata.get('vips',{}).get('vrouter',{}).get('node_roles').toSet())) > 0
|
||||
)
|
||||
) => [],
|
||||
true => [{
|
||||
name => 'virtual_ips',
|
||||
role => $.network_metadata.get('vips',{}).get('vrouter',{}).get('node_roles')
|
||||
}]
|
||||
)
|
||||
|
||||
- id: hiera-override
|
||||
type: puppet
|
||||
groups: [primary-controller, controller, compute, opendaylight]
|
||||
@ -108,7 +69,7 @@
|
||||
|
||||
- id: odl_vxgpe
|
||||
type: puppet
|
||||
version: 2.0.0
|
||||
version: 2.1.0
|
||||
groups: [compute, compute-vmware]
|
||||
required_for: [openstack-network-common-config]
|
||||
requires: [openstack-network-start]
|
||||
@ -146,11 +107,11 @@
|
||||
requires: [openstack-network-start, openstack-network-common-config, openstack-network-server-config]
|
||||
condition:
|
||||
yaql_exp: &network_plugins_l2 >
|
||||
changedAny($.configuration, $.fqdn, $.quantum_settings,
|
||||
$.network_metadata.nodes.values().where($.node_roles.any($.matches('controller'))),
|
||||
$.get('neutron_primary_controller_roles'), $.get('neutron_compute_nodes'),
|
||||
$.network_metadata.vips.get('management'), $.get('use_ssl'),
|
||||
$.get('region', 'RegionOne'), $.dpdk)
|
||||
changedAny($.quantum, $.configuration, $.fqdn,
|
||||
$.get('neutron_primary_controller_roles'),
|
||||
$.get('neutron_compute_nodes'), $.quantum_settings, $.public_ssl,
|
||||
$.get('use_ssl'), $.get('region'), $.network_scheme,
|
||||
$.neutron_advanced_configuration)
|
||||
refresh_on: [neutron_agent_ovs]
|
||||
parameters:
|
||||
puppet_manifest: puppet/manifests/odl-ml2-configuration.pp
|
||||
|
@ -3,11 +3,11 @@ name: opendaylight
|
||||
# Human-readable name for your plugin
|
||||
title: OpenDaylight plugin
|
||||
# Plugin version
|
||||
version: '0.9.0'
|
||||
version: '1.0.0'
|
||||
# Description
|
||||
description: 'This plugin provides OpenDaylight as a backend for neutron.'
|
||||
# Required fuel version
|
||||
fuel_version: ['9.0']
|
||||
fuel_version: ['10.0']
|
||||
# Specify license of your plugin
|
||||
licenses: ['Apache License Version 2.0']
|
||||
# Specify author or company name
|
||||
@ -21,7 +21,7 @@ groups: ['network']
|
||||
# The plugin is compatible with releases in the list
|
||||
releases:
|
||||
- os: ubuntu
|
||||
version: mitaka-9.0
|
||||
version: newton-10.0
|
||||
mode: ['ha']
|
||||
deployment_scripts_path: deployment_scripts/
|
||||
repository_path: repositories/ubuntu
|
||||
|
Loading…
Reference in New Issue
Block a user