Add support for encrypting backend Neutron API Server

Add TLS support for backend Neutron API Server communication using
HAProxy to perform TLS termination. When used in conjunction with
enabling TLS for service API endpoints, network communication will be
encrypted end to end, from client through HAProxy to the Neutron
service.

Change-Id: Ib333a1f1bd12491df72a9e52d961161210e2d330
Partially-Implements: blueprint add-ssl-internal-network
This commit is contained in:
James Kirsch 2020-10-06 10:01:32 -07:00 committed by Radosław Piliszek
parent 42d0b89361
commit 93ad57f47e
16 changed files with 284 additions and 10 deletions

View File

@ -380,6 +380,7 @@ neutron_internal_fqdn: "{{ kolla_internal_fqdn }}"
neutron_external_fqdn: "{{ kolla_external_fqdn }}"
neutron_server_port: "9696"
neutron_server_listen_port: "{{ neutron_server_port }}"
neutron_tls_proxy_stats_port: "9697"
nova_internal_fqdn: "{{ kolla_internal_fqdn }}"
nova_external_fqdn: "{{ kolla_external_fqdn }}"
@ -979,6 +980,7 @@ neutron_public_endpoint: "{{ public_protocol }}://{{ neutron_external_fqdn | put
# "sriovnet1" and tunnel_interface used here as placeholders
neutron_sriov_physnet_mappings:
sriovnet1: "{{ tunnel_interface }}"
neutron_enable_tls_backend: "{{ kolla_enable_tls_backend }}"
#######################
# Nova options

View File

@ -70,6 +70,7 @@ fluentd_image_full: "{{ fluentd_image }}:{{ fluentd_tag }}"
syslog_swift_facility: "local0"
syslog_haproxy_facility: "local1"
syslog_glance_tls_proxy_facility: "local2"
syslog_neutron_tls_proxy_facility: "local4"
kolla_toolbox_default_volumes:
- "{{ node_config_directory }}/kolla-toolbox/:{{ container_config_directory }}/:ro"
@ -153,6 +154,8 @@ fluentd_input_openstack_services:
enabled: "{{ enable_murano | bool }}"
- name: neutron
enabled: "{{ enable_neutron | bool }}"
- name: neutron-tls-proxy
enabled: "{{ neutron_enable_tls_backend | bool }}"
- name: nova
enabled: "{{ enable_nova | bool }}"
- name: octavia

View File

@ -203,6 +203,8 @@
- { name: "monasca", enabled: "{{ enable_monasca | bool }}" }
- { name: "murano", enabled: "{{ enable_murano | bool }}" }
- { name: "neutron", enabled: "{{ enable_neutron | bool }}" }
- { name: "neutron-tls-proxy", enabled: "{{ neutron_enable_tls_backend |
bool }}" }
- { name: "nova", enabled: "{{ enable_nova | bool }}" }
- { name: "octavia", enabled: "{{ enable_octavia | bool }}" }
- { name: "outward-rabbitmq", enabled: "{{ enable_outward_rabbitmq | bool }}" }

View File

@ -43,6 +43,13 @@
</record>
</filter>
<filter syslog.{{ syslog_neutron_tls_proxy_facility }}.**>
@type record_transformer
<record>
programname neutron-tls-proxy
</record>
</filter>
# Rename internal Fluent message field to match other logs. This removes
# all other fields by default, including the original message field. This is
# intented to avoid duplication of the log message and to prevent passing

View File

@ -180,3 +180,64 @@
{% endif %}
</match>
{% endif %}
{% if neutron_enable_tls_backend | bool %}
<match syslog.{{ syslog_neutron_tls_proxy_facility }}.**>
@type copy
<store>
@type file
path /var/log/kolla/neutron-tls-proxy/neutron-tls-proxy.*.log
output_tag false
output_time false
append true
compress gzip
<buffer>
timekey_use_utc
</buffer>
</store>
{% if log_direct_to_elasticsearch %}
<store>
type elasticsearch
host {{ elasticsearch_address }}
port {{ elasticsearch_port }}
scheme {{ fluentd_elasticsearch_scheme }}
{% if fluentd_elasticsearch_path != '' %}
path {{ fluentd_elasticsearch_path }}
{% endif %}
{% if fluentd_elasticsearch_scheme == 'https' %}
ssl_version {{ fluentd_elasticsearch_ssl_version }}
ssl_verify {{ fluentd_elasticsearch_ssl_verify }}
{% endif %}
{% if fluentd_elasticsearch_user != '' and fluentd_elasticsearch_password != ''%}
user {{ fluentd_elasticsearch_user }}
password {{ fluentd_elasticsearch_password }}
{% endif %}
logstash_format true
logstash_prefix {{ kibana_log_prefix }}
flush_interval 15s
reconnect_on_error true
buffer_type file
buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_neutron_tls_proxy_facility }}.*
</store>
{% elif enable_monasca | bool %}
<store>
@type monasca
keystone_url {{ keystone_internal_url }}
monasca_api {{ monasca_log_api_internal_endpoint }}
monasca_api_version v2.0
username {{ monasca_agent_user }}
password {{ monasca_agent_password }}
domain_id default
project_name {{ monasca_control_plane_project }}
message_field_name Payload
buffer_type file
buffer_path /var/lib/fluentd/data/monasca.buffer/{{ syslog_neutron_tls_proxy_facility }}.*
max_retry_wait 1800s
disable_retry_limit true
<buffer>
chunk_limit_size 8m
</buffer>
</store>
{% endif %}
</match>
{% endif %}

View File

@ -0,0 +1,3 @@
"/var/log/kolla/neutron-tls-proxy/*.log"
{
}

View File

@ -40,6 +40,13 @@
"owner": "{{ fluentd_user }}:{{ fluentd_user }}",
"recurse": true
},
{% endif %}
{% if neutron_enable_tls_backend | bool %}
{
"path": "/var/log/kolla/neutron-tls-proxy",
"owner": "{{ fluentd_user }}:{{ fluentd_user }}",
"recurse": true
},
{% endif %}
{
"path": "/var/log/kolla/swift",

View File

@ -13,13 +13,13 @@ neutron_services:
healthcheck: "{{ neutron_server_healthcheck }}"
haproxy:
neutron_server:
enabled: "{{ enable_neutron }}"
enabled: "{{ enable_neutron | bool and not neutron_enable_tls_backend | bool}}"
mode: "http"
external: false
port: "{{ neutron_server_port }}"
listen_port: "{{ neutron_server_listen_port }}"
neutron_server_external:
enabled: "{{ enable_neutron }}"
enabled: "{{ enable_neutron | bool and not neutron_enable_tls_backend | bool}}"
mode: "http"
external: true
port: "{{ neutron_server_port }}"
@ -174,6 +174,30 @@ neutron_services:
host_in_groups: "{{ inventory_hostname in groups['ironic-neutron-agent'] }}"
volumes: "{{ ironic_neutron_agent_default_volumes + ironic_neutron_agent_extra_volumes }}"
dimensions: "{{ ironic_neutron_agent_dimensions }}"
neutron-tls-proxy:
container_name: neutron_tls_proxy
group: neutron-server
host_in_groups: "{{ inventory_hostname in groups['neutron-server'] }}"
enabled: "{{ neutron_enable_tls_backend }}"
image: "{{ neutron_tls_proxy_image_full }}"
volumes: "{{ neutron_tls_proxy_default_volumes + neutron_tls_proxy_extra_volumes }}"
dimensions: "{{ neutron_tls_proxy_dimensions }}"
healthcheck: "{{ neutron_tls_proxy_healthcheck }}"
haproxy:
neutron_tls_proxy:
enabled: "{{ enable_neutron | bool and neutron_enable_tls_backend | bool }}"
mode: "http"
external: false
port: "{{ neutron_server_port }}"
listen_port: "{{ neutron_server_listen_port }}"
tls_backend: "yes"
neutron_tls_proxy_external:
enabled: "{{ enable_neutron | bool and neutron_enable_tls_backend | bool }}"
mode: "http"
external: true
port: "{{ neutron_server_port }}"
listen_port: "{{ neutron_server_listen_port }}"
tls_backend: "yes"
####################
# Database
@ -245,6 +269,10 @@ ironic_neutron_agent_image: "{{ docker_registry ~ '/' if docker_registry else ''
ironic_neutron_agent_tag: "{{ neutron_tag }}"
ironic_neutron_agent_image_full: "{{ ironic_neutron_agent_image }}:{{ ironic_neutron_agent_tag }}"
neutron_tls_proxy_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ neutron_install_type }}-haproxy"
neutron_tls_proxy_tag: "{{ neutron_tag }}"
neutron_tls_proxy_image_full: "{{ neutron_tls_proxy_image }}:{{ neutron_tls_proxy_tag }}"
neutron_agent_dimensions: "{{ default_container_dimensions }}"
neutron_dhcp_agent_dimensions: "{{ neutron_agent_dimensions }}"
@ -261,6 +289,7 @@ neutron_bgp_dragent_dimensions: "{{ default_container_dimensions }}"
neutron_infoblox_ipam_agent_dimensions: "{{ default_container_dimensions }}"
neutron_metering_agent_dimensions: "{{ neutron_agent_dimensions }}"
ironic_neutron_agent_dimensions: "{{ default_container_dimensions }}"
neutron_tls_proxy_dimensions: "{{ default_container_dimensions }}"
neutron_dhcp_agent_enable_healthchecks: "{{ enable_container_healthchecks }}"
neutron_dhcp_agent_healthcheck_interval: "{{ default_container_healthcheck_interval }}"
@ -344,7 +373,7 @@ neutron_server_enable_healthchecks: "{{ enable_container_healthchecks }}"
neutron_server_healthcheck_interval: "{{ default_container_healthcheck_interval }}"
neutron_server_healthcheck_retries: "{{ default_container_healthcheck_retries }}"
neutron_server_healthcheck_start_period: "{{ default_container_healthcheck_start_period }}"
neutron_server_healthcheck_test: ["CMD-SHELL", "healthcheck_curl http://{{ api_interface_address | put_address_in_context('url') }}:{{ neutron_server_listen_port }}"]
neutron_server_healthcheck_test: ["CMD-SHELL", "healthcheck_curl http://{% if neutron_enable_tls_backend | bool %}localhost{% else %}{{ api_interface_address | put_address_in_context('url') }}{% endif %}:{{ neutron_server_listen_port }}"]
neutron_server_healthcheck_timeout: "{{ default_container_healthcheck_timeout }}"
neutron_server_healthcheck:
interval: "{{ neutron_server_healthcheck_interval }}"
@ -353,6 +382,19 @@ neutron_server_healthcheck:
test: "{% if neutron_server_enable_healthchecks | bool %}{{ neutron_server_healthcheck_test }}{% else %}NONE{% endif %}"
timeout: "{{ neutron_server_healthcheck_timeout }}"
neutron_tls_proxy_enable_healthchecks: "{{ enable_container_healthchecks }}"
neutron_tls_proxy_healthcheck_interval: "{{ default_container_healthcheck_interval }}"
neutron_tls_proxy_healthcheck_retries: "{{ default_container_healthcheck_retries }}"
neutron_tls_proxy_healthcheck_start_period: "{{ default_container_healthcheck_start_period }}"
neutron_tls_proxy_healthcheck_test: ["CMD-SHELL", "healthcheck_curl -u {{ haproxy_user }}:{{ haproxy_password }} {{ api_interface_address | put_address_in_context('url') }}:{{ neutron_tls_proxy_stats_port }}"]
neutron_tls_proxy_healthcheck_timeout: "{{ default_container_healthcheck_timeout }}"
neutron_tls_proxy_healthcheck:
interval: "{{ neutron_tls_proxy_healthcheck_interval }}"
retries: "{{ neutron_tls_proxy_healthcheck_retries }}"
start_period: "{{ neutron_tls_proxy_healthcheck_start_period }}"
test: "{% if neutron_tls_proxy_enable_healthchecks | bool %}{{ neutron_tls_proxy_healthcheck_test }}{% else %}NONE{% endif %}"
timeout: "{{ neutron_tls_proxy_healthcheck_timeout }}"
neutron_sriov_agent_enable_healthchecks: "{{ enable_container_healthchecks }}"
neutron_sriov_agent_healthcheck_interval: "{{ default_container_healthcheck_interval }}"
neutron_sriov_agent_healthcheck_retries: "{{ default_container_healthcheck_retries }}"
@ -459,6 +501,11 @@ ironic_neutron_agent_default_volumes:
- "/etc/localtime:/etc/localtime:ro"
- "{{ '/etc/timezone:/etc/timezone:ro' if ansible_os_family == 'Debian' else '' }}"
- "kolla_logs:/var/log/kolla/"
neutron_tls_proxy_default_volumes:
- "{{ node_config_directory }}/neutron-tls-proxy/:{{ container_config_directory }}/:ro"
- "/etc/localtime:/etc/localtime:ro"
- "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}"
- "kolla_logs:/var/log/kolla/"
neutron_extra_volumes: "{{ default_extra_volumes }}"
neutron_dhcp_agent_extra_volumes: "{{ neutron_extra_volumes }}"
@ -475,6 +522,7 @@ neutron_bgp_dragent_extra_volumes: "{{ neutron_extra_volumes }}"
neutron_infoblox_ipam_agent_extra_volumes: "{{ neutron_extra_volumes }}"
neutron_metering_agent_extra_volumes: "{{ neutron_extra_volumes }}"
ironic_neutron_agent_extra_volumes: "{{ neutron_extra_volumes }}"
neutron_tls_proxy_extra_volumes: "{{ neutron_extra_volumes }}"
####################
# OpenStack
@ -712,3 +760,23 @@ neutron_ks_users:
# SRIOV
####################
neutron_sriov_physnets: "{{ neutron_sriov_physnet_mappings.items() | map('join', ':') | join(',') }}"
####################
# Backend TLS proxy
####################
syslog_server: "{{ api_interface_address }}"
syslog_neutron_tls_proxy_facility: "local4"
neutron_tls_proxy_max_connections: 40000
neutron_tls_proxy_processes: 1
neutron_tls_proxy_process_cpu_map: "no"
neutron_tls_proxy_defaults_max_connections: 10000
neutron_tls_proxy_http_request_timeout: "10s"
neutron_tls_proxy_http_keep_alive_timeout: "10s"
neutron_tls_proxy_queue_timeout: "1m"
neutron_tls_proxy_connect_timeout: "10s"
neutron_tls_proxy_client_timeout: "1m"
neutron_tls_proxy_server_timeout: "1m"
neutron_tls_proxy_check_timeout: "10s"
# Check http://www.haproxy.org/download/1.5/doc/configuration.txt for available options
neutron_tls_proxy_defaults_balance: "roundrobin"

View File

@ -154,6 +154,22 @@
when:
- kolla_action != "config"
- name: Restart neutron-tls-proxy container
vars:
service_name: "neutron-tls-proxy"
service: "{{ neutron_services[service_name] }}"
become: true
kolla_docker:
action: "recreate_or_restart_container"
common_options: "{{ docker_common_options }}"
name: "{{ service.container_name }}"
image: "{{ service.image }}"
volumes: "{{ service.volumes|reject('equalto', '')|list }}"
dimensions: "{{ service.dimensions }}"
healthcheck: "{{ service.healthcheck | default(omit) }}"
when:
- kolla_action != "config"
- name: Restart neutron-metadata-agent container
vars:
service_name: "neutron-metadata-agent"

View File

@ -22,7 +22,21 @@
- include_tasks: copy-certs.yml
when:
- kolla_copy_ca_into_containers | bool
- kolla_copy_ca_into_containers | bool or neutron_enable_tls_backend | bool
- name: Creating TLS backend PEM File
vars:
neutron_tls_proxy: "{{ neutron_services['neutron-tls-proxy'] }}"
assemble:
src: "{{ node_config_directory }}/neutron-tls-proxy/"
dest: "{{ node_config_directory }}/neutron-tls-proxy/neutron-cert-and-key.pem"
mode: "0660"
regexp: "^neutron-(cert|key)\\.pem$"
remote_src: true
become: true
when:
- neutron_tls_proxy.enabled | bool
- neutron_tls_proxy.host_in_groups | bool
- name: Check if policies shall be overwritten
stat:
@ -438,5 +452,24 @@
notify:
- "Restart {{ item.0 }} container"
- name: Copying over neutron-tls-proxy.cfg
vars:
neutron_tls_proxy: "{{ neutron_services['neutron-tls-proxy'] }}"
template:
src: "{{ item }}"
dest: "{{ node_config_directory }}/neutron-tls-proxy/neutron-tls-proxy.cfg"
mode: "0660"
become: true
with_first_found:
- "{{ node_custom_config }}/neutron/{{ inventory_hostname }}/neutron-tls-proxy.cfg"
- "{{ node_custom_config }}/neutron/neutron-tls-proxy.cfg"
- "neutron-tls-proxy.cfg.j2"
when:
- neutron_tls_proxy.enabled | bool
- neutron_tls_proxy.host_in_groups | bool
notify:
- Restart neutron-tls-proxy container
- import_tasks: check-containers.yml
when: kolla_action != "config"

View File

@ -0,0 +1,47 @@
#jinja2: lstrip_blocks: True
global
chroot /var/lib/haproxy
user neutron
group neutron
daemon
log {{ syslog_server }}:{{ syslog_udp_port }} {{ syslog_neutron_tls_proxy_facility }}
maxconn {{ neutron_tls_proxy_max_connections }}
nbproc {{ neutron_tls_proxy_processes }}
{% if (neutron_tls_proxy_processes | int > 1) and (neutron_tls_proxy_process_cpu_map | bool) %}
{% for cpu_idx in range(0, neutron_tls_proxy_processes) %}
cpu-map {{ cpu_idx + 1 }} {{ cpu_idx }}
{% endfor %}
{% endif %}
ssl-default-bind-ciphers DEFAULT:!MEDIUM:!3DES
ssl-default-bind-options no-sslv3 no-tlsv10 no-tlsv11
tune.ssl.default-dh-param 4096
defaults
log global
option redispatch
retries 3
timeout http-request {{ neutron_tls_proxy_http_request_timeout }}
timeout http-keep-alive {{ neutron_tls_proxy_http_keep_alive_timeout }}
timeout queue {{ neutron_tls_proxy_queue_timeout }}
timeout connect {{ neutron_tls_proxy_connect_timeout }}
timeout client {{ neutron_tls_proxy_client_timeout }}
timeout server {{ neutron_tls_proxy_server_timeout }}
timeout check {{ neutron_tls_proxy_check_timeout }}
balance {{ neutron_tls_proxy_defaults_balance }}
maxconn {{ neutron_tls_proxy_defaults_max_connections }}
listen stats
bind {{ api_interface_address }}:{{ neutron_tls_proxy_stats_port }}
mode http
stats enable
stats uri /
stats refresh 15s
stats realm Haproxy\ Stats
stats auth {{ haproxy_user }}:{{ haproxy_password }}
frontend neutron_backend_tls
bind {{ api_interface_address }}:{{ neutron_server_listen_port }} ssl crt /etc/neutron/certs/neutron-cert-and-key.pem
default_backend neutron_server
backend neutron_server
server neutron-server 127.0.0.1:{{ neutron_server_listen_port }} check

View File

@ -0,0 +1,17 @@
{
"command": "/usr/sbin/haproxy -W -db -p /run/haproxy.pid -f /etc/neutron/neutron-tls-proxy.cfg",
"config_files": [
{
"source": "{{ container_config_directory }}/neutron-tls-proxy.cfg",
"dest": "/etc/neutron/neutron-tls-proxy.cfg",
"owner": "neutron",
"perm": "0600"
},
{
"source": "{{ container_config_directory }}/neutron-cert-and-key.pem",
"dest": "/etc/neutron/certs/neutron-cert-and-key.pem",
"owner": "neutron",
"perm": "0600"
}
]
}

View File

@ -7,8 +7,12 @@ log_dir = /var/log/kolla/neutron
# NOTE(elemoine): set use_stderr to False or the logs will also be sent to
# stderr and collected by Docker
use_stderr = False
{% if neutron_enable_tls_backend | bool %}
bind_host = 127.0.0.1
{% else %}
bind_host = {{ api_interface_address }}
{% endif %}
bind_port = {{ neutron_server_listen_port }}
api_paste_config = /usr/share/neutron/api-paste.ini

View File

@ -264,6 +264,10 @@ If Glance TLS backend is enabled (``glance_enable_tls_backend``), the syslog
facility for the ``glance_tls_proxy`` service uses ``local2`` by default. This
can be set via ``syslog_glance_tls_proxy_facility``.
If Neutron TLS backend is enabled (``neutron_enable_tls_backend``), the syslog
facility for the ``neutron_tls_proxy`` service uses ``local4`` by default.
This can be set via ``syslog_neutron_tls_proxy_facility``.
Mount additional Docker volumes in containers
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

View File

@ -218,8 +218,8 @@ Enabling TLS on the backend services secures communication between the
HAProxy listing on the internal/external VIP and the OpenStack
services. It also enables secure end-to-end communication between OpenStack
services that support TLS termination. The OpenStack services that support
backend TLS termination in Ussuri are: Keystone, Glance, Heat, Placement,
Horizon, Barbican, and Cinder.
backend TLS termination in Victoria are: Nova, Ironic, Neutron, Keystone,
Glance, Heat, Placement, Horizon, Barbican, and Cinder.
The configuration variables that control back-end TLS for service endpoints
are:

View File

@ -2,6 +2,6 @@
features:
- |
Adds configuration options to enable backend TLS encryption from HAProxy
to the Nova and Ironic services. When used in conjunction with enabling TLS
for service API endpoints, network communcation will be encrypted end to
end, from client through HAProxy to the backend service.
to the Nova, Ironic, and Neutron services. When used in conjunction with
enabling TLS for service API endpoints, network communcation will be
encrypted end to end, from client through HAProxy to the backend service.