openstack/kolla-ansible
Code Issues Proposed changes

Add support for encrypting Glance api

Browse Source 
Add TLS support for Glance api using HAProxy to perform TLS termination.

Change-Id: I77051baaeb5d3f7dd9002262534e7d35f3926809
Partially-Implements: blueprint add-ssl-internal-network
This commit is contained in:
James Kirsch 2020-04-24 16:01:09 -07:00 committed by Mark Goddard
parent 70e7b1b0d8
commit f87814f794
15 changed files with 266 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
ansible/roles/common/defaults/main.yml
View File

@ -72,6 +72,7 @@ fluentd_image_full: "{{ fluentd_image }}:{{ fluentd_tag }}"
syslog_swift_facility: "local0"
syslog_haproxy_facility: "local1"
syslog_glance_tls_proxy_facility: "local2"
kolla_toolbox_default_volumes:
- "{{ node_config_directory }}/kolla-toolbox/:{{ container_config_directory }}/:ro"
@ -125,6 +126,8 @@ fluentd_input_openstack_services:
enabled: "{{ enable_freezer | bool }}"
- name: glance
enabled: "{{ enable_glance | bool }}"
- name: glance-tls-proxy
enabled: "{{ enable_glance | bool }}"
- name: gnocchi
enabled: "{{ enable_gnocchi | bool }}"
- name: heat

1
ansible/roles/common/tasks/config.yml
View File

@ -294,6 +294,7 @@
- { name: "etcd", enabled: "{{ enable_etcd }}" }
- { name: "freezer", enabled: "{{ enable_freezer }}" }
- { name: "glance", enabled: "{{ enable_glance }}" }
- { name: "glance-tls-proxy", enabled: "{{ enable_glance }}" }
- { name: "global", enabled: "yes" }
- { name: "gnocchi", enabled: "{{ enable_gnocchi }}" }
- { name: "grafana", enabled: "{{ enable_grafana }}" }

7
ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2
View File

@ -36,6 +36,13 @@
</record>
</filter>
<filter syslog.{{ syslog_glance_tls_proxy_facility }}.**>
@type record_transformer
<record>
programname glance-tls-proxy
</record>
</filter>
# Rename internal Fluent message field to match other logs. This removes
# all other fields by default, including the original message field. This is
# intented to avoid duplication of the log message and to prevent passing

55
ansible/roles/common/templates/conf/output/00-local.conf.j2
View File

@ -105,3 +105,58 @@
</store>
{% endif %}
</match>
<match syslog.{{ syslog_glance_tls_proxy_facility }}.**>
@type copy
<store>
@type file
path /var/log/kolla/glance-tls-proxy/glance-tls-proxy.*.log
output_tag false
output_time false
utc
append true
compress gzip
</store>
{% if log_direct_to_elasticsearch %}
<store>
type elasticsearch
host {{ elasticsearch_address }}
port {{ elasticsearch_port }}
scheme {{ fluentd_elasticsearch_scheme }}
{% if fluentd_elasticsearch_path != '' %}
path {{ fluentd_elasticsearch_path }}
{% endif %}
{% if fluentd_elasticsearch_scheme == 'https' %}
ssl_version {{ fluentd_elasticsearch_ssl_version }}
ssl_verify {{ fluentd_elasticsearch_ssl_verify }}
{% endif %}
{% if fluentd_elasticsearch_user != '' and fluentd_elasticsearch_password != ''%}
user {{ fluentd_elasticsearch_user }}
password {{ fluentd_elasticsearch_password }}
{% endif %}
logstash_format true
logstash_prefix {{ kibana_log_prefix }}
flush_interval 15s
reconnect_on_error true
buffer_type file
buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_glance_tls_proxy_facility }}.*
</store>
{% elif enable_monasca | bool %}
type copy
<store>
@type monasca
keystone_url {{ keystone_internal_url }}
monasca_log_api {{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ monasca_log_api_port }}
monasca_log_api_version v3.0
username {{ monasca_agent_user }}
password {{ monasca_agent_password }}
domain_id default
project_name {{ monasca_control_plane_project }}
message_field_name Payload
buffer_type file
buffer_path /var/lib/fluentd/data/monasca.buffer/{{ syslog_glance_tls_proxy_facility }}.*
max_retry_wait 1800s
disable_retry_limit true
</store>
{% endif %}
</match>

3
ansible/roles/common/templates/cron-logrotate-glance-tls-proxy.conf.j2 Normal file
View File

@ -0,0 +1,3 @@
"/var/log/kolla/glance-tls-proxy/*.log"
{
}

1
ansible/roles/common/templates/cron.json.j2
View File

@ -16,6 +16,7 @@
( 'etcd', enable_etcd ),
( 'freezer', enable_freezer ),
( 'glance', enable_glance ),
( 'glance-tls-proxy', enable_glance ),
( 'gnocchi', enable_gnocchi ),
( 'grafana', enable_grafana ),
( 'haproxy', enable_haproxy ),

72
ansible/roles/glance/defaults/main.yml
View File

@ -14,7 +14,7 @@ glance_services:
dimensions: "{{ glance_api_dimensions }}"
haproxy:
glance_api:
enabled: "{{ enable_glance }}"
enabled: "{{ enable_glance | bool and not glance_enable_tls_backend | bool }}"
mode: "http"
external: false
port: "{{ glance_api_port }}"
@ -24,7 +24,7 @@ glance_services:
- "timeout server {{ haproxy_glance_api_server_timeout }}"
custom_member_list: "{{ haproxy_members.split(';') }}"
glance_api_external:
enabled: "{{ enable_glance }}"
enabled: "{{ enable_glance | bool and not glance_enable_tls_backend | bool }}"
mode: "http"
external: true
port: "{{ glance_api_port }}"
@ -33,12 +33,43 @@ glance_services:
backend_http_extra:
- "timeout server {{ haproxy_glance_api_server_timeout }}"
custom_member_list: "{{ haproxy_members.split(';') }}"
glance-tls-proxy:
container_name: glance_tls_proxy
group: glance-api
host_in_groups: "{{ inventory_hostname in glance_api_hosts }}"
enabled: "{{ glance_enable_tls_backend }}"
image: "{{ glance_tls_proxy_image_full }}"
volumes: "{{ glance_tls_proxy_default_volumes + glance_tls_proxy_extra_volumes }}"
dimensions: "{{ glance_tls_proxy_dimensions }}"
haproxy:
glance_tls_proxy:
enabled: "{{ enable_glance | bool and glance_enable_tls_backend | bool }}"
mode: "http"
external: false
port: "{{ glance_api_port }}"
frontend_http_extra:
- "timeout client {{ haproxy_glance_api_client_timeout }}"
backend_http_extra:
- "timeout server {{ haproxy_glance_api_server_timeout }}"
custom_member_list: "{{ haproxy_tls_members.split(';') }}"
tls_backend: "yes"
glance_tls_proxy_external:
enabled: "{{ enable_glance | bool and glance_enable_tls_backend | bool }}"
mode: "http"
external: true
port: "{{ glance_api_port }}"
frontend_http_extra:
- "timeout client {{ haproxy_glance_api_client_timeout }}"
backend_http_extra:
- "timeout server {{ haproxy_glance_api_server_timeout }}"
custom_member_list: "{{ haproxy_tls_members.split(';') }}"
tls_backend: "yes"
####################
# HAProxy
####################
haproxy_members: "{% for host in glance_api_hosts %}server {{ hostvars[host]['ansible_hostname'] }} {{ 'api' | kolla_address(host) }}:{{ glance_api_listen_port }} check inter 2000 rise 2 fall 5;{% endfor %}"
haproxy_tls_members: "{% for host in glance_api_hosts %}server {{ hostvars[host]['ansible_hostname'] }} {{ 'api' | kolla_address(host) }}:{{ glance_api_listen_port }} check inter 2000 rise 2 fall 5 ssl verify required ca-file {{ haproxy_backend_cacert }};{% endfor %}"
####################
# Keystone
@ -92,7 +123,12 @@ glance_api_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ dock
glance_api_tag: "{{ glance_tag }}"
glance_api_image_full: "{{ glance_api_image }}:{{ glance_api_tag }}"
glance_tls_proxy_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ glance_install_type }}-haproxy"
glance_tls_proxy_tag: "{{ glance_tag }}"
glance_tls_proxy_image_full: "{{ glance_tls_proxy_image }}:{{ glance_tls_proxy_tag }}"
glance_api_dimensions: "{{ default_container_dimensions }}"
glance_tls_proxy_dimensions: "{{ default_container_dimensions }}"
glance_api_default_volumes:
- "{{ node_config_directory }}/glance-api/:{{ container_config_directory }}/:ro"
@ -104,9 +140,15 @@ glance_api_default_volumes:
# NOTE(yoctozepto): below to support Cinder iSCSI backends
- "{% if enable_cinder | bool and enable_cinder_backend_iscsi | bool %}iscsi_info:/etc/iscsi{% endif %}"
- "{% if enable_cinder | bool and enable_cinder_backend_iscsi | bool %}/dev:/dev{% endif %}"
glance_tls_proxy_default_volumes:
- "{{ node_config_directory }}/glance-tls-proxy/:{{ container_config_directory }}/:ro"
- "/etc/localtime:/etc/localtime:ro"
- "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}"
- "kolla_logs:/var/log/kolla/"
glance_extra_volumes: "{{ default_extra_volumes }}"
glance_api_extra_volumes: "{{ glance_extra_volumes }}"
glance_tls_proxy_extra_volumes: "{{ glance_extra_volumes }}"
####################
# Glance
@ -166,3 +208,27 @@ vmware_datastore_name:
###################
# Default maximum size of 10Gb
glance_cache_max_size: "10737418240"
####################
# TLS
####################
glance_enable_tls_backend: "{{ kolla_enable_tls_backend }}"
####################
# Backend TLS proxy
####################
syslog_server: "{{ api_interface_address }}"
syslog_glance_tls_proxy_facility: "local2"
glance_tls_proxy_max_connections: 40000
glance_tls_proxy_processes: 1
glance_tls_proxy_process_cpu_map: "no"
glance_tls_proxy_defaults_max_connections: 10000
glance_tls_proxy_http_request_timeout: "10s"
glance_tls_proxy_queue_timeout: "1m"
glance_tls_proxy_connect_timeout: "10s"
glance_tls_proxy_client_timeout: "{{ haproxy_glance_api_client_timeout}}"
glance_tls_proxy_server_timeout: "{{ haproxy_glance_api_server_timeout }}"
glance_tls_proxy_check_timeout: "10s"
# Check http://www.haproxy.org/download/1.5/doc/configuration.txt for available options
glance_tls_proxy_defaults_balance: "roundrobin"

15
ansible/roles/glance/handlers/main.yml
View File

@ -15,3 +15,18 @@
dimensions: "{{ service.dimensions }}"
when:
- kolla_action != "config"
- name: Restart glance-tls-proxy container
vars:
service_name: "glance-tls-proxy"
service: "{{ glance_services[service_name] }}"
become: true
kolla_docker:
action: "recreate_or_restart_container"
common_options: "{{ docker_common_options }}"
name: "{{ service.container_name }}"
image: "{{ service.image }}"
volumes: "{{ service.volumes|reject('equalto', '')|list }}"
dimensions: "{{ service.dimensions }}"
when:
- kolla_action != "config"

2
ansible/roles/glance/tasks/check-containers.yml
View File

@ -6,7 +6,7 @@
common_options: "{{ docker_common_options }}"
name: "{{ item.value.container_name }}"
image: "{{ item.value.image }}"
privileged: "{{ item.value.privileged }}"
privileged: "{{ item.value.privileged | default(omit) }}"
environment: "{{ item.value.environment | default(omit) }}"
volumes: "{{ item.value.volumes|reject('equalto', '')|list }}"
dimensions: "{{ item.value.dimensions }}"

61
ansible/roles/glance/tasks/config.yml
View File

@ -37,7 +37,18 @@
- include_tasks: copy-certs.yml
when:
- kolla_copy_ca_into_containers | bool
- kolla_copy_ca_into_containers | bool or glance_enable_tls_backend | bool
- name: Creating TLS backend PEM File
assemble:
src: "{{ node_config_directory }}/glance-tls-proxy/"
dest: "{{ node_config_directory }}/glance-tls-proxy/glance-cert-and-key.pem"
mode: "0660"
regexp: "^glance-(cert|key)\\.pem$"
remote_src: true
become: true
when:
- glance_enable_tls_backend | bool
- name: Copying over config.json files for services
template:
@ -52,25 +63,24 @@
notify:
- Restart {{ item.key }} container
- name: Copying over glance-*.conf
- name: Copying over glance-api.conf
vars:
service_name: "{{ item.key }}"
glance_api: "{{ glance_services['glance-api'] }}"
merge_configs:
sources:
- "{{ role_path }}/templates/{{ item.key }}.conf.j2"
- "{{ role_path }}/templates/glance-api.conf.j2"
- "{{ node_custom_config }}/global.conf"
- "{{ node_custom_config }}/glance.conf"
- "{{ node_custom_config }}/glance/{{ item.key }}.conf"
- "{{ node_custom_config }}/glance/{{ inventory_hostname }}/{{ item.key }}.conf"
dest: "{{ node_config_directory }}/{{ item.key }}/{{ item.key }}.conf"
- "{{ node_custom_config }}/glance/glance-api.conf"
- "{{ node_custom_config }}/glance/{{ inventory_hostname }}/glance-api.conf"
dest: "{{ node_config_directory }}/glance-api/glance-api.conf"
mode: "0660"
become: true
when:
- item.value.enabled | bool
- item.value.host_in_groups | bool
with_dict: "{{ glance_services }}"
- glance_api.enabled | bool
- glance_api.host_in_groups | bool
notify:
- Restart {{ item.key }} container
- Restart glance-api container
- name: Copying over glance-cache.conf for glance_api
vars:
@ -111,18 +121,37 @@
- Restart glance-api container
- name: Copying over existing policy file
vars:
glance_api: "{{ glance_services['glance-api'] }}"
template:
src: "{{ glance_policy_file_path }}"
dest: "{{ node_config_directory }}/{{ item.key }}/{{ glance_policy_file }}"
dest: "{{ node_config_directory }}/glance-api/{{ glance_policy_file }}"
mode: "0660"
become: true
when:
- glance_policy_file is defined
- item.value.host_in_groups | bool
- item.value.enabled | bool
with_dict: "{{ glance_services }}"
- glance_api.host_in_groups | bool
- glance_api.enabled | bool
notify:
- Restart {{ item.key }} container
- Restart glance-api container
- name: Copying over glance-haproxy-tls.cfg
vars:
glance_tls_proxy: "{{ glance_services['glance-tls-proxy'] }}"
template:
src: "{{ item }}"
dest: "{{ node_config_directory }}/glance-tls-proxy/glance-tls-proxy.cfg"
mode: "0660"
become: true
with_first_found:
- "{{ node_custom_config }}/glance/{{ inventory_hostname }}/glance-tls-proxy.cfg"
- "{{ node_custom_config }}/glance/glance-tls-proxy.cfg"
- "glance-tls-proxy.cfg.j2"
when:
- glance_tls_proxy.enabled | bool
- glance_tls_proxy.host_in_groups | bool
notify:
- Restart glance-tls-proxy container
- include_tasks: check-containers.yml
when: kolla_action != "config"

4
ansible/roles/glance/templates/glance-api.conf.j2
View File

@ -5,7 +5,11 @@ debug = {{ glance_logging_debug }}
log_file = /var/log/kolla/glance/glance-api.log
use_forwarded_for = true
{% if glance_enable_tls_backend | bool %}
bind_host = 127.0.0.1
{% else %}
bind_host = {{ api_interface_address }}
{% endif %}
bind_port = {{ glance_api_listen_port }}
workers = {{ openstack_service_workers }}

37
ansible/roles/glance/templates/glance-tls-proxy.cfg.j2 Normal file
View File

@ -0,0 +1,37 @@
#jinja2: lstrip_blocks: True
global
chroot /var/lib/haproxy
user glance
group glance
daemon
log {{ syslog_server }}:{{ syslog_udp_port }} {{ syslog_glance_tls_proxy_facility }}
maxconn {{ glance_tls_proxy_max_connections }}
nbproc {{ glance_tls_proxy_processes }}
{% if (glance_tls_proxy_processes | int > 1) and (glance_tls_proxy_process_cpu_map | bool) %}
{% for cpu_idx in range(0, glance_tls_proxy_processes) %}
cpu-map {{ cpu_idx + 1 }} {{ cpu_idx }}
{% endfor %}
{% endif %}
ssl-default-bind-ciphers DEFAULT:!MEDIUM:!3DES
ssl-default-bind-options no-sslv3 no-tlsv10 no-tlsv11
tune.ssl.default-dh-param 4096
defaults
log global
option redispatch
retries 3
timeout http-request {{ glance_tls_proxy_http_request_timeout }}
timeout queue {{ glance_tls_proxy_queue_timeout }}
timeout connect {{ glance_tls_proxy_connect_timeout }}
timeout client {{ glance_tls_proxy_client_timeout }}
timeout server {{ glance_tls_proxy_server_timeout }}
timeout check {{ glance_tls_proxy_check_timeout }}
balance {{ glance_tls_proxy_defaults_balance }}
maxconn {{ glance_tls_proxy_defaults_max_connections }}
frontend glance_backend_tls
bind {{ api_interface_address }}:{{ glance_api_listen_port }} ssl crt /etc/glance/certs/glance-cert-and-key.pem
default_backend glance_api
backend glance_api
server glance-api 127.0.0.1:{{ glance_api_listen_port }} check

17
ansible/roles/glance/templates/glance-tls-proxy.json.j2 Normal file
View File

@ -0,0 +1,17 @@
{
"command": "/usr/sbin/haproxy -W -db -p /run/haproxy.pid -f /etc/glance/glance-tls-proxy.cfg",
"config_files": [
{
"source": "{{ container_config_directory }}/glance-tls-proxy.cfg",
"dest": "/etc/glance/glance-tls-proxy.cfg",
"owner": "glance",
"perm": "0600"
},
{
"source": "{{ container_config_directory }}/glance-cert-and-key.pem",
"dest": "/etc/glance/certs/glance-cert-and-key.pem",
"owner": "glance",
"perm": "0600"
}
]
}

4
doc/source/admin/advanced-configuration.rst
View File

@ -382,6 +382,10 @@ By default, Swift and HAProxy use ``local0`` and ``local1``, respectively.
syslog_swift_facility: "local0"
syslog_haproxy_facility: "local1"
If Glance TLS backend is enabled (``glance_enable_tls_backend``), the syslog
facility for the ``glance_tls_proxy`` service uses ``local2`` by default. This
can be set via ``syslog_glance_tls_proxy_facility``.
Mount additional Docker volumes in containers
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

7
releasenotes/notes/encrypt-backend-haproxy-fb96285d74fb464c.yaml
View File

@ -2,6 +2,7 @@
features:
- |
Added configuration options to enable backend TLS encryption from HAProxy
to the Keystone, Heat, and cinder service. When used in conjunction with
enabling TLS for service API endpoints, network communcation will be
encrypted end to end, from client through HAProxy to the backend service.
to the Keystone, Glance, Heat, and Cinder services. When used in
conjunction with enabling TLS for service API endpoints, network
communcation will be encrypted end to end, from client through HAProxy to
the backend service.