From a9e7a6fa9232936aebbe449e75ac616fa9de2291 Mon Sep 17 00:00:00 2001 From: Alan Bishop Date: Mon, 13 Jul 2020 13:18:59 -0700 Subject: [PATCH] Revamp how etcd's cert and key are handled in containers Use kolla_config to merge etcd's cert and key files into containers, and set the ownership so the corresponding service can read the files. Previously, etcd's cert and key files were directly bind mounted in the etcd and cinder containers that need the files. An ACL was added to ensure the corresponding services had read access to the files on the host, which are owned by root. The ACL was cumbersome, and required hardcoding the UID of each service. Change-Id: Ic606e751cb046c34d33a94a2acd4313f4043441f Depends-On: I0ea26253355a57b3721bfa6ceef3972eaabc5b1d (cherry picked from commit 7bcdd2448b2aafe99e17e3719cf224d1009099ec) (cherry picked from commit 978c4e05dec23c7957af7ece44d4d551ca6d50e5) --- .../cinder/cinder-api-container-puppet.yaml | 9 ++++++ .../cinder-backup-container-puppet.yaml | 9 ++++++ .../cinder-backup-pacemaker-puppet.yaml | 9 ++++++ .../cinder-common-container-puppet.yaml | 4 +-- .../cinder-scheduler-container-puppet.yaml | 9 ++++++ .../cinder-volume-container-puppet.yaml | 26 ++++++---------- .../cinder-volume-pacemaker-puppet.yaml | 4 +++ deployment/etcd/etcd-container-puppet.yaml | 31 +++++++------------ 8 files changed, 63 insertions(+), 38 deletions(-) diff --git a/deployment/cinder/cinder-api-container-puppet.yaml b/deployment/cinder/cinder-api-container-puppet.yaml index 6e570bbf82..94434b9895 100644 --- a/deployment/cinder/cinder-api-container-puppet.yaml +++ b/deployment/cinder/cinder-api-container-puppet.yaml @@ -274,10 +274,19 @@ outputs: dest: "/" merge: true preserve_properties: true + - source: "/var/lib/kolla/config_files/src-tls/*" + dest: "/" + merge: true + preserve_properties: true + optional: true permissions: - path: /var/log/cinder owner: cinder:cinder recurse: true + - path: /etc/pki/tls/certs/etcd.crt + owner: cinder:cinder + - path: /etc/pki/tls/private/etcd.key + owner: cinder:cinder /var/lib/kolla/config_files/cinder_api_cron.json: command: /usr/sbin/crond -n config_files: diff --git a/deployment/cinder/cinder-backup-container-puppet.yaml b/deployment/cinder/cinder-backup-container-puppet.yaml index c8833e5fa0..081a8c92e2 100644 --- a/deployment/cinder/cinder-backup-container-puppet.yaml +++ b/deployment/cinder/cinder-backup-container-puppet.yaml @@ -165,6 +165,11 @@ outputs: dest: "/etc/iscsi/" merge: true preserve_properties: true + - source: "/var/lib/kolla/config_files/src-tls/*" + dest: "/" + merge: true + preserve_properties: true + optional: true permissions: - path: /var/lib/cinder owner: cinder:cinder @@ -180,6 +185,10 @@ outputs: USER: {get_param: CephClientUserName} owner: cinder:cinder perm: '0600' + - path: /etc/pki/tls/certs/etcd.crt + owner: cinder:cinder + - path: /etc/pki/tls/private/etcd.key + owner: cinder:cinder docker_config: step_3: cinder_backup_init_logs: diff --git a/deployment/cinder/cinder-backup-pacemaker-puppet.yaml b/deployment/cinder/cinder-backup-pacemaker-puppet.yaml index 1919908ba5..217963f1a9 100644 --- a/deployment/cinder/cinder-backup-pacemaker-puppet.yaml +++ b/deployment/cinder/cinder-backup-pacemaker-puppet.yaml @@ -163,6 +163,11 @@ outputs: dest: "/etc/iscsi/" merge: true preserve_properties: true + - source: "/var/lib/kolla/config_files/src-tls/*" + dest: "/" + merge: true + preserve_properties: true + optional: true permissions: - path: /var/lib/cinder owner: cinder:cinder @@ -170,6 +175,10 @@ outputs: - path: /var/log/cinder owner: cinder:cinder recurse: true + - path: /etc/pki/tls/certs/etcd.crt + owner: cinder:cinder + - path: /etc/pki/tls/private/etcd.key + owner: cinder:cinder container_config_scripts: {get_attr: [ContainersCommon, container_config_scripts]} docker_config: step_3: diff --git a/deployment/cinder/cinder-common-container-puppet.yaml b/deployment/cinder/cinder-common-container-puppet.yaml index 7aa993a3dd..06be5bda7b 100644 --- a/deployment/cinder/cinder-common-container-puppet.yaml +++ b/deployment/cinder/cinder-common-container-puppet.yaml @@ -114,8 +114,8 @@ outputs: if: - cvol_active_active_tls_enabled - - - /etc/pki/tls/certs/etcd.crt:/etc/pki/tls/certs/etcd.crt:ro - - /etc/pki/tls/private/etcd.key:/etc/pki/tls/private/etcd.key:ro + - /etc/pki/tls/certs/etcd.crt:/var/lib/kolla/config_files/src-tls/etc/pki/tls/certs/etcd.crt:ro + - /etc/pki/tls/private/etcd.key:/var/lib/kolla/config_files/src-tls/etc/pki/tls/private/etcd.key:ro - [] cinder_volume_host_prep_tasks: diff --git a/deployment/cinder/cinder-scheduler-container-puppet.yaml b/deployment/cinder/cinder-scheduler-container-puppet.yaml index d2bc561390..b976105f3a 100644 --- a/deployment/cinder/cinder-scheduler-container-puppet.yaml +++ b/deployment/cinder/cinder-scheduler-container-puppet.yaml @@ -101,10 +101,19 @@ outputs: dest: "/" merge: true preserve_properties: true + - source: "/var/lib/kolla/config_files/src-tls/*" + dest: "/" + merge: true + preserve_properties: true + optional: true permissions: - path: /var/log/cinder owner: cinder:cinder recurse: true + - path: /etc/pki/tls/certs/etcd.crt + owner: cinder:cinder + - path: /etc/pki/tls/private/etcd.key + owner: cinder:cinder docker_config: step_2: cinder_scheduler_init_logs: diff --git a/deployment/cinder/cinder-volume-container-puppet.yaml b/deployment/cinder/cinder-volume-container-puppet.yaml index b9cea65799..94603c8f34 100644 --- a/deployment/cinder/cinder-volume-container-puppet.yaml +++ b/deployment/cinder/cinder-volume-container-puppet.yaml @@ -310,6 +310,11 @@ outputs: dest: "/etc/iscsi/" merge: true preserve_properties: true + - source: "/var/lib/kolla/config_files/src-tls/*" + dest: "/" + merge: true + preserve_properties: true + optional: true permissions: - path: /var/log/cinder owner: cinder:cinder @@ -322,6 +327,10 @@ outputs: USER: {get_param: CephClientUserName} owner: cinder:cinder perm: '0600' + - path: /etc/pki/tls/certs/etcd.crt + owner: cinder:cinder + - path: /etc/pki/tls/private/etcd.key + owner: cinder:cinder docker_config: step_3: cinder_volume_init_logs: @@ -345,20 +354,3 @@ outputs: volumes: {get_attr: [CinderCommon, cinder_volume_volumes]} environment: {get_attr: [CinderCommon, cinder_volume_environment]} host_prep_tasks: {get_attr: [CinderCommon, cinder_volume_host_prep_tasks]} - deploy_steps_tasks: - - name: ensure cinder can access etcd's tls cert and key - become: true - acl: - path: "{{ item }}" - entity: "{{ 42407 | string }}" - etype: user - permissions: r - state: present - with_items: - - /etc/pki/tls/certs/etcd.crt - - /etc/pki/tls/private/etcd.key - vars: - cvol_active_active_tls_enabled: {if: [cvol_active_active_tls_enabled, true, false]} - when: - - cvol_active_active_tls_enabled|bool - - step|int == 3 diff --git a/deployment/cinder/cinder-volume-pacemaker-puppet.yaml b/deployment/cinder/cinder-volume-pacemaker-puppet.yaml index 6ca8ebca61..78317bc614 100644 --- a/deployment/cinder/cinder-volume-pacemaker-puppet.yaml +++ b/deployment/cinder/cinder-volume-pacemaker-puppet.yaml @@ -151,6 +151,10 @@ outputs: dest: "/etc/iscsi/" merge: true preserve_properties: true + # NOTE(abishop): no need to copy any src-tls/* files or set ownership + # of etcd's TLS certificate and key. The etcd service is only used by + # cinder-volume when it's running active/active, and *not* when it's + # under pcmk control. permissions: - path: /var/log/cinder owner: cinder:cinder diff --git a/deployment/etcd/etcd-container-puppet.yaml b/deployment/etcd/etcd-container-puppet.yaml index e17f05b786..67fe249158 100644 --- a/deployment/etcd/etcd-container-puppet.yaml +++ b/deployment/etcd/etcd-container-puppet.yaml @@ -131,6 +131,7 @@ outputs: "%{hiera('NETWORK')}" params: NETWORK: {get_param: [ServiceNetMap, EtcdNetwork]} + postsave_cmd: '/usr/bin/certmonger-etcd-refresh.sh' etcd::trusted_ca_file: {get_param: InternalTLSCAFile} etcd::peer_trusted_ca_file: {get_param: InternalTLSCAFile} - @@ -154,10 +155,19 @@ outputs: dest: "/" merge: true preserve_properties: true + - source: "/var/lib/kolla/config_files/src-tls/*" + dest: "/" + merge: true + preserve_properties: true + optional: true permissions: - path: /var/lib/etcd owner: etcd:etcd recurse: true + - path: /etc/pki/tls/certs/etcd.crt + owner: etcd:etcd + - path: /etc/pki/tls/private/etcd.key + owner: etcd:etcd docker_config: step_2: etcd: @@ -178,8 +188,8 @@ outputs: if: - internal_tls_enabled - - - /etc/pki/tls/certs/etcd.crt:/etc/pki/tls/certs/etcd.crt:ro - - /etc/pki/tls/private/etcd.key:/etc/pki/tls/private/etcd.key:ro + - /etc/pki/tls/certs/etcd.crt:/var/lib/kolla/config_files/src-tls/etc/pki/tls/certs/etcd.crt:ro + - /etc/pki/tls/private/etcd.key:/var/lib/kolla/config_files/src-tls/etc/pki/tls/private/etcd.key:ro - null environment: KOLLA_CONFIG_STRATEGY: COPY_ALWAYS @@ -200,23 +210,6 @@ outputs: path: /var/lib/etcd state: directory setype: svirt_sandbox_file_t - deploy_steps_tasks: - - name: ensure etcd can access its tls cert and key - become: true - acl: - path: "{{ item }}" - entity: "{{ 42413 | string }}" - etype: user - permissions: r - state: present - with_items: - - /etc/pki/tls/certs/etcd.crt - - /etc/pki/tls/private/etcd.key - vars: - internal_tls_enabled: {if: [internal_tls_enabled, true, false]} - when: - - internal_tls_enabled|bool - - step|int == 2 upgrade_tasks: [] metadata_settings: if: