Fix Redis TLS setup and its HA deployment

This patch reverts the revert of Redis TLS [1,2], and update the pacemaker redis template to configure Redis to encrypt the replication traffic between Redis nodes. [1] a3769c0317 [2] ebc8414cd0 Depends-On: I6cc818973fab25b4cd6f7a0d040aaa05a35c5bb1 Change-Id: I7f7be4bba6d41c04385f074857c82507cc8c2617 Closes-Bug: #1737707
2017-12-13 13:44:09 +00:00 · 2017-12-13 13:44:09 +00:00 · 91db2020df
parent 71d098e771
commit 91db2020df
6 changed files with 270 additions and 52 deletions
--- a/docker/services/database/redis.yaml
+++ b/docker/services/database/redis.yaml
@ -36,9 +36,19 @@ parameters:
    default: {}
    description: Parameters specific to the role
    type: json
+  EnableInternalTLS:
+    type: boolean
+    default: false
+
+conditions:
+
+  internal_tls_enabled: {equals: [{get_param: EnableInternalTLS}, true]}

 resources:

+  ContainersCommon:
+    type: ../containers-common.yaml
+
  RedisBase:
    type: ../../../puppet/services/database/redis.yaml
    properties:
@ -56,6 +66,8 @@ outputs:
        map_merge:
          - {get_attr: [RedisBase, role_data, config_settings]}
          - redis::daemonize: false
+            tripleo::stunnel::manage_service: false
+            tripleo::stunnel::foreground: 'yes'
      logging_source: {get_attr: [RedisBase, role_data, logging_source]}
      logging_groups: {get_attr: [RedisBase, role_data, logging_groups]}
      service_config_settings: {get_attr: [RedisBase, role_data, service_config_settings]}
@ -81,33 +93,64 @@ outputs:
            - path: /var/run/redis
              owner: redis:redis
              recurse: true
+        /var/lib/kolla/config_files/redis_tls_proxy.json:
+          command: stunnel /etc/stunnel/stunnel.conf
+          config_files:
+            - source: "/var/lib/kolla/config_files/src/*"
+              dest: "/"
+              merge: true
+              preserve_properties: true
      docker_config:
        step_1:
-          redis_init_logs:
-            start_order: 0
-            detach: false
-            image: &redis_image {get_param: DockerRedisImage}
-            privileged: false
-            user: root
-            volumes:
-              - /var/log/containers/redis:/var/log/redis
-            command: ['/bin/bash', '-c', 'chown -R redis:redis /var/log/redis']
-          redis:
-            start_order: 1
-            image: *redis_image
-            net: host
-            privileged: false
-            restart: always
-            healthcheck:
-              test: /openstack/healthcheck
-            volumes:
-              - /run:/run
-              - /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro
-              - /var/lib/config-data/puppet-generated/redis/:/var/lib/kolla/config_files/src:ro
-              - /etc/localtime:/etc/localtime:ro
-              - /var/log/containers/redis:/var/log/redis
-            environment:
-              - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+          map_merge:
+            - redis_init_logs:
+                start_order: 0
+                detach: false
+                image: &redis_image {get_param: DockerRedisImage}
+                privileged: false
+                user: root
+                volumes:
+                  - /var/log/containers/redis:/var/log/redis
+                command: ['/bin/bash', '-c', 'chown -R redis:redis /var/log/redis']
+            - redis:
+                start_order: 1
+                image: *redis_image
+                net: host
+                privileged: false
+                restart: always
+                healthcheck:
+                  test: /openstack/healthcheck
+                volumes:
+                  - /run:/run
+                  - /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro
+                  - /var/lib/config-data/puppet-generated/redis/:/var/lib/kolla/config_files/src:ro
+                  - /etc/localtime:/etc/localtime:ro
+                  - /var/log/containers/redis:/var/log/redis
+                environment:
+                  - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+            - if:
+                - internal_tls_enabled
+                - redis_tls_proxy:
+                    start_order: 2
+                    image: *redis_image
+                    net: host
+                    user: root
+                    restart: always
+                    healthcheck:
+                      test: /openstack/healthcheck
+                    volumes:
+                      list_concat:
+                        - {get_attr: [ContainersCommon, volumes]}
+                        -
+                          - /var/lib/kolla/config_files/redis_tls_proxy.json:/var/lib/kolla/config_files/config.json:ro
+                          - /var/lib/config-data/puppet-generated/redis/:/var/lib/kolla/config_files/src:ro
+                          - /etc/pki/tls/certs/redis.crt:/etc/pki/tls/certs/redis.crt:ro
+                          - /etc/pki/tls/private/redis.key:/etc/pki/tls/private/redis.key:ro
+                    environment:
+                      - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+                - {}
+      metadata_settings:
+        get_attr: [RedisBase, role_data, metadata_settings]
      host_prep_tasks:
        - name: create persistent directories
          file:
--- a/docker/services/pacemaker/database/redis.yaml
+++ b/docker/services/pacemaker/database/redis.yaml
@ -40,9 +40,18 @@ parameters:
    default: false
    description: Whether to run config management (e.g. Puppet) in debug mode.
    type: boolean
+  EnableInternalTLS:
+    type: boolean
+    default: false
+  RedisIPv6:
+    default: false
+    description: Enable IPv6 in Redis
+    type: boolean

 conditions:
  puppet_debug_enabled: {get_param: ConfigDebug}
+  internal_tls_enabled: {equals: [{get_param: EnableInternalTLS}, true]}
+  redis_ipv6: {get_param: RedisIPv6}

 resources:

@ -84,6 +93,28 @@ outputs:
                  - 3124
                  - 6379
                  - 26379
+            tripleo::stunnel::manage_service: false
+            tripleo::stunnel::foreground: 'yes'
+            tripleo::profile::pacemaker::database::redis_bundle::tls_proxy_bind_ip:
+              get_param: [ServiceNetMap, RedisNetwork]
+            tripleo::profile::pacemaker::database::redis_bundle::tls_proxy_fqdn:
+              str_replace:
+                template:
+                  "%{hiera('fqdn_$NETWORK')}"
+                params:
+                  $NETWORK: {get_param: [ServiceNetMap, RedisNetwork]}
+            tripleo::profile::pacemaker::database::redis_bundle::tls_proxy_port: 6379
+          - if:
+            - internal_tls_enabled
+            - redis::extra_config_file: "/etc/redis-tls.conf"
+              tripleo::profile::pacemaker::database::redis_bundle::extra_config_file: "/etc/redis-tls.conf"
+              tripleo::profile::pacemaker::database::redis_bundle::tls_tunnel_base_port: 6660
+              tripleo::profile::pacemaker::database::redis_bundle::tls_tunnel_local_name:
+                if:
+                - redis_ipv6
+                - '::1'
+                - '127.0.0.1'
+            - {}
      logging_source: {get_attr: [RedisBase, role_data, logging_source]}
      logging_groups: {get_attr: [RedisBase, role_data, logging_groups]}
      service_config_settings: {get_attr: [RedisBase, role_data, service_config_settings]}
@ -94,8 +125,7 @@ outputs:
        # /etc/redis.conf
        # https://github.com/arioch/puppet-redis/commit/1c004143223e660cbd433422ff8194508aab9763
        puppet_tags: 'exec'
-        step_config:
-          get_attr: [RedisBase, role_data, step_config]
+        step_config: 'include ::tripleo::profile::pacemaker::database::redis_bundle'
        config_image: &redis_config_image {get_param: DockerRedisConfigImage}
      kolla_config:
        /var/lib/kolla/config_files/redis.json:
@ -110,6 +140,11 @@ outputs:
              merge: true
              preserve_properties: true
              optional: true
+            - source: "/var/lib/kolla/config_files/src-tls/*"
+              dest: "/"
+              merge: true
+              optional: true
+              preserve_properties: true
          permissions:
            - path: /var/run/redis
              owner: redis:redis
@ -120,6 +155,21 @@ outputs:
            - path: /var/log/redis
              owner: redis:redis
              recurse: true
+            - path: /etc/pki/tls/certs/redis.crt
+              owner: redis:redis
+              perm: '0600'
+              optional: true
+            - path: /etc/pki/tls/private/redis.key
+              owner: redis:redis
+              perm: '0600'
+              optional: true
+        /var/lib/kolla/config_files/redis_tls_proxy.json:
+          command: stunnel /etc/stunnel/stunnel.conf
+          config_files:
+            - source: "/var/lib/kolla/config_files/src/*"
+              dest: "/"
+              merge: true
+              preserve_properties: true
      docker_config_scripts: {get_attr: [ContainersCommon, docker_config_scripts]}
      docker_config:
        step_1:
@ -146,28 +196,50 @@ outputs:
              - /usr/bin:/usr/bin:ro
              - /var/run/docker.sock:/var/run/docker.sock:rw
        step_2:
-          redis_init_bundle:
-            start_order: 2
-            detach: false
-            net: host
-            user: root
-            config_volume: 'redis_init_bundle'
-            command: # '/docker_puppet_apply.sh "STEP" "TAGS" "CONFIG" "DEBUG"'
-              list_concat:
-                - - '/docker_puppet_apply.sh'
-                  - '2'
-                  - 'file,file_line,concat,augeas,pacemaker::resource::bundle,pacemaker::property,pacemaker::resource::ocf,pacemaker::constraint::order,pacemaker::constraint::colocation'
-                  - 'include ::tripleo::profile::base::pacemaker;include ::tripleo::profile::pacemaker::database::redis_bundle'
-                - if:
-                  - puppet_debug_enabled
-                  - - '--debug'
-                  - - ''
-            image: *redis_config_image
-            volumes:
-              list_concat:
-                - {get_attr: [ContainersCommon, docker_puppet_apply_volumes]}
-                - - /etc/corosync/corosync.conf:/etc/corosync/corosync.conf:ro
-                  - /dev/shm:/dev/shm:rw
+          map_merge:
+            - redis_init_bundle:
+                start_order: 2
+                detach: false
+                net: host
+                user: root
+                config_volume: 'redis_init_bundle'
+                command: # '/docker_puppet_apply.sh "STEP" "TAGS" "CONFIG" "DEBUG"'
+                  list_concat:
+                    - - '/docker_puppet_apply.sh'
+                      - '2'
+                      - 'file,file_line,concat,augeas,pacemaker::resource::bundle,pacemaker::property,pacemaker::resource::ocf,pacemaker::constraint::order,pacemaker::constraint::colocation'
+                      - 'include ::tripleo::profile::base::pacemaker;include ::tripleo::profile::pacemaker::database::redis_bundle'
+                    - if:
+                      - puppet_debug_enabled
+                      - - '--debug'
+                      - - ''
+                image: *redis_config_image
+                volumes:
+                  list_concat:
+                    - {get_attr: [ContainersCommon, docker_puppet_apply_volumes]}
+                    - - /etc/corosync/corosync.conf:/etc/corosync/corosync.conf:ro
+                      - /dev/shm:/dev/shm:rw
+            - if:
+                - internal_tls_enabled
+                - redis_tls_proxy:
+                    start_order: 3
+                    image: *redis_image_pcmklatest
+                    net: host
+                    user: root
+                    restart: always
+                    volumes:
+                      list_concat:
+                        - {get_attr: [ContainersCommon, volumes]}
+                        -
+                          - /var/lib/kolla/config_files/redis_tls_proxy.json:/var/lib/kolla/config_files/config.json:ro
+                          - /var/lib/config-data/puppet-generated/redis/:/var/lib/kolla/config_files/src:ro
+                          - /etc/pki/tls/certs/redis.crt:/etc/pki/tls/certs/redis.crt:ro
+                          - /etc/pki/tls/private/redis.key:/etc/pki/tls/private/redis.key:ro
+                    environment:
+                      - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+                - {}
+      metadata_settings:
+        get_attr: [RedisBase, role_data, metadata_settings]
      host_prep_tasks:
        - name: create /var/run/redis
          file:
--- a/environments/network-isolation-v6.j2.yaml
+++ b/environments/network-isolation-v6.j2.yaml
@ -59,4 +59,5 @@ parameter_defaults:
  MysqlIPv6: True
  # Enable IPv6 environment for Manila
  ManilaIPv6: True
-
+  # Enable IPv6 environment for Redis.
+  RedisIPv6: True
--- a/puppet/services/database/redis-base.yaml
+++ b/puppet/services/database/redis-base.yaml
@ -38,6 +38,17 @@ parameters:
    description: Mapping of service endpoint -> protocol. Typically set
                 via parameter_defaults in the resource registry.
    type: json
+  EnableInternalTLS:
+    type: boolean
+    default: false
+  RedisIPv6:
+    default: false
+    description: Enable IPv6 in Redis
+    type: boolean
+
+conditions:
+  use_tls_proxy: {equals : [{get_param: EnableInternalTLS}, true]}
+  redis_ipv6: {get_param: RedisIPv6}

 outputs:
  role_data:
@ -48,15 +59,32 @@ outputs:
        redis::requirepass: {get_param: RedisPassword}
        redis::masterauth: {get_param: RedisPassword}
        redis::sentinel_auth_pass: {get_param: RedisPassword}
+        redis_ipv6: {get_param: RedisIPv6}
        # NOTE: bind IP is found in Heat replacing the network name with the local node IP
        # for the given network; replacement examples (eg. for internal_api):
        # internal_api -> IP
        # internal_api_uri -> [IP]
        # internal_api_subnet - > IP/CIDR
-        redis::bind: {get_param: [ServiceNetMap, RedisNetwork]}
+        # Bind to localhost if internal TLS is enabled, since we put a TLs
+        # proxy in front.
+        redis::bind:
+          if:
+          - use_tls_proxy
+          - if:
+            - redis_ipv6
+            - '::1'
+            - '127.0.0.1'
+          - {get_param: [ServiceNetMap, RedisNetwork]}
        redis::port: 6379
        redis::sentinel::master_name: "%{hiera('bootstrap_nodeid')}"
        redis::sentinel::redis_host: "%{hiera('bootstrap_nodeid_ip')}"
        redis::sentinel::notification_script: '/usr/local/bin/redis-notifications.sh'
-        redis::sentinel::sentinel_bind: {get_param: [ServiceNetMap, RedisNetwork]}
+        redis::sentinel::sentinel_bind:
+          if:
+          - use_tls_proxy
+          - if:
+            - redis_ipv6
+            - '::1'
+            - '127.0.0.1'
+          - {get_param: [ServiceNetMap, RedisNetwork]}
        redis::ulimit: {get_param: RedisFDLimit}
--- a/puppet/services/database/redis.yaml
+++ b/puppet/services/database/redis.yaml
@ -30,8 +30,15 @@ parameters:
    description: Mapping of service endpoint -> protocol. Typically set
                 via parameter_defaults in the resource registry.
    type: json
+  EnableInternalTLS:
+    type: boolean
+    default: false
+
+conditions:
+  use_tls_proxy: {equals : [{get_param: EnableInternalTLS}, true]}

 resources:
+
  RedisBase:
    type: ./redis-base.yaml
    properties:
@ -41,6 +48,7 @@ resources:
      EndpointMap: {get_param: EndpointMap}
      RoleName: {get_param: RoleName}
      RoleParameters: {get_param: RoleParameters}
+      EnableInternalTLS: {get_param: EnableInternalTLS}

 outputs:
  role_data:
@ -55,8 +63,41 @@ outputs:
                dport:
                  - 6379
                  - 26379
+            tripleo::profile::base::database::redis::tls_proxy_bind_ip:
+              get_param: [ServiceNetMap, RedisNetwork]
+            tripleo::profile::base::database::redis::tls_proxy_fqdn:
+              str_replace:
+                template:
+                  "%{hiera('fqdn_$NETWORK')}"
+                params:
+                  $NETWORK: {get_param: [ServiceNetMap, RedisNetwork]}
+            tripleo::profile::base::database::redis::tls_proxy_port: 6379
+          - if:
+            - use_tls_proxy
+            - redis_certificate_specs:
+                service_certificate: '/etc/pki/tls/certs/redis.crt'
+                service_key: '/etc/pki/tls/private/redis.key'
+                hostname:
+                  str_replace:
+                    template: "%{hiera('cloud_name_NETWORK')}"
+                    params:
+                      NETWORK: {get_param: [ServiceNetMap, RedisNetwork]}
+                principal:
+                  str_replace:
+                    template: "redis/%{hiera('cloud_name_NETWORK')}"
+                    params:
+                      NETWORK: {get_param: [ServiceNetMap, RedisNetwork]}
+            - {}
      step_config: |
        include ::tripleo::profile::base::database::redis
+      metadata_settings:
+        if:
+          - use_tls_proxy
+          -
+            - service: redis
+              network: {get_param: [ServiceNetMap, RabbitmqNetwork]}
+              type: vip
+          - null
      upgrade_tasks:
        - name: Check if redis is deployed
          command: systemctl is-enabled redis
--- a/puppet/services/pacemaker/database/redis.yaml
+++ b/puppet/services/pacemaker/database/redis.yaml
@ -30,6 +30,17 @@ parameters:
    description: Mapping of service endpoint -> protocol. Typically set
                 via parameter_defaults in the resource registry.
    type: json
+  EnableInternalTLS:
+    type: boolean
+    default: false
+  RedisIPv6:
+    default: false
+    description: Enable IPv6 in Redis
+    type: boolean
+
+conditions:
+  use_tls_proxy: {equals : [{get_param: EnableInternalTLS}, true]}
+  redis_ipv6: {get_param: RedisIPv6}

 resources:
  RedisBase:
@ -53,5 +64,27 @@ outputs:
          - redis::service_manage: false
            redis::notify_service: false
            redis::managed_by_cluster_manager: true
+            tripleo::profile::pacemaker::database::redis::tls_proxy_bind_ip:
+              get_param: [ServiceNetMap, RedisNetwork]
+            tripleo::profile::pacemaker::database::redis::tls_proxy_fqdn:
+              str_replace:
+                template:
+                  "%{hiera('fqdn_$NETWORK')}"
+                params:
+                  $NETWORK: {get_param: [ServiceNetMap, RedisNetwork]}
+            tripleo::profile::pacemaker::database::redis::tls_proxy_port: 6379
+          - if:
+            - use_tls_proxy
+            - redis::extra_config_file: "/etc/redis-tls.conf"
+              tripleo::profile::pacemaker::database::redis::extra_config_file: "/etc/redis-tls.conf"
+              tripleo::profile::pacemaker::database::redis::tls_tunnel_base_port: 6660
+              tripleo::profile::pacemaker::database::redis::tls_tunnel_local_name:
+                if:
+                - redis_ipv6
+                - '::1'
+                - '127.0.0.1'
+            - {}
      step_config: |
        include ::tripleo::profile::pacemaker::database::redis
+      metadata_settings:
+        get_attr: [RedisBase, role_data, metadata_settings]