Merge "Enable etcd with security setting."

playbookconfig/src/playbooks/enable_secured_etcd.yml

@@ -0,0 +1,97 @@
+---
+#
+# Copyright (c) 2020 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# ROLE DESCRIPTION:
+#   Enable secured etcd.
+# This file can be removed in the release after STX5.0
+
+- hosts: all
+  become: yes
+  become_user: root
+  tasks:
+    - name: Create cert for etcd server and client
+      import_role:
+        name: common/create-etcd-certs
+
+    - name: Create etcd cert permdir
+      file:
+        path: "{{ config_permdir + '/etcd' }}"
+        state: directory
+        mode: 0700
+
+    - name: Copy etcd certificates to config_permdir
+      copy:
+        src: "/etc/etcd/{{ item }}"
+        dest: "{{ config_permdir + '/etcd' }}/{{ item }}"
+        remote_src: yes
+        force: yes
+      with_items:
+        - "etcd-server.crt"
+        - "etcd-server.key"
+        - "etcd-client.crt"
+        - "etcd-client.key"
+        - "apiserver-etcd-client.crt"
+        - "apiserver-etcd-client.key"
+        - "ca.crt"
+        - "ca.key"
+
+    - name: Copy apiserver-etcd-client cert
+      copy:
+        src: "/etc/etcd/{{ item }}"
+        dest: "/etc/kubernetes/pki/{{ item }}"
+        remote_src: yes
+        force: yes
+      with_items:
+        - "apiserver-etcd-client.crt"
+        - "apiserver-etcd-client.key"
+
+    - name: Write security settings to hieradata
+      lineinfile:
+        path: "{{ puppet_permdir }}/hieradata/static.yaml"
+        line: "{{ item }}"
+      with_items:
+        - "platform::etcd::params::security_enabled: true"
+        - "platform::etcd::params::bind_address: {{ default_cluster_host_start_address }}"
+        - "platform::etcd::params::bind_address_version: {{ etcd_listen_address_version }}"
+
+    - name: Create list of etcd classes to pass to puppet
+      copy:
+        dest: "/tmp/etcd.yml"
+        content: |
+          classes:
+          - platform::etcd::upgrade::runtime
+
+    - name: Applying puppet for enabling etcd security
+      command: >
+        /usr/local/bin/puppet-manifest-apply.sh
+        {{ puppet_permdir }}/hieradata/
+        {{ ipaddress }}
+        controller runtime /tmp/etcd.yml
+      register: etcd_apply_result
+      environment:
+        LC_ALL: "en_US.UTF-8"
+
+    - block:
+      - name: Remove bind address and address version
+        lineinfile:
+          dest: "{{ puppet_permdir }}/hieradata/static.yaml"
+          regexp: "{{ item }}"
+          state: absent
+        with_items:
+          - "^platform::etcd::params::bind_address"
+          - "^platform::etcd::params::bind_address_version"
+
+      - name: Revert security_enable flag
+        lineinfile:
+          dest: "{{ puppet_permdir }}/hieradata/static.yaml"
+          regexp: "^platform::etcd::params::security_enabled"
+          line: "platform::etcd::params::security_enabled: false"
+
+      - name: Fail if puppet manifest apply script returns an error
+        fail:
+          msg: >-
+               Failed to apply etcd manifest!
+      when: etcd_apply_result.rc != 0

playbookconfig/src/playbooks/roles/backup/backup-system/tasks/main.yml

@@ -301,8 +301,15 @@
       set_fact:
         etcd_snapshot_file: "{{ etcd_snapshot_dir.path }}/etcd-snapshot.db"
 
+    - name: Get etcd endpoints
+      shell: |
+        source /etc/platform/openrc
+        system addrpool-list | awk '/cluster-host-subnet/{print$14}'
+      register: etcd_endpoint
+
     - name: Create etcd snapshot
-      command: "etcdctl snapshot save {{ etcd_snapshot_file }}"
+      command: "etcdctl --endpoints https://{{ etcd_endpoint.stdout }}:2379 --cert=/etc/etcd/etcd-client.crt
+                --key=/etc/etcd/etcd-client.key --cacert=/etc/etcd/ca.crt snapshot save {{ etcd_snapshot_file }}"
       environment:
         ETCDCTL_API: 3
 

playbookconfig/src/playbooks/roles/bootstrap/apply-bootstrap-manifest/tasks/main.yml

@@ -40,6 +40,9 @@
 
     when: distributed_cloud_role == 'subcloud'
 
+  - name: Create cert for etcd server and client
+    import_role:
+      name: common/create-etcd-certs
   when: mode == 'bootstrap'
 
 - block:
@@ -62,6 +65,47 @@
       warn: false
     when: migrate_platform_data is defined and migrate_platform_data
 
+  - name: Restore etcd certificates.
+    shell: tar -C / --overwrite -xpf {{ restore_data_file }} {{ item }}
+    args:
+      warn: false
+    with_items:
+      - "{{ '/etc/etcd' | regex_replace('^\\/', '') }}"
+    become_user: root
+
+  - name: Check if etcd certs are exist.
+    find:
+      paths: "/etc/etcd"
+      patterns:
+        - '*.crt'
+        - '*.key'
+    register: etcd_certs_find_output
+
+    # This is for simplex upgrade from STX 4.0 to 5.0
+  - block:
+    - name: set kubeadm_pki_dir
+      set_fact:
+        kubeadm_pki_dir: /etc/kubernetes/pki
+
+    - name: Create pki directory for kubernetes certificates
+      file:
+        path: "{{ kubeadm_pki_dir }}"
+        state: directory
+        mode: 0700
+
+    - name: Restore CA
+      shell: tar -C / --overwrite -xpf {{ restore_data_file }} {{ item }}
+      args:
+        warn: false
+      with_items:
+        - "{{ kubeadm_pki_dir | regex_replace('^\\/', '') }}"
+      become_user: root
+
+    - name: Create certs for etcd server and client for simplex upgrade
+      import_role:
+        name: common/create-etcd-certs
+    when: etcd_certs_find_output.matched == 0
+
   - name: Look for ssh_config dir in the backup tarball
     shell: "tar -tf {{ restore_data_file }} | grep 'opt/platform/config/.*/ssh_config'"
     args:
@@ -93,6 +137,26 @@
     when: search_ssh_config.rc == 0
   when: mode == 'restore'
 
+- name: Set the ip version of etcd listen address to its default value
+  set_fact:
+    etcd_listen_address_version: 4
+
+- name: Update the ip version of etcd listen address to ipv6
+  set_fact:
+    etcd_listen_address_version: 6
+  when: ipv6_addressing != False
+
+  # Add etcd security info to static hieradata so that etcd is configured with security
+  # when etc puppet manifest is applied before Kubernetes master is initialized in the later role.
+- name: Write security settings to static hieradata
+  lineinfile:
+    path: "{{ hieradata_workdir }}/static.yaml"
+    line: "{{ item }}"
+  with_items:
+    - "platform::etcd::params::security_enabled: true"
+    - "platform::etcd::params::bind_address: {{ default_cluster_host_start_address }}"
+    - "platform::etcd::params::bind_address_version: {{ etcd_listen_address_version }}"
+
 - name: Applying puppet bootstrap manifest
   command: >
     /usr/local/bin/puppet-manifest-apply.sh

playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/tasks/bringup_kubemaster.yml

@@ -58,20 +58,42 @@
     k8s_pki_files: { ca.crt: "{{k8s_root_ca_cert}}", ca.key: "{{k8s_root_ca_key}}" }
   when: (k8s_root_ca_cert)
 
-- block:
-  - name: Create pki directory for kubernetes certificates
-    file:
-      path: "{{ kubeadm_pki_dir }}"
-      state: directory
-      mode: 0700
+- name: Create pki directory for kubernetes certificates
+  file:
+    path: "{{ kubeadm_pki_dir }}"
+    state: directory
+    mode: 0700
 
+- block:
   - name: Copy kubernetes certificates
     copy:
       src: "{{ item.value }}"
       dest: "{{ kubeadm_pki_dir }}/{{item.key}}"
     with_dict: "{{ k8s_pki_files }}"
 
-  when: k8s_pki_files is defined and mode == 'bootstrap'
+  - name: Copy apiserver-etcd-client cert and key
+    copy:
+      src: "/etc/etcd/{{ item }}"
+      dest: "{{ kubeadm_pki_dir }}/{{ item }}"
+      remote_src: yes
+      force: yes
+    with_items:
+      - "apiserver-etcd-client.crt"
+      - "apiserver-etcd-client.key"
+  when: k8s_pki_files is defined
+
+- name: Copy ca, cert and key generated by etcd to kubeadm_pki_dir
+  copy:
+    src: "/etc/etcd/{{ item }}"
+    dest: "{{ kubeadm_pki_dir }}/{{ item }}"
+    remote_src: yes
+    force: yes
+  with_items:
+    - "ca.crt"
+    - "ca.key"
+    - "apiserver-etcd-client.crt"
+    - "apiserver-etcd-client.key"
+  when: k8s_pki_files is undefined
 
 - name: Set kubelet node configuration
   set_fact:
@@ -126,7 +148,7 @@
   environment:
     APISERVER_ADVERTISE_ADDRESS: "{{ controller_0_cluster_host }}"
     CONTROLPLANE_ENDPOINT: "{{ cluster_floating_address }}"
-    ETCD_ENDPOINT: "http://{{ cluster_floating_address | ipwrap }}:2379"
+    ETCD_ENDPOINT: "https://{{ cluster_floating_address | ipwrap }}:2379"
     POD_NETWORK_CIDR: "{{ cluster_pod_subnet }}"
     SERVICE_NETWORK_CIDR: "{{ cluster_service_subnet }}"
     VOLUME_PLUGIN_DIR: "{{ kubelet_vol_plugin_dir }}"

playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/tasks/main.yml

@@ -58,6 +58,31 @@
     import_role:
       name: common/push-docker-images
 
+  - name: Bring up etcd
+    systemd:
+      name: etcd
+      state: started
+
+  - name: Check if etcd-client crt was created.
+    find:
+      paths: "/etc/etcd"
+      patterns: "etcd-client.*"
+    register: etcd_client_find_output
+
+  - name: Create etcd client account for root, apiserver and enable etcd auth
+    command: "etcdctl --cert-file=$ETCD_CERT --key-file=$ETCD_KEY --ca-file=$ETCD_CA
+              --endpoint=$ETCD_ENDPOINT {{ item }}"
+    with_items:
+      - "user add root:sysadmin"
+      - "user add apiserver-etcd-client:sysadmin"
+      - "auth enable"
+    environment:
+      ETCD_ENDPOINT: "https://{{ default_cluster_host_start_address | ipwrap }}:2379"
+      ETCD_CERT: "/etc/etcd/etcd-client.crt"
+      ETCD_KEY: "/etc/etcd/etcd-client.key"
+      ETCD_CA: "/etc/etcd/ca.crt"
+    when: etcd_client_find_output.matched != 0
+
   - name: Bring up Kubernetes master
     import_tasks: bringup_kubemaster.yml
 

playbookconfig/src/playbooks/roles/bootstrap/persist-config/tasks/one_time_config_tasks.yml

@@ -18,6 +18,7 @@
     pxe_config_dir: "{{ config_permdir + '/pxelinux.cfg' }}"
     branding_config_dir: "{{ config_permdir + '/branding' }}"
     ssl_ca_certs_dir: "{{ config_permdir + '/ssl_ca' }}"
+    etcd_certs_dir: "{{ config_permdir + '/etcd' }}"
 
 - debug:
     msg: >-
@@ -25,6 +26,7 @@
       pxe_config_dir: {{ pxe_config_dir }}
       branding_config_dir: {{ branding_config_dir }}
       ssl_ca_certs_dir: {{ ssl_ca_certs_dir }}
+      etcd_certs_dir: {{ etcd_certs_dir }}
 
 - name: Ensure Postres, PXE config directories exist
   file:
@@ -38,13 +40,16 @@
     - "{{ postgres_config_dir }}"
     - "{{ pxe_config_dir }}"
 
-- name: Ensure SSL CA certificates directory exists
+- name: Ensure SSL CA and etcd certs directories exist
   file:
-    path: "{{ ssl_ca_certs_dir }}"
+    path: "{{ item }}"
     state: directory
     owner: root
     group: root
     mode: 0700
+  with_items:
+    - "{{ ssl_ca_certs_dir }}"
+    - "{{ etcd_certs_dir }}"
 
 - name: Get list of Postgres conf files
   find:
@@ -62,6 +67,29 @@
     remote_src: yes
   with_items: "{{ postgres_result.files }}"
 
+- name: Find etcd certs files
+  find:
+    paths: "/etc/etcd"
+    patterns:
+      - '*.crt'
+      - '*.key'
+  register: etcd_certs_find_output
+
+- name: Copy etcd certificates to etcd certs directory
+  copy:
+    src: "/etc/etcd/{{ item }}"
+    dest: "{{ etcd_certs_dir }}/{{ item }}"
+    remote_src: yes
+    force: yes
+  with_items:
+    - "etcd-server.crt"
+    - "etcd-server.key"
+    - "ca.crt"
+    - "ca.key"
+    - "etcd-client.crt"
+    - "etcd-client.key"
+  when: etcd_certs_find_output.matched != 0
+
 - name: Create a symlink to PXE config files
   file:
     src: "{{ pxe_config_dir }}"

playbookconfig/src/playbooks/roles/common/create-etcd-certs/tasks/main.yml

@@ -0,0 +1,126 @@
+---
+#
+# Copyright (c) 2020 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# ROLE DESCRIPTION:
+#   Create etcd server and client certs and key.
+
+- name: Generate private key for etcd server and client
+  openssl_privatekey:
+    path: "/etc/etcd/{{ item }}.key"
+    type: RSA
+    size: 4096
+    state: present
+    force: true
+  with_items:
+    - "etcd-server"
+    - "apiserver-etcd-client"
+    - "etcd-client"
+
+- name: Generate CSRs for etcd server and client
+  openssl_csr:
+    path: "/etc/etcd/{{ item }}.csr"
+    privatekey_path: "/etc/etcd/{{ item }}.key"
+    common_name: "{{ item }}"
+    key_usage:
+      - digitalSignature
+    extended_key_usage:
+      - serverAuth
+      - clientAuth
+    subject_alt_name:
+      - IP:{{ default_cluster_host_start_address }}
+      - IP:127.0.0.1
+    force: true
+  with_items:
+    - "etcd-server"
+    - "apiserver-etcd-client"
+
+- name: Generate CSRs for etcd root client
+  openssl_csr:
+    path: "/etc/etcd/{{ item }}.csr"
+    privatekey_path: "/etc/etcd/{{ item }}.key"
+    common_name: "root"
+    key_usage:
+      - digitalSignature
+    extended_key_usage:
+      - serverAuth
+      - clientAuth
+    force: true
+  with_items:
+    - "etcd-client"
+
+- name: Check if CA exist
+  shell: ls /etc/kubernetes/pki/ca.crt
+  ignore_errors: true
+  register: find_exist_ca_output
+
+- name: Copy existed CA
+  copy:
+    src: "/etc/kubernetes/pki/{{ item }}"
+    dest: "/etc/etcd/{{ item }}"
+    remote_src: yes
+    force: yes
+  with_items:
+    - "ca.crt"
+    - "ca.key"
+  when: find_exist_ca_output|succeeded
+
+- name: copy user specified CA
+  copy:
+    src: "{{ item }}"
+    dest: "/etc/etcd/{{ item }}"
+    remote_src: yes
+    force: yes
+  with_items:
+    - "{{k8s_root_ca_cert}}"
+    - "{{k8s_root_ca_key}}"
+  when: (k8s_root_ca_cert)
+
+- block:
+  - name: Generate private key for CA
+    openssl_privatekey:
+      path: "/etc/etcd/ca.key"
+      type: RSA
+      size: 4096
+      state: present
+      force: true
+
+  - name: Generate CSR for CA
+    openssl_csr:
+      path: "/etc/etcd/ca.csr"
+      privatekey_path: "/etc/etcd/ca.key"
+      common_name: ca
+      organization_name: "Etcd CA"
+      basic_constraints:
+        - CA:true
+        - pathlen:1
+      basic_constraints_critical: True
+      key_usage:
+        - keyCertSign
+        - digitalSignature
+      force: true
+
+  - name: Generate self-signed CA certificate
+    openssl_certificate:
+      path: "/etc/etcd/ca.crt"
+      privatekey_path: "/etc/etcd/ca.key"
+      csr_path: "/etc/etcd/ca.csr"
+      provider: selfsigned
+      force: true
+
+  when: find_exist_ca_output|failed and k8s_root_ca_cert == ''
+
+- name: Generate certs signed with etcd CA certificate"
+  openssl_certificate:
+    path: "/etc/etcd/{{ item }}.crt"
+    csr_path: "/etc/etcd/{{ item }}.csr"
+    ownca_path: "/etc/etcd/ca.crt"
+    ownca_privatekey_path: "/etc/etcd/ca.key"
+    provider: ownca
+    force: true
+  with_items:
+    - "etcd-server"
+    - "apiserver-etcd-client"
+    - "etcd-client"

playbookconfig/src/playbooks/roles/common/files/kubeadm.yaml.erb

@@ -40,6 +40,9 @@ etcd:
   external:
     endpoints:
     - <%= @etcd_endpoint %>
+    caFile: /etc/kubernetes/pki/ca.crt
+    certFile: /etc/kubernetes/pki/apiserver-etcd-client.crt
+    keyFile: /etc/kubernetes/pki/apiserver-etcd-client.key
 imageRepository: "registry.local:9001/k8s.gcr.io"
 kubernetesVersion: v1.18.1
 networking: