Merge "Enable etcd with security setting."

puppet-manifests/src/modules/platform/files/change_kube_apiserver_params.py

@@ -20,6 +20,11 @@ parser.add_argument("--oidc_client_id")
 parser.add_argument("--oidc_username_claim")
 parser.add_argument("--oidc_groups_claim")
 parser.add_argument("--admission_plugins")
+parser.add_argument("--etcd_cafile")
+parser.add_argument("--etcd_certfile")
+parser.add_argument("--etcd_keyfile")
+parser.add_argument("--etcd_servers")
+
 args = parser.parse_args()
 
 if args.configmap_file:
@@ -77,6 +82,34 @@ else:
     if plugins in cluster_config['apiServer']['extraArgs']:
         del cluster_config['apiServer']['extraArgs'][plugins]
 
+if args.etcd_cafile:
+    cluster_config['etcd']['external']['caFile'] = \
+        args.etcd_cafile
+else:
+    if 'caFile' in cluster_config['etcd']['external']:
+        del cluster_config['etcd']['external']['caFile']
+
+if args.etcd_certfile:
+    cluster_config['etcd']['external']['certFile'] = \
+        args.etcd_certfile
+else:
+    if 'certFile' in cluster_config['etcd']['external']:
+        del cluster_config['etcd']['external']['certFile']
+
+if args.etcd_keyfile:
+    cluster_config['etcd']['external']['keyFile'] = \
+        args.etcd_keyfile
+else:
+    if 'keyFile' in cluster_config['etcd']['external']:
+        del cluster_config['etcd']['external']['keyFile']
+
+if args.etcd_servers:
+    cluster_config['etcd']['external']['endpoints'] = \
+        args.etcd_servers.split(',')
+else:
+    if 'endpoints' in cluster_config['etcd']['external']:
+        del cluster_config['etcd']['external']['endpoints']
+
 cluster_config_string = yaml.dump(cluster_config, Dumper=yaml.RoundTripDumper,
                                   default_flow_style=False)
 # use yaml.scalarstring.PreservedScalarString to make sure the yaml is

puppet-manifests/src/modules/platform/files/etcd-override.conf

@@ -4,6 +4,14 @@ User=root
 NotifyAccess=all
 Type=notify
 ExecStart=
-ExecStart=-/bin/bash -c "GOMAXPROCS=$(nproc) /usr/bin/etcd --name=\"${ETCD_NAME}\" --data-dir=\"${ETCD_DATA_DIR}\" --listen-client-urls=\"${ETCD_LISTEN_CLIENT_URLS}\" 2>&1 | /usr/bin/forward-journald -tag etcd"
+ExecStart=-/bin/bash -c "GOMAXPROCS=$(nproc) /usr/bin/etcd \
+    --name=\"${ETCD_NAME}\" \
+    --data-dir=\"${ETCD_DATA_DIR}\" \
+    --listen-client-urls=\"${ETCD_LISTEN_CLIENT_URLS}\" \
+    --client-cert-auth=${ETCD_CLIENT_CERT_AUTH} \
+    --trusted-ca-file=\"${ETCD_TRUSTED_CA_FILE}\" \
+    --cert-file=\"${ETCD_CERT_FILE}\" \
+    --key-file=\"${ETCD_KEY_FILE}\" 2>&1 \
+    | /usr/bin/forward-journald -tag etcd"
 ExecStartPost=/bin/bash -c 'echo $MAINPID >/var/run/etcd.pid'
-ExecStopPost=/bin/bash/rm -f /var/run/etcd.pid
+ExecStopPost=/bin/bash -c 'rm -f /var/run/etcd.pid'

puppet-manifests/src/modules/platform/manifests/etcd.pp

@@ -1,7 +1,9 @@
 class platform::etcd::params (
   $bind_address = '0.0.0.0',
+  $bind_address_version = 4,
   $port    = 2379,
   $node   = 'controller',
+  $security_enabled = undef,
 )
 {
   include ::platform::params
@@ -42,8 +44,6 @@ class platform::etcd::init (
   $service_enabled = false,
 ) inherits ::platform::etcd::params {
 
-  $client_url = "http://${bind_address}:${port}"
-
   if $service_enabled {
     $service_ensure = 'running'
   }
@@ -51,6 +51,32 @@ class platform::etcd::init (
     $service_ensure = 'stopped'
   }
 
+  if $security_enabled {
+    $client_cert_auth = true
+    $cert_file = '/etc/etcd/etcd-server.crt'
+    $key_file = '/etc/etcd/etcd-server.key'
+    $trusted_ca_file = '/etc/etcd/ca.crt'
+    if $bind_address_version == $::platform::params::ipv6 {
+      $client_url = "https://[${bind_address}]:${port}"
+    }
+    else {
+      $client_url = "https://${bind_address}:${port}"
+    }
+  }
+  else {
+    # This else part can be removed after STX5.0
+    $client_cert_auth = false
+    $cert_file = undef
+    $key_file = undef
+    $trusted_ca_file = undef
+    if $bind_address_version == $::platform::params::ipv6 {
+      $client_url = "http://[${bind_address}]:${port}"
+    }
+    else {
+      $client_url = "http://${bind_address}:${port}"
+    }
+  }
+
   class { 'etcd':
     ensure                => 'present',
     etcd_name             => $node,
@@ -61,6 +87,10 @@ class platform::etcd::init (
     advertise_client_urls => $client_url,
     data_dir              => "${etcd_versioned_dir}/${node}.etcd",
     proxy                 => 'off',
+    client_cert_auth      => $client_cert_auth,
+    cert_file             => $cert_file,
+    key_file              => $key_file,
+    trusted_ca_file       => $trusted_ca_file,
   }
 }
 
@@ -94,6 +124,96 @@ class platform::etcd::datadir
   }
 }
 
+class platform::etcd::upgrade::runtime
+  inherits ::platform::etcd::params {
+
+  include ::platform::etcd::init
+
+  $server_url = $::platform::etcd::init::client_url
+  $etcd_cert = '/etc/etcd/etcd-client.crt'
+  $etcd_key = '/etc/etcd/etcd-client.key'
+  $etcd_ca = '/etc/etcd/ca.crt'
+
+  if ! str2bool($::is_controller_active) {
+    file { '/etc/etcd/etcd-server.crt':
+      ensure  => 'present',
+      replace => true,
+      source  => "/var/run/platform/config/${sw_version}/etcd/etcd-server.crt",
+    }
+
+    -> file { '/etc/etcd/etcd-server.key':
+      ensure  => 'present',
+      replace => true,
+      source  => "/var/run/platform/config/${sw_version}/etcd/etcd-server.key",
+    }
+
+    -> file { '/etc/etcd/etcd-client.crt':
+      ensure  => 'present',
+      replace => true,
+      source  => "/var/run/platform/config/${sw_version}/etcd/etcd-client.crt",
+    }
+
+    -> file { '/etc/etcd/etcd-client.key':
+      ensure  => 'present',
+      replace => true,
+      source  => "/var/run/platform/config/${sw_version}/etcd/etcd-client.key",
+    }
+
+    -> file { '/etc/etcd/ca.crt':
+      ensure  => 'present',
+      replace => true,
+      source  => "/var/run/platform/config/${sw_version}/etcd/ca.crt",
+    }
+
+    -> file { '/etc/kubernetes/pki/apiserver-etcd-client.crt':
+      ensure  => 'present',
+      replace => true,
+      source  => "/var/run/platform/config/${sw_version}/etcd/apiserver-etcd-client.crt",
+    }
+
+    -> file { '/etc/kubernetes/pki/apiserver-etcd-client.key':
+      ensure  => 'present',
+      replace => true,
+      source  => "/var/run/platform/config/${sw_version}/etcd/apiserver-etcd-client.key",
+    }
+
+    -> class { '::platform::kubernetes::master::change_apiserver_parameters':
+      etcd_cafile   => '/etc/kubernetes/pki/ca.crt',
+      etcd_certfile => '/etc/kubernetes/pki/apiserver-etcd-client.crt',
+      etcd_keyfile  => '/etc/kubernetes/pki/apiserver-etcd-client.key',
+      etcd_servers  => $server_url,
+    }
+  }
+  else {
+    class { '::platform::kubernetes::master::change_apiserver_parameters':
+      etcd_cafile   => '/etc/kubernetes/pki/ca.crt',
+      etcd_certfile => '/etc/kubernetes/pki/apiserver-etcd-client.crt',
+      etcd_keyfile  => '/etc/kubernetes/pki/apiserver-etcd-client.key',
+      etcd_servers  => $server_url,
+    }
+
+    -> exec { 'restart-etcd':
+      command => '/usr/bin/systemctl restart etcd.service',
+    }
+
+    -> exec { 'create-etcd-root-account':
+      command => "etcdctl --cert-file=${etcd_cert} --key-file=${etcd_key} --ca-file=${etcd_ca} --endpoint=${server_url} \
+                  user add root:sysadmin",
+    }
+
+    -> exec { 'create-etcd-user-account':
+      command => "etcdctl --cert-file=${etcd_cert} --key-file=${etcd_key} --ca-file=${etcd_ca} --endpoint=${server_url} \
+                  user add apiserver-etcd-client:sysadmin",
+    }
+
+    -> exec { 'enable-etcd-auth':
+      command => "etcdctl --cert-file=${etcd_cert} --key-file=${etcd_key} --ca-file=${etcd_ca} --endpoint=${server_url} \
+                  auth enable",
+      returns => [0,1]
+    }
+  }
+}
+
 class platform::etcd::datadir::bootstrap
   inherits ::platform::etcd::params {
 
@@ -117,6 +237,6 @@ class platform::etcd::bootstrap
   Class['::platform::etcd::datadir::bootstrap']
   -> Class['::platform::etcd::setup']
   -> class { '::platform::etcd::init':
-    service_enabled => true,
+    service_enabled => false,
   }
 }

puppet-manifests/src/modules/platform/manifests/kubernetes.pp

@@ -22,7 +22,11 @@ class platform::kubernetes::params (
   $oidc_client_id = undef,
   $oidc_username_claim = undef,
   $oidc_groups_claim = undef,
-  $admission_plugins = undef
+  $admission_plugins = undef,
+  $etcd_cafile = undef,
+  $etcd_certfile = undef,
+  $etcd_keyfile = undef,
+  $etcd_servers = undef,
 ) { }
 
 class platform::kubernetes::cgroup::params (
@@ -650,8 +654,12 @@ class platform::kubernetes::worker::upgrade_kubelet
   }
 }
 
-class platform::kubernetes::master::change_apiserver_parameters
-  inherits ::platform::kubernetes::params {
+class platform::kubernetes::master::change_apiserver_parameters (
+  $etcd_cafile = $platform::kubernetes::params::etcd_cafile,
+  $etcd_certfile = $platform::kubernetes::params::etcd_certfile,
+  $etcd_keyfile = $platform::kubernetes::params::etcd_keyfile,
+  $etcd_servers = $platform::kubernetes::params::etcd_servers,
+) inherits ::platform::kubernetes::params {
 
   $configmap_temp_file = '/tmp/cluster_configmap.yaml'
   $configview_temp_file = '/tmp/kubeadm_config_view.yaml'
@@ -659,7 +667,6 @@ class platform::kubernetes::master::change_apiserver_parameters
   exec { 'update kube-apiserver params':
     command => template('platform/kube-apiserver-change-params.erb')
   }
-
 }
 
 class platform::kubernetes::certsans::runtime

puppet-manifests/src/modules/platform/templates/kube-apiserver-change-params.erb

@@ -23,6 +23,18 @@ python /usr/share/puppet/modules/platform/files/change_kube_apiserver_params.py
 <%- if @admission_plugins -%>
 --admission_plugins <%= @admission_plugins %> \
 <%- end -%>
+<%- if @etcd_cafile -%>
+--etcd_cafile <%= @etcd_cafile %> \
+<%- end -%>
+<%- if @etcd_certfile -%>
+--etcd_certfile <%= @etcd_certfile %> \
+<%- end -%>
+<%- if @etcd_keyfile -%>
+--etcd_keyfile <%= @etcd_keyfile %> \
+<%- end -%>
+<%- if @etcd_servers -%>
+--etcd_servers <%= @etcd_servers %>
+<%- end -%>
 
 kubectl --kubeconfig=/etc/kubernetes/admin.conf -n kube-system patch configmap kubeadm-config -p "$(cat <%= @configmap_temp_file %>)"
 kubeadm --kubeconfig=/etc/kubernetes/admin.conf config view > <%= @configmap_temp_file %>