From 70917e77cf9d94cea0e2675177deb7b3a0c4adc0 Mon Sep 17 00:00:00 2001 From: Angie Wang Date: Mon, 26 Aug 2019 12:53:50 -0400 Subject: [PATCH] Puppet support for authenticated registries This commit supports to pull images from alternative authenticated registries that configured at Ansible bootstrap to bring up k8s pods at puppet time. At bootstrap time, barbican secrets are created to store credentials for accessing registry and alternative registries info are stored in service parameter. At puppet time, the barbican sercret is retrieved to get the credentials in order to pre-pull k8s images that required by kubeadm to bring up static pods(ie..kube-controller-manager, kube-apiserver, kube-scheduler..). The images for dynamic pods(kube-multus, kube-sriov-cni, calico..) and tiller are not needed to pre-pull, imagePullSecrets is added in their pod spec to pass credentials to kubelet. This is done in Ansible bootstrap https://review.opendev.org/#/c/679136/ This commit also updates to pull Armada image before creating Armada container if Armada image is not available in docker cache. Tests(AIO-SX, AIO-DX, Standard): - All types of system are installed successfully - Verified all k8s/gcr/docker images are downloaded from authenticated registry on controller-1 and worker nodes - Verified images from authenticated registries are used by k8s static/dynamic pods on controller-1 and worker nodes - Swact to controller-1, lock/unlock controller-0. Verified that tiller image is downloaded from authenticated registry and tiller pod is created on controller-1 - Swact to controller-1, apply application. Verified that Armada image is downloaded from authenticated registry and Armada container is created. Change-Id: Iaabef0f5d8a6a4640dcfde93a8c0449948f4a59f Depends-On: https://review.opendev.org/679335 Story: 2006274 Task: 36379 Signed-off-by: Angie Wang --- .../src/modules/platform/manifests/client.pp | 1 + .../src/modules/platform/manifests/docker.pp | 43 ++++++++++++ .../modules/platform/manifests/kubernetes.pp | 65 +++++++++++++++++-- .../platform/templates/get-secret-payload.erb | 11 ++++ 4 files changed, 114 insertions(+), 6 deletions(-) create mode 100644 puppet-manifests/src/modules/platform/templates/get-secret-payload.erb diff --git a/puppet-manifests/src/modules/platform/manifests/client.pp b/puppet-manifests/src/modules/platform/manifests/client.pp index 7590c0084..6fb1fdfa5 100644 --- a/puppet-manifests/src/modules/platform/manifests/client.pp +++ b/puppet-manifests/src/modules/platform/manifests/client.pp @@ -6,6 +6,7 @@ class platform::client::params ( $admin_user_domain = 'Default', $admin_project_domain = 'Default', $admin_project_name = 'admin', + $admin_password = undef, $keystone_identity_region = 'RegionOne', ) { } diff --git a/puppet-manifests/src/modules/platform/manifests/docker.pp b/puppet-manifests/src/modules/platform/manifests/docker.pp index c3755b331..1c18c3de2 100644 --- a/puppet-manifests/src/modules/platform/manifests/docker.pp +++ b/puppet-manifests/src/modules/platform/manifests/docker.pp @@ -7,6 +7,10 @@ class platform::docker::params ( $gcr_registry = undef, $quay_registry = undef, $docker_registry = undef, + $k8s_registry_secret = undef, + $gcr_registry_secret = undef, + $quay_registry_secret = undef, + $docker_registry_secret = undef, $insecure_registry = undef, ) { } @@ -85,3 +89,42 @@ class platform::docker::bootstrap include ::platform::docker::install include ::platform::docker::config::bootstrap } + +define platform::docker::login_registry ( + $registry_url, + $registry_secret, +) { + include ::platform::client::params + + $auth_url = $::platform::client::params::identity_auth_url + $username = $::platform::client::params::admin_username + $user_domain = $::platform::client::params::admin_user_domain + $project_name = $::platform::client::params::admin_project_name + $project_domain = $::platform::client::params::admin_project_domain + $region_name = $::platform::client::params::keystone_identity_region + $password = $::platform::client::params::admin_password + $interface = 'internal' + + # Registry credentials have been stored in Barbican secret at Ansible + # bootstrap time, retrieve Barbican secret to get the payload + notice("Get payload of Barbican secret ${registry_secret}") + $secret_payload = generate( + '/bin/sh', '-c', template('platform/get-secret-payload.erb')) + + if $secret_payload { + # Parse Barbican secret payload to get the registry username and password + $secret_payload_array = split($secret_payload, ' ') + $registry_username = split($secret_payload_array[0], 'username:')[1] + $registry_password = split($secret_payload_array[1], 'password:')[1] + + # Login to authenticated registry + if $registry_username and $registry_password { + exec { 'Login registry': + command => "docker login ${registry_url} -u ${registry_username} -p ${registry_password}", + logoutput => true, + } + } else { + notice('Registry username or/and password NOT FOUND') + } + } +} diff --git a/puppet-manifests/src/modules/platform/manifests/kubernetes.pp b/puppet-manifests/src/modules/platform/manifests/kubernetes.pp index 909d46a9d..2ed49b855 100644 --- a/puppet-manifests/src/modules/platform/manifests/kubernetes.pp +++ b/puppet-manifests/src/modules/platform/manifests/kubernetes.pp @@ -109,12 +109,6 @@ class platform::kubernetes::kubeadm { $iptables_file = "net.bridge.bridge-nf-call-ip6tables = 1 net.bridge.bridge-nf-call-iptables = 1" - if $::platform::docker::params::k8s_registry { - $k8s_registry = $::platform::docker::params::k8s_registry - } else { - $k8s_registry = 'k8s.gcr.io' - } - # Configure kubelet cpumanager options if str2bool($::is_worker_subfunction) and !('openstack-compute-node' @@ -335,6 +329,27 @@ class platform::kubernetes::master::init # This flag is created by Ansible on controller-0; # - Ansible replay is not impacted by flag creation. + # If alternative k8s registry requires the authentication, + # kubeadm required images need to be pre-pulled on controller + if $k8s_registry != 'k8s.gcr.io' and $::platform::docker::params::k8s_registry_secret != undef { + File['/etc/kubernetes/kubeadm.yaml'] + -> platform::docker::login_registry { 'login k8s registry': + registry_url => $k8s_registry, + registry_secret => $::platform::docker::params::k8s_registry_secret + } + + -> exec { 'kubeadm to pre pull images': + command => 'kubeadm config images pull --config /etc/kubernetes/kubeadm.yaml', + logoutput => true, + before => Exec['configure master node'] + } + + -> exec { 'logout k8s registry': + command => "docker logout ${k8s_registry}", + logoutput => true, + } + } + # Create necessary certificate files file { '/etc/kubernetes/pki': ensure => directory, @@ -467,6 +482,44 @@ class platform::kubernetes::worker::init Class['::platform::docker::config'] -> Class[$name] + if str2bool($::is_initial_config) { + include ::platform::params + + if $::platform::docker::params::k8s_registry { + $k8s_registry = $::platform::docker::params::k8s_registry + } else { + $k8s_registry = 'k8s.gcr.io' + } + + # If alternative k8s registry requires the authentication, + # k8s pause image needs to be pre-pulled on worker nodes + if $k8s_registry != 'k8s.gcr.io' and $::platform::docker::params::k8s_registry_secret != undef { + # Get the pause image tag from kubeadm required images + # list and replace with alternative k8s registry + $get_k8s_pause_img = "kubeadm config images list 2>/dev/null |\ + awk '/^k8s.gcr.io\\/pause:/{print \$1}' | sed 's/k8s.gcr.io/${k8s_registry}/'" + $k8s_pause_img = generate('/bin/sh', '-c', $get_k8s_pause_img) + + if k8s_pause_img { + platform::docker::login_registry { 'login k8s registry': + registry_url => $k8s_registry, + registry_secret => $::platform::docker::params::k8s_registry_secret + } + + -> exec { 'load k8s pause image': + command => "docker image pull ${k8s_pause_img}", + logoutput => true, + before => Exec['configure worker node'] + } + + -> exec { 'logout k8s registry': + command => "docker logout ${k8s_registry}", + logoutput => true, + } + } + } + } + # Configure the worker node. Only do this once, so check whether the # kubelet.conf file has already been created (by the join). exec { 'configure worker node': diff --git a/puppet-manifests/src/modules/platform/templates/get-secret-payload.erb b/puppet-manifests/src/modules/platform/templates/get-secret-payload.erb new file mode 100644 index 000000000..1b1eab67c --- /dev/null +++ b/puppet-manifests/src/modules/platform/templates/get-secret-payload.erb @@ -0,0 +1,11 @@ +# Retrieve barbican secret payload +openstack secret get <%=@registry_secret %> \ + --os-auth-url <%=@auth_url %> \ + --os-username <%=@username %> \ + --os-user-domain-name <%=@user_domain %> \ + --os-project-name <%=@project_name %> \ + --os-project-domain-name <%=@project_domain %> \ + --os-region-name <%=@region_name %> \ + --os-interface <%=@interface %> \ + --os-password <%=@password %> \ + -p -f value -c Payload