Merge "Ansible Support For Authenticated Registries"

This commit is contained in:
Zuul
2019-08-30 16:51:03 +00:00
committed by Gerrit Code Review
11 changed files with 343 additions and 30 deletions

View File

@@ -108,7 +108,13 @@ cluster_host_dynamic_address_allocation: true
#
# To overwrite a particular registry url, use the original registry
# value as the key followed by a custom IP address or domain for
# the value.
# the value. If the registry is authenticated, specify username and password
# e.g.
# docker_registries:
# k8s.gcr.io:
# url: my.k8sregistry.io
# username: k8sreguser
# password: K8sregPass*
#
# The "unified" is a special registry key. Defining and giving
# it a value implies all images are to be retrieved from this

View File

@@ -96,8 +96,23 @@
- name: Update Tiller and Armada image tags
set_fact:
tiller_img: "{{ tiller_img | regex_replace('gcr.io', '{{ gcr_registry }}') }}"
armada_img: "{{ armada_img | regex_replace('quay.io', '{{ quay_registry }}') }}"
tiller_img: "{{ tiller_img | regex_replace('gcr.io', '{{ gcr_registry.url }}') }}"
armada_img: "{{ armada_img | regex_replace('quay.io', '{{ quay_registry.url }}') }}"
- name: log in to gcr registry if credentials exist
docker_login:
registry: "{{ gcr_registry['url'] }}"
username: "{{ gcr_registry['username'] }}"
password: "{{ gcr_registry['password'] }}"
when: gcr_registry.username is defined
- name: log in to quay registry if credentials exist
docker_login:
registry: "{{ quay_registry['url'] }}"
username: "{{ quay_registry['username'] }}"
password: "{{ quay_registry['password'] }}"
when: quay_registry.username is defined
- name: Pull Tiller and Armada images
docker_image:
@@ -106,6 +121,18 @@
- "{{ tiller_img }}"
- "{{ armada_img }}"
- name: log out of gcr registry if credentials exist
docker_login:
registry: "{{ gcr_registry['url'] }}"
state: absent
when: gcr_registry.username is defined
- name: log out of quay registry if credentials exist
docker_login:
registry: "{{ quay_registry['url'] }}"
state: absent
when: quay_registry.username is defined
- name: Create source and target helm bind directories
file:
path: "{{ item }}"
@@ -133,6 +160,11 @@
kubectl --kubeconfig=/etc/kubernetes/admin.conf create serviceaccount
--namespace kube-system tiller
- name: Patch pull secret into tiller service account
command: >
kubectl --kubeconfig=/etc/kubernetes/admin.conf patch serviceaccount
tiller -p '{"imagePullSecrets": [{"name": "gcr-registry-secret"}]}' -n kube-system
- name: Create cluster role binding for Tiller service account
command: >
kubectl --kubeconfig=/etc/kubernetes/admin.conf create clusterrolebinding

View File

@@ -59,7 +59,7 @@
when: not is_secure_registry
environment:
DOCKER_REGISTRY_IP: "{{ docker_registry }}"
DOCKER_REGISTRY_IP: "{{ docker_registry.url }}"
when: use_unified_registry
- name: Update kernel parameters for iptables
@@ -140,7 +140,7 @@
ETCD_ENDPOINT: "http://{{ cluster_floating_address | ipwrap }}:2379"
POD_NETWORK_CIDR: "{{ cluster_pod_subnet }}"
SERVICE_NETWORK_CIDR: "{{ cluster_service_subnet }}"
K8S_REGISTRY: "{{ k8s_registry }}"
K8S_REGISTRY: "{{ k8s_registry.url }}"
- name: Add apiserver certificate SANs to kubeadm
replace:
@@ -172,6 +172,61 @@
- "sed -i -e '/<%= @apiserver_oidc_username_claim %>/d' /etc/kubernetes/kubeadm.yaml"
when: apiserver_oidc | length == 0
- name: log in to k8s registry if credentials exist
docker_login:
registry: "{{ k8s_registry['url'] }}"
username: "{{ k8s_registry['username'] }}"
password: "{{ k8s_registry['password'] }}"
when: k8s_registry.username is defined
- name: log in to gcr registry if credentials exist
docker_login:
registry: "{{ gcr_registry['url'] }}"
username: "{{ gcr_registry['username'] }}"
password: "{{ gcr_registry['password'] }}"
when: gcr_registry.username is defined
- name: log in to quay registry if credentials exist
docker_login:
registry: "{{ quay_registry['url'] }}"
username: "{{ quay_registry['username'] }}"
password: "{{ quay_registry['password'] }}"
when: quay_registry.username is defined
- name: log in to docker registry if credentials exist
docker_login:
registry: "{{ docker_registry['url'] }}"
username: "{{ docker_registry['username'] }}"
password: "{{ docker_registry['password'] }}"
when: docker_registry.username is defined
- name: prepull kubernetes images
command: kubeadm config images pull --config=/etc/kubernetes/kubeadm.yaml
- name: log out of k8s registry if credentials exist
docker_login:
registry: "{{ k8s_registry['url'] }}"
state: absent
when: k8s_registry.username is defined
- name: log out of gcr registry if credentials exist
docker_login:
registry: "{{ gcr_registry['url'] }}"
state: absent
when: gcr_registry.username is defined
- name: log out of quay registry if credentials exist
docker_login:
registry: "{{ quay_registry['url'] }}"
state: absent
when: quay_registry.username is defined
- name: log out of docker registry if credentials exist
docker_login:
registry: "{{ docker_registry['url'] }}"
state: absent
when: docker_registry.username is defined
- name: Initializing Kubernetes master
command: kubeadm init --config=/etc/kubernetes/kubeadm.yaml
@@ -187,6 +242,53 @@
dest: /etc/profile.d/kubeconfig.sh
remote_src: yes
- name: Patch pull secret into kube-proxy service account
command: >
kubectl --kubeconfig=/etc/kubernetes/admin.conf patch serviceaccount
kube-proxy -p '{"imagePullSecrets": [{"name": "k8s-registry-secret"}]}' -n kube-system
- name: Find old Kubernetes registry secrets
shell: "{{ item }}"
with_items:
- "kubectl --kubeconfig=/etc/kubernetes/admin.conf get secrets -n kube-system |
grep k8s-registry-secret | awk '{print $1}'"
- "kubectl --kubeconfig=/etc/kubernetes/admin.conf get secrets -n kube-system |
grep gcr-registry-secret | awk '{print $1}'"
- "kubectl --kubeconfig=/etc/kubernetes/admin.conf get secrets -n kube-system |
grep quay-registry-secret | awk '{print $1}'"
- "kubectl --kubeconfig=/etc/kubernetes/admin.conf get secrets -n kube-system |
grep docker-registry-secret | awk '{print $1}'"
register: old_kubernetes_secrets
- name: Delete old Kubernetes registry secrets
shell: "kubectl --kubeconfig=/etc/kubernetes/admin.conf delete secret -n kube-system {{ item }}"
with_items:
- "{{ old_kubernetes_secrets.results | map(attribute='stdout_lines') | flatten }}"
- name: Create k8s registry pull secret
command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf create secret docker-registry k8s-registry-secret
--docker-server={{ k8s_registry['url'] }} --docker-username={{ k8s_registry['username'] }}
--docker-password={{ k8s_registry['password'] }} -n kube-system"
when: k8s_registry['username'] is defined
- name: Create gcr registry pull secret
command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf create secret docker-registry gcr-registry-secret
--docker-server={{ gcr_registry['url'] }} --docker-username={{ gcr_registry['username'] }}
--docker-password={{ gcr_registry['password'] }} -n kube-system"
when: gcr_registry['username'] is defined
- name: Create quay registry pull secret
command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf create secret docker-registry quay-registry-secret
--docker-server={{ quay_registry['url'] }} --docker-username={{ quay_registry['username'] }}
--docker-password={{ quay_registry['password'] }} -n kube-system"
when: quay_registry['username'] is defined
- name: Create docker registry pull secret
command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf create secret docker-registry docker-registry-secret
--docker-server={{ docker_registry['url'] }} --docker-username={{ docker_registry['username'] }}
--docker-password={{ docker_registry['password'] }} -n kube-system"
when: docker_registry['username'] is defined
- name: Set Calico cluster configuration
set_fact:
cluster_network_ipv4: "{{ cluster_pod_subnet | ipv4 }}"

View File

@@ -525,12 +525,14 @@ spec:
# Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force
# deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.
terminationGracePeriodSeconds: 0
imagePullSecrets:
- name: quay-registry-secret
initContainers:
# This container performs upgrade from host-local IPAM to calico-ipam.
# It can be deleted if this is a fresh installation, or if you have already
# upgraded to use calico-ipam.
- name: upgrade-ipam
image: "{{ quay_registry }}/calico/cni:v3.6.2"
image: "{{ quay_registry.url }}/calico/cni:v3.6.2"
command: ["/opt/cni/bin/calico-ipam", "-upgrade"]
env:
- name: KUBERNETES_NODE_NAME
@@ -550,7 +552,7 @@ spec:
# This container installs the Calico CNI binaries
# and CNI network config file on each node.
- name: install-cni
image: "{{ quay_registry }}/calico/cni:v3.6.2"
image: "{{ quay_registry.url }}/calico/cni:v3.6.2"
command: ["/install-cni.sh"]
env:
# Name of the CNI config file to create.
@@ -586,7 +588,7 @@ spec:
# container programs network policy and routes on each
# host.
- name: calico-node
image: "{{ quay_registry }}/calico/node:v3.6.2"
image: "{{ quay_registry.url }}/calico/node:v3.6.2"
env:
# Configure inbound failsafe rules
- name: FELIX_FAILSAFEINBOUNDHOSTPORTS
@@ -790,9 +792,11 @@ spec:
- key: node-role.kubernetes.io/master
effect: NoSchedule
serviceAccountName: calico-kube-controllers
imagePullSecrets:
- name: quay-registry-secret
containers:
- name: calico-kube-controllers
image: "{{ quay_registry }}/calico/kube-controllers:v3.6.2"
image: "{{ quay_registry.url }}/calico/kube-controllers:v3.6.2"
env:
# Choose which controllers to run.
- name: ENABLED_CONTROLLERS

View File

@@ -155,9 +155,11 @@ spec:
- operator: Exists
effect: NoSchedule
serviceAccountName: multus
imagePullSecrets:
- name: docker-registry-secret
containers:
- name: kube-multus
image: "{{ docker_registry }}/nfvpe/multus:v3.2"
image: "{{ docker_registry.url }}/nfvpe/multus:v3.2"
env:
- name: KUBERNETES_NODE_NAME
valueFrom:

View File

@@ -27,9 +27,11 @@ spec:
tolerations:
- operator: Exists
effect: NoSchedule
imagePullSecrets:
- name: docker-registry-secret
containers:
- name: kube-sriov-cni
image: "{{ docker_registry }}/starlingx/k8s-cni-sriov:master-centos-stable-latest"
image: "{{ docker_registry.url }}/starlingx/k8s-cni-sriov:master-centos-stable-latest"
securityContext:
privileged: true
resources:

View File

@@ -38,9 +38,11 @@ spec:
- operator: Exists
effect: NoSchedule
serviceAccountName: sriov-device-plugin
imagePullSecrets:
- name: docker-registry-secret
containers:
- name: kube-sriovdp
image: "{{ docker_registry }}/starlingx/k8s-plugins-sriov-network-device:master-centos-stable-latest"
image: "{{ docker_registry.url }}/starlingx/k8s-plugins-sriov-network-device:master-centos-stable-latest"
args:
- --log-level=10
securityContext:

View File

@@ -590,6 +590,10 @@ def populate_docker_config(client):
gcr_url = CONF.get('BOOTSTRAP_CONFIG', 'GCR_REGISTRY')
quay_url = CONF.get('BOOTSTRAP_CONFIG', 'QUAY_REGISTRY')
docker_url = CONF.get('BOOTSTRAP_CONFIG', 'DOCKER_REGISTRY')
k8s_secret = CONF.get('BOOTSTRAP_CONFIG', 'K8S_REGISTRY_SECRET')
gcr_secret = CONF.get('BOOTSTRAP_CONFIG', 'GCR_REGISTRY_SECRET')
quay_secret = CONF.get('BOOTSTRAP_CONFIG', 'QUAY_REGISTRY_SECRET')
docker_secret = CONF.get('BOOTSTRAP_CONFIG', 'DOCKER_REGISTRY_SECRET')
parameters[
sysinv_constants.SERVICE_PARAM_SECTION_DOCKER_K8S_REGISTRY] = \
@@ -603,6 +607,27 @@ def populate_docker_config(client):
parameters[
sysinv_constants.SERVICE_PARAM_SECTION_DOCKER_DOCKER_REGISTRY] = \
{sysinv_constants.SERVICE_PARAM_NAME_DOCKER_URL: docker_url}
if k8s_secret != "none":
parameters[
sysinv_constants.SERVICE_PARAM_SECTION_DOCKER_K8S_REGISTRY][
sysinv_constants.SERVICE_PARAM_NAME_DOCKER_AUTH_SECRET] = \
k8s_secret.split('/')[-1]
# we need the split because we want the Barbican UUID, not the secret href
if gcr_secret != "none":
parameters[
sysinv_constants.SERVICE_PARAM_SECTION_DOCKER_GCR_REGISTRY][
sysinv_constants.SERVICE_PARAM_NAME_DOCKER_AUTH_SECRET] = \
gcr_secret.split('/')[-1]
if quay_secret != "none":
parameters[
sysinv_constants.SERVICE_PARAM_SECTION_DOCKER_QUAY_REGISTRY][
sysinv_constants.SERVICE_PARAM_NAME_DOCKER_AUTH_SECRET] = \
quay_secret.split('/')[-1]
if docker_secret != "none":
parameters[
sysinv_constants.SERVICE_PARAM_SECTION_DOCKER_DOCKER_REGISTRY][
sysinv_constants.SERVICE_PARAM_NAME_DOCKER_AUTH_SECRET] = \
docker_secret.split('/')[-1]
print("Populating/Updating docker registry config...")
for registry in parameters:

View File

@@ -91,6 +91,76 @@
include: shutdown_services.yml
when: restart_services
- name: Find old registry secrets in Barbican
shell: "{{ item }}"
with_items:
- "source /etc/platform/openrc; openstack secret list | grep k8s-registry-secret | awk '{print $2}'"
- "source /etc/platform/openrc; openstack secret list | grep gcr-registry-secret | awk '{print $2}'"
- "source /etc/platform/openrc; openstack secret list | grep quay-registry-secret | awk '{print $2}'"
- "source /etc/platform/openrc; openstack secret list | grep docker-registry-secret | awk '{print $2}'"
register: old_barbican_secrets
- name: Delete old registry secrets in Barbican
shell: "source /etc/platform/openrc; openstack secret delete {{ item }}"
with_items:
- "{{ old_barbican_secrets.results | map(attribute='stdout_lines') | flatten }}"
# need to do this here to get the barbican secret id for sysinv
- block:
- name: Create Barbican secret for k8s registry if credentials exist
shell: "source /etc/platform/openrc; openstack secret store -n k8s-registry-secret
-p 'username:{{ k8s_registry['username'] }} password:{{ k8s_registry['password'] }}' -c 'Secret href' -f value"
register: k8s_registry_secret_output
- set_fact:
k8s_registry_secret: "{{ k8s_registry_secret_output.stdout }}"
when: k8s_registry.username is defined
- block:
- name: Create Barbican secret for gcr registry if credentials exist
shell: "source /etc/platform/openrc; openstack secret store -n gcr-registry-secret
-p 'username:{{ gcr_registry['username'] }} password:{{ gcr_registry['password'] }}' -c 'Secret href' -f value"
register: gcr_registry_secret_output
- set_fact:
gcr_registry_secret: "{{ gcr_registry_secret_output.stdout }}"
when: gcr_registry.username is defined
- block:
- name: Create Barbican secret for quay registry if credentials exist
shell: "source /etc/platform/openrc; openstack secret store -n quay-registry-secret
-p 'username:{{ quay_registry['username'] }} password:{{ quay_registry['password'] }}' -c 'Secret href' -f value"
register: quay_registry_secret_output
- set_fact:
quay_registry_secret: "{{ quay_registry_secret_output.stdout }}"
when: quay_registry.username is defined
- block:
- name: Create Barbican secret for docker registry if credentials exist
shell: "source /etc/platform/openrc; openstack secret store -n docker-registry-secret
-p 'username:{{ docker_registry['username'] }} password:{{ docker_registry['password'] }}'
-c 'Secret href' -f value"
register: docker_registry_secret_output
- set_fact:
docker_registry_secret: "{{ docker_registry_secret_output.stdout }}"
when: docker_registry.username is defined
- name: Append config ini file with Barbican secret uuid
lineinfile:
path: "{{ config_permdir + '/' + bootstrap_config_file|basename }}"
line: "{{ item }}"
with_items:
- "K8S_REGISTRY_SECRET={{ k8s_registry_secret | default('none') }}"
- "GCR_REGISTRY_SECRET={{ gcr_registry_secret | default('none') }}"
- "QUAY_REGISTRY_SECRET={{ quay_registry_secret | default('none') }}"
- "DOCKER_REGISTRY_SECRET={{ docker_registry_secret | default('none') }}"
- include: update_sysinv_database.yml
when: save_config_to_db

View File

@@ -193,6 +193,10 @@
external_oam_node_0_address: "{{ external_oam_node_0_address | default('derived') }}"
external_oam_node_1_address: "{{ external_oam_node_1_address | default('derived') }}"
- set_fact:
docker_registries: "{{ vault_docker_registries }}"
when: vault_docker_registries is defined
- name: Set default registries dictionary
set_fact:
default_docker_registries:
@@ -259,12 +263,16 @@
when: (docker_http_proxy is defined and docker_http_proxy is not none) or
(docker_https_proxy is defined and docker_https_proxy is not none)
- name: Set default values for platform registries
- name: Set default values for individual platform registries and registry secrets
set_fact:
default_k8s_registry: k8s.gcr.io
default_gcr_registry: gcr.io
default_quay_registry: quay.io
default_docker_registry: docker.io
default_k8s_registry:
url: k8s.gcr.io
default_gcr_registry:
url: gcr.io
default_quay_registry:
url: quay.io
default_docker_registry:
url: docker.io
- name: Set default values for OpenID connect
set_fact:

View File

@@ -8,6 +8,63 @@
# This role is to validate and save host (non secure) config.
#
# error check the password section of docker registries
# check password parameters before trying to hide the password
# we need to do that here as opposed to with the other docker registry
# stuff because of the debug log statement.
# we need to do this all before the debug log statement to not log passwords.
- name: Check k8s_registry credentials
fail:
msg: "k8s registry username and password must both be specified or not at all"
when: (docker_registries[default_k8s_registry.url].username is defined and
docker_registries[default_k8s_registry.url].password is not defined) or
(docker_registries[default_k8s_registry.url].username is not defined and
docker_registries[default_k8s_registry.url].password is defined)
- name: Check gcr_registry credentials
fail:
msg: "gcr registry username and password must both be specified or not at all"
when: (docker_registries[default_gcr_registry.url].username is defined and
docker_registries[default_gcr_registry.url].password is not defined) or
(docker_registries[default_gcr_registry.url].username is not defined and
docker_registries[default_gcr_registry.url].password is defined)
- name: Check quay_registry credentials
fail:
msg: "quay registry username and password must both be specified or not at all"
when: (docker_registries[default_quay_registry.url].username is defined and
docker_registries[default_quay_registry.url].password is not defined) or
(docker_registries[default_quay_registry.url].username is not defined and
docker_registries[default_quay_registry.url].password is defined)
- name: Check docker_registry credentials
fail:
msg: "docker registry username and password must both be specified or not at all"
when: (docker_registries[default_docker_registry.url].username is defined and
docker_registries[default_docker_registry.url].password is not defined) or
(docker_registries[default_docker_registry.url].username is not defined and
docker_registries[default_docker_registry.url].password is defined)
- name: Check unified registry credentials
fail:
msg: "unified registry username and password must both be specified or not at all"
when: docker_registries['unified'] is defined and
((docker_registries['unified'].username is defined and
docker_registries['unified'].password is not defined) or
(docker_registries['unified'].username is not defined and
docker_registries['unified'].password is defined))
# create a copy of docker_registries without passwords for debug logging
- set_fact:
docker_registries_with_secrets: "{{ docker_registries }}"
- set_fact:
docker_registries: "{{ docker_registries | combine(hide_pw, recursive=true) }}"
vars:
hide_pw: "{ '{{ item.key }}': { 'password': 'secret' } }"
with_dict: "{{ docker_registries }}"
no_log: true
- debug:
msg:
- System mode is {{ system_mode }}
@@ -369,17 +426,20 @@
- set_fact:
use_default_registries: true
k8s_registry:
"{{ docker_registries[default_k8s_registry]['url'] if docker_registries[default_k8s_registry]['url'] is not none
"{{ docker_registries_with_secrets[default_k8s_registry.url]
if docker_registries[default_k8s_registry.url]['url'] is not none
else default_k8s_registry }}"
gcr_registry:
"{{ docker_registries[default_gcr_registry]['url'] if docker_registries[default_gcr_registry]['url'] is not none
"{{ docker_registries_with_secrets[default_gcr_registry.url]
if docker_registries[default_gcr_registry.url]['url'] is not none
else default_gcr_registry }}"
quay_registry:
"{{ docker_registries[default_quay_registry]['url'] if docker_registries[default_quay_registry]['url'] is not none
"{{ docker_registries_with_secrets[default_quay_registry.url]
if docker_registries[default_quay_registry.url]['url'] is not none
else default_quay_registry }}"
docker_registry:
"{{ docker_registries[default_docker_registry]['url']
if docker_registries[default_docker_registry]['url'] is not none
"{{ docker_registries_with_secrets[default_docker_registry.url]
if docker_registries[default_docker_registry.url]['url'] is not none
else default_docker_registry }}"
default_no_proxy:
- localhost
@@ -434,10 +494,10 @@
- name: Turn on use_unified_registry flag
set_fact:
use_unified_registry: true
k8s_registry: "{{ docker_registries['unified']['url'] }}"
gcr_registry: "{{ docker_registries['unified']['url'] }}"
quay_registry: "{{ docker_registries['unified']['url'] }}"
docker_registry: "{{ docker_registries['unified']['url'] }}"
k8s_registry: "{{ docker_registries_with_secrets['unified'] }}"
gcr_registry: "{{ docker_registries_with_secrets['unified'] }}"
quay_registry: "{{ docker_registries_with_secrets['unified'] }}"
docker_registry: "{{ docker_registries_with_secrets['unified'] }}"
when: docker_registries['unified'] is defined and docker_registries['unified'] is not none
@@ -584,10 +644,10 @@
- "DOCKER_HTTP_PROXY={{ docker_http_proxy }}"
- "DOCKER_HTTPS_PROXY={{ docker_https_proxy }}"
- "DOCKER_NO_PROXY={{ docker_no_proxy_combined | join(',') }}"
- "K8S_REGISTRY={{ k8s_registry }}"
- "GCR_REGISTRY={{ gcr_registry }}"
- "QUAY_REGISTRY={{ quay_registry }}"
- "DOCKER_REGISTRY={{ docker_registry }}"
- "K8S_REGISTRY={{ k8s_registry.url }}"
- "GCR_REGISTRY={{ gcr_registry.url }}"
- "QUAY_REGISTRY={{ quay_registry.url }}"
- "DOCKER_REGISTRY={{ docker_registry.url }}"
- "USE_DEFAULT_REGISTRIES={{ use_default_registries }}"
- "IS_SECURE_REGISTRY={{ is_secure_registry | default(True) }}"
- "RECONFIGURE_ENDPOINTS={{ reconfigure_endpoints }}"