diff --git a/playbookconfig/playbookconfig/playbooks/bootstrap/host_vars/default.yml b/playbookconfig/playbookconfig/playbooks/bootstrap/host_vars/default.yml index 3157255ac5..8c62f823b6 100644 --- a/playbookconfig/playbookconfig/playbooks/bootstrap/host_vars/default.yml +++ b/playbookconfig/playbookconfig/playbooks/bootstrap/host_vars/default.yml @@ -92,7 +92,23 @@ dynamic_address_allocation: True # DOCKER REGISTRIES # ================= # -# The default list of registries can be extended with new entries. +# The docker_registries is a map of known registry keys and their +# source values. Each key is a fully scoped registry name and the +# same name is used as the default value. For instance, +# k8s.gcr.io registry which hosts Kubernetes related images, has +# the default registry value of k8s.gcr.io +# +# To overwrite a particular registry, use the original registry +# value as the key followed by a custom IP address or domain for +# the value. +# +# The "unified" is a special registry key. Defining and giving +# it a value implies all images are to be retrieved from this +# single source. Hence, registry values of all other registry keys +# if specified will be ignored. +# +# The docker registries map can be extended with new custom keys in +# the near future. # # The valid formats for a registry value are: # - domain (e.g. example.domain) @@ -102,17 +118,15 @@ dynamic_address_allocation: True # - IPv6 address (e.g. FD01::0100) # - IPv6 address with port (e.g. [FD01::0100]:5000 # -# If the registry list contains only a single item, the specified registry is -# considered a unified registry and will replace all default registries. -# # Parameter is_secure_registry is only relevant when a unified registry is # used. docker_registries: -# - k8s.gcr.io -# - gcr.io -# - quay.io -# - docker.io + k8s.gcr.io: + gcr.io: + quay.io: + docker.io: + # unified: example.domain #is_secure_registry: True diff --git a/playbookconfig/playbookconfig/playbooks/bootstrap/roles/bringup-essential-services/tasks/bringup_helm.yml b/playbookconfig/playbookconfig/playbooks/bootstrap/roles/bringup-essential-services/tasks/bringup_helm.yml index f02f096ee0..24deb4fe43 100644 --- a/playbookconfig/playbookconfig/playbooks/bootstrap/roles/bringup-essential-services/tasks/bringup_helm.yml +++ b/playbookconfig/playbookconfig/playbooks/bootstrap/roles/bringup-essential-services/tasks/bringup_helm.yml @@ -94,11 +94,10 @@ regexp: "var.pxeboot_ip_network =.*$" replace: 'var.pxeboot_ip_network = "{{ pxeboot_subnet }}"' -- name: Update tiller image tag if using unified registry +- name: Update Tiller and Armada image tags set_fact: - tiller_img: "{{ tiller_img | regex_replace('gcr.io', '{{ docker_registries[0] }}') }}" - armada_img: "{{ armada_img | regex_replace('quay.io', '{{ docker_registries[0] }}') }}" - when: use_unified_registry + tiller_img: "{{ tiller_img | regex_replace('gcr.io', '{{ gcr_registry }}') }}" + armada_img: "{{ armada_img | regex_replace('quay.io', '{{ quay_registry }}') }}" - name: Pull Tiller and Armada images docker_image: diff --git a/playbookconfig/playbookconfig/playbooks/bootstrap/roles/bringup-essential-services/tasks/bringup_kubemaster.yml b/playbookconfig/playbookconfig/playbooks/bootstrap/roles/bringup-essential-services/tasks/bringup_kubemaster.yml index 6494fc661e..048b76408a 100644 --- a/playbookconfig/playbookconfig/playbooks/bootstrap/roles/bringup-essential-services/tasks/bringup_kubemaster.yml +++ b/playbookconfig/playbookconfig/playbooks/bootstrap/roles/bringup-essential-services/tasks/bringup_kubemaster.yml @@ -59,7 +59,7 @@ when: not is_secure_registry environment: - DOCKER_REGISTRY_IP: "{{ docker_registries[0] }}" + DOCKER_REGISTRY_IP: "{{ docker_registry }}" when: use_unified_registry - name: Update kernel parameters for iptables @@ -136,14 +136,7 @@ ETCD_PORT: 2379 POD_NETWORK_CIDR: "{{ cluster_pod_subnet }}" SERVICE_NETWORK_CIDR: "{{ cluster_service_subnet }}" - K8S_REGISTRY: "{{ default_k8s_registry }}" - -- name: Update image repo in admin yaml if unified registry is used - replace: - path: /etc/kubernetes/kubeadm.yaml - regexp: "imageRepository: .*$" - replace: 'imageRepository: "{{ docker_registries[0] }}"' - when: use_unified_registry + K8S_REGISTRY: "{{ k8s_registry }}" - name: Initializing Kubernetes master command: kubeadm init --config=/etc/kubernetes/kubeadm.yaml @@ -167,21 +160,11 @@ remote_src: yes - name: Update Multus config file - command: "{{ item }}" - args: - warn: false - with_items: - - "sed -i -e 's|<%= @docker_registry %>|'$DOCKER_REGISTRY'|g' /etc/kubernetes/multus.yaml" - environment: - DOCKER_REGISTRY: "{{ default_docker_registry }}" - -- name: Update Multus yaml file with new registry info if unified registry is used - command: "sed -i -e 's|{{ default_docker_registry }}|'$DOCKER_REGISTRY'|g' /etc/kubernetes/multus.yaml" + command: "sed -i -e 's|<%= @docker_registry %>|'$DOCKER_REGISTRY'|g' /etc/kubernetes/multus.yaml" args: warn: false environment: - DOCKER_REGISTRY: "{{ docker_registries[0] }}" - when: use_unified_registry + DOCKER_REGISTRY: "{{ docker_registry }}" - name: Activate Multus Networking command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /etc/kubernetes/multus.yaml" @@ -204,15 +187,7 @@ environment: CLUSTER_IP: "{{ cluster_floating_address }}" POD_NETWORK_CIDR: "{{ cluster_pod_subnet }}" - QUAY_REGISTRY: "{{ default_quay_registry }}" - -- name: Update Calico yaml file with new registry info if unified registry is used - command: "sed -i -e 's|{{ default_quay_registry }}|'$QUAY_REGISTRY'|g' /etc/kubernetes/calico.yaml" - args: - warn: false - environment: - QUAY_REGISTRY: "{{ docker_registries[0] }}" - when: use_unified_registry + QUAY_REGISTRY: "{{ quay_registry }}" - name: Activate Calico Networking command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /etc/kubernetes/calico.yaml" @@ -224,21 +199,11 @@ remote_src: yes - name: Update SRIOV Networking config file - command: "{{ item }}" - args: - warn: false - with_items: - - "sed -i -e 's|<%= @docker_registry %>|'$DOCKER_REGISTRY'|g' /etc/kubernetes/sriov-cni.yaml" - environment: - DOCKER_REGISTRY: "{{ default_docker_registry }}" - -- name: Update SRIOV Networking yaml file with new registry info if unified registry is used - command: "sed -i -e 's|{{ default_docker_registry }}|'$DOCKER_REGISTRY'|g' /etc/kubernetes/sriov-cni.yaml" + command: "sed -i -e 's|<%= @docker_registry %>|'$DOCKER_REGISTRY'|g' /etc/kubernetes/sriov-cni.yaml" args: warn: false environment: - DOCKER_REGISTRY: "{{ docker_registries[0] }}" - when: use_unified_registry + DOCKER_REGISTRY: "{{ docker_registry }}" - name: Activate SRIOV Networking command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /etc/kubernetes/sriov-cni.yaml" @@ -250,21 +215,11 @@ remote_src: yes - name: Update SRIOV device plugin config file - command: "{{ item }}" - args: - warn: false - with_items: - - "sed -i -e 's|<%= @docker_registry %>|'$DOCKER_REGISTRY'|g' /etc/kubernetes/sriovdp-daemonset.yaml" - environment: - DOCKER_REGISTRY: "{{ default_docker_registry }}" - -- name: Update SRIOV device plugin yaml file with new registry info if unified registry is used - command: "sed -i -e 's|{{ default_docker_registry }}|'$DOCKER_REGISTRY'|g' /etc/kubernetes/sriovdp-daemonset.yaml" + command: "sed -i -e 's|<%= @docker_registry %>|'$DOCKER_REGISTRY'|g' /etc/kubernetes/sriovdp-daemonset.yaml" args: warn: false environment: - DOCKER_REGISTRY: "{{ docker_registries[0] }}" - when: use_unified_registry + DOCKER_REGISTRY: "{{ docker_registry }}" - name: Activate SRIOV device plugin command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /etc/kubernetes/sriovdp-daemonset.yaml" diff --git a/playbookconfig/playbookconfig/playbooks/bootstrap/roles/bringup-essential-services/tasks/main.yml b/playbookconfig/playbookconfig/playbooks/bootstrap/roles/bringup-essential-services/tasks/main.yml index a3dcdfabeb..de31518ad7 100644 --- a/playbookconfig/playbookconfig/playbooks/bootstrap/roles/bringup-essential-services/tasks/main.yml +++ b/playbookconfig/playbookconfig/playbooks/bootstrap/roles/bringup-essential-services/tasks/main.yml @@ -96,7 +96,6 @@ - "prev_external_oam_floating_address: {{ external_oam_floating_address }}" - "prev_management_multicast_subnet: {{ management_multicast_subnet }}" - "prev_dns_servers: {{ dns_servers | join(',') }}" - - "prev_docker_registries: {{ docker_registries | join(',') }}" - "prev_docker_http_proxy: {{ docker_http_proxy }}" - "prev_docker_https_proxy: {{ docker_https_proxy }}" - "prev_docker_no_proxy: {{ docker_no_proxy | join(',') }}" @@ -121,4 +120,16 @@ - "prev_external_oam_node_0_address: {{ external_oam_node_0_address }}" - "prev_external_oam_node_1_address: {{ external_oam_node_1_address }}" + # Need to save the dictionary this way for proper comparison during replay + - name: Save previous docker registries header + lineinfile: + path: "{{ last_bootstrap_config_file }}" + line: "prev_docker_registries:" + + - name: Save previous docker registries content + lineinfile: + path: "{{ last_bootstrap_config_file }}" + line: " {{ item.key }}: {{ item.value }}" + with_dict: "{{ docker_registries }}" + when: save_config diff --git a/playbookconfig/playbookconfig/playbooks/bootstrap/roles/persist-config/files/populate_initial_config.py b/playbookconfig/playbookconfig/playbooks/bootstrap/roles/persist-config/files/populate_initial_config.py index 3f55c094fa..44d36b71d9 100644 --- a/playbookconfig/playbookconfig/playbooks/bootstrap/roles/persist-config/files/populate_initial_config.py +++ b/playbookconfig/playbookconfig/playbooks/bootstrap/roles/persist-config/files/populate_initial_config.py @@ -450,11 +450,17 @@ def populate_docker_config(client): 'BOOTSTRAP_CONFIG', 'USE_DEFAULT_REGISTRIES') if not use_default_registries: - registries = CONF.get('BOOTSTRAP_CONFIG', 'DOCKER_REGISTRIES') secure_registry = CONF.getboolean('BOOTSTRAP_CONFIG', 'IS_SECURE_REGISTRY') parameters = {} - parameters['registries'] = registries + + # TODO(tngo): The following 4 service parameters will be removed when + # we switch to the long term solution using a single "registries" + # service parameter that is extensible. + parameters['k8s'] = CONF.get('BOOTSTRAP_CONFIG', 'K8S_REGISTRY') + parameters['gcr'] = CONF.get('BOOTSTRAP_CONFIG', 'GCR_REGISTRY') + parameters['quay'] = CONF.get('BOOTSTRAP_CONFIG', 'QUAY_REGISTRY') + parameters['docker'] = CONF.get('BOOTSTRAP_CONFIG', 'DOCKER_REGISTRY') if not secure_registry: parameters['insecure_registry'] = "True" @@ -469,7 +475,10 @@ def populate_docker_config(client): if RECONFIGURE_SERVICE: parameters = client.sysinv.service_parameter.list() for parameter in parameters: - if (parameter.name == 'registries' or + if (parameter.name == 'k8s' or + parameter.name == 'gcr' or + parameter.name == 'quay' or + parameter.name == 'docker' or parameter.name == 'insecure_registry'): client.sysinv.service_parameter.delete( parameter.uuid) diff --git a/playbookconfig/playbookconfig/playbooks/bootstrap/roles/prepare-env/tasks/main.yml b/playbookconfig/playbookconfig/playbooks/bootstrap/roles/prepare-env/tasks/main.yml index c30fd3a600..dc270cfa8a 100644 --- a/playbookconfig/playbookconfig/playbooks/bootstrap/roles/prepare-env/tasks/main.yml +++ b/playbookconfig/playbookconfig/playbooks/bootstrap/roles/prepare-env/tasks/main.yml @@ -135,14 +135,17 @@ external_oam_node_0_address: "{{ external_oam_node_0_address | default('derived') }}" external_oam_node_1_address: "{{ external_oam_node_1_address | default('derived') }}" - - name: Set docker registries to default values if not specified + - name: Set default registries dictionary set_fact: - docker_registries: - - k8s.gcr.io - - gcr.io - - quay.io - - docker.io - when: docker_registries is none + default_docker_registries: + k8s.gcr.io: k8s.gcr.io + gcr.io: gcr.io + quay.io: quay.io + docker.io: docker.io + + - name: Merge user and default registries dictionaries + set_fact: + docker_registries: "{{ default_docker_registries | combine(docker_registries) }}" - name: Initialize some flags to be used in subsequent roles/tasks set_fact: @@ -186,6 +189,13 @@ when: (docker_http_proxy is defined and docker_http_proxy is not none) or (docker_https_proxy is defined and docker_https_proxy is not none) + - name: Set default values for platform registries + set_fact: + default_k8s_registry: k8s.gcr.io + default_gcr_registry: gcr.io + default_quay_registry: quay.io + default_docker_registry: docker.io + - name: Set default values for docker proxies if not defined set_fact: docker_http_proxy: "{{ docker_http_proxy | default('undef') }}" @@ -277,7 +287,7 @@ - name: Turn on docker reconfiguration flag if docker config is changed set_fact: docker_config_update: true - when: (prev_docker_registries.split(',') | sort != docker_registries | sort) or + when: (prev_docker_registries != docker_registries) or ((use_docker_proxy) and (prev_docker_http_proxy != docker_http_proxy or prev_docker_https_proxy != docker_https_proxy or diff --git a/playbookconfig/playbookconfig/playbooks/bootstrap/roles/validate-config/tasks/main.yml b/playbookconfig/playbookconfig/playbooks/bootstrap/roles/validate-config/tasks/main.yml index c118e97c5d..e469eb50b8 100644 --- a/playbookconfig/playbookconfig/playbooks/bootstrap/roles/validate-config/tasks/main.yml +++ b/playbookconfig/playbookconfig/playbooks/bootstrap/roles/validate-config/tasks/main.yml @@ -257,11 +257,10 @@ - block: - set_fact: use_default_registries: true - # Define these just in case we need them later - default_k8s_registry: k8s.gcr.io - default_gcr_registry: gcr.io - default_quay_registry: quay.io - default_docker_registry: docker.io + k8s_registry: "{{ docker_registries[default_k8s_registry] if docker_registries[default_k8s_registry] is not none else default_k8s_registry }}" + gcr_registry: "{{ docker_registries[default_gcr_registry] if docker_registries[default_gcr_registry] is not none else default_gcr_registry }}" + quay_registry: "{{ docker_registries[default_quay_registry] if docker_registries[default_quay_registry] is not none else default_quay_registry }}" + docker_registry: "{{ docker_registries[default_docker_registry] if docker_registries[default_docker_registry] is not none else default_docker_registry }}" default_no_proxy: - localhost - 127.0.0.1 @@ -314,23 +313,26 @@ - name: Turn on use_unified_registry flag set_fact: use_unified_registry: true - unified_registry: "{{ docker_registries }}" + k8s_registry: "{{ docker_registries['unified'] }}" + gcr_registry: "{{ docker_registries['unified'] }}" + quay_registry: "{{ docker_registries['unified'] }}" + docker_registry: "{{ docker_registries['unified'] }}" - when: docker_registries|length == 1 + when: docker_registries['unified'] is defined and docker_registries['unified'] is not none - name: Update use_default_registries flag set_fact: use_default_registries: false when: use_unified_registry or docker_registries|length != 4 or - default_k8s_registry not in docker_registries or - default_gcr_registry not in docker_registries or - default_quay_registry not in docker_registries or - default_docker_registry not in docker_registries + k8s_registry != default_k8s_registry or + gcr_registry != default_gcr_registry or + quay_registry != default_quay_registry or + docker_registry != default_docker_registry - block: - - include: validate_address.yml input_address={{ item }} - with_items: "{{ docker_registries }}" + - include: validate_address.yml input_address={{ item.value }} + with_dict: "{{ docker_registries }}" when: not use_default_registries @@ -417,7 +419,10 @@ - "DOCKER_HTTP_PROXY={{ docker_http_proxy }}" - "DOCKER_HTTPS_PROXY={{ docker_https_proxy }}" - "DOCKER_NO_PROXY={{ docker_no_proxy_combined | join(',') }}" - - "DOCKER_REGISTRIES={{ docker_registries | join(',') }}" + - "K8S_REGISTRY={{ k8s_registry }}" + - "GCR_REGISTRY={{ gcr_registry }}" + - "QUAY_REGISTRY={{ quay_registry }}" + - "DOCKER_REGISTRY={{ docker_registry }}" - "USE_DEFAULT_REGISTRIES={{ use_default_registries }}" - "IS_SECURE_REGISTRY={{ is_secure_registry | default(True) }}" - "RECONFIGURE_ENDPOINTS={{ reconfigure_endpoints }}"