starlingx
/
config
Code Issues Proposed changes

Allow user specified registries for config_controller 

Currently docker images were pulled from public registries during
config_controller. For some users, the connection to the public
docker registry may be slow such that installing the containerized
services images may timeout or the system simply does not have
access to the public internet.

This change allows users to specify alternative public/private
registries to replace k8s.gcr.io, gcr.io, quay.io and docker.io.
Insecure registry is supported if all default registries were
replaced by one unified registry. It lowers the complexity for
those who build his own registry without internet access.

Docker doesn't support ipv6 addr as registry name, instead
hostname or domain name in ipv6 network is allowed.

Test:
AIO-SX/AIO-DX/Standard(2+2):
  Alternative public registry (ipv4/domain) with proxy
    - config_controller pass
  Private registry (ipv4/ipv6/domain) without internet
    - config_controller pass
  Default registry with/without proxy
    - config_controller pass

Story: 2004711
Task: 28742

Change-Id: I4fee3f4e0637863b9b5ef4ef556082ac75f62a1d
Signed-off-by: Mingyuan Qi <mingyuan.qi@intel.com>
This commit is contained in:
Mingyuan Qi 2019-01-30 09:41:27 +08:00 committed by Sun Austin
parent 234c04aee2
commit 611a68a96a
18 changed files with 569 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
configutilities/configutilities/configutilities/common/utils.py
View File

@ -182,16 +182,38 @@ def is_valid_ipv6(address):
def is_valid_domain_or_ip(url_str):
if is_valid_domain(url_str):
return True
elif is_valid_ipv4(url_str):
return True
elif is_valid_ipv6(url_str):
return True
if url_str:
if is_valid_domain(url_str):
return True
ip_with_port = url_str.split(':')
if len(ip_with_port) <= 2:
# check ipv4 or ipv4 with port
return is_valid_ipv4(ip_with_port[0])
else:
# check ipv6 with port
if '[' in url_str:
try:
bkt_idx = url_str.index(']')
if bkt_idx + 1 == len(url_str):
# brackets without port
return False
else:
return is_valid_ipv6(url_str[1:bkt_idx])
except Exception:
return False
else:
# check ipv6 without port
return is_valid_ipv6(url_str)
else:
return False
def is_valid_bool_str(val):
"""Check if the provided string is a valid bool string or not."""
boolstrs = ('true', 'false')
return str(val).lower() in boolstrs
def validate_address_str(ip_address_str, network):
"""Determine whether an address is valid."""
try:

74
configutilities/configutilities/configutilities/common/validator.py
View File

@ -26,6 +26,7 @@ from configutilities.common.utils import validate_address_str
from configutilities.common.utils import validate_nameserver_address_str
from configutilities.common.utils import is_valid_url
from configutilities.common.utils import is_valid_domain_or_ip
from configutilities.common.utils import is_valid_bool_str
from configutilities.common.exceptions import ConfigFail
from configutilities.common.exceptions import ValidateFail
@ -1010,6 +1011,77 @@ class ConfigValidator(object):
self.cgcs_conf.set('cDOCKER_PROXY', 'DOCKER_NO_PROXY',
docker_no_proxy_list_str)
def validate_docker_registry(self):
if not self.conf.has_section('DOCKER_REGISTRY'):
return
if self.cgcs_conf is not None:
self.cgcs_conf.add_section('cDOCKER_REGISTRY')
# check k8s_registry
if self.conf.has_option('DOCKER_REGISTRY', 'DOCKER_K8S_REGISTRY'):
docker_k8s_registry_str = self.conf.get(
'DOCKER_REGISTRY', 'DOCKER_K8S_REGISTRY')
if is_valid_domain_or_ip(docker_k8s_registry_str):
if self.cgcs_conf is not None:
self.cgcs_conf.set('cDOCKER_REGISTRY',
'DOCKER_K8S_REGISTRY',
docker_k8s_registry_str)
else:
raise ConfigFail(
"Invalid DOCKER_K8S_REGISTRY value of %s." %
docker_k8s_registry_str)
# check gcr_registry
if self.conf.has_option('DOCKER_REGISTRY', 'DOCKER_GCR_REGISTRY'):
docker_gcr_registry_str = self.conf.get(
'DOCKER_REGISTRY', 'DOCKER_GCR_REGISTRY')
if is_valid_domain_or_ip(docker_gcr_registry_str):
if self.cgcs_conf is not None:
self.cgcs_conf.set('cDOCKER_REGISTRY',
'DOCKER_GCR_REGISTRY',
docker_gcr_registry_str)
else:
raise ConfigFail(
"Invalid DOCKER_GCR_REGISTRY value of %s." %
docker_gcr_registry_str)
# check quay_registry
if self.conf.has_option('DOCKER_REGISTRY', 'DOCKER_QUAY_REGISTRY'):
docker_quay_registry_str = self.conf.get(
'DOCKER_REGISTRY', 'DOCKER_QUAY_REGISTRY')
if is_valid_domain_or_ip(docker_quay_registry_str):
if self.cgcs_conf is not None:
self.cgcs_conf.set('cDOCKER_REGISTRY',
'DOCKER_QUAY_REGISTRY',
docker_quay_registry_str)
else:
raise ConfigFail(
"Invalid DOCKER_QUAY_REGISTRY value of %s." %
docker_quay_registry_str)
# check docker_registry
if self.conf.has_option('DOCKER_REGISTRY', 'DOCKER_DOCKER_REGISTRY'):
docker_docker_registry_str = self.conf.get(
'DOCKER_REGISTRY', 'DOCKER_DOCKER_REGISTRY')
if is_valid_domain_or_ip(docker_docker_registry_str):
if self.cgcs_conf is not None:
self.cgcs_conf.set('cDOCKER_REGISTRY',
'DOCKER_DOCKER_REGISTRY',
docker_docker_registry_str)
else:
raise ConfigFail(
"Invalid DOCKER_DOCKER_REGISTRY value of %s." %
docker_docker_registry_str)
# check is_secure_registry
if self.conf.has_option('DOCKER_REGISTRY', 'IS_SECURE_REGISTRY'):
docker_is_secure_registry_str = self.conf.get(
'DOCKER_REGISTRY', 'IS_SECURE_REGISTRY')
if is_valid_bool_str(docker_is_secure_registry_str):
if self.cgcs_conf is not None:
self.cgcs_conf.set('cDOCKER_REGISTRY',
'IS_SECURE_REGISTRY',
docker_is_secure_registry_str)
else:
raise ConfigFail(
"Invalid IS_SECURE_REGISTRY value of %s." %
docker_is_secure_registry_str)
def validate_ntp(self):
if self.conf.has_section('NTP'):
raise ConfigFail("NTP Configuration is no longer supported")
@ -1468,6 +1540,8 @@ def validate(system_config, config_type=REGION_CONFIG, cgcs_config=None,
validator.validate_dns()
# Docker Proxy configuration
validator.validate_docker_proxy()
# Docker Registry configuration
validator.validate_docker_registry()
# NTP configuration
validator.validate_ntp()
# Network configuration

290
controllerconfig/controllerconfig/controllerconfig/configassistant.py
View File

@ -470,6 +470,14 @@ class ConfigAssistant():
self.docker_https_proxy = ""
self.docker_no_proxy = ""
# Docker registry config
self.docker_use_default_registry = True
self.docker_k8s_registry = ""
self.docker_gcr_registry = ""
self.docker_quay_registry = ""
self.docker_docker_registry = ""
self.is_secure_registry = True
# SDN config
self.enable_sdn = False
@ -2741,7 +2749,7 @@ class ConfigAssistant():
"""Allow user to input docker proxy config."""
print("\nDocker Proxy:")
print("-------------------------\n")
print("-------------\n")
print(textwrap.fill(
"Docker proxy is needed if host OAM network is behind a proxy.",
80))
@ -2762,7 +2770,7 @@ class ConfigAssistant():
self.docker_http_proxy = user_input
break
else:
print("Please input a valid url")
print("Please enter a valid url")
continue
else:
self.docker_http_proxy = ""
@ -2778,7 +2786,7 @@ class ConfigAssistant():
self.docker_https_proxy = user_input
break
else:
print("Please input a valid url")
print("Please enter a valid url")
continue
else:
self.docker_https_proxy = ""
@ -2849,6 +2857,145 @@ class ConfigAssistant():
print("Invalid choice")
continue
def input_docker_registry_config(self):
"""Allow user to input docker registry config."""
print("\nDocker Registry:")
print("----------------\n")
print("Configure docker registries to pull images from.\n"
"Default registries are:\n"
"k8s.gcr.io, gcr.io, quay.io, docker.io\n"
)
while True:
user_input = input(
"Use default docker registries [Y/n]: ")
if user_input.lower() == 'q':
raise UserQuit
elif user_input.lower() == 'n':
# unexpected newline displayed if textwrap.fill with
# '\n' included
print("\nEach registry can be specified as one of the"
"following:\n"
" - domain (e.g. example.domain)\n"
" - domain with port (e.g. example.domain:5000)\n"
" - IPv4 address (e.g. 1.2.3.4)\n"
" - IPv4 address with port (e.g. 1.2.3.4:5000)\n"
)
while True:
user_input = input(
"Use a unified registry replacing all "
"default registries [y/n]: ")
if user_input.lower() == 'q':
raise UserQuit
elif user_input.lower() == 'y':
# Input a unified registry to avoid
# inputing the same registry repeatly
while True:
user_input = input(
"Enter a unified registry: ")
if user_input.lower() == 'q':
raise UserQuit
if is_valid_domain_or_ip(user_input):
self.docker_k8s_registry = user_input
self.docker_gcr_registry = user_input
self.docker_quay_registry = user_input
self.docker_docker_registry = user_input
self.docker_use_default_registry = False
break
else:
print("Please enter a valid registry address")
continue
# Only if a unified registry set, it could be
# an insecure registry
while True:
user_input = input(
"Is '" + self.docker_k8s_registry +
"' a secure registry (https) [Y/n]: ")
if user_input.lower() == 'q':
raise UserQuit
elif user_input.lower() in ('y', ''):
self.is_secure_registry = True
break
elif user_input.lower() == 'n':
self.is_secure_registry = False
break
else:
print("Invalid choice")
continue
break
elif user_input.lower() == 'n':
# Input alternative registries separately
while True:
user_input = input(
"Alternative registry to k8s.gcr.io: ")
if user_input.lower() == 'q':
raise UserQuit
if is_valid_domain_or_ip(user_input):
self.docker_k8s_registry = user_input
break
else:
print("Please enter a valid registry address")
continue
while True:
user_input = input(
"Alternative registry to gcr.io: ")
if user_input.lower() == 'q':
raise UserQuit
if is_valid_domain_or_ip(user_input):
self.docker_gcr_registry = user_input
break
else:
print("Please enter a valid registry address")
continue
while True:
user_input = input(
"Alternative registry to quay.io: ")
if user_input.lower() == 'q':
raise UserQuit
if is_valid_domain_or_ip(user_input):
self.docker_quay_registry = user_input
break
else:
print("Please enter a valid registry address")
continue
while True:
user_input = input(
"Alternative registry to docker.io: ")
if user_input.lower() == 'q':
raise UserQuit
if is_valid_domain_or_ip(user_input):
self.docker_docker_registry = user_input
break
else:
print("Please enter a valid registry address")
continue
if (self.docker_k8s_registry or
self.docker_gcr_registry or
self.docker_quay_registry or
self.docker_docker_registry):
self.docker_use_default_registry = False
break
else:
print("At least one registry is required")
continue
else:
print("Invalid choice")
continue
break
elif user_input.lower() in ('y', ''):
self.docker_use_default_registry = True
break
else:
print("Invalid choice")
continue
def input_authentication_config(self):
"""Allow user to input authentication config and perform validation.
"""
@ -2936,6 +3083,7 @@ class ConfigAssistant():
self.input_dns_config()
# Docker proxy is only used in kubernetes config
self.input_docker_proxy_config()
self.input_docker_registry_config()
self.input_authentication_config()
def is_valid_management_multicast_subnet(self, ip_subnet):
@ -3298,6 +3446,32 @@ class ConfigAssistant():
self.docker_no_proxy = config.get(
'cDOCKER_PROXY', 'DOCKER_NO_PROXY')
# Docker Registry Configuration
if config.has_section('cDOCKER_REGISTRY'):
self.docker_use_default_registry = False
if config.has_option('cDOCKER_REGISTRY',
'DOCKER_K8S_REGISTRY'):
self.docker_k8s_registry = config.get(
'cDOCKER_REGISTRY', 'DOCKER_K8S_REGISTRY')
if config.has_option('cDOCKER_REGISTRY',
'DOCKER_GCR_REGISTRY'):
self.docker_gcr_registry = config.get(
'cDOCKER_REGISTRY', 'DOCKER_GCR_REGISTRY')
if config.has_option('cDOCKER_REGISTRY',
'DOCKER_QUAY_REGISTRY'):
self.docker_quay_registry = config.get(
'cDOCKER_REGISTRY', 'DOCKER_QUAY_REGISTRY')
if config.has_option('cDOCKER_REGISTRY',
'DOCKER_DOCKER_REGISTRY'):
self.docker_docker_registry = config.get(
'cDOCKER_REGISTRY', 'DOCKER_DOCKER_REGISTRY')
if config.has_option('cDOCKER_REGISTRY',
'IS_SECURE_REGISTRY'):
self.is_secure_registry = config.getboolean(
'cDOCKER_REGISTRY', 'IS_SECURE_REGISTRY')
else:
self.is_secure_registry = True
# SDN Network configuration
if config.has_option('cSDN', 'ENABLE_SDN'):
raise ConfigFail("The option ENABLE_SDN is no longer "
@ -3742,14 +3916,31 @@ class ConfigAssistant():
if not dns_config:
print("External DNS servers not configured")
if self.enable_docker_proxy:
print("\nDocker Proxy Configuraton")
print("----------------------")
print("\nDocker Proxy Configuration")
print("--------------------------")
if self.docker_http_proxy:
print("Docker HTTP proxy: " + self.docker_http_proxy)
if self.docker_https_proxy:
print("Docker HTTPS proxy: " + self.docker_https_proxy)
if self.docker_no_proxy:
print("Docker NO proxy: " + self.docker_no_proxy)
if not self.docker_use_default_registry:
print("\nDocker Registry Configuration")
print("-----------------------------")
if self.docker_k8s_registry:
print("Alternative registry to k8s.gcr.io: " +
self.docker_k8s_registry)
if self.docker_gcr_registry:
print("Alternative registry to gcr.io: " +
self.docker_gcr_registry)
if self.docker_quay_registry:
print("Alternative registry to quay.io: " +
self.docker_quay_registry)
if self.docker_docker_registry:
print("Alternative registry to docker.io: " +
self.docker_docker_registry)
print("Is registries secure: " +
str(self.is_secure_registry))
if self.region_config:
print("\nRegion Configuration")
@ -4064,6 +4255,30 @@ class ConfigAssistant():
"DOCKER_NO_PROXY=" +
str(self.docker_no_proxy) + "\n")
# Docker registry configuration
if not self.docker_use_default_registry:
f.write("\n[cDOCKER_REGISTRY]")
f.write("\n# Docker Registry Configuration\n")
if self.docker_k8s_registry:
f.write(
"DOCKER_K8S_REGISTRY=" +
str(self.docker_k8s_registry) + "\n")
if self.docker_gcr_registry:
f.write(
"DOCKER_GCR_REGISTRY=" +
str(self.docker_gcr_registry) + "\n")
if self.docker_quay_registry:
f.write(
"DOCKER_QUAY_REGISTRY=" +
str(self.docker_quay_registry) + "\n")
if self.docker_docker_registry:
f.write(
"DOCKER_DOCKER_REGISTRY=" +
str(self.docker_docker_registry) + "\n")
f.write(
"IS_SECURE_REGISTRY=" +
str(self.is_secure_registry) + "\n")
# Network configuration
f.write("\n[cNETWORK]")
f.write("\n# Data Network Configuration\n")
@ -5468,22 +5683,54 @@ class ConfigAssistant():
client.sysinv.idns.update(dns_record.uuid, patch)
def _populate_docker_config(self, client):
parameter = {}
if self.docker_http_proxy:
parameter['http_proxy'] = self.docker_http_proxy
if self.docker_https_proxy:
parameter['https_proxy'] = self.docker_https_proxy
if self.docker_no_proxy:
parameter['no_proxy'] = self.docker_no_proxy
if self.enable_docker_proxy:
proxy_parameter = {}
if self.docker_http_proxy:
proxy_parameter['http_proxy'] = self.docker_http_proxy
if self.docker_https_proxy:
proxy_parameter['https_proxy'] = self.docker_https_proxy
if self.docker_no_proxy:
proxy_parameter['no_proxy'] = self.docker_no_proxy
if parameter:
client.sysinv.service_parameter.create(
sysinv_constants.SERVICE_TYPE_DOCKER,
sysinv_constants.SERVICE_PARAM_SECTION_DOCKER_PROXY,
None,
None,
parameter
)
if proxy_parameter:
client.sysinv.service_parameter.create(
sysinv_constants.SERVICE_TYPE_DOCKER,
sysinv_constants.SERVICE_PARAM_SECTION_DOCKER_PROXY,
None,
None,
proxy_parameter
)
if not self.docker_use_default_registry:
registry_parameter = {}
if self.docker_k8s_registry:
registry_parameter['k8s'] = \
self.docker_k8s_registry
if self.docker_gcr_registry:
registry_parameter['gcr'] = \
self.docker_gcr_registry
if self.docker_quay_registry:
registry_parameter['quay'] = \
self.docker_quay_registry
if self.docker_docker_registry:
registry_parameter['docker'] = \
self.docker_docker_registry
if not self.is_secure_registry:
registry_parameter['insecure_registry'] = "True"
if registry_parameter:
client.sysinv.service_parameter.create(
sysinv_constants.SERVICE_TYPE_DOCKER,
sysinv_constants.
SERVICE_PARAM_SECTION_DOCKER_REGISTRY,
None,
None,
registry_parameter
)
def populate_initial_config(self):
"""Populate initial system inventory configuration"""
@ -5494,8 +5741,7 @@ class ConfigAssistant():
self._populate_network_config(client)
if self.kubernetes:
self._populate_dns_config(client)
if self.enable_docker_proxy:
self._populate_docker_config(client)
self._populate_docker_config(client)
controller = self._populate_controller_config(client)
# ceph_mon config requires controller host to be created
self._inventory_config_complete_wait(client, controller)

8
controllerconfig/controllerconfig/controllerconfig/tests/files/cgcs_config.kubernetes
View File

@ -79,6 +79,14 @@ DOCKER_HTTP_PROXY=http://proxy.com:123
DOCKER_HTTPS_PROXY=https://proxy.com:123
DOCKER_NO_PROXY=localhost,127.0.0.1,192.168.204.2
[cDOCKER_REGISTRY]
# Docker Registry Configuration
DOCKER_K8S_REGISTRY=my.registry.com:5000
DOCKER_GCR_REGISTRY=my.registry.com
DOCKER_QUAY_REGISTRY=1.2.3.4:5000
DOCKER_DOCKER_REGISTRY=[1:2:3:4:a:b:c:d]:5000
IS_SECURE_REGISTRY=False
[cNETWORK]
# Data Network Configuration
VSWITCH_TYPE=ovs-dpdk

8
controllerconfig/controllerconfig/controllerconfig/tests/files/system_config.kubernetes
View File

@ -55,6 +55,14 @@ DOCKER_HTTP_PROXY=http://proxy.com:123
DOCKER_HTTPS_PROXY=https://proxy.com:123
DOCKER_NO_PROXY=localhost,127.0.0.1,192.168.204.2
[DOCKER_REGISTRY]
# Docker Registry Configuration
DOCKER_K8S_REGISTRY=my.registry.com:5000
DOCKER_GCR_REGISTRY=my.registry.com
DOCKER_QUAY_REGISTRY=1.2.3.4:5000
DOCKER_DOCKER_REGISTRY=[1:2:3:4:a:b:c:d]:5000
IS_SECURE_REGISTRY=False
;[PXEBOOT_NETWORK]
;PXEBOOT_CIDR=192.168.203.0/24

7
controllerconfig/controllerconfig/controllerconfig/tests/test_system_config.py
View File

@ -654,3 +654,10 @@ def test_kubernetes():
cr.create_cgcs_config_file(None, system_config, None, None, None, 0,
validate_only=True)
validate(system_config, DEFAULT_CONFIG, None, False)
# Test absence of optional docker registry configuration
system_config = cr.parse_system_config(systemfile)
system_config.remove_section('DOCKER_REGISTRY')
cr.create_cgcs_config_file(None, system_config, None, None, None, 0,
validate_only=True)
validate(system_config, DEFAULT_CONFIG, None, False)

5
puppet-manifests/src/modules/platform/manifests/docker.pp
View File

@ -3,6 +3,11 @@ class platform::docker::params (
$http_proxy = undef,
$https_proxy = undef,
$no_proxy = undef,
$k8s_registry = undef,
$gcr_registry = undef,
$quay_registry = undef,
$docker_registry = undef,
$insecure_registry = undef,
) { }
class platform::docker::config

18
puppet-manifests/src/modules/platform/manifests/dockerdistribution.pp
View File

@ -7,9 +7,18 @@ class platform::dockerdistribution::config
if $enabled {
include ::platform::network::mgmt::params
include ::platform::docker::params
$docker_registry_ip = $::platform::network::mgmt::params::controller_address
# check insecure registries
if $::platform::docker::params::insecure_registry {
# insecure registry is true means unified registry was set
$insecure_registries = "\"${::platform::docker::params::k8s_registry}\", \"${docker_registry_ip}:9001\""
} else {
$insecure_registries = "\"${docker_registry_ip}:9001\""
}
# currently docker registry is running insecure mode
# when proper authentication is implemented, this would go away
file { '/etc/docker':
@ -52,9 +61,18 @@ class platform::dockerdistribution::compute
$enabled = $::platform::kubernetes::params::enabled
if $enabled {
include ::platform::network::mgmt::params
include ::platform::docker::params
$docker_registry_ip = $::platform::network::mgmt::params::controller_address
# check insecure registries
if $::platform::docker::params::insecure_registry {
# insecure registry is true means unified registry was set
$insecure_registries = "\"${::platform::docker::params::k8s_registry}\", \"${docker_registry_ip}:9001\""
} else {
$insecure_registries = "\"${docker_registry_ip}:9001\""
}
# currently docker registry is running insecure mode
# when proper authentication is implemented, this would go away
file { '/etc/docker':

19
puppet-manifests/src/modules/platform/manifests/helm.pp
View File

@ -7,6 +7,7 @@ class platform::helm
inherits ::platform::helm::repository::params {
include ::platform::kubernetes::params
include ::platform::docker::params
if $::platform::kubernetes::params::enabled {
file {$source_helm_repo_dir:
@ -27,17 +28,29 @@ class platform::helm
if str2bool($::is_initial_config_primary) {
if $::platform::docker::params::gcr_registry {
$gcr_registry = $::platform::docker::params::gcr_registry
} else {
$gcr_registry = 'gcr.io'
}
if $::platform::docker::params::quay_registry {
$quay_registry = $::platform::docker::params::quay_registry
} else {
$quay_registry = 'quay.io'
}
Class['::platform::kubernetes::master']
# TODO(jrichard): Upversion tiller image to v2.11.1 once released.
-> exec { 'load tiller docker image':
command => 'docker image pull gcr.io/kubernetes-helm/tiller:v2.12.1',
command => "docker image pull ${gcr_registry}/kubernetes-helm/tiller:v2.12.1",
logoutput => true,
}
# TODO(tngo): If and when tiller image is upversioned, please ensure armada compatibility as part of the test
-> exec { 'load armada docker image':
command => 'docker image pull quay.io/airshipit/armada:f807c3a1ec727c883c772ffc618f084d960ed5c9',
command => "docker image pull ${quay_registry}/airshipit/armada:f807c3a1ec727c883c772ffc618f084d960ed5c9",
logoutput => true,
}
@ -54,7 +67,7 @@ class platform::helm
# TODO(jrichard): Upversion tiller image to v2.11.1 once released.
-> exec { 'initialize helm':
environment => [ 'KUBECONFIG=/etc/kubernetes/admin.conf', 'HOME=/home/wrsroot' ],
command => 'helm init --skip-refresh --service-account tiller --node-selectors "node-role.kubernetes.io/master"="" --tiller-image=gcr.io/kubernetes-helm/tiller:v2.12.1', # lint:ignore:140chars
command => "helm init --skip-refresh --service-account tiller --node-selectors \"node-role.kubernetes.io/master\"=\"\" --tiller-image=${gcr_registry}/kubernetes-helm/tiller:v2.12.1", # lint:ignore:140chars
logoutput => true,
user => 'wrsroot',
group => 'wrs',

36
puppet-manifests/src/modules/platform/manifests/kubernetes.pp
View File

@ -13,9 +13,27 @@ class platform::kubernetes::params (
) { }
class platform::kubernetes::kubeadm {
include ::platform::docker::params
$iptables_file = "net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1"
if $::platform::docker::params::k8s_registry {
$k8s_registry = $::platform::docker::params::k8s_registry
} else {
$k8s_registry = undef
}
# kubelet use --pod-infra-container-image to indentify the specified image
# TODO: this is not needed after kubernetes upgraded to 1.13
# because the imageRepository setting will be used
if $k8s_registry {
file { '/etc/sysconfig/kubelet':
ensure => file,
content => template('platform/kubelet.conf.erb'),
}
}
# Update iptables config. This is required based on:
# https://kubernetes.io/docs/tasks/tools/install-kubeadm
# This probably belongs somewhere else - initscripts package?
@ -52,6 +70,21 @@ class platform::kubernetes::master::init
inherits ::platform::kubernetes::params {
include ::platform::params
include ::platform::docker::params
# This is used for imageRepository in template kubeadm.yaml.erb
if $::platform::docker::params::k8s_registry {
$k8s_registry = $::platform::docker::params::k8s_registry
} else {
$k8s_registry = undef
}
# This is used for calico image in template calico.yaml.erb
if $::platform::docker::params::quay_registry {
$quay_registry = $::platform::docker::params::quay_registry
} else {
$quay_registry = 'quay.io'
}
if str2bool($::is_initial_config_primary) {
# For initial controller install, configure kubernetes from scratch.
@ -347,10 +380,11 @@ class platform::kubernetes::worker
}
# TODO: remove port 9001 once we have a public docker image registry using standard ports.
# add 5000 as the default port for private registry
class platform::kubernetes::firewall::params (
$transport = 'tcp',
$table = 'nat',
$dports = [80, 443, 9001],
$dports = [80, 443, 9001, 5000],
$chain = 'POSTROUTING',
$jump = 'SNAT',
) {}

6
puppet-manifests/src/modules/platform/templates/calico.yaml.erb
View File

@ -107,7 +107,7 @@ spec:
# as a host-networked pod.
serviceAccountName: calico-node
containers:
- image: quay.io/calico/typha:v3.1.4
- image: <%= @quay_registry %>/calico/typha:v3.1.4
name: calico-typha
ports:
- containerPort: 5473
@ -198,7 +198,7 @@ spec:
# container programs network policy and routes on each
# host.
- name: calico-node
image: quay.io/calico/node:v3.1.4
image: <%= @quay_registry %>/calico/node:v3.1.4
env:
# Use Kubernetes API as the backing datastore.
- name: DATASTORE_TYPE
@ -283,7 +283,7 @@ spec:
# This container installs the Calico CNI binaries
# and CNI network config file on each node.
- name: install-cni
image: quay.io/calico/cni:v3.1.4
image: <%= @quay_registry %>/calico/cni:v3.1.4
command: ["/install-cni.sh"]
env:
# Name of the CNI config file to create.

2
puppet-manifests/src/modules/platform/templates/insecuredockerregistry.conf.erb
View File

@ -1,3 +1,3 @@
{
"insecure-registries" : [ "<%= @docker_registry_ip %>:9001" ]
"insecure-registries" : [ <%= @insecure_registries %> ]
}

3
puppet-manifests/src/modules/platform/templates/kubeadm.yaml.erb
View File

@ -21,6 +21,9 @@ controllerManagerExtraArgs:
node-monitor-period: "2s"
node-monitor-grace-period: "20s"
pod-eviction-timeout: "30s"
<%- if @k8s_registry -%>
imageRepository: "<%= @k8s_registry %>"
<%- end -%>
---
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1

2
puppet-manifests/src/modules/platform/templates/kubelet.conf.erb Normal file
View File

@ -0,0 +1,2 @@
# Overrides config file for kubelet
KUBELET_EXTRA_ARGS=--pod-infra-container-image=<%= @k8s_registry %>/pause:3.1

9
puppet-modules-wrs/puppet-sysinv/src/sysinv/manifests/init.pp
View File

@ -77,6 +77,7 @@ class sysinv (
include sysinv::params
include ::platform::kubernetes::params
include ::platform::docker::params
Package['sysinv'] -> Sysinv_config<||>
Package['sysinv'] -> Sysinv_api_paste_ini<||>
@ -217,7 +218,13 @@ class sysinv (
}
if $::platform::kubernetes::params::enabled == true {
$armada_img_tag = 'quay.io/airshipit/armada:f807c3a1ec727c883c772ffc618f084d960ed5c9'
if $::platform::docker::params::quay_registry {
$quay_registry = $::platform::docker::params::quay_registry
} else {
$quay_registry = 'quay.io'
}
$armada_img_tag = "${quay_registry}/airshipit/armada:f807c3a1ec727c883c772ffc618f084d960ed5c9"
sysinv_config {
'DEFAULT/armada_image_tag': value => $armada_img_tag;
}

6
sysinv/sysinv/sysinv/sysinv/common/constants.py
View File

@ -1103,6 +1103,12 @@ SERVICE_PARAM_SECTION_DOCKER_PROXY = 'proxy'
SERVICE_PARAM_NAME_DOCKER_HTTP_PROXY = 'http_proxy'
SERVICE_PARAM_NAME_DOCKER_HTTPS_PROXY = 'https_proxy'
SERVICE_PARAM_NAME_DOCKER_NO_PROXY = 'no_proxy'
SERVICE_PARAM_SECTION_DOCKER_REGISTRY = 'registry'
SERVICE_PARAM_NAME_DOCKER_K8S_REGISTRY = 'k8s'
SERVICE_PARAM_NAME_DOCKER_GCR_REGISTRY = 'gcr'
SERVICE_PARAM_NAME_DOCKER_QUAY_REGISTRY = 'quay'
SERVICE_PARAM_NAME_DOCKER_DOCKER_REGISTRY = 'docker'
SERVICE_PARAM_NAME_DOCKER_INSECURE_REGISTRY = 'insecure_registry'
# default filesystem size to 25 MB
SERVICE_PARAM_SWIFT_FS_SIZE_MB_DEFAULT = 25

57
sysinv/sysinv/sysinv/sysinv/common/service_parameter.py
View File

@ -660,11 +660,24 @@ def _validate_docker_no_proxy_address(name, value):
values = value.split(',')
for item in values:
# will extend to more cases if CIDR notation is supported
if not cutils.is_valid_domain(item):
if not cutils.is_valid_ip(item):
raise wsme.exc.ClientSideError(_(
"Parameter '%s' includes an invalid address '%s'." %
(name, item)))
if not cutils.is_valid_domain_or_ip(item):
raise wsme.exc.ClientSideError(_(
"Parameter '%s' includes an invalid address '%s'." %
(name, item)))
def _validate_docker_registry_address(name, value):
"""Check if registry address is valid"""
if not cutils.is_valid_domain_or_ip(value):
raise wsme.exc.ClientSideError(_(
"Parameter '%s' must be a valid address." % name))
def _validate_docker_insecure_registry_bool(name, value):
"""Check if insecure registry is a valid bool"""
if not cutils.is_valid_boolstr(value):
raise wsme.exc.ClientSideError(_(
"Parameter '%s' must be a valid bool string." % name))
# LDAP Identity Service Parameters (mandatory)
@ -1516,6 +1529,35 @@ DOCKER_PROXY_PARAMETER_RESOURCE = {
'platform::docker::params::no_proxy',
}
DOCKER_REGISTRY_PARAMETER_OPTIONAL = [
constants.SERVICE_PARAM_NAME_DOCKER_K8S_REGISTRY,
constants.SERVICE_PARAM_NAME_DOCKER_GCR_REGISTRY,
constants.SERVICE_PARAM_NAME_DOCKER_QUAY_REGISTRY,
constants.SERVICE_PARAM_NAME_DOCKER_DOCKER_REGISTRY,
constants.SERVICE_PARAM_NAME_DOCKER_INSECURE_REGISTRY,
]
DOCKER_REGISTRY_PARAMETER_VALIDATOR = {
constants.SERVICE_PARAM_NAME_DOCKER_K8S_REGISTRY: _validate_docker_registry_address,
constants.SERVICE_PARAM_NAME_DOCKER_GCR_REGISTRY: _validate_docker_registry_address,
constants.SERVICE_PARAM_NAME_DOCKER_QUAY_REGISTRY: _validate_docker_registry_address,
constants.SERVICE_PARAM_NAME_DOCKER_DOCKER_REGISTRY: _validate_docker_registry_address,
constants.SERVICE_PARAM_NAME_DOCKER_INSECURE_REGISTRY: _validate_docker_insecure_registry_bool,
}
DOCKER_REGISTRY_PARAMETER_RESOURCE = {
constants.SERVICE_PARAM_NAME_DOCKER_K8S_REGISTRY:
'platform::docker::params::k8s_registry',
constants.SERVICE_PARAM_NAME_DOCKER_GCR_REGISTRY:
'platform::docker::params::gcr_registry',
constants.SERVICE_PARAM_NAME_DOCKER_QUAY_REGISTRY:
'platform::docker::params::quay_registry',
constants.SERVICE_PARAM_NAME_DOCKER_DOCKER_REGISTRY:
'platform::docker::params::docker_registry',
constants.SERVICE_PARAM_NAME_DOCKER_INSECURE_REGISTRY:
'platform::docker::params::insecure_registry',
}
HTTPD_PORT_PARAMETER_OPTIONAL = [
constants.SERVICE_PARAM_HTTP_PORT_HTTP,
constants.SERVICE_PARAM_HTTP_PORT_HTTPS,
@ -1715,6 +1757,11 @@ SERVICE_PARAMETER_SCHEMA = {
SERVICE_PARAM_VALIDATOR: DOCKER_PROXY_PARAMETER_VALIDATOR,
SERVICE_PARAM_RESOURCE: DOCKER_PROXY_PARAMETER_RESOURCE,
},
constants.SERVICE_PARAM_SECTION_DOCKER_REGISTRY: {
SERVICE_PARAM_OPTIONAL: DOCKER_REGISTRY_PARAMETER_OPTIONAL,
SERVICE_PARAM_VALIDATOR: DOCKER_REGISTRY_PARAMETER_VALIDATOR,
SERVICE_PARAM_RESOURCE: DOCKER_REGISTRY_PARAMETER_RESOURCE,
},
},
constants.SERVICE_TYPE_HTTP: {
constants.SERVICE_PARAM_SECTION_HTTP_CONFIG: {

27
sysinv/sysinv/sysinv/sysinv/common/utils.py
View File

@ -1832,6 +1832,33 @@ def is_valid_domain(url_str):
return False
def is_valid_domain_or_ip(url_str):
if url_str:
if is_valid_domain(url_str):
return True
ip_with_port = url_str.split(':')
if len(ip_with_port) <= 2:
# check ipv4 or ipv4 with port
return is_valid_ipv4(ip_with_port[0])
else:
# check ipv6 with port
if '[' in url_str:
try:
bkt_idx = url_str.index(']')
if bkt_idx + 1 == len(url_str):
# brackets without port
return False
else:
return is_valid_ipv6(url_str[1:bkt_idx])
except Exception:
return False
else:
# check ipv6 without port
return is_valid_ipv6(url_str)
else:
return False
def verify_checksum(path):
""" Find and validate the checksum file in a given directory. """
rc = True