starlingx
/
config
Code Issues Proposed changes

Support to pull container images from authenticated registries 

If alternative registries configured during bootstrap,
the application requires container images to be pulled
from user specified registries. However, the user
specified registries may be authenticated. This commit
supports to pull from authenticated registries.

At ansible bootstrap time, the credentials for accessing
the registries will be stored in barbican secrets and the
created barbican secrets refs will be stored in service
parameters. This will be done in task 35974.

During application apply, registries auth info in service
parameters are retrieved/parsed and used for images pull
from authenticated registries.

This commit also updates the data model that stores the
user specified registries in service parameter.

Tests:
 - application apply stx-openstack/custom app from
   both authenticated and non-authenticated registries

Story: 2006274
Task: 35998
Change-Id: I395623f0ba3a7a1c60936df8aca48a80447ca52d
Signed-off-by: Angie Wang <angie.wang@windriver.com>
This commit is contained in:
Angie Wang 2019-07-26 00:32:19 -04:00
parent 8d39303a5b
commit eec9b6c1fa
4 changed files with 250 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
sysinv/sysinv/sysinv/sysinv/common/constants.py
View File

@ -963,14 +963,46 @@ SERVICE_PARAM_SECTION_DOCKER_PROXY = 'proxy'
SERVICE_PARAM_NAME_DOCKER_HTTP_PROXY = 'http_proxy'
SERVICE_PARAM_NAME_DOCKER_HTTPS_PROXY = 'https_proxy'
SERVICE_PARAM_NAME_DOCKER_NO_PROXY = 'no_proxy'
SERVICE_PARAM_SECTION_DOCKER_REGISTRY = 'registry'
SERVICE_PARAM_NAME_DOCKER_K8S_REGISTRY = 'k8s'
SERVICE_PARAM_NAME_DOCKER_GCR_REGISTRY = 'gcr'
SERVICE_PARAM_NAME_DOCKER_QUAY_REGISTRY = 'quay'
SERVICE_PARAM_NAME_DOCKER_DOCKER_REGISTRY = 'docker'
SERVICE_PARAM_NAME_DOCKER_REGISTRIES = 'registries'
SERVICE_PARAM_NAME_DOCKER_INSECURE_REGISTRY = 'insecure_registry'
SERVICE_PARAM_SECTION_DOCKER_DOCKER_REGISTRY = 'docker-registry'
SERVICE_PARAM_SECTION_DOCKER_GCR_REGISTRY = 'gcr-registry'
SERVICE_PARAM_SECTION_DOCKER_K8S_REGISTRY = 'k8s-registry'
SERVICE_PARAM_SECTION_DOCKER_QUAY_REGISTRY = 'quay-registry'
SERVICE_PARAM_NAME_DOCKER_URL = 'url'
SERVICE_PARAM_NAME_DOCKER_AUTH_SECRET = 'auth-secret'
# default docker registries
DEFAULT_DOCKER_K8S_REGISTRY = 'k8s.gcr.io'
DEFAULT_DOCKER_GCR_REGISTRY = 'gcr.io'
DEFAULT_DOCKER_QUAY_REGISTRY = 'quay.io'
DEFAULT_DOCKER_DOCKER_REGISTRY = 'docker.io'
DEFAULT_REGISTRIES_INFO = {
SERVICE_PARAM_SECTION_DOCKER_K8S_REGISTRY: {
'registry_default': DEFAULT_DOCKER_K8S_REGISTRY,
'registry_replaced': None,
'registry_auth': None
},
SERVICE_PARAM_SECTION_DOCKER_GCR_REGISTRY: {
'registry_default': DEFAULT_DOCKER_GCR_REGISTRY,
'registry_replaced': None,
'registry_auth': None
},
SERVICE_PARAM_SECTION_DOCKER_QUAY_REGISTRY: {
'registry_default': DEFAULT_DOCKER_QUAY_REGISTRY,
'registry_replaced': None,
'registry_auth': None
},
SERVICE_PARAM_SECTION_DOCKER_DOCKER_REGISTRY: {
'registry_default': DEFAULT_DOCKER_DOCKER_REGISTRY,
'registry_replaced': None,
'registry_auth': None
}
}
# kubernetes parameters
SERVICE_PARAM_SECTION_KUBERNETES_CERTIFICATES = 'certificates'
SERVICE_PARAM_NAME_KUBERNETES_API_SAN_LIST = 'apiserver_certsan'

69
sysinv/sysinv/sysinv/sysinv/common/service_parameter.py
View File

@ -451,38 +451,47 @@ DOCKER_PROXY_PARAMETER_RESOURCE = {
}
DOCKER_REGISTRY_PARAMETER_OPTIONAL = [
constants.SERVICE_PARAM_NAME_DOCKER_K8S_REGISTRY,
constants.SERVICE_PARAM_NAME_DOCKER_GCR_REGISTRY,
constants.SERVICE_PARAM_NAME_DOCKER_QUAY_REGISTRY,
constants.SERVICE_PARAM_NAME_DOCKER_DOCKER_REGISTRY,
constants.SERVICE_PARAM_NAME_DOCKER_REGISTRIES,
constants.SERVICE_PARAM_NAME_DOCKER_INSECURE_REGISTRY,
]
DOCKER_REGISTRY_PARAMETER_VALIDATOR = {
constants.SERVICE_PARAM_NAME_DOCKER_K8S_REGISTRY: _validate_docker_registry_address,
constants.SERVICE_PARAM_NAME_DOCKER_GCR_REGISTRY: _validate_docker_registry_address,
constants.SERVICE_PARAM_NAME_DOCKER_QUAY_REGISTRY: _validate_docker_registry_address,
constants.SERVICE_PARAM_NAME_DOCKER_DOCKER_REGISTRY: _validate_docker_registry_address,
constants.SERVICE_PARAM_NAME_DOCKER_REGISTRIES: _validate_docker_registry_address,
constants.SERVICE_PARAM_NAME_DOCKER_INSECURE_REGISTRY: _validate_docker_insecure_registry_bool,
}
DOCKER_REGISTRY_PARAMETER_RESOURCE = {
constants.SERVICE_PARAM_NAME_DOCKER_K8S_REGISTRY:
'platform::docker::params::k8s_registry',
constants.SERVICE_PARAM_NAME_DOCKER_GCR_REGISTRY:
'platform::docker::params::gcr_registry',
constants.SERVICE_PARAM_NAME_DOCKER_QUAY_REGISTRY:
'platform::docker::params::quay_registry',
constants.SERVICE_PARAM_NAME_DOCKER_DOCKER_REGISTRY:
'platform::docker::params::docker_registry',
constants.SERVICE_PARAM_NAME_DOCKER_REGISTRIES:
'platform::docker::params::docker_registries',
constants.SERVICE_PARAM_NAME_DOCKER_INSECURE_REGISTRY:
'platform::docker::params::insecure_registry',
}
DOCKER_REGISTRIES_PARAMETER_OPTIONAL = [
constants.SERVICE_PARAM_NAME_DOCKER_URL,
constants.SERVICE_PARAM_NAME_DOCKER_AUTH_SECRET,
]
DOCKER_REGISTRIES_PARAMETER_VALIDATOR = {
constants.SERVICE_PARAM_NAME_DOCKER_URL: _validate_docker_registry_address,
}
DOCKER_DOCKER_REGISTRY_PARAMETER_RESOURCE = {
constants.SERVICE_PARAM_NAME_DOCKER_URL:
'platform::docker::params::docker_registry',
}
DOCKER_GCR_REGISTRY_PARAMETER_RESOURCE = {
constants.SERVICE_PARAM_NAME_DOCKER_URL:
'platform::docker::params::gcr_registry'
}
DOCKER_K8S_REGISTRY_PARAMETER_RESOURCE = {
constants.SERVICE_PARAM_NAME_DOCKER_URL:
'platform::docker::params::k8s_registry'
}
DOCKER_QUAY_REGISTRY_PARAMETER_RESOURCE = {
constants.SERVICE_PARAM_NAME_DOCKER_URL:
'platform::docker::params::quay_registry'
}
KUBERNETES_CERTIFICATES_PARAMETER_OPTIONAL = [
constants.SERVICE_PARAM_NAME_KUBERNETES_API_SAN_LIST,
]
@ -583,6 +592,26 @@ SERVICE_PARAMETER_SCHEMA = {
SERVICE_PARAM_VALIDATOR: DOCKER_REGISTRY_PARAMETER_VALIDATOR,
SERVICE_PARAM_RESOURCE: DOCKER_REGISTRY_PARAMETER_RESOURCE,
},
constants.SERVICE_PARAM_SECTION_DOCKER_DOCKER_REGISTRY: {
SERVICE_PARAM_OPTIONAL: DOCKER_REGISTRIES_PARAMETER_OPTIONAL,
SERVICE_PARAM_VALIDATOR: DOCKER_REGISTRIES_PARAMETER_VALIDATOR,
SERVICE_PARAM_RESOURCE: DOCKER_DOCKER_REGISTRY_PARAMETER_RESOURCE
},
constants.SERVICE_PARAM_SECTION_DOCKER_GCR_REGISTRY: {
SERVICE_PARAM_OPTIONAL: DOCKER_REGISTRIES_PARAMETER_OPTIONAL,
SERVICE_PARAM_VALIDATOR: DOCKER_REGISTRIES_PARAMETER_VALIDATOR,
SERVICE_PARAM_RESOURCE: DOCKER_GCR_REGISTRY_PARAMETER_RESOURCE
},
constants.SERVICE_PARAM_SECTION_DOCKER_K8S_REGISTRY: {
SERVICE_PARAM_OPTIONAL: DOCKER_REGISTRIES_PARAMETER_OPTIONAL,
SERVICE_PARAM_VALIDATOR: DOCKER_REGISTRIES_PARAMETER_VALIDATOR,
SERVICE_PARAM_RESOURCE: DOCKER_K8S_REGISTRY_PARAMETER_RESOURCE
},
constants.SERVICE_PARAM_SECTION_DOCKER_QUAY_REGISTRY: {
SERVICE_PARAM_OPTIONAL: DOCKER_REGISTRIES_PARAMETER_OPTIONAL,
SERVICE_PARAM_VALIDATOR: DOCKER_REGISTRIES_PARAMETER_VALIDATOR,
SERVICE_PARAM_RESOURCE: DOCKER_QUAY_REGISTRY_PARAMETER_RESOURCE
}
},
constants.SERVICE_TYPE_KUBERNETES: {
constants.SERVICE_PARAM_SECTION_KUBERNETES_CERTIFICATES: {

209
sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py
View File

@ -10,6 +10,7 @@
""" System Inventory Kubernetes Application Operator."""
import base64
import copy
import docker
import grp
import keyring
@ -38,6 +39,7 @@ from sysinv.common import exception
from sysinv.common import kubernetes
from sysinv.common import utils as cutils
from sysinv.common.storage_backend_conf import K8RbdProvisioner
from sysinv.conductor import openstack
from sysinv.helm import common
from sysinv.helm import helm
from sysinv.helm import utils as helm_utils
@ -729,11 +731,26 @@ class AppOperator(object):
start = time.time()
pool = greenpool.GreenPool(size=threads)
f = lambda x: self._docker.download_an_image(app.name, x)
try:
local_registry_auth = get_local_docker_registry_auth()
with self._lock:
self._docker._retrieve_specified_registries()
except Exception as e:
raise exception.KubeAppApplyFailure(
name=app.name,
version=app.version,
reason=str(e))
f = lambda x: self._docker.download_an_image(
app.name, local_registry_auth, x)
for tag, rc in pool.imap(f, images_to_download):
if not rc:
failed_downloads.append(tag)
with self._lock:
self._docker._reset_registries_info()
elapsed = time.time() - start
failed_count = len(failed_downloads)
if failed_count > 0:
if not AppOperator.is_app_aborted(app.name):
@ -2263,37 +2280,117 @@ class DockerHelper(object):
def __init__(self, dbapi):
self._dbapi = dbapi
self._lock = threading.Lock()
self.k8s_registry = None
self.gcr_registry = None
self.quay_registry = None
self.docker_registry = None
self.registries_info = \
copy.deepcopy(constants.DEFAULT_REGISTRIES_INFO)
def _parse_barbican_secret(self, secret_ref):
"""Get the registry credentials from the
barbican secret payload
The format of the credentials stored in
barbican secret:
username:xxx password:xxx
:param secret_ref: barbican secret ref/uuid
:return: dict of registry credentials
"""
operator = openstack.OpenStackOperator(self._dbapi)
payload = operator.get_barbican_secret_payload(secret_ref)
if not payload:
raise exception.SysinvException(_(
"Unable to get the payload of Barbican secret "
"%s" % secret_ref))
def _get_registry_parameters(self):
try:
registry = self._dbapi.service_parameter_get_all(
service=constants.SERVICE_TYPE_DOCKER,
section=constants.SERVICE_PARAM_SECTION_DOCKER_REGISTRY,
)
return registry
except Exception:
return None
username, password = payload.split()
username = username.split('username:')[1]
password = password.split('password:')[1]
return dict(username=username, password=password)
except Exception as e:
LOG.error("Unable to parse the secret payload, "
"unknown format of the registry secret: %s" % e)
raise exception.SysinvException(_(
"Unable to parse the secret payload"))
def _retrieve_specified_registries(self):
registry_params = self._get_registry_parameters()
if registry_params:
for param in registry_params:
if param.name == \
constants.SERVICE_PARAM_NAME_DOCKER_K8S_REGISTRY:
self.k8s_registry = str(param.value)
if param.name == \
constants.SERVICE_PARAM_NAME_DOCKER_GCR_REGISTRY:
self.gcr_registry = str(param.value)
if param.name == \
constants.SERVICE_PARAM_NAME_DOCKER_QUAY_REGISTRY:
self.quay_registry = str(param.value)
if param.name == \
constants.SERVICE_PARAM_NAME_DOCKER_DOCKER_REGISTRY:
self.docker_registry = str(param.value)
registries = self._dbapi.service_parameter_get_all(
service=constants.SERVICE_TYPE_DOCKER,
name=constants.SERVICE_PARAM_NAME_DOCKER_URL)
if not registries:
# return directly if no user specified registries
return
registries_auth_db = self._dbapi.service_parameter_get_all(
service=constants.SERVICE_TYPE_DOCKER,
name=constants.SERVICE_PARAM_NAME_DOCKER_AUTH_SECRET)
registries_auth = {r.section: r.value for r in registries_auth_db}
for r in registries:
try:
self.registries_info[r.section]['registry_replaced'] = str(r.value)
if r.section in registries_auth:
secret_ref = str(registries_auth[r.section])
if secret_ref != 'None':
# If user specified registry requires the
# authentication, get the registry auth
# from barbican secret
auth = self._parse_barbican_secret(secret_ref)
self.registries_info[r.section]['registry_auth'] = auth
except exception.SysinvException:
raise exception.SysinvException(_(
"Unable to get the credentials to access "
"registry %s" % str(r.value)))
except KeyError:
# Unexpected
pass
def _reset_registries_info(self):
# Set cached registries information
# back to default
if self.registries_info != \
constants.DEFAULT_REGISTRIES_INFO:
self.registries_info = copy.deepcopy(
constants.DEFAULT_REGISTRIES_INFO)
def _get_img_tag_with_registry(self, pub_img_tag):
"""Regenerate public image tag with user specified registries
"""
if self.registries_info == constants.DEFAULT_REGISTRIES_INFO:
# return if no user specified registries
return pub_img_tag, None
# An example of passed public image tag:
# docker.io/starlingx/stx-keystone:latest
# extracted registry_name = docker.io
# extracted img_name = starlingx/stx-keystone:latest
registry_name = pub_img_tag[0:1 + pub_img_tag.find('/')].replace('/', '')
img_name = pub_img_tag[1 + pub_img_tag.find('/'):]
for registry_info in self.registries_info.values():
if registry_name == registry_info['registry_default']:
registry = registry_info['registry_replaced']
registry_auth = registry_info['registry_auth']
if registry:
return registry + '/' + img_name, registry_auth
return pub_img_tag, registry_auth
# If extracted registry_name is none of k8s.gcr.io, gcr.io,
# quay.io and docker.io or no registry_name specified in image
# tag, use user specified docker registry as default
registry = self.registries_info[
constants.SERVICE_PARAM_NAME_DOCKER_DOCKER_REGISTRY]['registry_replaced']
registry_auth = self.registries_info[
constants.SERVICE_PARAM_NAME_DOCKER_DOCKER_REGISTRY]['registry_auth']
if registry:
LOG.info("Registry %s not recognized or docker.io repository "
"detected. Pulling from public/private registry"
% registry_name)
return registry + '/' + pub_img_tag, registry_auth
return pub_img_tag, registry_auth
def _start_armada_service(self, client):
try:
@ -2492,50 +2589,9 @@ class DockerHelper(object):
# Failed to get a docker client
LOG.error("Failed to stop Armada service : %s " % e)
def _get_img_tag_with_registry(self, pub_img_tag):
registry_name = pub_img_tag[0:1 + pub_img_tag.find('/')]
img_name = pub_img_tag[1 + pub_img_tag.find('/'):]
if registry_name:
if 'k8s.gcr.io' in registry_name:
registry = self.k8s_registry
elif 'gcr.io' in registry_name:
registry = self.gcr_registry
elif 'quay.io' in registry_name:
registry = self.quay_registry
elif 'docker.io' in registry_name:
registry = self.docker_registry
else:
# try docker.io registry as default
# if other registries newly added
# or docker.io repository detected
LOG.info("Registry %s not recognized or docker.io repository "
"detected. Pulling from public/private registry"
% registry_name)
registry = self.docker_registry
if registry:
return str(registry) + '/' + pub_img_tag
else:
return pub_img_tag
# replace registry
if registry:
return str(registry) + '/' + img_name
else:
return pub_img_tag
else:
# docker.io registry as default
# if no registries specified in img tag
registry = self.docker_registry
if registry:
return str(registry) + '/' + pub_img_tag
else:
return pub_img_tag
def download_an_image(self, app_name, img_tag):
def download_an_image(self, app_name, local_registry_auth, img_tag):
rc = True
# retrieve user specified registries first
self._retrieve_specified_registries()
start = time.time()
if img_tag.startswith(constants.DOCKER_REGISTRY_HOST):
@ -2545,19 +2601,18 @@ class DockerHelper(object):
return img_tag, False
LOG.info("Image %s download started from local registry" % img_tag)
local_registry_auth = get_local_docker_registry_auth()
client = docker.APIClient(timeout=INSTALLATION_TIMEOUT)
client.pull(img_tag, auth_config=local_registry_auth)
except docker.errors.NotFound:
try:
# Pull the image from the public registry
# Pull the image from the public/private registry
LOG.info("Image %s is not available in local registry, "
"download started from public/private registry"
% img_tag)
pub_img_tag = img_tag.replace(
constants.DOCKER_REGISTRY_SERVER + "/", "")
target_img_tag = self._get_img_tag_with_registry(pub_img_tag)
client.pull(target_img_tag)
target_img_tag, registry_auth = self._get_img_tag_with_registry(pub_img_tag)
client.pull(target_img_tag, auth_config=registry_auth)
except Exception as e:
rc = False
LOG.error("Image %s download failed from public/private"
@ -2579,12 +2634,12 @@ class DockerHelper(object):
try:
LOG.info("Image %s download started from public/private registry" % img_tag)
client = docker.APIClient(timeout=INSTALLATION_TIMEOUT)
target_img_tag = self._get_img_tag_with_registry(img_tag)
client.pull(target_img_tag)
target_img_tag, registry_auth = self._get_img_tag_with_registry(img_tag)
client.pull(target_img_tag, auth_config=registry_auth)
client.tag(target_img_tag, img_tag)
except Exception as e:
rc = False
LOG.error("Image %s download failed from public registry: %s" % (img_tag, e))
LOG.error("Image %s download failed from public/private registry: %s" % (img_tag, e))
elapsed_time = time.time() - start
if rc:

37
sysinv/sysinv/sysinv/sysinv/conductor/openstack.py
View File

@ -129,6 +129,7 @@ class OpenStackOperator(object):
def __init__(self, dbapi):
self.dbapi = dbapi
self.barbican_client = None
self.openstack_barbican_client = None
self.cinder_client = None
self.keystone_client = None
self.keystone_session = None
@ -757,13 +758,39 @@ class OpenStackOperator(object):
#################
# Barbican
#################
def _get_barbicanclient(self):
if not self.barbican_client:
self.barbican_client = barbican_client_v1.Client(
session=self._get_keystone_session(PLATFORM_CONFIG),
def _get_cached_barbican_client(self, service_config):
if service_config == PLATFORM_CONFIG:
return self.barbican_client
else:
return self.openstack_barbican_client
def _set_cached_barbican_client(self, service_config, client):
if service_config == PLATFORM_CONFIG:
self.barbican_client = client
else:
self.openstack_barbican_client = client
def _get_barbicanclient(self, service_config=PLATFORM_CONFIG):
client = self._get_cached_barbican_client(service_config)
if not client:
client = barbican_client_v1.Client(
session=self._get_keystone_session(service_config),
interface='internalURL',
region_name=cfg.CONF[OPENSTACK_CONFIG].barbican_region_name)
return self.barbican_client
self._set_cached_barbican_client(service_config, client)
return client
def get_barbican_secret_payload(self, secret_ref):
try:
client = self._get_barbicanclient(
service_config=OPENSTACK_CONFIG)
secret = client.secrets.get(secret_ref)
payload = secret.payload
return payload
except Exception:
LOG.error("Unable to find Barbican secret %s or secret does not "
"have payload", secret_ref)
return None
def get_barbican_secret_by_name(self, context, name):
try: