starlingx
/
config
Code Issues Proposed changes

Support authenticated registries 

This commit updates to support pulling application images from
aws ecr at application-apply time.

In commit https://review.opendev.org/686057, all system images
are pushed to local registry at Ansible bootstrap time. This
commit also updates armada container to use armada image from
local registry.

Allow to add "elastic-registry" in service parameter to be able
to deploy stx-monitor from private registries.

Tests conducted:
 - deploy platform-app-integ from public/aws ecr registry
 - deploy stx-monitor from public/aws ecr registry
 - tiller image upgrade/downgrade test

Change-Id: I7a597a2bfdacf969462f166e8f416e570f243076
Story: 2006274
Task: 36874
Depends-On: https://review.opendev.org/686057
Signed-off-by: Angie Wang <angie.wang@windriver.com>
This commit is contained in:
Angie Wang 2019-09-26 14:38:19 -04:00
parent 8e51a4bb54
commit 75b9156ea5
10 changed files with 142 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
sysinv/sysinv/centos/sysinv.spec
View File

@ -14,6 +14,7 @@ BuildRequires: python2-pip
BuildRequires: python2-wheel
BuildRequires: systemd
Requires: pyparted
Requires: python-boto3
Requires: python-docker
Requires: python-eventlet
Requires: python-ipaddr

1
sysinv/sysinv/debian/control
View File

@ -15,6 +15,7 @@ Package: sysinv
Architecture: all
Depends: ${misc:Depends},
${python:Depends},
python-boto3,
python-django,
python-docker,
python-parted,

1
sysinv/sysinv/opensuse/sysinv.spec
View File

@ -11,6 +11,7 @@ BuildRequires: python-setuptools
BuildRequires: python-pbr
BuildRequires: python2-pip
BuildRequires: systemd
Requires: python-boto3
Requires: python-parted
Requires: python2-docker
Requires: python2-eventlet

1
sysinv/sysinv/sysinv/requirements.txt
View File

@ -3,6 +3,7 @@ SQLAlchemy
amqplib>=0.6.1
anyjson>=0.3.3
argparse
boto3
cryptography!=2.0 # BSD/Apache-2.0
eventlet==0.20.0
greenlet>=0.3.2 # MIT

14
sysinv/sysinv/sysinv/sysinv/common/constants.py
View File

@ -976,14 +976,21 @@ SERVICE_PARAM_SECTION_DOCKER_DOCKER_REGISTRY = 'docker-registry'
SERVICE_PARAM_SECTION_DOCKER_GCR_REGISTRY = 'gcr-registry'
SERVICE_PARAM_SECTION_DOCKER_K8S_REGISTRY = 'k8s-registry'
SERVICE_PARAM_SECTION_DOCKER_QUAY_REGISTRY = 'quay-registry'
SERVICE_PARAM_SECTION_DOCKER_ELASTIC_REGISTRY = 'elastic-registry'
SERVICE_PARAM_NAME_DOCKER_URL = 'url'
SERVICE_PARAM_NAME_DOCKER_AUTH_SECRET = 'auth-secret'
SERVICE_PARAM_NAME_DOCKER_TYPE = 'type'
SERVICE_PARAM_NAME_DOCKER_ADDITIONAL_OVERRIDES = 'additional-overrides'
DOCKER_REGISTRY_TYPE_AWS_ECR = 'aws-ecr'
DOCKER_REGISTRY_TYPE_DOCKER = 'docker'
# default docker registries
DEFAULT_DOCKER_K8S_REGISTRY = 'k8s.gcr.io'
DEFAULT_DOCKER_GCR_REGISTRY = 'gcr.io'
DEFAULT_DOCKER_QUAY_REGISTRY = 'quay.io'
DEFAULT_DOCKER_DOCKER_REGISTRY = 'docker.io'
DEFAULT_DOCKER_ELASTIC_REGISTRY = 'docker.elastic.co'
DEFAULT_REGISTRIES_INFO = {
SERVICE_PARAM_SECTION_DOCKER_K8S_REGISTRY: {
@ -1005,6 +1012,11 @@ DEFAULT_REGISTRIES_INFO = {
'registry_default': DEFAULT_DOCKER_DOCKER_REGISTRY,
'registry_replaced': None,
'registry_auth': None
},
SERVICE_PARAM_SECTION_DOCKER_ELASTIC_REGISTRY: {
'registry_default': DEFAULT_DOCKER_ELASTIC_REGISTRY,
'registry_replaced': None,
'registry_auth': None
}
}
@ -1225,6 +1237,8 @@ SSL_PEM_FILE = os.path.join(SSL_CERT_DIR, SSL_CERT_FILE)
SSL_PEM_SS_FILE = os.path.join(SSL_CERT_DIR, SSL_CERT_SS_FILE)
SSL_PEM_FILE_SHARED = os.path.join(tsc.CONFIG_PATH, SSL_CERT_FILE)
DOCKER_REGISTRY_USER = 'admin'
DOCKER_REGISTRY_SERVICE = 'CGCS'
DOCKER_REGISTRY_HOST = 'registry.local'
DOCKER_REGISTRY_PORT = '9001'
DOCKER_REGISTRY_SERVER = '%s:%s' % (DOCKER_REGISTRY_HOST, DOCKER_REGISTRY_PORT)

20
sysinv/sysinv/sysinv/sysinv/common/service_parameter.py
View File

@ -306,6 +306,16 @@ def _validate_docker_registry_auth_secret(name, value):
"Parameter '%s' must be a valid UUID." % name))
def _validate_docker_registry_type(name, value):
"""Check if registry type is supported or not"""
if value not in [constants.DOCKER_REGISTRY_TYPE_DOCKER,
constants.DOCKER_REGISTRY_TYPE_AWS_ECR]:
raise wsme.exc.ClientSideError(_(
"%s is not supported. Parameter %s must be one of %s or %s") %
(value, constants.DOCKER_REGISTRY_TYPE_DOCKER,
constants.DOCKER_REGISTRY_TYPE_AWS_ECR))
def _validate_docker_insecure_registry_bool(name, value):
"""Check if insecure registry is a valid bool"""
if not cutils.is_valid_boolstr(value):
@ -481,11 +491,15 @@ DOCKER_REGISTRY_PARAMETER_RESOURCE = {
DOCKER_REGISTRIES_PARAMETER_OPTIONAL = [
constants.SERVICE_PARAM_NAME_DOCKER_URL,
constants.SERVICE_PARAM_NAME_DOCKER_AUTH_SECRET,
constants.SERVICE_PARAM_NAME_DOCKER_TYPE,
constants.SERVICE_PARAM_NAME_DOCKER_ADDITIONAL_OVERRIDES
]
DOCKER_REGISTRIES_PARAMETER_VALIDATOR = {
constants.SERVICE_PARAM_NAME_DOCKER_URL: _validate_docker_registry_address,
constants.SERVICE_PARAM_NAME_DOCKER_AUTH_SECRET: _validate_docker_registry_auth_secret
constants.SERVICE_PARAM_NAME_DOCKER_AUTH_SECRET: _validate_docker_registry_auth_secret,
constants.SERVICE_PARAM_NAME_DOCKER_TYPE: _validate_docker_registry_type,
constants.SERVICE_PARAM_NAME_DOCKER_ADDITIONAL_OVERRIDES: _validate_docker_registry_address
}
DOCKER_DOCKER_REGISTRY_PARAMETER_RESOURCE = {
@ -635,6 +649,10 @@ SERVICE_PARAMETER_SCHEMA = {
SERVICE_PARAM_OPTIONAL: DOCKER_REGISTRIES_PARAMETER_OPTIONAL,
SERVICE_PARAM_VALIDATOR: DOCKER_REGISTRIES_PARAMETER_VALIDATOR,
SERVICE_PARAM_RESOURCE: DOCKER_QUAY_REGISTRY_PARAMETER_RESOURCE
},
constants.SERVICE_PARAM_SECTION_DOCKER_ELASTIC_REGISTRY: {
SERVICE_PARAM_OPTIONAL: DOCKER_REGISTRIES_PARAMETER_OPTIONAL,
SERVICE_PARAM_VALIDATOR: DOCKER_REGISTRIES_PARAMETER_VALIDATOR
}
},
constants.SERVICE_TYPE_KUBERNETES: {

38
sysinv/sysinv/sysinv/sysinv/common/utils.py
View File

@ -24,6 +24,7 @@
"""Utilities and helper functions."""
import boto3
import collections
import contextlib
import datetime
@ -35,6 +36,7 @@ import grp
import hashlib
import itertools as it
import json
import keyring
import math
import os
import pwd
@ -2116,3 +2118,39 @@ def app_reapply_pending_fault_entity(app_name):
return "%s=%s" % (
fm_constants.FM_ENTITY_TYPE_APPLICATION,
app_name)
def get_local_docker_registry_auth():
registry_password = keyring.get_password(
constants.DOCKER_REGISTRY_SERVICE, constants.DOCKER_REGISTRY_USER)
if not registry_password:
raise exception.DockerRegistryCredentialNotFound(
name=constants.DOCKER_REGISTRY_USER)
return dict(username=constants.DOCKER_REGISTRY_USER,
password=registry_password)
def get_aws_ecr_registry_credentials(registry, username, password):
region = re.compile("[0-9]*.dkr.ecr.(.*).amazonaws.com.*").match(registry)
if region:
ecr_region = region.groups()[0]
else:
ecr_region = 'us-west-2'
try:
client = boto3.client(
'ecr',
region_name=ecr_region,
aws_access_key_id=username,
aws_secret_access_key=password)
response = client.get_authorization_token()
token = response['authorizationData'][0]['authorizationToken']
username, password = token.decode('base64').split(':')
except Exception as e:
raise exception.SysinvException(_(
"Failed to get AWS ECR credentials: %s" % e))
return dict(username=username, password=password)

115
sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py
View File

@ -14,7 +14,6 @@ import copy
import docker
import grp
import functools
import keyring
import os
import pwd
import re
@ -64,8 +63,6 @@ MAX_DOWNLOAD_ATTEMPTS = 3
DOWNLOAD_WAIT_BEFORE_RETRY = 30
TARFILE_DOWNLOAD_CONNECTION_TIMEOUT = 60
TARFILE_TRANSFER_CHUNK_SIZE = 1024 * 512
DOCKER_REGISTRY_USER = 'admin'
DOCKER_REGISTRY_SERVICE = 'CGCS'
DOCKER_REGISTRY_SECRET = 'default-registry-key'
CHARTS_PENDING_INSTALL_ITERATIONS = 60
@ -117,17 +114,6 @@ def get_app_install_root_path_ownership():
return (uid, gid)
def get_local_docker_registry_auth():
registry_password = keyring.get_password(
DOCKER_REGISTRY_SERVICE, DOCKER_REGISTRY_USER)
if not registry_password:
raise exception.DockerRegistryCredentialNotFound(
name=DOCKER_REGISTRY_USER)
return dict(username=DOCKER_REGISTRY_USER,
password=registry_password)
Chart = namedtuple('Chart', 'metadata_name name namespace location release labels sequenced')
@ -765,7 +751,7 @@ class AppOperator(object):
start = time.time()
try:
local_registry_auth = get_local_docker_registry_auth()
local_registry_auth = cutils.get_local_docker_registry_auth()
with self._lock:
self._docker._retrieve_specified_registries()
except Exception as e:
@ -1064,7 +1050,7 @@ class AppOperator(object):
continue
try:
local_registry_auth = get_local_docker_registry_auth()
local_registry_auth = cutils.get_local_docker_registry_auth()
auth = '{0}:{1}'.format(local_registry_auth['username'],
local_registry_auth['password'])
@ -2460,34 +2446,51 @@ class DockerHelper(object):
"Unable to parse the secret payload"))
def _retrieve_specified_registries(self):
registries = self._dbapi.service_parameter_get_all(
service=constants.SERVICE_TYPE_DOCKER,
name=constants.SERVICE_PARAM_NAME_DOCKER_URL)
registries_url = {}
registries_type = {}
registries_auth = {}
registries_overrides = {}
if not registries:
registries = self._dbapi.service_parameter_get_all(
service=constants.SERVICE_TYPE_DOCKER)
for r in registries:
if r.name == constants.SERVICE_PARAM_NAME_DOCKER_URL:
registries_url.update({r.section: str(r.value)})
elif r.name == constants.SERVICE_PARAM_NAME_DOCKER_TYPE:
registries_type.update({r.section: str(r.value)})
elif r.name == constants.SERVICE_PARAM_NAME_DOCKER_AUTH_SECRET:
registries_auth.update({r.section: str(r.value)})
elif r.name == constants.SERVICE_PARAM_NAME_DOCKER_ADDITIONAL_OVERRIDES:
registries_overrides.update({r.section: str(r.value)})
if not registries_url:
# return directly if no user specified registries
return
registries_auth_db = self._dbapi.service_parameter_get_all(
service=constants.SERVICE_TYPE_DOCKER,
name=constants.SERVICE_PARAM_NAME_DOCKER_AUTH_SECRET)
registries_auth = {r.section: r.value for r in registries_auth_db}
for r in registries:
for section, url in registries_url.items():
try:
self.registries_info[r.section]['registry_replaced'] = str(r.value)
if r.section in registries_auth:
secret_ref = str(registries_auth[r.section])
self.registries_info[section]['registry_replaced'] = str(url)
if section in registries_overrides:
self.registries_info[section]['registry_default'] = \
registries_overrides[section]
if section in registries_auth:
secret_ref = registries_auth[section]
if secret_ref != 'None':
# If user specified registry requires the
# authentication, get the registry auth
# from barbican secret
auth = self._parse_barbican_secret(secret_ref)
self.registries_info[r.section]['registry_auth'] = auth
if (section in registries_type and
registries_type[section] == constants.DOCKER_REGISTRY_TYPE_AWS_ECR):
auth = cutils.get_aws_ecr_registry_credentials(
url, auth['username'], auth['password'])
self.registries_info[section]['registry_auth'] = auth
except exception.SysinvException:
raise exception.SysinvException(_(
"Unable to get the credentials to access "
"registry %s" % str(r.value)))
"registry %s" % url))
except KeyError:
# Unexpected
pass
@ -2502,38 +2505,39 @@ class DockerHelper(object):
def _get_img_tag_with_registry(self, pub_img_tag):
"""Regenerate public image tag with user specified registries
An example of passed public image reference:
docker.io/starlingx/stx-keystone:latest
"""
if self.registries_info == constants.DEFAULT_REGISTRIES_INFO:
# return if no user specified registries
return pub_img_tag, None
# An example of passed public image tag:
# docker.io/starlingx/stx-keystone:latest
# extracted registry_name = docker.io
# extracted img_name = starlingx/stx-keystone:latest
registry_name = pub_img_tag[0:1 + pub_img_tag.find('/')].replace('/', '')
img_name = pub_img_tag[1 + pub_img_tag.find('/'):]
for registry_info in self.registries_info.values():
if registry_name == registry_info['registry_default']:
registry_auth = registry_info['registry_auth']
if pub_img_tag.startswith(registry_info['registry_default']):
registry = registry_info['registry_replaced']
registry_auth = registry_info['registry_auth']
if registry:
return registry + '/' + img_name, registry_auth
return pub_img_tag, registry_auth
elif registry_name == registry_info['registry_replaced']:
registry_auth = registry_info['registry_auth']
img_name = pub_img_tag.split(
registry_info['registry_default'])[1]
return registry + img_name, registry_auth
return pub_img_tag, registry_auth
# If extracted registry_name is none of k8s.gcr.io, gcr.io,
# quay.io and docker.io or no registry_name specified in image
# tag, use user specified docker registry as default
elif pub_img_tag.startswith(registry_info['registry_replaced']):
return pub_img_tag, registry_auth
# If the image is not from any of the known registries
# (ie..k8s.gcr.io, gcr.io, quay.io, docker.io. docker.elastic.co)
# or no registry name specified in image tag, use user specified
# docker registry as default
registry = self.registries_info[
constants.SERVICE_PARAM_SECTION_DOCKER_DOCKER_REGISTRY]['registry_replaced']
registry_auth = self.registries_info[
constants.SERVICE_PARAM_SECTION_DOCKER_DOCKER_REGISTRY]['registry_auth']
registry_name = pub_img_tag[:pub_img_tag.find('/')]
if registry:
LOG.info("Registry %s not recognized or docker.io repository "
@ -2588,28 +2592,17 @@ class DockerHelper(object):
else:
quay_url = constants.DEFAULT_DOCKER_QUAY_REGISTRY
armada_image_tag = quay_url + \
armada_image_tag = constants.DOCKER_REGISTRY_SERVER + '/' + quay_url + \
image_versions.ARMADA_IMAGE_NAME + ":" + \
image_versions.ARMADA_IMAGE_VERSION
armada_image = client.images.list(armada_image_tag)
# Pull Armada image if it's not available
if not armada_image:
LOG.info("Downloading Armada image %s ..." % armada_image_tag)
quay_registry_secret = self._dbapi.service_parameter_get_all(
service=constants.SERVICE_TYPE_DOCKER,
section=constants.SERVICE_PARAM_SECTION_DOCKER_QUAY_REGISTRY,
name=constants.SERVICE_PARAM_NAME_DOCKER_AUTH_SECRET)
if quay_registry_secret:
quay_registry_auth = self._parse_barbican_secret(
quay_registry_secret[0].value)
else:
quay_registry_auth = None
local_registry_auth = cutils.get_local_docker_registry_auth()
client.images.pull(armada_image_tag,
auth_config=quay_registry_auth)
auth_config=local_registry_auth)
LOG.info("Armada image %s downloaded!" % armada_image_tag)
container = client.containers.run(

7
sysinv/sysinv/sysinv/sysinv/conductor/manager.py
View File

@ -5144,9 +5144,8 @@ class ConductorManager(service.PeriodicService):
# Grab the version from the image name. Version is preceded
# by a ":" e.g.
# gcr.io/kubernetes-helm/tiller:v2.13.0
running_image_name = running_image.split(":")[0]
running_version = running_image.split(":")[1]
# registry.local:9001/gcr.io/kubernetes-helm/tiller:v2.13.0
running_image_name, running_version = running_image.rsplit(":", 1)
if not running_version:
LOG.warning("Failed to get version from tiller image")
return
@ -5158,7 +5157,7 @@ class ConductorManager(service.PeriodicService):
"Upgrade in progress."
% image_versions.TILLER_IMAGE_VERSION)
download_image = running_image_name + ":" + image_versions.TILLER_IMAGE_VERSION
local_registry_auth = kube_app.get_local_docker_registry_auth()
local_registry_auth = cutils.get_local_docker_registry_auth()
self._docker._retrieve_specified_registries()
# download the image, retry if it fails

10
sysinv/sysinv/sysinv/sysinv/puppet/dockerdistribution.py
View File

@ -4,6 +4,7 @@
# SPDX-License-Identifier: Apache-2.0
#
from sysinv.common import utils
from sysinv.puppet import base
@ -17,3 +18,12 @@ class DockerDistributionPuppet(base.BasePuppet):
}
return config
def get_secure_system_config(self):
registry_credentials = utils.get_local_docker_registry_auth()
config = {
'platform::dockerdistribution::params::registry_username': registry_credentials['username'],
'platform::dockerdistribution::params::registry_password': registry_credentials['password']
}
return config