openstack/solum
Code Issues Proposed changes

Merge "Support operator LP download with swiftclient"

Browse Source
This commit is contained in:
Jenkins 2015-06-30 17:29:52 +00:00 committed by Gerrit Code Review
commit 716b6d9965
7 changed files with 129 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
contrib/lp-cedarish/docker/build-app
View File

@ -40,6 +40,10 @@ TEMP_URL_SECRET=${TEMP_URL_SECRET:-null}
TEMP_URL_PROTOCOL=${TEMP_URL_PROTOCOL:-null}
TEMP_URL_TTL=${TEMP_URL_TTL:-null}
OPR_LP_DOWNLOAD_STRATEGY=${OPR_LP_DOWNLOAD_STRATEGY:-null}
OPER_AUTH_TOKEN=${OPER_AUTH_TOKEN:-null}
OPER_OS_STORAGE_URL=${OPER_OS_STORAGE_URL:-null}
# TLOG, PRUN, etc. defined in common/utils
HERE=$(dirname $0)
source $HERE/../../common/utils
@ -210,7 +214,12 @@ else
python $HERE/swift-handler.py $OS_REGION_NAME $OS_AUTH_TOKEN $OS_STORAGE_URL download solum_lp $IMG_EXTERNAL_REF $OUTPUT \
> >(while read ALINE; do TLOG $ALINE; done)
elif [[ $LP_ACCESS == "operator" ]]; then
wget -q "$IMG_EXTERNAL_REF" --output-document=$OUTPUT
if [[ $OPR_LP_DOWNLOAD_STRATEGY == "swift-client" ]]; then
python $HERE/swift-handler.py $OS_REGION_NAME $OPER_AUTH_TOKEN $OPER_OS_STORAGE_URL download solum_lp $IMG_EXTERNAL_REF $OUTPUT \
> >(while read ALINE; do TLOG $ALINE; done)
elif [[ $OPR_LP_DOWNLOAD_STRATEGY == "wget" ]]; then
wget -q "$IMG_EXTERNAL_REF" --output-document=$OUTPUT
fi
fi
if [[ $? != 0 ]]; then
TLOG Failed to download image $IMG_EXTERNAL_REF from swift. && exit 1

30
contrib/lp-cedarish/docker/build-lp
View File

@ -18,6 +18,8 @@ TEMP_URL_SECRET=${TEMP_URL_SECRET:-null}
TEMP_URL_PROTOCOL=${TEMP_URL_PROTOCOL:-null}
TEMP_URL_TTL=${TEMP_URL_TTL:-null}
OPR_LP_DOWNLOAD_STRATEGY=${OPR_LP_DOWNLOAD_STRATEGY:-null}
# TLOG, PRUN, etc. defined in common/utils
HERE=$(dirname $0)
source $HERE/../../common/utils
@ -128,22 +130,26 @@ if [[ -d "$TMP_LP_DIR/build" ]]; then
if [[ $LP_ACCESS == "custom" ]]; then
image_id="${STORAGE_OBJ_NAME}"
elif [[ $LP_ACCESS == "operator" ]]; then
ACCOUNT=$(echo $OS_STORAGE_URL | sed 's/\// /'g | awk '{print $4}')
TLOG "ACCOUNT=$ACCOUNT"
STORAGE_HOST=$(echo $OS_STORAGE_URL | sed 's/\// /'g | awk '{print $2}')
TLOG "STORAGE_HOST=$STORAGE_HOST"
if [[ $OPR_LP_DOWNLOAD_STRATEGY == "swift-client" ]]; then
image_id="${STORAGE_OBJ_NAME}"
elif [[ $OPR_LP_DOWNLOAD_STRATEGY == "wget" ]]; then
ACCOUNT=$(echo $OS_STORAGE_URL | sed 's/\// /'g | awk '{print $4}')
TLOG "ACCOUNT=$ACCOUNT"
STORAGE_HOST=$(echo $OS_STORAGE_URL | sed 's/\// /'g | awk '{print $2}')
TLOG "STORAGE_HOST=$STORAGE_HOST"
TLOG "STORAGE_URL:$OS_STORAGE_URL"
TLOG "REGION:$OS_REGION_NAME"
TLOG "AUTH_TOKEN:$OS_AUTH_TOKEN"
TLOG "STORAGE_URL:$OS_STORAGE_URL"
TLOG "REGION:$OS_REGION_NAME"
TLOG "AUTH_TOKEN:$OS_AUTH_TOKEN"
curl -i -X POST -H X-Auth-Token:$OS_AUTH_TOKEN -H X-Account-Meta-Temp-URL-Key:$TEMP_URL_SECRET $TEMP_URL_PROTOCOL://$STORAGE_HOST/v1/$ACCOUNT
TLOG "HERE:$HERE"
curl -i -X POST -H X-Auth-Token:$OS_AUTH_TOKEN -H X-Account-Meta-Temp-URL-Key:$TEMP_URL_SECRET $TEMP_URL_PROTOCOL://$STORAGE_HOST/v1/$ACCOUNT
TLOG "HERE:$HERE"
TEMP_URL=$(python $HERE/get-temp-url.py $STORAGE_HOST solum_lp $STORAGE_OBJ_NAME $ACCOUNT $TEMP_URL_SECRET $TEMP_URL_TTL $TEMP_URL_PROTOCOL)
TLOG "TEMP_URL:$TEMP_URL"
TEMP_URL=$(python $HERE/get-temp-url.py $STORAGE_HOST solum_lp $STORAGE_OBJ_NAME $ACCOUNT $TEMP_URL_SECRET $TEMP_URL_TTL $TEMP_URL_PROTOCOL)
TLOG "TEMP_URL:$TEMP_URL"
image_id="${TEMP_URL}"
image_id="${TEMP_URL}"
fi
fi
TLOG ===== finished uploading LP to $IMAGE_STORAGE

11
contrib/lp-cedarish/docker/unittest-app
View File

@ -35,6 +35,10 @@ OS_AUTH_TOKEN=${OS_AUTH_TOKEN:-null}
OS_REGION_NAME=${OS_REGION_NAME:-null}
OS_STORAGE_URL=${OS_STORAGE_URL:-null}
OPR_LP_DOWNLOAD_STRATEGY=${OPR_LP_DOWNLOAD_STRATEGY:-null}
OPER_AUTH_TOKEN=${OPER_AUTH_TOKEN:-null}
OPER_OS_STORAGE_URL=${OPER_OS_STORAGE_URL:-null}
# TLOG, PRUN, ENSURE_LOGFILE, and elapsed defined in app-common
HERE=$(dirname $0)
source $HERE/../../common/utils
@ -135,7 +139,12 @@ if [[ $IMG_EXTERNAL_REF != "auto" ]]; then
python $HERE/swift-handler.py $OS_REGION_NAME $OS_AUTH_TOKEN $OS_STORAGE_URL download solum_lp $IMG_EXTERNAL_REF $OUTPUT \
> >(while read ALINE; do TLOG $ALINE; done)
elif [[ $LP_ACCESS == "operator" ]]; then
wget -q "$IMG_EXTERNAL_REF" --output-document=$OUTPUT
if [[ $OPR_LP_DOWNLOAD_STRATEGY == "swift-client" ]]; then
python $HERE/swift-handler.py $OS_REGION_NAME $OPER_AUTH_TOKEN $OPER_OS_STORAGE_URL download solum_lp $IMG_EXTERNAL_REF $OUTPUT \
> >(while read ALINE; do TLOG $ALINE; done)
elif [[ $OPR_LP_DOWNLOAD_STRATEGY == "wget" ]]; then
wget -q "$IMG_EXTERNAL_REF" --output-document=$OUTPUT
fi
fi
if [[ $? != 0 ]]; then
TLOG Failed to download image $IMG_EXTERNAL_REF from swift. && exit 1

35
solum/common/solum_keystoneclient.py
View File

@ -43,6 +43,12 @@ AUTH_OPTS = [
cfg.CONF.register_opts(AUTH_OPTS)
cfg.CONF.import_opt('lp_operator_user', 'solum.worker.config', group='worker')
cfg.CONF.import_opt('lp_operator_password',
'solum.worker.config', group='worker')
cfg.CONF.import_opt('lp_operator_tenant_name',
'solum.worker.config', group='worker')
class KeystoneClient(object):
"""Keystone client wrapper to initialize the right version of the client.
@ -79,8 +85,9 @@ class KeystoneClientV3(object):
self.context = context
self._client = None
self._admin_client = None
self._lp_admin_client = None
if self.context.auth_url:
if self.context and self.context.auth_url:
self.endpoint = self.context.auth_url.replace('v2.0', 'v3')
else:
# Import auth_token to have keystone_authtoken settings setup.
@ -88,7 +95,7 @@ class KeystoneClientV3(object):
self.endpoint = cfg.CONF.keystone_authtoken.auth_uri.replace(
'v2.0', 'v3')
if self.context.trust_id:
if self.context and self.context.trust_id:
# Create a client with the specified trust_id, this
# populates self.context.auth_token with a trust-scoped token
self._client = self._v3_client_init()
@ -113,6 +120,19 @@ class KeystoneClientV3(object):
raise exception.AuthorizationFailure()
return self._admin_client
@property
def lp_admin_client(self):
if not self._lp_admin_client:
# Create lp operator client connection to v3 API
lp_operator_creds = self._lp_operator_creds()
c = kc_v3.Client(**lp_operator_creds)
if c.authenticate():
self._lp_admin_client = c
else:
LOG.error("LP Operator client authentication failed")
raise exception.AuthorizationFailure()
return self._lp_admin_client
def _v3_client_init(self):
kwargs = {
'auth_url': self.endpoint,
@ -176,7 +196,16 @@ class KeystoneClientV3(object):
'auth_url': self.endpoint,
'endpoint': self.endpoint,
'project_name': cfg.CONF.keystone_authtoken.admin_tenant_name}
LOG.info('admin creds %s' % creds)
return creds
def _lp_operator_creds(self):
# Get LP Operator creds from config.
creds = {
'username': cfg.CONF.worker.lp_operator_user,
'password': cfg.CONF.worker.lp_operator_password,
'auth_url': self.endpoint,
'endpoint': self.endpoint,
'project_name': cfg.CONF.worker.lp_operator_tenant_name}
return creds
def create_trust_context(self):

24
solum/tests/common/test_solum_keystoneclient.py
View File

@ -179,6 +179,30 @@ class KeystoneClientTest(base.BaseTestCase):
self.assertRaises(exception.AuthorizationFailure,
get_admin_client)
def test_init_lp_admin_client_denied(self, mock_ks):
"""Test the get_lp_admin_client property, auth failure path."""
self.ctx.username = None
self.ctx.password = None
self.ctx.trust_id = None
mock_ks.return_value.authenticate.return_value = False
solum_ks_client = solum_keystoneclient.KeystoneClientV3(self.ctx)
# Define wrapper for property or the property raises the exception
# outside of the assertRaises which fails the test
def get_lp_admin_client():
solum_ks_client.lp_admin_client
self.assertRaises(exception.AuthorizationFailure,
get_lp_admin_client)
def test_init_with_no_context(self, mock_ks):
"""Init with no context."""
mock_ks.return_value.authenticate.return_value = False
solum_ks_client = solum_keystoneclient.KeystoneClientV3(None)
self.assertEqual(solum_ks_client.endpoint,
'http://server.test:5000/v3')
def test_trust_init_fail(self, mock_ks):
"""Test consuming a trust when initializing, error scoping."""
self.ctx.username = None

13
solum/worker/config.py
View File

@ -76,6 +76,19 @@ SERVICE_OPTS = [
cfg.StrOpt('lp_location_url',
default="",
help='url to the container where LPs are stored.'),
cfg.StrOpt('operator_lp_download_strategy',
default="swift-client",
help='Options for downloading operator LPs.'
'Possible values are "wget" or "swift-client"'),
cfg.StrOpt('lp_operator_user',
default="",
help='LP operator username.'),
cfg.StrOpt('lp_operator_password',
default="",
help='LP operator password.'),
cfg.StrOpt('lp_operator_tenant_name',
default="",
help='LP operator tenant name.'),
]
opt_group = cfg.OptGroup(

22
solum/worker/handlers/shell.py
View File

@ -173,6 +173,16 @@ class Handler(object):
user_env['TEMP_URL_SECRET'] = cfg.CONF.worker.temp_url_secret
user_env['TEMP_URL_PROTOCOL'] = cfg.CONF.worker.temp_url_protocol
user_env['TEMP_URL_TTL'] = cfg.CONF.worker.temp_url_ttl
user_env['OPR_LP_DOWNLOAD_STRATEGY'] = (
cfg.CONF.worker.operator_lp_download_strategy)
# Get LP Operator context for downloading operator LPs
lp_kc = clients.OpenStackClients(None).keystone().lp_admin_client
user_env['OPER_AUTH_TOKEN'] = lp_kc.auth_token
user_env['OPER_OS_STORAGE_URL'] = lp_kc.service_catalog.url_for(
service_type='object-store',
endpoint_type='publicURL',
region_name=client_region_name)
if test_cmd is not None:
user_env['TEST_CMD'] = test_cmd
@ -369,6 +379,10 @@ class Handler(object):
log_env = user_env.copy()
if 'OS_AUTH_TOKEN' in log_env:
del log_env['OS_AUTH_TOKEN']
if 'OPER_AUTH_TOKEN' in log_env:
del log_env['OPER_AUTH_TOKEN']
if 'OPER_OS_STORAGE_URL' in log_env:
del log_env['OPER_OS_STORAGE_URL']
solum.TLS.trace.support_info(environment=log_env)
job_update_notification(ctxt, build_id, IMAGE_STATES.BUILDING,
@ -484,6 +498,10 @@ class Handler(object):
log_env = user_env.copy()
if 'OS_AUTH_TOKEN' in log_env:
del log_env['OS_AUTH_TOKEN']
if 'OPER_AUTH_TOKEN' in log_env:
del log_env['OPER_AUTH_TOKEN']
if 'OPER_OS_STORAGE_URL' in log_env:
del log_env['OPER_OS_STORAGE_URL']
solum.TLS.trace.support_info(environment=log_env)
logpath = "%s/%s-%s.log" % (user_env['SOLUM_TASK_DIR'],
@ -584,6 +602,10 @@ class Handler(object):
log_env = user_env.copy()
if 'OS_AUTH_TOKEN' in log_env:
del log_env['OS_AUTH_TOKEN']
if 'OPER_AUTH_TOKEN' in log_env:
del log_env['OPER_AUTH_TOKEN']
if 'OPER_OS_STORAGE_URL' in log_env:
del log_env['OPER_OS_STORAGE_URL']
solum.TLS.trace.support_info(environment=log_env)
logpath = "%s/%s-%s.log" % (user_env['SOLUM_TASK_DIR'],