openstack/solum
Code Issues Proposed changes

Added support for private GitHub repos

Browse Source 
Implements: blueprint support-private-github-repos
Closes-Bug: 1319604
Change-Id: I6f0181a59165e5ab7e0b2d38aab07c8b06ec7462
This commit is contained in:
Ravi Sankar Penta
2014-06-09 16:05:59 -07:00
parent 3c3a51ede6
commit e640ec3943
32 changed files with 637 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@@ -31,6 +31,9 @@ cover
nosetests.xml
solum.sqlite
# functional tests
functionaltests/tempest.log
# Translations
*.mo

18
contrib/common/README.md Normal file
View File

@@ -0,0 +1,18 @@
# This directory contains common reusable code for lp-cedarish and lp-dockerfile.
# TODO(ravips): Reorganize language pack directory structure
# solum/contrib--> language-pack
# |
# --> common (code common to all lp formats)
# |
# --> formats (supported formats)
# |
# --> cedarish
# | |
# | --> common (code common to cedarish formats)
# | |
# | --> docker
# | |
# | --> vm
# |
# --> dockerfile

30
contrib/lp-cedarish/docker/app-common → contrib/common/utils
View File

@@ -98,3 +98,33 @@ function PRUN () {
return $EXIT_STATUS
}
# Register ssh private key with ssh-agent.
# SSH_AUTH_SOCK env variable will be unique to this process and
# it restricts other apps to access current ssh credentials.
function add_ssh_creds () {
local SSH_PRIVATE_KEY=$1
local APP_DIR=$2
if [ -n "$SSH_PRIVATE_KEY" ]; then
eval `ssh-agent -s`
SSH_PRIVATE_KEY_FILE=$APP_DIR/.creds
echo "$SSH_PRIVATE_KEY" > $SSH_PRIVATE_KEY_FILE
chmod 600 $SSH_PRIVATE_KEY_FILE
ssh-add $SSH_PRIVATE_KEY_FILE ; EXIT_STATUS=$?
rm -f $SSH_PRIVATE_KEY_FILE
if [ "$EXIT_STATUS" != "0" ] ; then
TLOG "FAILED: ssh key register with ssh-agent (EXIT_STATUS=$EXIT_STATUS)"
exit 1
fi
fi
}
# De-register ssh private key with ssh-agent.
function remove_ssh_creds () {
local SSH_PRIVATE_KEY=$1
if [ -n "$SSH_PRIVATE_KEY" ]; then
ssh-agent -k
fi
}

12
contrib/devstack/lib/solum
View File

@@ -109,6 +109,15 @@ function configure_solum() {
fi
sudo chown $STACK_USER $SOLUM_CONF_DIR
# To support private github repos, do not perform host key check for github.com
# Need this change on solum-worker instances
STACK_USER_SSH_DIR=/home/$STACK_USER/.ssh
if [[ ! -d $STACK_USER_SSH_DIR ]]; then
sudo mkdir -p $STACK_USER_SSH_DIR
fi
sudo chown $STACK_USER $STACK_USER_SSH_DIR
echo -e "Host github.com\n\tStrictHostKeyChecking no\n" > $STACK_USER_SSH_DIR/config
# Copy over solum configuration file and configure common parameters.
cp $SOLUM_DIR/etc/solum/solum.conf.sample $SOLUM_CONF_DIR/$SOLUM_CONF_FILE
@@ -200,6 +209,9 @@ function install_solumclient {
# install_solum() - Collect source and prepare
function install_solum() {
# Install package requirements
install_package expect
git_clone $SOLUM_REPO $SOLUM_DIR $SOLUM_BRANCH
# When solum is re-listed in openstack/requirements/projects.txt we
# should change setup_package back to setup_develop.

18
contrib/lp-cedarish/docker/build-app
View File

@@ -23,9 +23,9 @@ BUILD_ID=${BUILD_ID:-null}
TASKNAME=build
REUSE_IMAGES_IF_REPO_UNCHANGED=${REUSE_IMAGES_IF_REPO_UNCHANGED:="1"}
# TLOG, PRUN, ENSURE_LOGFILE, and elapsed defined in app-common
# TLOG, PRUN, etc. defined in common/utils
HERE=$(dirname $0)
source $HERE/app-common
source $HERE/../../common/utils
LOG_FILE=$(GET_LOGFILE)
@@ -37,14 +37,14 @@ function app_glance_id () {
TLOG ===== Starting Build Script $0 $*
# Make sure tenant auth credentials were passed in.
if [[ -z $OS_AUTH_TOKEN ]]; then
TLOG openstack credentials not passed via ENV.
if [[ -z "$OS_AUTH_TOKEN" ]]; then
TLOG OpenStack credentials not passed via ENV.
exit 1
fi
# Check command line arguments
if [ $# -ne 4 ]; then
TLOG Usage: $0 git_url appname project_id base_image
if [ $# -lt 4 ]; then
TLOG Usage: $0 git_url appname project_id base_image [git_private_key]
exit 1
fi
@@ -64,11 +64,14 @@ TENANT=$1
shift
BASE_IMAGE=$1
shift
GIT_PRIVATE_KEY=$1
shift
BASE_DIR=/dev/shm
GIT_CHECKSUM=$(echo $GIT | md5sum | awk '{print $1;}')
APP_DIR=$BASE_DIR/apps/$TENANT/$GIT_CHECKSUM
PRUN silent mkdir -p $APP_DIR
add_ssh_creds "$GIT_PRIVATE_KEY" "$APP_DIR"
if [ -d "$APP_DIR/build" ] ; then
cd $APP_DIR/build
@@ -107,6 +110,7 @@ if [ ! -f "$APP_DIR/slug.tgz" ] ; then
exit
fi
PRUN sudo docker rm $BUILD_ID
remove_ssh_creds "$GIT_PRIVATE_KEY"
# Build the application image by injecting slug into runner
# and push to docker-registry ( which is tied to glance )
@@ -124,7 +128,7 @@ EOF
cd $APP_DIR
PRUN sudo docker build -t $APP .
sudo docker save "$APP" | glance image-create --container-format=docker --disk-format=raw --name "$APP"
PRUN silent sudo docker save "$APP" | glance image-create --container-format=docker --disk-format=raw --name "$APP"
image_id=$(app_glance_id $APP)

33
contrib/lp-cedarish/docker/unittest-app
View File

@@ -24,15 +24,16 @@ TASKNAME=unittest
DOCKER_REGISTRY=${DOCKER_REGISTRY:-'127.0.0.1:5042'}
# TLOG, PRUN, ENSURE_LOGFILE, and elapsed defined in app-common
source $(dirname $0)/app-common
HERE=$(dirname $0)
source $HERE/../../common/utils
LOG_FILE=$(GET_LOGFILE)
TLOG ===== Starting Test Script $0 $*
# Check command line arguments
if [ $# -lt 4 ]; then
TLOG Usage: $0 git_url git_branch tenant unit_test_entry_point
if [ $# -lt 5 ]; then
TLOG Usage: $0 git_url git_branch tenant git_private_key unit_test_entry_point
exit 1
fi
@@ -45,6 +46,8 @@ GIT_BRANCH=$1
shift
TENANT=$1
shift
GIT_PRIVATE_KEY=$1
shift
ENTRYPOINT="$@"
shift
@@ -58,10 +61,12 @@ APP_DIR=$BASE_DIR/solum/$DIR_NAME
rm -rf $APP_DIR
PRUN mkdir -p $APP_DIR
add_ssh_creds "$GIT_PRIVATE_KEY" "$APP_DIR"
PRUN git clone $GIT --single-branch -b $GIT_BRANCH $APP_DIR/code
cd $APP_DIR/code
COMMIT_ID=$(git log -1 --pretty=%H)
echo "$GIT_PRIVATE_KEY" > $APP_DIR/code/id_rsa
# Test the application code
TLOG "===>" Testing App
@@ -69,7 +74,7 @@ if [[ $(which drone) ]]; then
TLOG "===>" Using Drone
if [[ ! -e $APP_DIR/code/.drone.yml ]]; then
TLOG "===>" Creating .drone.yml
cat << EOF > $APP_DIR/code/.drone.yml
cat << EOF > $APP_DIR/code/.drone.yml
image: $DOCKER_REGISTRY/slugtester
script:
- $ENTRYPOINT
@@ -80,10 +85,13 @@ EOF
sudo /usr/local/bin/drone build $APP_DIR/code 2>&1 > >(while read LINE; do TLOG $LINE; done)
else
TLOG Creating Dockerfile
cat << EOF > $APP_DIR/Dockerfile
cat << EOF > $APP_DIR/Dockerfile
# SOLUM APP BUILDER
FROM $DOCKER_REGISTRY/slugtester
ADD code /code
ADD code/id_rsa /root/.ssh/id_rsa
RUN chmod 0600 /root/.ssh/id_rsa
RUN echo "Host github.com\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile=/dev/null" > /root/.ssh/config
WORKDIR /code
RUN ${ENTRYPOINT}
EOF
@@ -93,7 +101,18 @@ EOF
fi
SUCCESS=$?
TLOG Tests finished with status $SUCCESS.
remove_ssh_creds "$GIT_PRIVATE_KEY"
echo Docker finished with status $SUCCESS.
if [[ $SUCCESS == 0 ]]; then
TLOG ==== Status: SUCCESS
else
TLOG ==== Status: FAIL
fi
[[ $SUCCESS == 0 ]] && sudo docker rmi $DIR_NAME
cd /tmp
rm -rf $APP_DIR
TOTAL_TIME=$(elapsed $SCRIPT_START_TIME)
TLOG ===== Total elapsed time: $TOTAL_TIME sec

56
contrib/lp-cedarish/vm-slug/build-app
View File

@@ -1,20 +1,20 @@
#!/bin/bash
LOG=${SOLUM_BUILD_LOG:="/opt/stack/logs/solum_build.log"}
IMAGE_DIR=/opt/solum/cedarish/image
BASE_IMAGE=
# TLOG, PRUN, etc. defined in common/utils
HERE=$(dirname $0)
source $HERE/../../common/utils
function check_docker () {
sudo docker ps 2> /dev/null > /dev/null
if [[ $? != 0 ]]; then
echo "Cannot talk to Docker." >&2
exit 1
fi
PRUN silent docker ps
[[ $? != 0 ]] && TLOG cannot talk to docker. && exit 1
}
function check_os_credentials () {
if [[ -z "$OS_AUTH_TOKEN" ]]; then
echo "OpenStack credentials not passed via ENV."
TLOG OpenStack credentials not passed via ENV.
exit 1
fi
}
@@ -22,13 +22,12 @@ function check_os_credentials () {
function check_glance_access () {
glance image-list 2> /dev/null > /dev/null
if [ $? != 0 ]; then
echo "Cannot talk to Glance. Check your OpenStack credentials." >&2
exit 1
TLOG Cannot talk to Glance. Check your OpenStack credentials. && exit 1
fi
}
function build_app () {
echo "Building app..."
TLOG "===>" Building App
local GIT_URL=$1
local APP_DIR=$2
@@ -37,7 +36,7 @@ function build_app () {
[[ -d build ]] && rm -rf build
git clone $GIT_URL build
if [[ ! -d build ]]; then
echo "Git clone failed." >&2
TLOG Git clone failed.
exit 1
fi
pushd build
@@ -47,7 +46,7 @@ function build_app () {
-v /opt/solum/buildpacks:/tmp/buildpacks:rw \
solum/slugbuilder)
if [[ -z "$BUILD_ID" ]]; then
echo "Docker build failed. Did not get a build ID." >&2
TLOG Docker build failed. Did not get a build ID.
exit 1
fi
sudo docker attach $BUILD_ID
@@ -57,28 +56,28 @@ function build_app () {
sudo docker rm $BUILD_ID
rm -rf build
popd
echo "...done building."
TLOG Done building
}
function inject_app_into_image () {
echo "Injecting app into image..."
TLOG Injecting app into image
local APP_DIR=$1
pushd $APP_DIR
[[ -f image.qcow2 ]] || glance image-download --file image.qcow2 cedarish
echo "Mounting image."
TLOG Mounting image
mkdir -p mnt
sudo guestmount -a image.qcow2 -i --rw mnt
PRUN sudo guestmount -a image.qcow2 -i --rw mnt
sudo mkdir -p mnt/app
echo "Injecting app."
sudo tar xzf slug.tgz -C mnt/app
TLOG Injecting app
PRUN sudo tar xzf slug.tgz -C mnt/app
sudo umount mnt
popd
echo "...done injecting."
TLOG Done injecting
}
function upload_image_to_glance () {
echo "Uploading to Glance..."
TLOG Uploading to Glance
local APP=$1
local APP_DIR=$2
@@ -89,8 +88,8 @@ function upload_image_to_glance () {
local IMAGE_ID=$(glance image-show $APP | grep " id " | cut -d"|" -f3 | tr -d " ")
echo "created_image_id=$IMAGE_ID"
echo "...done uploading."
TLOG created_image_id=$image_id
TLOG Done uploading
}
@@ -101,7 +100,9 @@ function main () {
shift
local TENANT=$1
shift
BASE_IMAGE=$1
local BASE_IMAGE=$1
shift
local GIT_PRIVATE_KEY=$1
shift
local APP_DIR=/opt/solum/apps/$TENANT/$APP
@@ -109,13 +110,16 @@ function main () {
check_docker
check_os_credentials
check_glance_access
add_ssh_creds "$GIT_PRIVATE_KEY" "$APP_DIR"
build_app $GIT_URL $APP_DIR
remove_ssh_creds "$GIT_PRIVATE_KEY"
inject_app_into_image $APP_DIR
upload_image_to_glance $APP $APP_DIR
}
if [[ -z "$1" ]] || [[ -z "$2" ]] || [[ -z "$3" ]]; then
echo "Usage: $0 git_url appname tenant_id" >&2
# Check command line arguments
if [ $# -lt 4 ]; then
TLOG Usage: $0 git_url appname project_id base_image [git_private_key]
exit 1
fi

36
contrib/lp-dockerfile/docker/build-app
View File

@@ -1,13 +1,17 @@
#!/bin/bash
docker ps 2> /dev/null > /dev/null
[[ $? != 0 ]] && echo "cannot talk to docker." && exit 1
LOG=${SOLUM_BUILD_LOG:="/opt/stack/logs/solum_build.log"}
# tool that calls this has specific variabes, even though
# we don't use base image, it still has an arg that we need
# to shift out.
if [[ -z $1 ]] || [[ -z $2 ]] || [[ -z $3 ]] || [[ -z $4 ]]; then
echo "Usage: build git_url appname project_id base_image"
# TLOG, PRUN, etc. defined in common/utils
HERE=$(dirname $0)
source $HERE/../../common/utils
PRUN silent docker ps
[[ $? != 0 ]] && TLOG cannot talk to docker. && exit 1
# Check command line arguments
if [ $# -lt 4 ]; then
TLOG Usage: $0 git_url appname project_id base_image [git_private_key]
exit 1
fi
@@ -19,31 +23,35 @@ TENANT=$1
shift
BASE_IMAGE=$1
shift
GIT_PRIVATE_KEY=$1
shift
DOCKER_REGISTRY=${DOCKER_REGISTRY:-'127.0.0.1:5042'}
if [[ -z $OS_USERNAME ]]; then
echo 'openstack credentials not passed via ENV. hunting for openrc.'
TLOG openstack credentials not passed via ENV.
[[ -f ./openrc ]] && . ./openrc
[[ -f ~/devstack/openrc ]] && . ~/devstack/openrc
fi
APP_DIR=/opt/solum/apps/$TENANT/$APP
mkdir -p $APP_DIR
PRUN mkdir -p $APP_DIR
add_ssh_creds "$GIT_PRIVATE_KEY" "$APP_DIR"
[[ -d $APP_DIR/build ]] && rm -rf $APP_DIR/build
git clone $GIT $APP_DIR/build
PRUN git clone $GIT $APP_DIR/build
echo '===> building App'
remove_ssh_creds "$GIT_PRIVATE_KEY"
TLOG "===>" Building App
cd $APP_DIR/build
sudo docker build -t $DOCKER_REGISTRY/$APP .
PRUN sudo docker build -t $DOCKER_REGISTRY/$APP .
sudo docker push $DOCKER_REGISTRY/$APP
PRUN sudo docker push $DOCKER_REGISTRY/$APP
image_id=$(glance image-show $APP:latest | grep " id " | cut -d"|" -f3 | tr -d " ")
echo "created_image_id=$image_id"
TLOG created_image_id=$image_id
exit 0

6
doc/source/getting_started/index.rst
View File

@@ -25,7 +25,7 @@ __ https://wiki.openstack.org/wiki/Solum/Demo
Create your app
---------------
Solum clones code from the user's public Git repository. Before you begin, push your code to a public Git repo. From within your devstack host, you can now run solum commands to build and deploy your application.
Solum clones code from the user's public Git repository or user's public/private GitHub repository. Before you begin, push your code to a Git repo. From within your devstack host, you can now run solum commands to build and deploy your application.
2. To register an app with Solum, you will need to write a planfile to describe it.
We provide an example planfile at :code:`examples/plans/ex1.yaml`
@@ -166,12 +166,12 @@ At present it is a Stackforge project, and its repository is available on the Op
$ cd Solum
$ git clone git://git.openstack.org/stackforge/solum
In addition to Solum, your environment will also need Devstack to configure and run the requisite Openstack components, including Keystone, Glance, Nova, Neutron, and Heat.
In addition to Solum, your environment will also need Devstack to configure and run the requisite OpenStack components, including Keystone, Glance, Nova, Neutron, and Heat.
Vagrant Dev Environment (optional, for developers)
--------------------------------------------------
2. We have provided a Vagrant environment to deploy Solum and its required Openstack components via Devstack. We recommend using this approach if you are planning to contribute to Solum. This takes about the same amount of time as setting up Devstack manually, but it automates the setup for you.
2. We have provided a Vagrant environment to deploy Solum and its required OpenStack components via Devstack. We recommend using this approach if you are planning to contribute to Solum. This takes about the same amount of time as setting up Devstack manually, but it automates the setup for you.
By default, it uses virtualbox as its provisioner. We have tested this with Vagrant 1.5.4.
The environment will need to know where your Solum code is, via the environment variable :code:`SOLUM`.

44
etc/solum/solum.conf.sample
View File

@@ -14,10 +14,6 @@
# Size of RPC connection pool. (integer value)
#rpc_conn_pool_size=30
# Modules of exceptions that are permitted to be recreated
# upon receiving exception data from an rpc call. (list value)
#allowed_rpc_exception_modules=oslo.messaging.exceptions,nova.exception,cinder.exception,exceptions
# Qpid broker hostname. (string value)
#qpid_hostname=localhost
@@ -47,6 +43,10 @@
# Whether to disable the Nagle algorithm. (boolean value)
#qpid_tcp_nodelay=true
# The number of prefetched messages held by receiver. (integer
# value)
#qpid_receiver_capacity=1
# The qpid topology version to use. Version 1 is what was
# originally used by impl_qpid. Version 2 includes some
# backwards-incompatible changes that allow broker federation
@@ -156,15 +156,6 @@
# Heartbeat time-to-live. (integer value)
#matchmaker_heartbeat_ttl=600
# Host to locate redis. (string value)
#host=127.0.0.1
# Use this port to connect to redis host. (integer value)
#port=6379
# Password for Redis server (optional). (string value)
#password=<None>
# Size of RPC greenthread pool. (integer value)
#rpc_thread_pool_size=64
@@ -357,6 +348,17 @@
#source_format=heroku
[barbican_client]
#
# Options defined in solum.common.clients
#
# If set, then the server's certificate for barbican will not
# be verified. (boolean value)
#insecure=false
[builder]
#
@@ -727,6 +729,22 @@
#hash_algorithms=md5
[matchmaker_redis]
#
# Options defined in oslo.messaging
#
# Host to locate redis. (string value)
#host=127.0.0.1
# Use this port to connect to redis host. (integer value)
#port=6379
# Password for Redis server (optional). (string value)
#password=<None>
[matchmaker_ring]
#

12
functionaltests/api/base.py
View File

@@ -16,6 +16,7 @@
import copy
import json
import os
import time
from tempest import auth
@@ -45,6 +46,7 @@ plan_sample_data = {"version": "1",
"artifact_type": "heroku",
"content": {
"href": "https://example.com/git/a.git",
"private": False
},
"language_pack": "auto",
}]}
@@ -92,10 +94,8 @@ class SolumClient(rest_client.RestClient):
self.created_assemblies.append(uuid)
return assembly_resp
def create_plan(self, data=None, private_github_repo=False):
def create_plan(self, data=None):
plan_data = copy.deepcopy(data or plan_sample_data)
if private_github_repo:
plan_data['artifacts'][0]['content']['private'] = True
yaml_data = yaml.dump(plan_data)
resp, body = self.post('v1/plans', yaml_data,
headers={'content-type': 'application/x-yaml'})
@@ -172,3 +172,9 @@ class SolumCredentials(auth.KeystoneV2Credentials):
)
super(SolumCredentials, self).__init__(**creds)
def is_fedora():
if os.path.exists("/etc/redhat-release"):
return True
return False

42
functionaltests/api/v1/test_plan.py
View File

@@ -25,11 +25,25 @@ sample_data = {"version": "1",
"name": "No deus",
"artifact_type": "heroku",
"content": {
"href": "https://example.com/git/a.git"
"href": "https://example.com/git/a.git",
"private": False,
},
"language_pack": "auto",
}]}
sample_data_private = {"version": "1",
"name": "test_plan",
"description": "A test to create plan",
"artifacts": [{
"name": "No deus",
"artifact_type": "heroku",
"content": {
"href": "https://example.com/git/a.git",
"private": True,
},
"language_pack": "auto",
}]}
class TestPlanController(base.TestCase):
def setUp(self):
@@ -63,6 +77,14 @@ class TestPlanController(base.TestCase):
self.assertEqual(resp.status, 201)
self._assert_output_expected(resp.data, sample_data)
def test_plans_create_with_private_github_repo(self):
# FIXME(ravips): remove this when bug #1371252 is fixed
if base.is_fedora():
self.skipTest("Fails on Fedora 20, bug: #1371252")
resp = self.client.create_plan(data=sample_data_private)
self.assertEqual(resp.status, 201)
self._assert_output_expected(resp.data, sample_data)
def test_plans_create_empty_yaml(self):
# NOTE(stannie): tempest rest_client raises InvalidContentType and not
# BadRequest because yaml content-type is not supported in their
@@ -87,6 +109,7 @@ class TestPlanController(base.TestCase):
create_resp = self.client.create_plan(data=sample_data)
self.assertEqual(create_resp.status, 201)
uuid = create_resp.uuid
resp, body = self.client.get(
'v1/plans/%s' % uuid,
headers={'content-type': 'application/x-yaml'})
@@ -94,6 +117,23 @@ class TestPlanController(base.TestCase):
yaml_data = yaml.load(body)
self._assert_output_expected(yaml_data, sample_data)
def test_plans_get_with_private_github_repo(self):
# FIXME(ravips): remove this when bug #1371252 is fixed
if base.is_fedora():
self.skipTest("Fails on Fedora 20, bug: #1371252")
create_resp = self.client.create_plan(data=sample_data_private)
self.assertEqual(create_resp.status, 201)
uuid = create_resp.uuid
resp, body = self.client.get(
'v1/plans/%s' % uuid,
headers={'content-type': 'application/x-yaml'})
self.assertEqual(resp.status, 200)
yaml_data = yaml.load(body)
public_key = yaml_data['artifacts'][0]['content']['public_key']
self.assertNotEqual(None, public_key)
self._assert_output_expected(yaml_data, sample_data)
def test_plans_get_not_found(self):
# NOTE(stannie): tempest rest_client raises InvalidContentType and not
# NotFound because yaml content-type is not supported in their

2
requirements.txt
View File

@@ -9,6 +9,8 @@ oslo.db>=0.2.0 # Apache-2.0
oslo.messaging>=1.3.0
pbr>=0.6,!=0.7,<1.0
pecan>=0.5.0
pycrypto>=2.6
python-barbicanclient>=2.1.0
python-glanceclient>=0.13.1
python-heatclient>=0.2.9
python-zaqarclient>=0.0.3

7
solum/api/controllers/v1/datamodel/plan.py
View File

@@ -14,6 +14,7 @@
import uuid
import six
import wsme
from wsme.rest import json as wjson
from wsme import types as wtypes
@@ -51,7 +52,8 @@ class Artifact(wtypes.Base):
artifact_type = wtypes.text
"Type of artifact."
content = {wtypes.text: wtypes.text}
content = {wtypes.text: api_types.MultiType(wtypes.text,
six.types.BooleanType)}
"Type specific content as a flat dict."
language_pack = wtypes.text
@@ -109,7 +111,8 @@ class Plan(api_types.Base):
artifacts=[{
'name': 'My-python-app',
'artifact_type': 'git_pull',
'content': {'href': 'git://example.com/project.git'},
'content': {'href': 'git://example.com/project.git',
'private': False},
'language_pack': str(uuid.uuid4()),
'requirements': [{
'requirement_type': 'git_pull',

26
solum/api/controllers/v1/datamodel/types.py
View File

@@ -80,3 +80,29 @@ class Base(wtypes.Base):
for k in keys
if hasattr(self, k) and
getattr(self, k) != wsme.Unset)
class MultiType(wtypes.UserType):
"""A complex type that represents one or more types.
Used for validating that a value is an instance of one of the types.
:param *types: Variable-length list of types.
"""
def __init__(self, *types):
self.types = types
def __str__(self):
return ' | '.join(map(str, self.types))
def validate(self, value):
for t in self.types:
if t is wsme.types.text and isinstance(value, wsme.types.bytes):
value = value.decode()
if isinstance(value, t):
return value
else:
raise ValueError(
_("Wrong type. Expected '%(type)s', got '%(value)s'")
% {'type': self.types, 'value': type(value)})

21
solum/api/handlers/assembly_handler.py
View File

@@ -79,7 +79,9 @@ class AssemblyHandler(handler.Handler):
artifacts = plan_obj.raw_content.get('artifacts', [])
for arti in artifacts:
self._build_artifact(db_obj, arti, branch_name, status_url)
self._build_artifact(assem=db_obj, artifact=arti,
branch_name=branch_name,
status_url=status_url)
def update(self, id, data):
"""Modify a resource."""
@@ -123,11 +125,13 @@ class AssemblyHandler(handler.Handler):
artifacts = plan_obj.raw_content.get('artifacts', [])
for arti in artifacts:
self._build_artifact(db_obj, arti)
self._build_artifact(assem=db_obj, artifact=arti,
deploy_keys_ref=plan_obj.deploy_keys_uri)
return db_obj
def _unittest_artifact(self, assem, artifact, branch_name='master',
status_url=None):
status_url=None, deploy_keys_ref=None):
test_cmd = artifact.get('unittest_cmd')
status_token = artifact.get('status_token')
git_info = {
@@ -140,10 +144,12 @@ class AssemblyHandler(handler.Handler):
api.API(context=self.context).unittest(
assembly_id=assem.id,
git_info=git_info,
test_cmd=test_cmd)
test_cmd=test_cmd,
source_creds_ref=deploy_keys_ref)
def _build_artifact(self, assem, artifact, branch_name='master',
status_url=None):
status_url=None, deploy_keys_ref=None):
# This is a tempory hack so we don't need the build client
# in the requirments.
image = objects.registry.Image()
@@ -165,7 +171,7 @@ class AssemblyHandler(handler.Handler):
'source_url': image.source_uri,
'branch_name': branch_name,
'status_token': status_token,
'status_url': status_url,
'status_url': status_url
}
api.API(context=self.context).build(
@@ -176,7 +182,8 @@ class AssemblyHandler(handler.Handler):
source_format=image.source_format,
image_format=image.image_format,
assembly_id=assem.id,
test_cmd=test_cmd)
test_cmd=test_cmd,
source_creds_ref=deploy_keys_ref)
def get_all(self):
"""Return all assemblies, based on the query provided."""

35
solum/api/handlers/plan_handler.py
View File

@@ -12,9 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
import base64
import uuid
from Crypto.PublicKey import RSA
from solum.api.handlers import handler
from solum.common import clients
from solum import objects
@@ -26,7 +30,7 @@ class PlanHandler(handler.Handler):
return objects.registry.Plan.get_by_uuid(self.context, id)
def update(self, id, data):
"""Modify a resource."""
"""Modify existing plan."""
db_obj = objects.registry.Plan.get_by_uuid(self.context, id)
if 'name' in data:
db_obj.name = data['name']
@@ -35,19 +39,42 @@ class PlanHandler(handler.Handler):
return db_obj
def delete(self, id):
"""Delete a resource."""
"""Delete existing plan."""
db_obj = objects.registry.Plan.get_by_uuid(self.context, id)
if db_obj.deploy_keys_uri:
barbican = clients.OpenStackClients(None).barbican().admin_client
barbican.secrets.delete(db_obj.deploy_keys_uri)
db_obj.destroy(self.context)
def create(self, data):
"""Create a new resource."""
"""Create a new plan."""
db_obj = objects.registry.Plan()
if 'name' in data:
db_obj.name = data['name']
db_obj.raw_content = data
db_obj.uuid = str(uuid.uuid4())
db_obj.user_id = self.context.user
db_obj.project_id = self.context.tenant
deploy_keys = []
for artifact in data.get('artifacts', []):
if (('content' not in artifact) or
('private' not in artifact['content']) or
(not artifact['content']['private'])):
continue
new_key = RSA.generate(2048)
public_key = new_key.publickey().exportKey("OpenSSH")
private_key = new_key.exportKey("PEM")
artifact['content']['public_key'] = public_key
deploy_keys.append({'source_url': artifact['content']['href'],
'private_key': private_key})
if deploy_keys:
barbican = clients.OpenStackClients(None).barbican().admin_client
encoded_payload = base64.b64encode(bytes(str(deploy_keys)))
db_obj.deploy_keys_uri = barbican.secrets.Secret(
name=db_obj.uuid,
payload=encoded_payload,
payload_content_type='application/octet-stream',
payload_content_encoding='base64').store()
db_obj.raw_content = data
db_obj.create(self.context)
return db_obj

18
solum/common/clients.py
View File

@@ -21,6 +21,7 @@ from swiftclient import client as swiftclient
from zaqarclient.queues.v1 import client as zaqarclient
from solum.common import exception
from solum.common import solum_barbicanclient
from solum.common import solum_keystoneclient
from solum.openstack.common.gettextutils import _
from solum.openstack.common import log as logging
@@ -29,6 +30,12 @@ from solum.openstack.common import log as logging
LOG = logging.getLogger(__name__)
barbican_client_opts = [
cfg.BoolOpt('insecure',
default=False,
help=_("If set, then the server's certificate for barbican "
"will not be verified."))]
# Note: this config is duplicated in many projects that use OpenStack
# clients. This should really be in the client.
# There is a place holder bug here:
@@ -109,6 +116,7 @@ mistral_client_opts = [
help=_("If set the server certificate will not be verified "
"while using Mistral."))]
cfg.CONF.register_opts(barbican_client_opts, group='barbican_client')
cfg.CONF.register_opts(glance_client_opts, group='glance_client')
cfg.CONF.register_opts(heat_client_opts, group='heat_client')
cfg.CONF.register_opts(zaqar_client_opts, group='zaqar_client')
@@ -122,6 +130,7 @@ class OpenStackClients(object):
def __init__(self, context):
self.context = context
self._barbican = None
self._keystone = None
self._glance = None
self._heat = None
@@ -141,6 +150,15 @@ class OpenStackClients(object):
def auth_token(self):
return self.context.auth_token or self.keystone().auth_token
@exception.wrap_keystone_exception
def barbican(self):
if self._barbican:
return self._barbican
insecure = self._get_client_option('barbican', 'insecure')
self._barbican = solum_barbicanclient.BarbicanClient(insecure)
return self._barbican
def keystone(self):
if self._keystone:
return self._keystone

45
solum/common/solum_barbicanclient.py Normal file
View File

@@ -0,0 +1,45 @@
# Copyright 2014 - Red Hat Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from barbicanclient import client as barbicanclient
from barbicanclient.common import auth as barbicanauth
from oslo.config import cfg
from solum.openstack.common import importutils
class BarbicanClient(object):
"""Barbican client wrapper so we can encapsulate logic in one place."""
def __init__(self, insecure=False):
self.insecure = insecure
self._admin_client = None
@property
def admin_client(self):
if not self._admin_client:
# Create connection to API
self._admin_client = self._barbican_admin_init()
return self._admin_client
def _barbican_admin_init(self):
# Import auth_token to have keystone_authtoken settings setup.
importutils.import_module('keystoneclient.middleware.auth_token')
keystone = barbicanauth.KeystoneAuthV2(
auth_url=cfg.CONF.keystone_authtoken.auth_uri,
username=cfg.CONF.keystone_authtoken.admin_user,
password=cfg.CONF.keystone_authtoken.admin_password,
tenant_name=cfg.CONF.keystone_authtoken.admin_tenant_name)
return barbicanclient.Client(auth_plugin=keystone,
insecure=self.insecure)

1
solum/objects/sqlalchemy/migration/alembic_migrations/versions/498adc6185ae_create_initial_db_schema.py
View File

@@ -108,6 +108,7 @@ def upgrade():
sa.Column('raw_content', models.YAMLEncodedDict(2048)),
sa.Column('description', sa.String(length=255)),
sa.Column('name', sa.String(255)),
sa.Column('deploy_keys_uri', sa.String(length=1024)),
)
op.create_table(

1
solum/objects/sqlalchemy/plan.py
View File

@@ -33,6 +33,7 @@ class Plan(sql.Base, abstract.Plan):
name = sqlalchemy.Column(sqlalchemy.String(255))
description = sqlalchemy.Column(sqlalchemy.String(255))
raw_content = sqlalchemy.Column(sql.YAMLEncodedDict(2048))
deploy_keys_uri = sqlalchemy.Column(sqlalchemy.String(1024))
def refined_content(self):
if self.raw_content and self.uuid:

26
solum/tests/api/controllers/v1/test_types.py
View File

@@ -10,8 +10,11 @@
# License for the specific language governing permissions and limitations
# under the License.
import six
from wsme import types as wtypes
from solum.api.controllers.v1.datamodel import component as component_api
from solum.api.controllers.v1.datamodel import types as api_types
from solum import objects
from solum.objects.sqlalchemy import component as component_model
from solum.tests import base
@@ -74,6 +77,29 @@ class TestTypes(base.BaseTestCase):
self.assertEqual(assembly_data['uuid'], c.assembly_uuid)
class TestMultiType(base.BaseTestCase):
def test_valid_values(self):
vt = api_types.MultiType(wtypes.text, six.types.BooleanType)
value = vt.validate("somestring")
self.assertEqual("somestring", value)
value = vt.validate(True)
self.assertEqual(True, value)
def test_invalid_values(self):
vt = api_types.MultiType(wtypes.text, six.types.BooleanType)
self.assertRaises(ValueError, vt.validate, 10)
self.assertRaises(ValueError, vt.validate, 0.10)
self.assertRaises(ValueError, vt.validate, object())
def test_multitype_tostring(self):
vt = api_types.MultiType(wtypes.text, six.types.BooleanType)
vts = str(vt)
self.assertIn(str(wtypes.text), vts)
self.assertIn(str(six.types.BooleanType), vts)
self.assertNotIn(str(six.integer_types[0]), vts)
class TestTypeNames(base.BaseTestCase):
scenarios = [

60
solum/tests/api/handlers/test_assembly.py
View File

@@ -75,9 +75,10 @@ class TestAssemblyHandler(base.BaseTestCase):
'name': 'theplan',
'artifacts': [{'name': 'nodeus',
'artifact_type': 'heroku',
'content': {
'href': 'https://example.com/ex.git'},
'content': {'private': False,
'href': 'https://example.com/ex.git'},
'language_pack': 'auto'}]}
mock_registry.Image.return_value = fakes.FakeImage()
trust_ctx = utils.dummy_context()
trust_ctx.trust_id = '12345'
@@ -97,7 +98,54 @@ class TestAssemblyHandler(base.BaseTestCase):
mock_build.assert_called_once_with(
build_id=8, name='nodeus', assembly_id=8,
git_info=git_info, test_cmd=None,
base_image_id='auto', source_format='heroku', image_format='qcow2')
base_image_id='auto', source_format='heroku',
source_creds_ref=None, image_format='qcow2')
mock_kc.return_value.create_trust_context.assert_called_once_with()
@mock.patch('solum.worker.api.API.build')
@mock.patch('solum.common.solum_keystoneclient.KeystoneClientV3')
def test_create_with_private_github_repo(self, mock_kc, mock_build,
mock_registry):
data = {'user_id': 'new_user_id',
'uuid': 'input_uuid',
'plan_uuid': 'input_plan_uuid'}
db_obj = fakes.FakeAssembly()
mock_registry.Assembly.return_value = db_obj
fp = fakes.FakePlan()
mock_registry.Plan.get_by_id.return_value = fp
fp.raw_content = {
'name': 'theplan',
'artifacts': [{'name': 'nodeus',
'artifact_type': 'heroku',
'content': {'private': True,
'href': 'https://example.com/ex.git',
'public_key': 'ssh-rsa abc'},
'language_pack': 'auto'}]}
fp.deploy_keys_uri = 'secret_ref_uri'
mock_registry.Image.return_value = fakes.FakeImage()
trust_ctx = utils.dummy_context()
trust_ctx.trust_id = '12345'
mock_kc.return_value.create_trust_context.return_value = trust_ctx
handler = assembly_handler.AssemblyHandler(self.ctx)
res = handler.create(data)
db_obj.update.assert_called_once_with(data)
db_obj.create.assert_called_once_with(self.ctx)
self.assertEqual(db_obj, res)
git_info = {
'source_url': "https://example.com/ex.git",
'branch_name': 'master',
'status_token': None,
'status_url': None,
}
mock_build.assert_called_once_with(
build_id=8, name='nodeus', assembly_id=8,
git_info=git_info,
test_cmd=None, base_image_id='auto', source_format='heroku',
source_creds_ref='secret_ref_uri', image_format='qcow2')
mock_kc.return_value.create_trust_context.assert_called_once_with()
@mock.patch('solum.common.solum_keystoneclient.KeystoneClientV3')
@@ -130,8 +178,10 @@ class TestAssemblyHandler(base.BaseTestCase):
handler._build_artifact = mock.MagicMock()
handler._context_from_trust_id = mock.MagicMock(return_value=self.ctx)
handler.trigger_workflow(trigger_id)
handler._build_artifact.assert_called_once_with(db_obj, artifacts[0],
'master', None)
handler._build_artifact.assert_called_once_with(assem=db_obj,
artifact=artifacts[0],
branch_name='master',
status_url=None)
handler._context_from_trust_id.assert_called_once_with('trust_worthy')
mock_registry.Assembly.get_by_trigger_id.assert_called_once_with(
None, trigger_id)

42
solum/tests/common/test_clients.py
View File

@@ -10,11 +10,15 @@
# License for the specific language governing permissions and limitations
# under the License.
from barbicanclient import client as barbicanclient
from barbicanclient.common import auth as barbicanauth
from glanceclient import client as glanceclient
from heatclient import client as heatclient
from keystoneclient.openstack.common.apiclient import exceptions
from mistralclient.api import client as mistralclient
import mock
from neutronclient.neutron import client as neutronclient
from oslo.config import cfg
from swiftclient import client as swiftclient
from zaqarclient.queues.v1 import client as zaqarclient
@@ -34,6 +38,44 @@ class ClientsTest(base.BaseTestCase):
mock_cat.url_for.assert_called_once_with(service_type='fake_service',
endpoint_type='fake_endpoint')
@mock.patch.object(barbicanclient, 'Client')
@mock.patch.object(barbicanauth, 'KeystoneAuthV2')
def test_clients_barbican(self, mock_auth, mock_call):
mock_auth.return_value = "keystone_auth_handle"
mock_call.return_value = "barbican_client_handle"
obj = clients.OpenStackClients(None)
self.assertEqual(None, obj._barbican)
obj.barbican().admin_client
self.assertNotEqual(None, obj._barbican)
mock_call.assert_called_once_with(auth_plugin='keystone_auth_handle',
insecure=False)
def test_clients_barbican_noauth(self):
dummy_url = 'http://server.test:5000/v2.0'
cfg.CONF.set_override('auth_uri', dummy_url,
group='keystone_authtoken')
cfg.CONF.set_override('admin_user', 'solum',
group='keystone_authtoken')
cfg.CONF.set_override('admin_password', 'verybadpass',
group='keystone_authtoken')
cfg.CONF.set_override('admin_tenant_name', 'service',
group='keystone_authtoken')
obj = clients.OpenStackClients(None)
self.assertRaises(exceptions.AuthorizationFailure,
lambda: obj.barbican().admin_client)
@mock.patch.object(barbicanclient, 'Client')
@mock.patch.object(barbicanauth, 'KeystoneAuthV2')
def test_clients_barbican_cached(self, mock_auth, mock_call):
mock_auth.return_value = "keystone_auth_handle"
mock_call.return_value = "barbican_client_handle"
obj = clients.OpenStackClients(None)
barbican_admin = obj.barbican().admin_client
barbican_admin_cached = obj.barbican().admin_client
self.assertEqual(barbican_admin, barbican_admin_cached)
mock_call.assert_called_once_with(auth_plugin='keystone_auth_handle',
insecure=False)
@mock.patch.object(glanceclient, 'Client')
@mock.patch.object(clients.OpenStackClients, 'url_for')
def test_clients_glance(self, mock_url, mock_call):

1
solum/tests/fakes.py
View File

@@ -248,6 +248,7 @@ class FakePlan(mock.Mock):
self.uuid = 'test_uuid'
self.id = 8
self.name = 'faker'
self.deploy_keys_uri = None
def as_dict(self):
return dict(raw_content=self.raw_content,

8
solum/tests/worker/handlers/test_noop.py
View File

@@ -35,15 +35,15 @@ class HandlerTest(base.BaseTestCase):
def test_build(self, fake_LOG):
git_info = test_shell.mock_git_info()
args = [5, git_info, 'new_app',
'1-2-3-4', 'heroku', 'docker', 44, None]
'1-2-3-4', 'heroku', 'docker', 44, None, 'fake-private-key']
noop_handler.Handler().build(self.ctx, *args)
message = 'Build %s %s %s %s %s %s %s %s' % tuple(args)
message = 'Build %s %s %s %s %s %s %s %s %s' % tuple(args)
fake_LOG.debug.assert_called_once_with(_("%s") % message)
@mock.patch('solum.worker.handlers.noop.LOG')
def test_unittest(self, fake_LOG):
git_info = test_shell.mock_git_info()
args = [5, git_info, 'pep8']
args = [5, git_info, 'pep8', 'fake-private-key']
noop_handler.Handler().unittest(self.ctx, *args)
message = 'Unittest %s %s %s' % tuple(args)
message = 'Unittest %s %s %s %s' % tuple(args)
fake_LOG.debug.assert_called_once_with(_("%s") % message)

84
solum/tests/worker/handlers/test_shell.py
View File

@@ -101,14 +101,60 @@ class HandlerTest(base.BaseTestCase):
handler.build(self.ctx, build_id=5, git_info=git_info,
name='new_app', base_image_id='1-2-3-4',
source_format='heroku', image_format='docker',
assembly_id=44, test_cmd=None)
assembly_id=44, test_cmd=None,
source_creds_ref=None)
proj_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
'..', '..', '..', '..'))
script = os.path.join(proj_dir, 'contrib/lp-cedarish/docker/build-app')
mock_popen.assert_called_once_with([script, 'git://example.com/foo',
'new_app', self.ctx.tenant,
'1-2-3-4'],
'1-2-3-4', ''], env=test_env,
stdout=-1)
expected = [mock.call(5, 'BUILDING', 'Starting the image build',
None, 44),
mock.call(5, 'COMPLETE', 'built successfully',
fake_glance_id, 44)]
self.assertEqual(expected, mock_b_update.call_args_list)
expected = [mock.call(assembly_id=44, image_id=fake_glance_id)]
self.assertEqual(expected, mock_deploy.call_args_list)
@mock.patch('solum.worker.handlers.shell.Handler._get_environment')
@mock.patch('solum.objects.registry')
@mock.patch('solum.conductor.api.API.build_job_update')
@mock.patch('solum.deployer.api.API.deploy')
@mock.patch('subprocess.Popen')
@mock.patch('solum.common.clients.OpenStackClients.barbican')
@mock.patch('ast.literal_eval')
def test_build_with_private_github_repo(
self, mock_ast, mock_barbican, mock_popen, mock_deploy,
mock_b_update, mock_registry, mock_get_env):
handler = shell_handler.Handler()
fake_assembly = fakes.FakeAssembly()
fake_glance_id = str(uuid.uuid4())
mock_registry.Assembly.get_by_id.return_value = fake_assembly
handler._update_assembly_status = mock.MagicMock()
mock_popen.return_value.communicate.return_value = [
'foo\ncreated_image_id=%s' % fake_glance_id, None]
test_env = mock_environment()
mock_get_env.return_value = test_env
mock_ast.return_value = [{'source_url': 'git://example.com/foo',
'private_key': 'some-private-key'}]
git_info = mock_git_info()
handler.build(self.ctx, build_id=5,
git_info=git_info, name='new_app',
base_image_id='1-2-3-4', source_format='heroku',
image_format='docker', assembly_id=44,
test_cmd=None, source_creds_ref='secret_ref_uri')
proj_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
'..', '..', '..', '..'))
script = os.path.join(proj_dir, 'contrib/lp-cedarish/docker/build-app')
mock_popen.assert_called_once_with([script, 'git://example.com/foo',
'new_app', self.ctx.tenant,
'1-2-3-4', 'some-private-key'],
env=test_env, stdout=-1)
expected = [mock.call(5, 'BUILDING', 'Starting the image build',
None, 44),
@@ -143,8 +189,9 @@ class HandlerTest(base.BaseTestCase):
script = os.path.join(proj_dir, 'contrib/lp-cedarish/docker/build-app')
mock_popen.assert_called_once_with([script, 'git://example.com/foo',
'new_app', self.ctx.tenant,
'1-2-3-4'],
'1-2-3-4', ''],
env=test_env, stdout=-1)
expected = [mock.call(5, 'BUILDING', 'Starting the image build',
None, 44),
mock.call(5, 'ERROR', 'image not created', None, 44)]
@@ -165,15 +212,16 @@ class HandlerTest(base.BaseTestCase):
mock_popen.return_value.wait.return_value = 0
git_info = mock_git_info()
handler.unittest(self.ctx, assembly_id=fake_assembly.id,
git_info=git_info, test_cmd='tox')
git_info=git_info, test_cmd='tox',
source_creds_ref=None)
proj_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
'..', '..', '..', '..'))
script = os.path.join(proj_dir,
'contrib/lp-cedarish/docker/unittest-app')
mock_popen.assert_called_once_with([script, 'git://example.com/foo',
'master', self.ctx.tenant, 'tox'],
env=test_env, stdout=-1)
'master', self.ctx.tenant, '',
'tox'], env=test_env, stdout=-1)
expected = [mock.call(self.ctx, 8, 'UNIT_TESTING')]
self.assertEqual(expected, mock_a_update.call_args_list)
@@ -182,8 +230,8 @@ class HandlerTest(base.BaseTestCase):
@mock.patch('solum.objects.registry')
@mock.patch('subprocess.Popen')
@mock.patch('solum.worker.handlers.shell.update_assembly_status')
def test_unittest_failure(self, mock_a_update, mock_popen, mock_registry,
mock_get_env):
def test_unittest_failure(self, mock_a_update, mock_popen,
mock_registry, mock_get_env):
handler = shell_handler.Handler()
fake_assembly = fakes.FakeAssembly()
mock_registry.Assembly.get_by_id.return_value = fake_assembly
@@ -192,15 +240,16 @@ class HandlerTest(base.BaseTestCase):
mock_popen.return_value.wait.return_value = 1
git_info = mock_git_info()
handler.unittest(self.ctx, assembly_id=fake_assembly.id,
git_info=git_info, test_cmd='tox')
git_info=git_info, test_cmd='tox',
source_creds_ref=None)
proj_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
'..', '..', '..', '..'))
script = os.path.join(proj_dir,
'contrib/lp-cedarish/docker/unittest-app')
mock_popen.assert_called_once_with([script, 'git://example.com/foo',
'master', self.ctx.tenant, 'tox'],
env=test_env, stdout=-1)
'master', self.ctx.tenant, '',
'tox'], env=test_env, stdout=-1)
expected = [mock.call(self.ctx, 8, 'UNIT_TESTING'),
mock.call(self.ctx, 8, 'UNIT_TESTING_FAILED')]
@@ -229,7 +278,7 @@ class HandlerTest(base.BaseTestCase):
handler.build(self.ctx, build_id=5, git_info=git_info, name='new_app',
base_image_id='1-2-3-4', source_format='heroku',
image_format='docker', assembly_id=44,
test_cmd='faketests')
test_cmd='faketests', source_creds_ref=None)
proj_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
'..', '..', '..', '..'))
@@ -239,9 +288,11 @@ class HandlerTest(base.BaseTestCase):
expected = [
mock.call([u_script, 'git://example.com/foo', 'master',
self.ctx.tenant, 'faketests'], env=test_env, stdout=-1),
self.ctx.tenant, '', 'faketests'], env=test_env,
stdout=-1),
mock.call([b_script, 'git://example.com/foo', 'new_app',
self.ctx.tenant, '1-2-3-4'], env=test_env, stdout=-1)]
self.ctx.tenant, '1-2-3-4', ''], env=test_env,
stdout=-1)]
self.assertEqual(expected, mock_popen.call_args_list)
expected = [mock.call(5, 'BUILDING', 'Starting the image build',
@@ -269,7 +320,7 @@ class HandlerTest(base.BaseTestCase):
handler.build(self.ctx, build_id=5, git_info=git_info, name='new_app',
base_image_id='1-2-3-4', source_format='heroku',
image_format='docker', assembly_id=44,
test_cmd='faketests')
test_cmd='faketests', source_creds_ref=None)
proj_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
'..', '..', '..', '..'))
@@ -278,7 +329,8 @@ class HandlerTest(base.BaseTestCase):
expected = [
mock.call([u_script, 'git://example.com/foo', 'master',
self.ctx.tenant, 'faketests'], env=test_env, stdout=-1)]
self.ctx.tenant, '', 'faketests'], env=test_env,
stdout=-1)]
self.assertEqual(expected, mock_popen.call_args_list)
expected = [mock.call(self.ctx, 44, 'UNIT_TESTING'),

12
solum/tests/worker/handlers/test_shell_nobuild.py
View File

@@ -59,7 +59,7 @@ class HandlerTest(base.BaseTestCase):
handler.build(self.ctx, build_id=5, git_info=git_info, name='new_app',
base_image_id='1-2-3-4', source_format='heroku',
image_format='docker', assembly_id=44,
test_cmd='faketests')
test_cmd='faketests', source_creds_ref=None)
expected = [
mock.call(status_url, 'POST',
@@ -70,6 +70,7 @@ class HandlerTest(base.BaseTestCase):
headers=test_shell.mock_request_hdr(status_token),
body=test_shell.mock_req_success_body(
'https://log.com/commit/SHA'))]
self.assertEqual(expected, mock_req.call_args_list)
proj_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
@@ -79,7 +80,8 @@ class HandlerTest(base.BaseTestCase):
expected = [
mock.call([u_script, 'git://example.com/foo', 'master',
self.ctx.tenant, 'faketests'], env=test_env, stdout=-1)]
self.ctx.tenant, '', 'faketests'], env=test_env,
stdout=-1)]
self.assertEqual(expected, mock_popen.call_args_list)
# The UNIT_TESTING update happens from shell...
@@ -112,7 +114,8 @@ class HandlerTest(base.BaseTestCase):
handler.build(self.ctx, build_id=5, git_info=git_info,
name='new_app', base_image_id='1-2-3-4',
source_format='heroku', image_format='docker',
assembly_id=44, test_cmd='faketests')
assembly_id=44, test_cmd='faketests',
source_creds_ref=None)
expected = [
mock.call(status_url, 'POST',
@@ -132,7 +135,8 @@ class HandlerTest(base.BaseTestCase):
expected = [
mock.call([u_script, 'git://example.com/foo', 'master',
self.ctx.tenant, 'faketests'], env=test_env, stdout=-1)]
self.ctx.tenant, '', 'faketests'], env=test_env,
stdout=-1)]
self.assertEqual(expected, mock_popen.call_args_list)
expected = [mock.call(self.ctx, 44, 'UNIT_TESTING'),

10
solum/worker/api.py
View File

@@ -27,12 +27,14 @@ class API(service.API):
topic=cfg.CONF.worker.topic)
def build(self, build_id, git_info, name, base_image_id,
source_format, image_format, assembly_id=None, test_cmd=None):
source_format, image_format, assembly_id=None,
test_cmd=None, source_creds_ref=None):
self._cast('build', build_id=build_id, git_info=git_info,
name=name, base_image_id=base_image_id,
source_format=source_format, image_format=image_format,
assembly_id=assembly_id, test_cmd=test_cmd)
assembly_id=assembly_id, test_cmd=test_cmd,
source_creds_ref=source_creds_ref)
def unittest(self, assembly_id, git_info, test_cmd):
def unittest(self, assembly_id, git_info, test_cmd, source_creds_ref=None):
self._cast('unittest', assembly_id=assembly_id, git_info=git_info,
test_cmd=test_cmd)
test_cmd=test_cmd, source_creds_ref=source_creds_ref)

15
solum/worker/handlers/noop.py
View File

@@ -24,13 +24,16 @@ class Handler(object):
LOG.debug("%s" % message)
def build(self, ctxt, build_id, source_uri, name, base_image_id,
source_format, image_format, assembly_id, test_cmd):
message = ("Build %s %s %s %s %s %s %s %s" %
source_format, image_format, assembly_id,
test_cmd, source_creds_ref=None):
message = ("Build %s %s %s %s %s %s %s %s %s" %
(build_id, source_uri, name, base_image_id, source_format,
image_format, assembly_id, test_cmd))
image_format, assembly_id,
test_cmd, (source_creds_ref or '')))
LOG.debug("%s" % message)
def unittest(self, ctxt, assembly_id, git_info, test_cmd):
message = ("Unittest %s %s %s" %
(assembly_id, git_info, test_cmd))
def unittest(self, ctxt, assembly_id, git_info, test_cmd,
source_creds_ref=None):
message = ("Unittest %s %s %s %s" %
(assembly_id, git_info, test_cmd, (source_creds_ref or '')))
LOG.debug("%s" % message)

47
solum/worker/handlers/shell.py
View File

@@ -14,6 +14,7 @@
"""Solum Worker shell handler."""
import ast
import json
import os
import subprocess
@@ -22,6 +23,7 @@ import httplib2
from oslo.config import cfg
import solum
from solum.common import clients
from solum.common import exception
from solum.common import solum_keystoneclient
from solum.conductor import api as conductor_api
@@ -97,7 +99,7 @@ class Handler(object):
'..', '..', '..'))
def _get_build_command(self, ctxt, source_uri, name, base_image_id,
source_format, image_format):
source_format, image_format, source_creds_ref=None):
# map the input formats to script paths.
# TODO(asalkeld) we need an "auto".
pathm = {'heroku': 'lp-cedarish',
@@ -111,8 +113,10 @@ class Handler(object):
pathm.get(source_format, 'lp-cedarish'),
pathm.get(image_format, 'vm-slug'),
'build-app')
return [build_app, source_uri, name, ctxt.tenant, base_image_id]
source_private_key = self._get_private_key(source_creds_ref,
source_uri)
return [build_app, source_uri, name, ctxt.tenant,
base_image_id, source_private_key]
def _send_status(self, test_result, status_url, status_token,
pending=False):
@@ -148,11 +152,13 @@ class Handler(object):
LOG.debug("No url or token available to send back status")
def build(self, ctxt, build_id, git_info, name, base_image_id,
source_format, image_format, assembly_id, test_cmd):
source_format, image_format, assembly_id,
test_cmd, source_creds_ref=None):
# TODO(datsun180b): This is only temporary, until Mistral becomes our
# workflow engine.
if self._run_unittest(ctxt, assembly_id, git_info, test_cmd) != 0:
if self._run_unittest(ctxt, assembly_id, git_info, test_cmd,
source_creds_ref) != 0:
return
update_assembly_status(ctxt, assembly_id, ASSEMBLY_STATES.BUILDING)
@@ -162,8 +168,9 @@ class Handler(object):
source_uri = git_info['source_url']
build_cmd = self._get_build_command(ctxt, source_uri, name,
base_image_id, source_format,
image_format)
base_image_id,
source_format, image_format,
source_creds_ref)
solum.TLS.trace.support_info(build_cmd=' '.join(build_cmd),
assembly_id=assembly_id)
@@ -174,7 +181,6 @@ class Handler(object):
job_update_notification(ctxt, build_id, IMAGE_STATES.ERROR,
description=str(env_ex),
assembly_id=assembly_id)
return
log_env = user_env.copy()
if 'OS_AUTH_TOKEN' in log_env:
@@ -218,7 +224,8 @@ class Handler(object):
deployer_api.API(context=ctxt).deploy(assembly_id=assembly_id,
image_id=created_image_id)
def _run_unittest(self, ctxt, assembly_id, git_info, test_cmd):
def _run_unittest(self, ctxt, assembly_id, git_info, test_cmd,
source_creds_ref=None):
if test_cmd is None:
LOG.debug("Unit test command is None; skipping unittests.")
return 0
@@ -231,7 +238,9 @@ class Handler(object):
test_app = os.path.join(self.proj_dir, 'contrib', 'lp-cedarish',
'docker', 'unittest-app')
git_url = git_info['source_url']
command = [test_app, git_url, git_branch, ctxt.tenant, test_cmd]
source_private_key = self._get_private_key(source_creds_ref, git_url)
command = [test_app, git_url, git_branch, ctxt.tenant,
source_private_key, test_cmd]
solum.TLS.trace.clear()
solum.TLS.trace.import_context(ctxt)
@@ -262,5 +271,19 @@ class Handler(object):
return returncode
def unittest(self, ctxt, assembly_id, git_info, test_cmd):
self._run_unittest(ctxt, assembly_id, git_info, test_cmd)
def unittest(self, ctxt, assembly_id, git_info, test_cmd,
source_creds_ref=None):
self._run_unittest(ctxt, assembly_id, git_info, test_cmd,
source_creds_ref)
def _get_private_key(self, source_creds_ref, source_url):
source_private_key = ''
if source_creds_ref:
barbican = clients.OpenStackClients(None).barbican().admin_client
secret = barbican.secrets.Secret(secret_ref=source_creds_ref)
deploy_keys_str = secret.payload
deploy_keys = ast.literal_eval(deploy_keys_str)
for dk in deploy_keys:
if source_url == dk['source_url']:
source_private_key = dk['private_key']
return source_private_key

6
solum/worker/handlers/shell_nobuild.py
View File

@@ -28,7 +28,8 @@ update_assembly_status = shell_handler.update_assembly_status
class Handler(shell_handler.Handler):
def build(self, ctxt, build_id, git_info, name, base_image_id,
source_format, image_format, assembly_id, test_cmd):
source_format, image_format, assembly_id,
test_cmd, source_creds_ref=None):
# TODO(datsun180b): This is only temporary, until Mistral becomes our
# workflow engine.
@@ -37,7 +38,8 @@ class Handler(shell_handler.Handler):
status_token = git_info.get('status_token')
self._send_status(ret_code, status_url, status_token, pending=True)
ret_code = self._run_unittest(ctxt, assembly_id, git_info, test_cmd)
ret_code = self._run_unittest(ctxt, assembly_id, git_info, test_cmd,
source_creds_ref)
self._send_status(ret_code, status_url, status_token)
# Deployer is normally in charge of declaring an assembly READY.