openstack
/
barbican
Code Issues Proposed changes

Add files to create worker RPM and get api/db/queue/worker nodes deployed. 

As part of building out the Barbican network of nodes, this commit adds files
and configuration to create an RPM package for the Barbican worker nodes. Files
needed to make this work as part of a network of Barbican API, queue, worker
and database nodes are also added.

Change-Id: I87a36b7c09bbd3fec7661ce6adbcbb5b0c6df689
This commit is contained in:
jfwood 2013-09-11 18:30:57 -05:00
parent a9dfab1778
commit 2e65939bfd
17 changed files with 161 additions and 238 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
barbican/crypto/extension_manager.py
View File

@ -251,6 +251,12 @@ class CryptoExtensionManager(named.NamedExtensionManager):
datum.cypher_text, datum.kek_meta_extended = encrypting_plugin.encrypt(
unencrypted, kek_meta_dto, tenant.keystone_id
)
# Convert binary data into a text-based format.
#TODO(jwood) Figure out by storing binary (BYTEA) data in Postgres
# isn't working.
datum.cypher_text = base64.b64encode(datum.cypher_text)
return datum
def decrypt(self, content_type, secret, tenant):
@ -269,9 +275,14 @@ class CryptoExtensionManager(named.NamedExtensionManager):
# wrap the KEKDatum instance in our DTO
kek_meta_dto = plugin_mod.KEKMetaDTO(datum.kek_meta_tenant)
# Convert from text-based storage format to binary.
#TODO(jwood) Figure out by storing binary (BYTEA) data in
# Postgres isn't working.
encrypted = base64.b64decode(datum.cypher_text)
# Decrypt the secret.
unencrypted = decrypting_plugin \
.decrypt(datum.cypher_text,
.decrypt(encrypted,
kek_meta_dto,
datum.kek_meta_extended,
tenant.keystone_id)

26
barbican/model/migration/alembic_migrations/versions/153ed0150d78_change_test_column_b.py
View File

@ -1,26 +0,0 @@
"""change test column back to not null
Revision ID: 153ed0150d78
Revises: 40a9c7408b51
Create Date: 2013-06-18 17:33:20.281076
"""
# revision identifiers, used by Alembic.
revision = '153ed0150d78'
down_revision = '40a9c7408b51'
from alembic import op
import sqlalchemy as sa
def upgrade():
op.alter_column('test', u'name',
existing_type=sa.String(50),
nullable=False)
def downgrade():
op.alter_column('test', u'name',
existing_type=sa.String(50),
nullable=True)

27
barbican/model/migration/alembic_migrations/versions/1a0c2cdafb38_create_test_table.py
View File

@ -1,27 +0,0 @@
"""create test table
Revision ID: 1a0c2cdafb38
Revises: 40a9c7408b51
Create Date: 2013-06-17 16:42:13.634746
"""
# revision identifiers, used by Alembic.
revision = '1a0c2cdafb38'
down_revision = None
from alembic import op
import sqlalchemy as sa
def upgrade():
op.create_table(
'test',
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('name', sa.String(50), nullable=False),
sa.Column('description', sa.Unicode(200)),
)
def downgrade():
op.drop_table('test')

19
barbican/model/migration/alembic_migrations/versions/1a0c2cdafb38_initial_version.py Normal file
View File

@ -0,0 +1,19 @@
"""create test table
Revision ID: 1a0c2cdafb38
Revises: None
Create Date: 2013-06-17 16:42:13.634746
"""
# revision identifiers, used by Alembic.
revision = '1a0c2cdafb38'
down_revision = None
def upgrade():
pass
def downgrade():
pass

22
barbican/model/migration/alembic_migrations/versions/1cfb48928f42_test_column_add.py
View File

@ -1,22 +0,0 @@
"""test column add
Revision ID: 1cfb48928f42
Revises: 153ed0150d78
Create Date: 2013-06-19 00:15:03.656628
"""
# revision identifiers, used by Alembic.
revision = '1cfb48928f42'
down_revision = '153ed0150d78'
from alembic import op
from sqlalchemy import Column, String
def upgrade():
op.add_column('secrets', Column('dummy_column', String()))
def downgrade():
op.drop_column('secrets', 'dummy_column')

26
barbican/model/migration/alembic_migrations/versions/40a9c7408b51_test_auto_migration.py
View File

@ -1,26 +0,0 @@
"""Test auto migration
Revision ID: 40a9c7408b51
Revises: None
Create Date: 2013-06-17 10:42:20.078204
"""
# revision identifiers, used by Alembic.
revision = '40a9c7408b51'
down_revision = '1a0c2cdafb38'
from alembic import op
import sqlalchemy as sa
def upgrade():
op.alter_column('test', u'name',
existing_type=sa.String(50),
nullable=True)
def downgrade():
op.alter_column('test', u'name',
existing_type=sa.String(50),
nullable=False)

33
barbican/model/migration/alembic_migrations/versions/53c2ae2df15d_test_change.py
View File

@ -1,33 +0,0 @@
"""test change
Revision ID: 53c2ae2df15d
Revises: 1cfb48928f42
Create Date: 2013-06-19 23:35:52.802639
"""
# revision identifiers, used by Alembic.
revision = '53c2ae2df15d'
down_revision = '1cfb48928f42'
from alembic import op
import sqlalchemy as sa
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.drop_table(u'test')
op.drop_column('secrets', u'dummy_column')
### end Alembic commands ###
def downgrade():
op.add_column('secrets', sa.Column(u'dummy_column', sa.VARCHAR(),
nullable=True))
op.create_table(
'test',
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('name', sa.String(50), nullable=False),
sa.Column('description', sa.Unicode(200)),
)

40
barbican/model/migration/alembic_migrations/versions/b4ba6a2a663_bp_mime_type_revamp.py
View File

@ -1,40 +0,0 @@
"""bp/mime-type-revamp
Revision ID: b4ba6a2a663
Revises: 53c2ae2df15d
Create Date: 2013-07-26 15:25:44.193889
"""
# revision identifiers, used by Alembic.
revision = 'b4ba6a2a663'
down_revision = '53c2ae2df15d'
from alembic import op
import sqlalchemy as sa
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.add_column('encrypted_data', sa.Column('content_type',
sa.String(length=255),
nullable=True))
op.alter_column('secrets', u'mime_type',
existing_type=sa.VARCHAR(length=255),
nullable=True)
op.alter_column('orders', u'secret_mime_type',
existing_type=sa.VARCHAR(length=255),
nullable=True)
### end Alembic commands ###
def downgrade():
### commands auto generated by Alembic - please adjust! ###
op.alter_column('secrets', u'mime_type',
existing_type=sa.VARCHAR(length=255),
nullable=False)
op.alter_column('orders', u'secret_mime_type',
existing_type=sa.VARCHAR(length=255),
nullable=True)
op.drop_column('encrypted_data', 'content_type')
### end Alembic commands ###

5
barbican/model/models.py
View File

@ -233,7 +233,10 @@ class EncryptedDatum(BASE, ModelBase):
kek_id = sa.Column(sa.String(36), sa.ForeignKey('kek_data.id'),
nullable=False)
content_type = sa.Column(sa.String(255))
cypher_text = sa.Column(sa.LargeBinary)
# TODO(jwood) Why LargeBinary on Postgres (BYTEA) not work correctly?
cypher_text = sa.Column(sa.Text)
kek_meta_extended = sa.Column(sa.Text)
kek_meta_tenant = orm.relationship("KEKDatum")

8
barbican/tests/api/test_resources.py
View File

@ -207,7 +207,7 @@ class BaseSecretsResource(unittest.TestCase):
args, kwargs = self.datum_repo.create_from.call_args
datum = args[0]
self.assertIsInstance(datum, models.EncryptedDatum)
self.assertEqual('cypher_text', datum.cypher_text)
self.assertEqual(base64.b64encode('cypher_text'), datum.cypher_text)
self.assertEqual(self.payload_content_type, datum.content_type)
validate_datum(self, datum)
@ -576,7 +576,7 @@ class WhenGettingPuttingOrDeletingSecretUsingSecretResource(unittest.TestCase):
self.datum.kek_id = kek_id
self.datum.kek_meta_tenant = self.kek_tenant
self.datum.content_type = "text/plain"
self.datum.cypher_text = "cypher_text"
self.datum.cypher_text = "aaaa" # base64 value.
self.secret = create_secret(id=secret_id,
name=self.name,
@ -653,6 +653,7 @@ class WhenGettingPuttingOrDeletingSecretUsingSecretResource(unittest.TestCase):
def test_should_get_secret_meta_for_binary(self):
self.req.accept = 'application/json'
self.datum.content_type = "application/octet-stream"
self.datum.cypher_text = 'aaaa'
self.resource.on_get(self.req, self.resp, self.keystone_id,
self.secret.id)
@ -675,6 +676,7 @@ class WhenGettingPuttingOrDeletingSecretUsingSecretResource(unittest.TestCase):
# mock Content-Encoding header
self.req.get_header.return_value = None
self.datum.content_type = 'application/octet-stream'
self.datum.cypher_text = 'aaaa'
self.resource.on_get(self.req, self.resp, self.keystone_id,
self.secret.id)
@ -721,7 +723,7 @@ class WhenGettingPuttingOrDeletingSecretUsingSecretResource(unittest.TestCase):
args, kwargs = self.datum_repo.create_from.call_args
datum = args[0]
self.assertIsInstance(datum, models.EncryptedDatum)
self.assertEqual('cypher_text', datum.cypher_text)
self.assertEqual(base64.b64encode('cypher_text'), datum.cypher_text)
validate_datum(self, datum)

2
barbican/tests/crypto/test_plugin.py
View File

@ -24,7 +24,7 @@ class TestCryptoPlugin(plugin.CryptoPluginBase):
"""Crypto plugin implementation for testing the plugin manager."""
def encrypt(self, unencrypted, kek_meta_dto, keystone_id):
cypher_text = 'cypher_text'
cypher_text = b'cypher_text'
return cypher_text, None
def decrypt(self, encrypted, kek_meta_dto, kek_meta_extended, keystone_id):

58
bin/barbican-worker
View File

@ -1,57 +1,5 @@
#!/usr/bin/env python
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#!/bin/bash
# Copyright (c) 2013 Rackspace, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# barbican-worker - Script run Cloudkeep's Barbican worker app.
"""
Barbican worker server.
"""
import gettext
import os
import sys
# 'Borrowed' from the Glance project:
# If ../barbican/__init__.py exists, add ../ to Python search path, so that
# it will override what happens to be installed in /usr/(local/)lib/python...
possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
os.pardir,
os.pardir))
if os.path.exists(os.path.join(possible_topdir, 'barbican', '__init__.py')):
sys.path.insert(0, possible_topdir)
gettext.install('barbican', unicode=1)
from barbican.common import config
from barbican.common import exception
from barbican.openstack.common import log
from barbican.queue.celery.resources import celery
def fail(returncode, e):
sys.stderr.write("ERROR: {0}\n".format(e))
sys.exit(returncode)
if __name__ == '__main__':
try:
config.parse_args()
log.setup('barbican')
celery.worker_main()
except RuntimeError as e:
fail(1, e)
python bin/barbican-worker.py

63
bin/barbican-worker.py Executable file
View File

@ -0,0 +1,63 @@
#!/usr/bin/env python
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (c) 2013 Rackspace, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Barbican worker server.
"""
import gettext
import os
import sys
# 'Borrowed' from the Glance project:
# If ../barbican/__init__.py exists, add ../ to Python search path, so that
# it will override what happens to be installed in /usr/(local/)lib/python...
possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
os.pardir,
os.pardir))
if os.path.exists(os.path.join(possible_topdir, 'barbican', '__init__.py')):
sys.path.insert(0, possible_topdir)
gettext.install('barbican', unicode=1)
from barbican.common import config
from barbican.openstack.common import log
def fail(returncode, e):
sys.stderr.write("ERROR: {0}\n".format(e))
sys.exit(returncode)
if __name__ == '__main__':
try:
config.parse_args()
# Import and configure logging.
log.setup('barbican')
LOG = log.getLogger(__name__)
LOG.debug("Booting up Barbican worker node...")
# Import after the 'parse_args()' call, to ensure we apply configuration
# parameters to the Celery server used below.
from barbican.queue.celery.resources import celery
celery.worker_main()
except RuntimeError as e:
fail(1, e)

10
debian/barbican-worker.upstart vendored Normal file
View File

@ -0,0 +1,10 @@
# Barbican Worker upstart script
# Used in deb build. Keep in sync with etc/init/barbican-worker.conf
description "Barbican Key Management Workers"
start on runlevel [2345]
stop on runlevel [06]
script
/usr/bin/barbican-worker
end script

10
etc/init/barbican-worker.conf Normal file
View File

@ -0,0 +1,10 @@
# Barbican Worker node upstart script
# Used in rpm build. Keep in sync with debian/barbican-worker.upstart
description "Barbican Key Management Workers"
start on runlevel [2345]
stop on runlevel [06]
script
barbican-worker.py >> /var/log/barbican/barbican_worker.log 2>&1
end script

34
rpmbuild/SPECS/barbican.spec
View File

@ -22,7 +22,7 @@ Requires: python-psycopg2, python-pysqlite, python-sqlalchemy, python-stevedore
Requires: python-uwsgi, python-webob, python-netaddr, python-babel
%description
Common files for Barbican Key Management API (barbican-api) and
Common files for Barbican Key Management API (barbican-api) and
Barbican Worker (barbican-worker)
%prep
@ -33,11 +33,12 @@ python setup.py build
%install
python setup.py install -O1 --root $RPM_BUILD_ROOT
mkdir -p $RPM_BUILD_ROOT/etc/barbican/vassals
mkdir -p $RPM_BUILD_ROOT/etc/init
mkdir -p $RPM_BUILD_ROOT/etc/barbican/vassals
mkdir -p $RPM_BUILD_ROOT/var/l{ib,og}/barbican
install etc/barbican/policy.json $RPM_BUILD_ROOT/etc/barbican
install etc/init/barbican-api.conf $RPM_BUILD_ROOT/etc/init
install etc/init/barbican-worker.conf $RPM_BUILD_ROOT/etc/init
install -D etc/barbican/barbican* $RPM_BUILD_ROOT/etc/barbican
install -D etc/barbican/vassals/*.ini $RPM_BUILD_ROOT/etc/barbican/vassals
touch $RPM_BUILD_ROOT/var/log/barbican/barbican-api.log
@ -62,7 +63,10 @@ rm -rf $RPM_BUILD_ROOT
%{python_sitelib}/*
%dir /var/lib/barbican
# ------------------
# API package
# ------------------
%package -n barbican-api
Summary: Barbican Key Manager API daemon
Requires: barbican-common
@ -84,3 +88,29 @@ if [ $1 -eq 0 ] ; then
/sbin/stop barbican-api >/dev/null 2>&1 || :
fi
# ------------------
# Worker package
# ------------------
%package -n barbican-worker
Summary: Barbican Key Manager worker daemon
Requires: barbican-common
%description -n barbican-worker
Barbican Key Manager worker daemon
%files -n barbican-worker
%defattr(-,root,root)
%dir /var/lib/barbican
%verify(not md5 size mtime) %attr(0750, barbican,root) /var/log/barbican/barbican-api.log
/etc/logrotate.d/barbican-api
%attr(0755,root,root) /usr/bin/barbican-worker.py
%config(noreplace) /etc/init/barbican-worker.conf
%config(noreplace) /etc/barbican/*
%preun -n barbican-worker
if [ $1 -eq 0 ] ; then
# Package removal, not upgrade
/sbin/stop barbican-worker >/dev/null 2>&1 || :
fi

3
setup.py
View File

@ -71,7 +71,8 @@ setup(
'Programming Language :: Python :: 2.7',
'Environment :: No Input/Output (Daemon)',
],
scripts=['bin/barbican-all'],
scripts=['bin/barbican-all', 'bin/barbican-worker',
'bin/barbican-worker.py'],
py_modules=[],
entry_points="""
[barbican.crypto.plugin]