Add Sync template feature to KB-server

This commit from kingbird server is to enhance the existing "sync create" job.
Existing feature "sync create" is to sync single resource-type
to single target region i.e one resource off one resource-type
to one region.
Issue with "sync create":
Kingbird cannot sync more than one resource-type to multiple
target regions in a single request.
This issue is solved by "sync template" feature.
Working of sync template feature:
User have to provide a input file with .yaml or .yml or .json extension.
This input file should consists of
    ->resource-type
    ->resources
    ->source_region
    ->target_region.

Change-Id: Iaa8e9568f97581f74e688d15a11c7e7fd832e019
This commit is contained in:
mounikasreeram 2017-11-22 19:09:14 +05:30
parent fd6b4cba10
commit a364c4b11c
12 changed files with 111 additions and 40 deletions

View File

@ -55,11 +55,12 @@ class ResourceSyncController(object):
pass
def _entries_to_database(self, context, target_regions, source_region,
resources, resource_type, job_id):
resources, resource_type, job_id, job_name):
"""Manage the entries to database for both Keypair and image."""
# Insert into the parent table
try:
result = db_api.sync_job_create(context, job_id=job_id)
result = db_api.sync_job_create(context, job_name, job_id=job_id)
except exceptions.InternalError:
pecan.abort(500, _('Internal Server Error.'))
# Insert into the child table
@ -107,17 +108,32 @@ class ResourceSyncController(object):
context = restcomm.extract_context_from_environ()
if not uuidutils.is_uuid_like(project) or project != context.project:
pecan.abort(400, _('Invalid request URL'))
payload = eval(request.body)
if not payload:
request_data = eval(request.body)
if not request_data:
pecan.abort(400, _('Body required'))
payload = payload.get('resource_set')
if not payload:
request_data = request_data.get('resource_set')
if not request_data:
pecan.abort(400, _('resource_set required'))
job_name = None
if 'name' in request_data.keys():
job_name = request_data.get('name')
for iteration in range(len(request_data['Sync'])):
payload = request_data['Sync'][iteration]
response = self._get_post_data(payload,
context, job_name)
else:
response = self._get_post_data(request_data,
context, job_name)
return response
def _get_post_data(self, payload, context, job_name):
resource_type = payload.get('resource_type')
target_regions = payload.get('target')
if not target_regions or not isinstance(target_regions, list):
pecan.abort(400, _('Target regions required'))
source_region = payload.get('source')
if(isinstance(source_region, list)):
source_region = "".join(source_region)
if not source_region or not isinstance(source_region, str):
pecan.abort(400, _('Source region required'))
source_resources = payload.get('resources')
@ -138,7 +154,8 @@ class ResourceSyncController(object):
result = self._entries_to_database(context, target_regions,
source_region,
source_resources,
resource_type, job_id)
resource_type,
job_id, job_name)
return self._keypair_sync(job_id, payload, context, result)
elif resource_type == consts.IMAGE:
@ -152,7 +169,8 @@ class ResourceSyncController(object):
result = self._entries_to_database(context, target_regions,
source_region,
source_resources,
resource_type, job_id)
resource_type,
job_id, job_name)
return self._image_sync(job_id, payload, context, result)
elif resource_type == consts.FLAVOR:
@ -169,7 +187,8 @@ class ResourceSyncController(object):
result = self._entries_to_database(context, target_regions,
source_region,
source_resources,
resource_type, job_id)
resource_type,
job_id, job_name)
return self._flavor_sync(job_id, payload, context, result)
else:
@ -212,8 +231,8 @@ class ResourceSyncController(object):
:param result: Result object to return an output.
"""
self.rpc_client.keypair_sync_for_user(context, job_id, payload)
return {'job_status': {'id': result.id, 'status': result.sync_status,
return {'job_status': {'name': result.name, 'id': result.id,
'status': result.sync_status,
'created_at': result.created_at}}
def _image_sync(self, job_id, payload, context, result):
@ -226,7 +245,8 @@ class ResourceSyncController(object):
:param result: Result object to return an output.
"""
self.rpc_client.image_sync(context, job_id, payload)
return {'job_status': {'id': result.id, 'status': result.sync_status,
return {'job_status': {'name': result.name, 'id': result.id,
'status': result.sync_status,
'created_at': result.created_at}}
def _flavor_sync(self, job_id, payload, context, result):
@ -239,5 +259,6 @@ class ResourceSyncController(object):
:param result: Result object to return an output.
"""
self.rpc_client.flavor_sync(context, job_id, payload)
return {'job_status': {'id': result.id, 'status': result.sync_status,
return {'job_status': {'name': result.name, 'id': result.id,
'status': result.sync_status,
'created_at': result.created_at}}

View File

@ -383,9 +383,11 @@ def service_get_all(context):
##########################
@require_context
def sync_job_create(context, job_id):
def sync_job_create(context, job_name, job_id):
with write_session() as session:
sjc = models.SyncJob()
if job_name is not None:
sjc.name = job_name
sjc.id = job_id
sjc.user_id = context.user
sjc.project_id = context.project

View File

@ -22,6 +22,7 @@ def upgrade(migrate_engine):
sync_job = sqlalchemy.Table(
'sync_job', meta,
sqlalchemy.Column('name', sqlalchemy.String(255)),
sqlalchemy.Column('id', sqlalchemy.String(36),
primary_key=True),
sqlalchemy.Column('sync_status', sqlalchemy.String(length=36),

View File

@ -154,6 +154,8 @@ class SyncJob(BASE, KingbirdBase):
__tablename__ = 'sync_job'
name = Column('name', String(255))
id = Column('id', String(36), primary_key=True)
sync_status = Column(String(36), default=consts.JOB_PROGRESS,

View File

@ -136,6 +136,7 @@ class NovaClient(base.DriverBase):
def get_flavor(self, res_id):
"""Get Flavor for a specified context."""
try:
res_id = self.nova_client.flavors.find(name=res_id)
flavor = self.nova_client.flavors.get(res_id)
LOG.info("Source Flavor: %s", flavor.name)
return flavor

View File

@ -58,7 +58,7 @@ class FlavorSyncManager(object):
% {'flavor': source_flavor.name, 'region': region})
try:
db_api.resource_sync_update(context, job_id, region,
source_flavor.id,
source_flavor.name,
consts.JOB_SUCCESS)
except exceptions.JobNotFound():
raise
@ -67,7 +67,7 @@ class FlavorSyncManager(object):
% {'msg': exc.message, 'region': region})
try:
db_api.resource_sync_update(context, job_id, region,
source_flavor.id,
source_flavor.name,
consts.JOB_FAILURE)
except exceptions.JobNotFound():
raise
@ -87,6 +87,8 @@ class FlavorSyncManager(object):
force = eval(str(payload.get('force', False)))
resource_ids = payload.get('resources')
source_region = payload['source']
if(isinstance(source_region, list)):
source_region = "".join(source_region)
session = EndpointCache().get_session_from_token(
context.auth_token, context.project)
# Create Source Region object

View File

@ -186,6 +186,8 @@ class ImageSyncManager(object):
force = eval(str(payload.get('force', False)))
resource_ids = payload.get('resources')
source_region = payload['source']
if(isinstance(source_region, list)):
source_region = "".join(source_region)
for resource in resource_ids:
thread = threading.Thread(target=self.create_resources_in_region,
args=(job_id, target_regions,

View File

@ -84,6 +84,8 @@ class KeypairSyncManager(object):
force = eval(str(payload.get('force', False)))
resource_ids = payload.get('resources')
source_region = payload['source']
if(isinstance(source_region, list)):
source_region = "".join(source_region)
session = EndpointCache().get_session_from_token(
context.auth_token, context.project)
# Create Source Region object

View File

@ -25,6 +25,7 @@ from kingbird.tests.unit.api import test_root_controller as testroot
from kingbird.tests import utils
DEFAULT_FORCE = False
JOB_NAME = 'fake_job_name'
SOURCE_KEYPAIR = 'fake_key1'
SOURCE_FLAVOR = 'fake_flavor1'
SOURCE_IMAGE_NAME = 'fake_image'
@ -70,15 +71,17 @@ class FakeImage(object):
class Result(object):
def __init__(self, job_id, status, time):
def __init__(self, job_id, name, status, time):
self.job_id = job_id
self.name = name
self.status = status
self.time = time
class SyncJob(object):
def __init__(self, id, sync_status, created_at):
def __init__(self, id, name, sync_status, created_at):
self.id = id
self.name = name
self.sync_status = sync_status
self.created_at = created_at
@ -88,6 +91,21 @@ class TestResourceManager(testroot.KBApiTest):
super(TestResourceManager, self).setUp()
self.ctx = utils.dummy_context()
@mock.patch.object(rpc_client, 'EngineClient')
@mock.patch.object(sync_manager, 'NovaClient')
@mock.patch.object(sync_manager, 'EndpointCache')
@mock.patch.object(sync_manager, 'db_api')
def test_post_request_data(self, mock_db_api, mock_endpoint_cache,
mock_nova, mock_rpc_client):
payload = {"resources": [SOURCE_KEYPAIR],
"resource_type": "keypair",
"source": FAKE_SOURCE_REGION,
"target": [FAKE_TARGET_REGION]}
result = sync_manager.ResourceSyncController().\
_get_post_data(payload, self.ctx, JOB_NAME)
self.assertEqual(result['job_status'].get('status'),
mock_db_api.sync_job_create().sync_status)
@mock.patch.object(rpc_client, 'EngineClient')
@mock.patch.object(sync_manager, 'NovaClient')
@mock.patch.object(sync_manager, 'EndpointCache')
@ -101,7 +119,8 @@ class TestResourceManager(testroot.KBApiTest):
"source": FAKE_SOURCE_REGION,
"target": [FAKE_TARGET_REGION]}}
fake_key = FakeKeypair('fake_name', 'fake-rsa')
sync_job_result = SyncJob(FAKE_JOB, consts.JOB_PROGRESS, time_now)
sync_job_result = SyncJob(JOB_NAME, FAKE_JOB,
consts.JOB_PROGRESS, time_now)
mock_endpoint_cache().get_session_from_token.\
return_value = 'fake_session'
mock_nova().get_keypairs.return_value = fake_key
@ -129,8 +148,10 @@ class TestResourceManager(testroot.KBApiTest):
"force": "True",
"source": FAKE_SOURCE_REGION,
"target": [FAKE_TARGET_REGION]}}
fake_flavor = Fake_Flavor('fake_id', 512, 2, 30, 'fake_flavor', 1)
sync_job_result = SyncJob(FAKE_JOB, consts.JOB_PROGRESS, time_now)
fake_flavor = Fake_Flavor('fake_id', 512, 2,
30, 'fake_flavor', 1)
sync_job_result = SyncJob(JOB_NAME, FAKE_JOB,
consts.JOB_PROGRESS, time_now)
mock_endpoint_cache().get_session_from_token.\
return_value = 'fake_session'
mock_nova().get_flavor.return_value = fake_flavor
@ -172,7 +193,8 @@ class TestResourceManager(testroot.KBApiTest):
"source": FAKE_SOURCE_REGION,
"target": [FAKE_TARGET_REGION]}}
fake_image = FakeImage(SOURCE_IMAGE_ID, SOURCE_IMAGE_NAME)
sync_job_result = SyncJob(FAKE_JOB, consts.JOB_PROGRESS, time_now)
sync_job_result = SyncJob(JOB_NAME, FAKE_JOB,
consts.JOB_PROGRESS, time_now)
mock_glance().check_image.return_value = fake_image.id
mock_db_api.sync_job_create.return_value = sync_job_result
response = self.app.post_json(FAKE_URL,
@ -368,11 +390,11 @@ class TestResourceManager(testroot.KBApiTest):
@mock.patch.object(sync_manager, 'db_api')
def test_entries_to_database(self, mock_db_api, mock_rpc_client):
time_now = timeutils.utcnow()
result = Result(FAKE_JOB, FAKE_STATUS, time_now)
result = Result(JOB_NAME, FAKE_JOB, FAKE_STATUS, time_now)
mock_db_api.sync_job_create.return_value = result
sync_manager.ResourceSyncController()._entries_to_database(
self.ctx, FAKE_TARGET_REGION, FAKE_SOURCE_REGION,
FAKE_RESOURCE_ID, FAKE_RESOURCE_TYPE, FAKE_JOB)
FAKE_RESOURCE_ID, FAKE_RESOURCE_TYPE, FAKE_TENANT, JOB_NAME)
mock_db_api.resource_sync_create.assert_called_once_with(
self.ctx, result, FAKE_TARGET_REGION[0], FAKE_SOURCE_REGION,
FAKE_RESOURCE_ID[0], FAKE_RESOURCE_TYPE)

View File

@ -69,7 +69,8 @@ class DBAPIResourceSyncTest(base.KingbirdTestCase):
self.ctx = utils.dummy_context()
def test_create_sync_job(self):
job = self.sync_job_create(self.ctx, job_id=UUID1)
job = self.sync_job_create(self.ctx, job_name='fake_job_name',
job_id=UUID1)
self.assertIsNotNone(job)
self.assertEqual(consts.JOB_PROGRESS, job.sync_status)
created_job = db_api.sync_job_list(self.ctx, "active")
@ -77,38 +78,44 @@ class DBAPIResourceSyncTest(base.KingbirdTestCase):
created_job[0].get('sync_status'))
def test_primary_key_sync_job(self):
self.sync_job_create(self.ctx, job_id=UUID1)
self.sync_job_create(self.ctx, job_name='fake_job_name', job_id=UUID1)
self.assertRaises(oslo_db.exception.DBDuplicateEntry,
self.sync_job_create, self.ctx, job_id=UUID1)
self.sync_job_create, self.ctx,
job_name='fake_job_name', job_id=UUID1)
def test_sync_job_update(self):
job = self.sync_job_create(self.ctx, job_id=UUID1)
job = self.sync_job_create(self.ctx, job_name='fake_job_name',
job_id=UUID1)
self.assertIsNotNone(job)
db_api.sync_job_update(self.ctx, UUID1, consts.JOB_SUCCESS)
updated_job = db_api.sync_job_list(self.ctx)
self.assertEqual(consts.JOB_SUCCESS, updated_job[0].get('sync_status'))
def test_active_jobs(self):
job = self.sync_job_create(self.ctx, job_id=UUID1)
job = self.sync_job_create(self.ctx, job_name='fake_job_name',
job_id=UUID1)
self.assertIsNotNone(job)
query = db_api.sync_job_list(self.ctx, 'active')
self.assertEqual(query[0].get('sync_status'), job.sync_status)
def test_sync_job_status(self):
job = self.sync_job_create(self.ctx, job_id=UUID1)
job = self.sync_job_create(self.ctx, job_name='fake_job_name',
job_id=UUID1)
self.assertIsNotNone(job)
query = db_api.sync_job_status(self.ctx, job_id=UUID1)
self.assertEqual(query, consts.JOB_PROGRESS)
def test_update_invalid_job(self):
job = self.sync_job_create(self.ctx, job_id=UUID1)
job = self.sync_job_create(self.ctx, job_name='fake_job_name',
job_id=UUID1)
self.assertIsNotNone(job)
self.assertRaises(exceptions.JobNotFound,
db_api.sync_job_update, self.ctx, 'fake_job',
consts.JOB_SUCCESS)
def test_resource_sync_create(self):
job = self.sync_job_create(self.ctx, job_id=UUID1)
job = self.sync_job_create(self.ctx, job_name='fake_job_name',
job_id=UUID1)
resource_sync_create = self.resource_sync_create(
self.ctx, job=job, region='Fake_region',
source_region='Fake_region2', resource='fake_key',
@ -117,7 +124,8 @@ class DBAPIResourceSyncTest(base.KingbirdTestCase):
self.assertEqual(consts.JOB_PROGRESS, resource_sync_create.sync_status)
def test_resource_sync_status(self):
job = self.sync_job_create(self.ctx, job_id=UUID1)
job = self.sync_job_create(self.ctx, job_name='fake_job_name',
job_id=UUID1)
resource_sync_create = self.resource_sync_create(
self.ctx, job=job, region='Fake_region',
source_region='Fake_region2', resource='fake_key',
@ -127,7 +135,8 @@ class DBAPIResourceSyncTest(base.KingbirdTestCase):
self.assertEqual(consts.JOB_PROGRESS, status[0])
def test_resource_sync_update(self):
job = self.sync_job_create(self.ctx, job_id=UUID1)
job = self.sync_job_create(self.ctx, job_name='fake_job_name',
job_id=UUID1)
resource_sync_create = self.resource_sync_create(
self.ctx, job=job, region='Fake_region',
source_region='Fake_region2', resource='fake_key',
@ -140,7 +149,8 @@ class DBAPIResourceSyncTest(base.KingbirdTestCase):
self.assertEqual(consts.JOB_SUCCESS, updated_job[0].get('sync_status'))
def test_foreign_key(self):
job = self.sync_job_create(self.ctx, job_id=UUID1)
job = self.sync_job_create(self.ctx, job_name='fake_job_name',
job_id=UUID1)
self.assertIsNotNone(job)
resource_sync_create = self.resource_sync_create(
self.ctx, job=job, region='Fake_region',
@ -151,7 +161,8 @@ class DBAPIResourceSyncTest(base.KingbirdTestCase):
def test_delete_sync_job(self):
job_id = UUID1
job = self.sync_job_create(self.ctx, job_id=UUID1)
job = self.sync_job_create(self.ctx, job_name='fake_job_name',
job_id=UUID1)
self.assertIsNotNone(job)
self.resource_sync_create(
self.ctx, job=job, region='Fake_region',
@ -162,7 +173,8 @@ class DBAPIResourceSyncTest(base.KingbirdTestCase):
self.assertEqual(0, len(updated_job))
def test_composite_primary_key(self):
job = self.sync_job_create(self.ctx, job_id=UUID1)
job = self.sync_job_create(self.ctx, job_name='fake_job_name',
job_id=UUID1)
self.resource_sync_create(
self.ctx, job=job, region='Fake_region',
source_region='Fake_region2', resource='fake_key',

View File

@ -175,11 +175,15 @@ class TestNovaClient(base.KingbirdTestCase):
@mock.patch.object(nova_v2, 'client')
def test_get_flavor(self, mock_novaclient):
"""Test get_flavor method of nova."""
fake_flavor = Fake_Flavor('fake_id', 512, 2, 30, 'fake_flavor', 1,
1.0, False)
nv_client = nova_v2.NovaClient('fake_region', self.session,
DISABLED_QUOTAS)
nv_client.get_flavor('fake_id')
nv_client.get_flavor(fake_flavor.name)
result = mock_novaclient.Client().flavors.find(fake_flavor.name)
mock_novaclient.Client().flavors.get.\
assert_called_once_with('fake_id')
assert_called_once_with(result)
@mock.patch.object(nova_v2, 'client')
def test_get_flavor_access_tenants(self, mock_novaclient):

View File

@ -212,5 +212,5 @@ class TestFlavorSyncManager(base.KingbirdTestCase):
mock_nova().create_flavor.assert_called_once_with(
payload['force'], fake_flavor, access_tenants)
mock_db_api.resource_sync_update.assert_called_once_with(
self.ctxt, FAKE_JOB_ID, payload['target'][0], fake_flavor.id,
self.ctxt, FAKE_JOB_ID, payload['target'][0], fake_flavor.name,
JOB_RESULT)