openstack/solum
Code Issues Proposed changes

Deployment related cleanup

Browse Source 
- Changing assembly status to correctly reflect failure
  to create an assembly
- Reordering setting of assembly status and uploading of
  tenant's deployer log
- Removing tenant-id from location of tenant's deployer
  logs. This is to be consistent with how we are naming
  logs for unittest and build stages.

Co-Authored-By: Ed Cranford <ed.cranford@rackspace.com>
Change-Id: I98af988ad9726513127f4478d47f26f050e639c7
This commit is contained in:
Devdatta Kulkarni
2015-04-14 14:29:39 -05:00
parent 731a7bf9c3
commit fa7f1bc1d8
8 changed files with 132 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
solum/deployer/handlers/heat.py
View File

@@ -106,6 +106,7 @@ class Handler(object):
t_logger = tlog.TenantLogger(ctxt, assem, deployer_log_dir, 'delete')
msg = "Deleting Assembly %s" % assem.uuid
t_logger.log(logging.DEBUG, msg)
LOG.debug(msg)
if stack_id is None:
assem.destroy(ctxt)
@@ -139,18 +140,19 @@ class Handler(object):
# Must use stack_name for expecting a 404
osc.heat().stacks.get(stack_name)
except exc.HTTPNotFound:
assem.destroy(ctxt)
t_logger.log(logging.DEBUG, "Stack delete successful.")
t_logger.upload()
assem.destroy(ctxt)
return
time.sleep(wait_interval)
wait_interval *= growth_factor
t_logger.log(logging.ERROR, "Error deleting heat stack.")
t_logger.upload()
update_assembly(ctxt, assem_id,
{'status': STATES.ERROR_STACK_DELETE_FAILED})
t_logger.log(logging.ERROR, "Error deleting heat stack.")
t_logger.upload()
def destroy_app(self, ctxt, app_id):
# Destroy a plan's assemblies, and then the plan.
plan = objects.registry.Plan.get_by_id(ctxt, app_id)
@@ -261,7 +263,8 @@ class Handler(object):
LOG.error("Error creating Heat Stack for,"
" assembly %s" % assembly_id)
LOG.exception(exp)
update_assembly(ctxt, assembly_id, {'status': STATES.ERROR})
update_assembly(ctxt, assembly_id,
{'status': STATES.ERROR_STACK_CREATE_FAILED})
t_logger.log(logging.ERROR, "Error creating heat stack.")
t_logger.upload()
return
@@ -285,9 +288,12 @@ class Handler(object):
return
update_assembly(ctxt, assembly_id, {'status': STATES.DEPLOYING})
self._check_stack_status(ctxt, assembly_id, osc, stack_id, ports)
self._check_stack_status(ctxt, assembly_id, osc, stack_id, ports,
t_logger)
t_logger.upload()
def _check_stack_status(self, ctxt, assembly_id, osc, stack_id, ports):
def _check_stack_status(self, ctxt, assembly_id, osc, stack_id, ports,
t_logger):
wait_interval = cfg.CONF.deployer.wait_interval
growth_factor = cfg.CONF.deployer.growth_factor
@@ -308,11 +314,15 @@ class Handler(object):
elif stack.status == 'FAILED':
update_assembly(ctxt, assembly_id,
{'status': STATES.ERROR_STACK_CREATE_FAILED})
lg_msg = "App deployment failed: Heat stack creation failure"
t_logger.log(logging.ERROR, lg_msg)
return
if stack is None or (stack and stack.status == ""):
update_assembly(ctxt, assembly_id,
{'status': STATES.ERROR_STACK_CREATE_FAILED})
lg_msg = "App deployment failed: Heat stack is in unexpected state"
t_logger.log(logging.ERROR, lg_msg)
return
host_ip = self._parse_server_url(stack)
@@ -320,6 +330,9 @@ class Handler(object):
LOG.exception("Could not parse url from heat stack.")
update_assembly(ctxt, assembly_id,
{'status': STATES.ERROR})
lg_msg = ("App deployment failed: "
"container IP address not available")
t_logger.log(logging.ERROR, lg_msg)
return
du_is_up = False
@@ -350,11 +363,17 @@ class Handler(object):
except Exception as exp:
LOG.exception(exp)
update_assembly(ctxt, assembly_id, {'status': STATES.ERROR})
lg_msg = ("App deployment error: unexpected error when trying"
" to reach app endpoint")
t_logger.log(logging.ERROR, lg_msg)
return
if du_is_up:
to_update = {'status': STATES.READY, 'application_uri': host_ip}
else:
to_update = {'status': STATES.ERROR_CODE_DEPLOYMENT}
lg_msg = ("App deployment error: unreachable server or port, "
" please check your port config.")
t_logger.log(logging.ERROR, lg_msg)
update_assembly(ctxt, assembly_id, to_update)
def _parse_server_url(self, heat_output):

9
solum/tests/deployer/handlers/test_heat.py
View File

@@ -216,9 +216,10 @@ class HandlerTest(base.BaseTestCase):
cfg.CONF.deployer.du_attempts = 1
mock_logger = mock.MagicMock()
handler._parse_server_url = mock.MagicMock(return_value=('xyz'))
handler._check_stack_status(self.ctx, fake_assembly.id, mock_clients,
'fake_id', [80])
'fake_id', [80], mock_logger)
c1 = mock.call(fake_assembly.id,
{'status': STATES.WAITING_FOR_DOCKER_DU,
@@ -240,8 +241,9 @@ class HandlerTest(base.BaseTestCase):
stack = mock.MagicMock()
stack.status = 'FAILED'
mock_clients.heat().stacks.get.return_value = stack
mock_logger = mock.MagicMock()
handler._check_stack_status(self.ctx, fake_assembly.id, mock_clients,
'fake_id', [80])
'fake_id', [80], mock_logger)
mock_ua.assert_called_once_with(fake_assembly.id,
{'status':
STATES.ERROR_STACK_CREATE_FAILED})
@@ -258,8 +260,9 @@ class HandlerTest(base.BaseTestCase):
cfg.CONF.set_override('growth_factor', 1, group='deployer')
cfg.CONF.set_override('max_attempts', 1, group='deployer')
mock_logger = mock.MagicMock()
handler._check_stack_status(self.ctx, fake_assembly.id, mock_clients,
'fake_id', [80])
'fake_id', [80], mock_logger)
mock_ua.assert_called_once_with(fake_assembly.id,
{'status':
STATES.ERROR_STACK_CREATE_FAILED})

0
solum/tests/uploaders/__init__.py Normal file
View File

9
solum/tests/uploaders/test_common.py
View File

@@ -12,6 +12,8 @@
# License for the specific language governing permissions and limitations
# under the License.
import mock
from solum.tests import base
from solum.tests import fakes
from solum.tests import utils
@@ -27,8 +29,13 @@ class CommonTest(base.BaseTestCase):
orig_path = "original path"
assembly = fakes.FakeAssembly()
build_id = "5678"
baseuploader = uploader.UploaderBase(ctxt, orig_path,
assembly, build_id,
"fakestage")
self.assertEqual(0, baseuploader.write_userlog_row.call_count)
baseuploader.write_userlog_row = mock.MagicMock()
baseuploader.upload_log()
self.assertEqual(0, baseuploader.write_userlog_row.call_count)

104
solum/tests/uploaders/test_swift.py
View File

@@ -13,6 +13,7 @@
# under the License.
import mock
from oslo.config import cfg
from solum.tests import base
from solum.tests import fakes
@@ -24,58 +25,97 @@ class SwiftUploadTest(base.BaseTestCase):
def setUp(self):
super(SwiftUploadTest, self).setUp()
@mock.patch('open')
@mock.patch('clients.OpenStackClients')
@mock.patch('oslo.config.cfg.CONF.worker')
def test_upload(self, mock_config, mock_client, mock_open):
def test_upload_on_assembly_delete(self):
ctxt = utils.dummy_context()
orig_path = "original path"
assembly = fakes.FakeAssembly()
build_id = "5678"
container = 'fake-container'
mock_config.log_upload_swift_container.return_value = container
mock_swift = mock.MagicMock()
mock_client.return_value.swift.return_value = mock_swift
fake_file = mock.MagicMock()
mock_open.return_value = fake_file
cfg.CONF.worker.log_upload_swift_container = container
stage = "fakestage"
swiftupload = uploader.SwiftUpload(ctxt, orig_path,
assembly, build_id,
"fakestage")
stage)
swiftupload._open = mock.MagicMock()
swiftupload._upload = mock.MagicMock()
swiftupload.transform_jsonlog = mock.MagicMock()
swiftupload.write_userlog_row = mock.MagicMock()
rs_before_delete = swiftupload.resource
name_before = swiftupload.resource.name
uuid_before = swiftupload.resource.uuid
# Delete the assembly object before calling upload_log()
del assembly
swiftupload.upload_log()
self.assertEqual(name_before, swiftupload.resource.name)
self.assertEqual(uuid_before, swiftupload.resource.uuid)
# Asserts
swift_info = {'container': container}
swiftupload.write_userlog_row.assert_called_once_with(orig_path,
swift_info)
swiftupload.transform_jsonlog.assert_called_once()
mock_swift.put_container.assert_called_once_with(container)
mock_swift.put_object.assert_called_once_with(container,
orig_path,
fake_file)
filename = "%s-%s/%s-%s.log" % (rs_before_delete.name,
rs_before_delete.uuid,
stage, build_id)
swiftupload.transform_jsonlog.assert_called_once()
swiftupload._upload.assert_called_once()
swiftupload.write_userlog_row.assert_called_once_with(filename,
swift_info)
def test_upload(self):
ctxt = utils.dummy_context()
orig_path = "original path"
assembly = fakes.FakeAssembly()
build_id = "5678"
container = 'fake-container'
cfg.CONF.worker.log_upload_swift_container = container
stage = "fakestage"
swiftupload = uploader.SwiftUpload(ctxt, orig_path,
assembly, build_id,
stage)
swiftupload._open = mock.MagicMock()
swiftupload._upload = mock.MagicMock()
swiftupload.transform_jsonlog = mock.MagicMock()
swiftupload.write_userlog_row = mock.MagicMock()
resource = swiftupload.resource
swiftupload.upload_log()
swift_info = {'container': container}
filename = "%s-%s/%s-%s.log" % (resource.name,
resource.uuid,
stage, build_id)
swiftupload.transform_jsonlog.assert_called_once()
swiftupload._upload.assert_called_once()
swiftupload.write_userlog_row.assert_called_once_with(filename,
swift_info)
@mock.patch('swiftclient.client.Connection')
def test_upload_image(self, mock_conn):
@mock.patch('open')
@mock.patch('swiftclient.Connection')
def test_upload_image(self, mock_conn, mock_open):
mock_swift = mock.MagicMock()
fake_file = mock.MagicMock()
mock_open.return_value = fake_file
mock_conn.return_value = mock_swift
container = "solum_du"
name = "test_file"
swiftupload = self._get_connection_handle(container, name)
swiftupload.upload_image()
mock_swift.put_container.assert_called_once_with(container)
mock_swift.put_object.assert_called_once_with(container,
name, fake_file)
swiftupload._open = mock.MagicMock()
swiftupload._upload = mock.MagicMock()
@mock.patch('swiftclient.Connection')
swiftupload.upload_image()
swiftupload._upload.assert_called_once()
@mock.patch('swiftclient.client.Connection')
def test_stat(self, mock_conn):
mock_swift = mock.MagicMock()
mock_conn.return_value = mock_swift
swift_client = self._get_connection_handle('', '')
swift_client.stat()
mock_swift.stat.assert_called_once()
def _get_connection_handle(self, container, name):
client_args = {"region_name": "RegionOne",

7
solum/uploaders/common.py
View File

@@ -65,9 +65,12 @@ class UploaderBase(object):
def stat(self):
pass
def _open(self, filepath, readwrite):
return open(filepath, readwrite)
def transform_jsonlog(self):
with open(self.original_file_path, 'r') as logfile:
with open(self.transformed_path, 'w') as tflogfile:
with self._open(self.original_file_path, 'r') as logfile:
with self._open(self.transformed_path, 'w') as tflogfile:
for line in logfile.readlines():
try:
# Log lines generated from Python logger

22
solum/uploaders/swift.py
View File

@@ -30,19 +30,26 @@ cfg.CONF.import_opt('log_upload_swift_container', 'solum.worker.config',
class SwiftUpload(solum.uploaders.common.UploaderBase):
strategy = "swift"
def _upload(self, container, filename, filelike, client_args=None):
swift = None
if client_args:
swift = clients.OpenStackClients(**client_args).swift()
else:
swift = clients.OpenStackClients(self.context).swift()
swift.put_container(container)
swift.put_object(container, filename, filelike)
def upload_log(self):
container = cfg.CONF.worker.log_upload_swift_container
filename = "%s-%s/%s-%s.log" % (self.resource.name, self.resource.uuid,
self.stage_name, self.stage_id)
self.transform_jsonlog()
with open(self.transformed_path, 'r') as logfile:
with self._open(self.transformed_path, 'r') as logfile:
try:
LOG.debug("Uploading log to Swift. %s, %s" %
(container, filename))
swift = clients.OpenStackClients(self.context).swift()
swift.put_container(container)
swift.put_object(container, filename, logfile)
self._upload(container, filename, logfile)
except swiftexceptions.ClientException:
LOG.exception("Failed to upload logfile to Swift.")
return
@@ -55,7 +62,7 @@ class SwiftUpload(solum.uploaders.common.UploaderBase):
self.write_userlog_row(filename, swift_info)
def upload_image(self):
with open(self.path, 'r') as du_file:
with self._open(self.path, 'r') as du_file:
try:
client_args = {
'auth_version': '2.0',
@@ -63,9 +70,8 @@ class SwiftUpload(solum.uploaders.common.UploaderBase):
'preauthurl': self.storage_url,
'os_options': {'region_name': self.region_name},
}
swift = swiftclient.Connection(**client_args)
swift.put_container(self.container)
swift.put_object(self.container, self.name, du_file)
self._upload(self.container, self.name, du_file,
client_args=client_args)
except swiftexceptions.ClientException as e:
LOG.exception("Failed to upload %s to Swift." % self.name)
raise e

4
solum/uploaders/tenant_logger.py
View File

@@ -43,14 +43,14 @@ class TenantLogger(object):
# Note: assembly type is used by uploader
self.assem.type = 'app'
tenant_log_file = "%s-%s-%s" % (stage, ctxt.tenant, assem.uuid)
tenant_log_file = "%s-%s" % (stage, assem.uuid)
self.path = "%s/%s.log" % (deployer_log_dir, tenant_log_file)
LOG.debug("Deployer logs stored at %s" % self.path)
uploadr = {
'local': local_uploader.LocalStorage,
'swift': swift_uploader.SwiftUpload,
}.get(strategy, 'local')
}.get(strategy, local_uploader.LocalStorage)
self.uploader = uploadr(ctxt, self.path, assem, assem.uuid, stage)