Merge "System application - dual controller support"
This commit is contained in:
commit
37f0d72439
@ -1474,7 +1474,7 @@ K8S_RBD_PROV_STOR_CLASS_NAME = 'general'
|
||||
##################################
|
||||
# Working paths
|
||||
APP_INSTALL_PATH = '/scratch/apps'
|
||||
APP_MANIFEST_PATH = os.path.join(tsc.PLATFORM_PATH, 'armada', tsc.SW_VERSION)
|
||||
APP_SYNCED_DATA_PATH = os.path.join(tsc.PLATFORM_PATH, 'armada', tsc.SW_VERSION)
|
||||
|
||||
# State constants
|
||||
APP_UPLOAD_IN_PROGRESS = 'uploading'
|
||||
|
@ -49,6 +49,7 @@ MAX_DOWNLOAD_THREAD = 20
|
||||
INSTALLATION_TIMEOUT = 3600
|
||||
APPLY_SEARCH_PATTERN = 'Processing Chart,'
|
||||
DELETE_SEARCH_PATTERN = 'Deleting release'
|
||||
ARMADA_MANIFEST_APPLY_SUCCESS_MSG = 'Done applying manifest'
|
||||
CONTAINER_ABNORMAL_EXIT_CODE = 137
|
||||
|
||||
|
||||
@ -72,10 +73,12 @@ class AppOperator(object):
|
||||
os.path.exists(os.path.join(app.path, 'metadata.yaml'))):
|
||||
self._process_node_labels(app, op=constants.LABEL_REMOVE_OP)
|
||||
if app.system_app and app.status != constants.APP_UPLOAD_FAILURE:
|
||||
self._remove_chart_overrides(app.mfile_abs)
|
||||
self._remove_chart_overrides(app.armada_mfile_abs)
|
||||
|
||||
os.unlink(app.armada_mfile_abs)
|
||||
shutil.rmtree(app.path)
|
||||
os.unlink(app.imgfile_abs)
|
||||
if os.path.exists(app.path):
|
||||
shutil.rmtree(app.path)
|
||||
except OSError as e:
|
||||
LOG.exception(e)
|
||||
|
||||
@ -94,7 +97,7 @@ class AppOperator(object):
|
||||
def _abort_operation(self, app, operation):
|
||||
if (app.status == constants.APP_UPLOAD_IN_PROGRESS):
|
||||
self._update_app_status(app, constants.APP_UPLOAD_FAILURE,
|
||||
constants.APP_PRORESS_ABORTED)
|
||||
constants.APP_PROGRESS_ABORTED)
|
||||
elif (app.status == constants.APP_APPLY_IN_PROGRESS):
|
||||
self._update_app_status(app, constants.APP_APPLY_FAILURE,
|
||||
constants.APP_PROGRESS_ABORTED)
|
||||
@ -109,10 +112,13 @@ class AppOperator(object):
|
||||
name=app.name,
|
||||
reason="failed to extract tarfile content.")
|
||||
try:
|
||||
# One time set up of install path per controller
|
||||
if not os.path.isdir(constants.APP_INSTALL_PATH):
|
||||
# One time set up
|
||||
os.makedirs(constants.APP_INSTALL_PATH)
|
||||
os.makedirs(constants.APP_MANIFEST_PATH)
|
||||
|
||||
# One time set up of Armada manifest path for the system
|
||||
if not os.path.isdir(constants.APP_SYNCED_DATA_PATH):
|
||||
os.makedirs(constants.APP_SYNCED_DATA_PATH)
|
||||
|
||||
if not os.path.isdir(app.path):
|
||||
os.makedirs(app.path)
|
||||
@ -171,9 +177,10 @@ class AppOperator(object):
|
||||
# Either this chart does not have overrides file or image tags are
|
||||
# not in its overrides file, walk the chart path to find image tags
|
||||
chart_path = os.path.join(app_path, chart.name)
|
||||
tags = self._get_image_tags_by_path(chart_path)
|
||||
if tags:
|
||||
image_tags.extend(tags)
|
||||
if os.path.exists(chart_path):
|
||||
tags = self._get_image_tags_by_path(chart_path)
|
||||
if tags:
|
||||
image_tags.extend(tags)
|
||||
|
||||
return list(set(image_tags))
|
||||
|
||||
@ -193,11 +200,15 @@ class AppOperator(object):
|
||||
name=app.name,
|
||||
reason="embedded images are not yet supported.")
|
||||
|
||||
def _download_images(self, app):
|
||||
if os.path.isdir(app.images_dir):
|
||||
return self._register_embedded_images(app)
|
||||
|
||||
def _save_images_list(self, app):
|
||||
# Extract the list of images from the charts and overrides where
|
||||
# applicable. Save the list to the same location as the armada manifest
|
||||
# so it can be sync'ed.
|
||||
if app.system_app:
|
||||
LOG.info("Generating application overrides...")
|
||||
self._helm.generate_helm_application_overrides(
|
||||
app.name, cnamespace=None, armada_format=True, combined=True)
|
||||
app.charts = self._get_list_of_charts(app.armada_mfile_abs)
|
||||
# Grab the image tags from the overrides. If they don't exist
|
||||
# then mine them from the chart paths.
|
||||
images_to_download = self._get_image_tags_by_charts(app.charts_dir,
|
||||
@ -206,11 +217,40 @@ class AppOperator(object):
|
||||
# For custom apps, mine image tags from application path
|
||||
images_to_download = self._get_image_tags_by_path(app.path)
|
||||
|
||||
if images_to_download is None:
|
||||
raise exception.KubeAppApplyFailure(
|
||||
if not images_to_download:
|
||||
raise exception.KubeAppUploadFailure(
|
||||
name=app.name,
|
||||
reason="charts specify no docker images.")
|
||||
|
||||
with open(app.imgfile_abs, 'wb') as f:
|
||||
yaml.safe_dump(images_to_download, f, explicit_start=True,
|
||||
default_flow_style=False)
|
||||
|
||||
def _retrieve_images_list(self, app_images_file):
|
||||
with open(app_images_file, 'rb') as f:
|
||||
images_list = yaml.load(f)
|
||||
return images_list
|
||||
|
||||
def _download_images(self, app):
|
||||
if os.path.isdir(app.images_dir):
|
||||
return self._register_embedded_images(app)
|
||||
|
||||
if app.system_app:
|
||||
# Some images could have been overwritten via user overrides
|
||||
# between upload and apply, or between applies. Refresh the
|
||||
# saved images list.
|
||||
saved_images_list = self._retrieve_images_list(app.imgfile_abs)
|
||||
combined_images_list = list(saved_images_list)
|
||||
combined_images_list.extend(
|
||||
self._get_image_tags_by_charts(app.charts_dir, app.charts))
|
||||
images_to_download = list(set(combined_images_list))
|
||||
if saved_images_list != images_to_download:
|
||||
with open(app.imgfile_abs, 'wb') as f:
|
||||
yaml.safe_dump(images_to_download, f, explicit_start=True,
|
||||
default_flow_style=False)
|
||||
else:
|
||||
images_to_download = self._retrieve_images_list(app.imgfile_abs)
|
||||
|
||||
total_count = len(images_to_download)
|
||||
threads = min(MAX_DOWNLOAD_THREAD, total_count)
|
||||
failed_downloads = []
|
||||
@ -577,6 +617,8 @@ class AppOperator(object):
|
||||
# user can upload helm charts
|
||||
os.chmod('/scratch', 0o755)
|
||||
self._upload_helm_charts(app)
|
||||
|
||||
self._save_images_list(app)
|
||||
self._update_app_status(app, constants.APP_UPLOAD_SUCCESS)
|
||||
LOG.info("Application (%s) upload completed." % app.name)
|
||||
except Exception as e:
|
||||
@ -613,7 +655,7 @@ class AppOperator(object):
|
||||
overrides_str = ''
|
||||
ready = True
|
||||
try:
|
||||
app.charts = self._get_list_of_charts(app.mfile_abs)
|
||||
app.charts = self._get_list_of_charts(app.armada_mfile_abs)
|
||||
if app.system_app:
|
||||
self._update_app_status(
|
||||
app, new_progress=constants.APP_PROGRESS_GENERATE_OVERRIDES)
|
||||
@ -670,7 +712,7 @@ class AppOperator(object):
|
||||
app = AppOperator.Application(rpc_app)
|
||||
LOG.info("Application (%s) remove started." % app.name)
|
||||
|
||||
app.charts = self._get_list_of_charts(app.mfile_abs)
|
||||
app.charts = self._get_list_of_charts(app.armada_mfile_abs)
|
||||
self._update_app_status(
|
||||
app, new_progress=constants.APP_PROGRESS_DELETE_MANIFEST)
|
||||
|
||||
@ -743,13 +785,16 @@ class AppOperator(object):
|
||||
os.path.join('/manifests', self._kube_app.get('name') + "-" +
|
||||
self._kube_app.get('manifest_file'))
|
||||
self.armada_mfile_abs =\
|
||||
os.path.join(constants.APP_MANIFEST_PATH,
|
||||
os.path.join(constants.APP_SYNCED_DATA_PATH,
|
||||
self._kube_app.get('name') + "-" +
|
||||
self._kube_app.get('manifest_file'))
|
||||
self.mfile_abs =\
|
||||
os.path.join(constants.APP_INSTALL_PATH,
|
||||
self._kube_app.get('name'),
|
||||
self._kube_app.get('manifest_file'))
|
||||
self.imgfile_abs =\
|
||||
os.path.join(constants.APP_SYNCED_DATA_PATH,
|
||||
self._kube_app.get('name') + "-images.yaml")
|
||||
self.charts = []
|
||||
|
||||
@property
|
||||
@ -790,13 +835,13 @@ class DockerHelper(object):
|
||||
try:
|
||||
# First make kubernetes config accessible to Armada. This
|
||||
# is a work around the permission issue in Armada container.
|
||||
install_dir = constants.APP_INSTALL_PATH
|
||||
kube_config = os.path.join(install_dir, 'admin.conf')
|
||||
kube_config = os.path.join(constants.APP_SYNCED_DATA_PATH,
|
||||
'admin.conf')
|
||||
shutil.copy('/etc/kubernetes/admin.conf', kube_config)
|
||||
os.chown(kube_config, 1000, grp.getgrnam("wrs").gr_gid)
|
||||
|
||||
overrides_dir = common.HELM_OVERRIDES_PATH
|
||||
manifests_dir = constants.APP_MANIFEST_PATH
|
||||
manifests_dir = constants.APP_SYNCED_DATA_PATH
|
||||
LOG.info("kube_config=%s, manifests_dir=%s, "
|
||||
"overrides_dir=%s." % (kube_config, manifests_dir,
|
||||
overrides_dir))
|
||||
@ -857,8 +902,14 @@ class DockerHelper(object):
|
||||
LOG.info("Armada apply command = %s" % cmd)
|
||||
(exit_code, exec_logs) = armada_svc.exec_run(cmd)
|
||||
if exit_code == 0:
|
||||
LOG.info("Application manifest %s was successfully "
|
||||
"applied/re-applied." % manifest_file)
|
||||
if ARMADA_MANIFEST_APPLY_SUCCESS_MSG in exec_logs:
|
||||
LOG.info("Application manifest %s was successfully "
|
||||
"applied/re-applied." % manifest_file)
|
||||
else:
|
||||
rc = False
|
||||
LOG.error("Received a false positive response from "
|
||||
"Docker/Armada. Failed to apply application "
|
||||
"manifest %s: %s." % (manifest_file, exec_logs))
|
||||
else:
|
||||
rc = False
|
||||
if exit_code == CONTAINER_ABNORMAL_EXIT_CODE:
|
||||
|
Loading…
Reference in New Issue
Block a user