Merge "System application - dual controller support"
This commit is contained in:
commit
37f0d72439
|
@ -1474,7 +1474,7 @@ K8S_RBD_PROV_STOR_CLASS_NAME = 'general'
|
||||||
##################################
|
##################################
|
||||||
# Working paths
|
# Working paths
|
||||||
APP_INSTALL_PATH = '/scratch/apps'
|
APP_INSTALL_PATH = '/scratch/apps'
|
||||||
APP_MANIFEST_PATH = os.path.join(tsc.PLATFORM_PATH, 'armada', tsc.SW_VERSION)
|
APP_SYNCED_DATA_PATH = os.path.join(tsc.PLATFORM_PATH, 'armada', tsc.SW_VERSION)
|
||||||
|
|
||||||
# State constants
|
# State constants
|
||||||
APP_UPLOAD_IN_PROGRESS = 'uploading'
|
APP_UPLOAD_IN_PROGRESS = 'uploading'
|
||||||
|
|
|
@ -49,6 +49,7 @@ MAX_DOWNLOAD_THREAD = 20
|
||||||
INSTALLATION_TIMEOUT = 3600
|
INSTALLATION_TIMEOUT = 3600
|
||||||
APPLY_SEARCH_PATTERN = 'Processing Chart,'
|
APPLY_SEARCH_PATTERN = 'Processing Chart,'
|
||||||
DELETE_SEARCH_PATTERN = 'Deleting release'
|
DELETE_SEARCH_PATTERN = 'Deleting release'
|
||||||
|
ARMADA_MANIFEST_APPLY_SUCCESS_MSG = 'Done applying manifest'
|
||||||
CONTAINER_ABNORMAL_EXIT_CODE = 137
|
CONTAINER_ABNORMAL_EXIT_CODE = 137
|
||||||
|
|
||||||
|
|
||||||
|
@ -72,10 +73,12 @@ class AppOperator(object):
|
||||||
os.path.exists(os.path.join(app.path, 'metadata.yaml'))):
|
os.path.exists(os.path.join(app.path, 'metadata.yaml'))):
|
||||||
self._process_node_labels(app, op=constants.LABEL_REMOVE_OP)
|
self._process_node_labels(app, op=constants.LABEL_REMOVE_OP)
|
||||||
if app.system_app and app.status != constants.APP_UPLOAD_FAILURE:
|
if app.system_app and app.status != constants.APP_UPLOAD_FAILURE:
|
||||||
self._remove_chart_overrides(app.mfile_abs)
|
self._remove_chart_overrides(app.armada_mfile_abs)
|
||||||
|
|
||||||
os.unlink(app.armada_mfile_abs)
|
os.unlink(app.armada_mfile_abs)
|
||||||
shutil.rmtree(app.path)
|
os.unlink(app.imgfile_abs)
|
||||||
|
if os.path.exists(app.path):
|
||||||
|
shutil.rmtree(app.path)
|
||||||
except OSError as e:
|
except OSError as e:
|
||||||
LOG.exception(e)
|
LOG.exception(e)
|
||||||
|
|
||||||
|
@ -94,7 +97,7 @@ class AppOperator(object):
|
||||||
def _abort_operation(self, app, operation):
|
def _abort_operation(self, app, operation):
|
||||||
if (app.status == constants.APP_UPLOAD_IN_PROGRESS):
|
if (app.status == constants.APP_UPLOAD_IN_PROGRESS):
|
||||||
self._update_app_status(app, constants.APP_UPLOAD_FAILURE,
|
self._update_app_status(app, constants.APP_UPLOAD_FAILURE,
|
||||||
constants.APP_PRORESS_ABORTED)
|
constants.APP_PROGRESS_ABORTED)
|
||||||
elif (app.status == constants.APP_APPLY_IN_PROGRESS):
|
elif (app.status == constants.APP_APPLY_IN_PROGRESS):
|
||||||
self._update_app_status(app, constants.APP_APPLY_FAILURE,
|
self._update_app_status(app, constants.APP_APPLY_FAILURE,
|
||||||
constants.APP_PROGRESS_ABORTED)
|
constants.APP_PROGRESS_ABORTED)
|
||||||
|
@ -109,10 +112,13 @@ class AppOperator(object):
|
||||||
name=app.name,
|
name=app.name,
|
||||||
reason="failed to extract tarfile content.")
|
reason="failed to extract tarfile content.")
|
||||||
try:
|
try:
|
||||||
|
# One time set up of install path per controller
|
||||||
if not os.path.isdir(constants.APP_INSTALL_PATH):
|
if not os.path.isdir(constants.APP_INSTALL_PATH):
|
||||||
# One time set up
|
|
||||||
os.makedirs(constants.APP_INSTALL_PATH)
|
os.makedirs(constants.APP_INSTALL_PATH)
|
||||||
os.makedirs(constants.APP_MANIFEST_PATH)
|
|
||||||
|
# One time set up of Armada manifest path for the system
|
||||||
|
if not os.path.isdir(constants.APP_SYNCED_DATA_PATH):
|
||||||
|
os.makedirs(constants.APP_SYNCED_DATA_PATH)
|
||||||
|
|
||||||
if not os.path.isdir(app.path):
|
if not os.path.isdir(app.path):
|
||||||
os.makedirs(app.path)
|
os.makedirs(app.path)
|
||||||
|
@ -171,9 +177,10 @@ class AppOperator(object):
|
||||||
# Either this chart does not have overrides file or image tags are
|
# Either this chart does not have overrides file or image tags are
|
||||||
# not in its overrides file, walk the chart path to find image tags
|
# not in its overrides file, walk the chart path to find image tags
|
||||||
chart_path = os.path.join(app_path, chart.name)
|
chart_path = os.path.join(app_path, chart.name)
|
||||||
tags = self._get_image_tags_by_path(chart_path)
|
if os.path.exists(chart_path):
|
||||||
if tags:
|
tags = self._get_image_tags_by_path(chart_path)
|
||||||
image_tags.extend(tags)
|
if tags:
|
||||||
|
image_tags.extend(tags)
|
||||||
|
|
||||||
return list(set(image_tags))
|
return list(set(image_tags))
|
||||||
|
|
||||||
|
@ -193,11 +200,15 @@ class AppOperator(object):
|
||||||
name=app.name,
|
name=app.name,
|
||||||
reason="embedded images are not yet supported.")
|
reason="embedded images are not yet supported.")
|
||||||
|
|
||||||
def _download_images(self, app):
|
def _save_images_list(self, app):
|
||||||
if os.path.isdir(app.images_dir):
|
# Extract the list of images from the charts and overrides where
|
||||||
return self._register_embedded_images(app)
|
# applicable. Save the list to the same location as the armada manifest
|
||||||
|
# so it can be sync'ed.
|
||||||
if app.system_app:
|
if app.system_app:
|
||||||
|
LOG.info("Generating application overrides...")
|
||||||
|
self._helm.generate_helm_application_overrides(
|
||||||
|
app.name, cnamespace=None, armada_format=True, combined=True)
|
||||||
|
app.charts = self._get_list_of_charts(app.armada_mfile_abs)
|
||||||
# Grab the image tags from the overrides. If they don't exist
|
# Grab the image tags from the overrides. If they don't exist
|
||||||
# then mine them from the chart paths.
|
# then mine them from the chart paths.
|
||||||
images_to_download = self._get_image_tags_by_charts(app.charts_dir,
|
images_to_download = self._get_image_tags_by_charts(app.charts_dir,
|
||||||
|
@ -206,11 +217,40 @@ class AppOperator(object):
|
||||||
# For custom apps, mine image tags from application path
|
# For custom apps, mine image tags from application path
|
||||||
images_to_download = self._get_image_tags_by_path(app.path)
|
images_to_download = self._get_image_tags_by_path(app.path)
|
||||||
|
|
||||||
if images_to_download is None:
|
if not images_to_download:
|
||||||
raise exception.KubeAppApplyFailure(
|
raise exception.KubeAppUploadFailure(
|
||||||
name=app.name,
|
name=app.name,
|
||||||
reason="charts specify no docker images.")
|
reason="charts specify no docker images.")
|
||||||
|
|
||||||
|
with open(app.imgfile_abs, 'wb') as f:
|
||||||
|
yaml.safe_dump(images_to_download, f, explicit_start=True,
|
||||||
|
default_flow_style=False)
|
||||||
|
|
||||||
|
def _retrieve_images_list(self, app_images_file):
|
||||||
|
with open(app_images_file, 'rb') as f:
|
||||||
|
images_list = yaml.load(f)
|
||||||
|
return images_list
|
||||||
|
|
||||||
|
def _download_images(self, app):
|
||||||
|
if os.path.isdir(app.images_dir):
|
||||||
|
return self._register_embedded_images(app)
|
||||||
|
|
||||||
|
if app.system_app:
|
||||||
|
# Some images could have been overwritten via user overrides
|
||||||
|
# between upload and apply, or between applies. Refresh the
|
||||||
|
# saved images list.
|
||||||
|
saved_images_list = self._retrieve_images_list(app.imgfile_abs)
|
||||||
|
combined_images_list = list(saved_images_list)
|
||||||
|
combined_images_list.extend(
|
||||||
|
self._get_image_tags_by_charts(app.charts_dir, app.charts))
|
||||||
|
images_to_download = list(set(combined_images_list))
|
||||||
|
if saved_images_list != images_to_download:
|
||||||
|
with open(app.imgfile_abs, 'wb') as f:
|
||||||
|
yaml.safe_dump(images_to_download, f, explicit_start=True,
|
||||||
|
default_flow_style=False)
|
||||||
|
else:
|
||||||
|
images_to_download = self._retrieve_images_list(app.imgfile_abs)
|
||||||
|
|
||||||
total_count = len(images_to_download)
|
total_count = len(images_to_download)
|
||||||
threads = min(MAX_DOWNLOAD_THREAD, total_count)
|
threads = min(MAX_DOWNLOAD_THREAD, total_count)
|
||||||
failed_downloads = []
|
failed_downloads = []
|
||||||
|
@ -577,6 +617,8 @@ class AppOperator(object):
|
||||||
# user can upload helm charts
|
# user can upload helm charts
|
||||||
os.chmod('/scratch', 0o755)
|
os.chmod('/scratch', 0o755)
|
||||||
self._upload_helm_charts(app)
|
self._upload_helm_charts(app)
|
||||||
|
|
||||||
|
self._save_images_list(app)
|
||||||
self._update_app_status(app, constants.APP_UPLOAD_SUCCESS)
|
self._update_app_status(app, constants.APP_UPLOAD_SUCCESS)
|
||||||
LOG.info("Application (%s) upload completed." % app.name)
|
LOG.info("Application (%s) upload completed." % app.name)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
@ -613,7 +655,7 @@ class AppOperator(object):
|
||||||
overrides_str = ''
|
overrides_str = ''
|
||||||
ready = True
|
ready = True
|
||||||
try:
|
try:
|
||||||
app.charts = self._get_list_of_charts(app.mfile_abs)
|
app.charts = self._get_list_of_charts(app.armada_mfile_abs)
|
||||||
if app.system_app:
|
if app.system_app:
|
||||||
self._update_app_status(
|
self._update_app_status(
|
||||||
app, new_progress=constants.APP_PROGRESS_GENERATE_OVERRIDES)
|
app, new_progress=constants.APP_PROGRESS_GENERATE_OVERRIDES)
|
||||||
|
@ -670,7 +712,7 @@ class AppOperator(object):
|
||||||
app = AppOperator.Application(rpc_app)
|
app = AppOperator.Application(rpc_app)
|
||||||
LOG.info("Application (%s) remove started." % app.name)
|
LOG.info("Application (%s) remove started." % app.name)
|
||||||
|
|
||||||
app.charts = self._get_list_of_charts(app.mfile_abs)
|
app.charts = self._get_list_of_charts(app.armada_mfile_abs)
|
||||||
self._update_app_status(
|
self._update_app_status(
|
||||||
app, new_progress=constants.APP_PROGRESS_DELETE_MANIFEST)
|
app, new_progress=constants.APP_PROGRESS_DELETE_MANIFEST)
|
||||||
|
|
||||||
|
@ -743,13 +785,16 @@ class AppOperator(object):
|
||||||
os.path.join('/manifests', self._kube_app.get('name') + "-" +
|
os.path.join('/manifests', self._kube_app.get('name') + "-" +
|
||||||
self._kube_app.get('manifest_file'))
|
self._kube_app.get('manifest_file'))
|
||||||
self.armada_mfile_abs =\
|
self.armada_mfile_abs =\
|
||||||
os.path.join(constants.APP_MANIFEST_PATH,
|
os.path.join(constants.APP_SYNCED_DATA_PATH,
|
||||||
self._kube_app.get('name') + "-" +
|
self._kube_app.get('name') + "-" +
|
||||||
self._kube_app.get('manifest_file'))
|
self._kube_app.get('manifest_file'))
|
||||||
self.mfile_abs =\
|
self.mfile_abs =\
|
||||||
os.path.join(constants.APP_INSTALL_PATH,
|
os.path.join(constants.APP_INSTALL_PATH,
|
||||||
self._kube_app.get('name'),
|
self._kube_app.get('name'),
|
||||||
self._kube_app.get('manifest_file'))
|
self._kube_app.get('manifest_file'))
|
||||||
|
self.imgfile_abs =\
|
||||||
|
os.path.join(constants.APP_SYNCED_DATA_PATH,
|
||||||
|
self._kube_app.get('name') + "-images.yaml")
|
||||||
self.charts = []
|
self.charts = []
|
||||||
|
|
||||||
@property
|
@property
|
||||||
|
@ -790,13 +835,13 @@ class DockerHelper(object):
|
||||||
try:
|
try:
|
||||||
# First make kubernetes config accessible to Armada. This
|
# First make kubernetes config accessible to Armada. This
|
||||||
# is a work around the permission issue in Armada container.
|
# is a work around the permission issue in Armada container.
|
||||||
install_dir = constants.APP_INSTALL_PATH
|
kube_config = os.path.join(constants.APP_SYNCED_DATA_PATH,
|
||||||
kube_config = os.path.join(install_dir, 'admin.conf')
|
'admin.conf')
|
||||||
shutil.copy('/etc/kubernetes/admin.conf', kube_config)
|
shutil.copy('/etc/kubernetes/admin.conf', kube_config)
|
||||||
os.chown(kube_config, 1000, grp.getgrnam("wrs").gr_gid)
|
os.chown(kube_config, 1000, grp.getgrnam("wrs").gr_gid)
|
||||||
|
|
||||||
overrides_dir = common.HELM_OVERRIDES_PATH
|
overrides_dir = common.HELM_OVERRIDES_PATH
|
||||||
manifests_dir = constants.APP_MANIFEST_PATH
|
manifests_dir = constants.APP_SYNCED_DATA_PATH
|
||||||
LOG.info("kube_config=%s, manifests_dir=%s, "
|
LOG.info("kube_config=%s, manifests_dir=%s, "
|
||||||
"overrides_dir=%s." % (kube_config, manifests_dir,
|
"overrides_dir=%s." % (kube_config, manifests_dir,
|
||||||
overrides_dir))
|
overrides_dir))
|
||||||
|
@ -857,8 +902,14 @@ class DockerHelper(object):
|
||||||
LOG.info("Armada apply command = %s" % cmd)
|
LOG.info("Armada apply command = %s" % cmd)
|
||||||
(exit_code, exec_logs) = armada_svc.exec_run(cmd)
|
(exit_code, exec_logs) = armada_svc.exec_run(cmd)
|
||||||
if exit_code == 0:
|
if exit_code == 0:
|
||||||
LOG.info("Application manifest %s was successfully "
|
if ARMADA_MANIFEST_APPLY_SUCCESS_MSG in exec_logs:
|
||||||
"applied/re-applied." % manifest_file)
|
LOG.info("Application manifest %s was successfully "
|
||||||
|
"applied/re-applied." % manifest_file)
|
||||||
|
else:
|
||||||
|
rc = False
|
||||||
|
LOG.error("Received a false positive response from "
|
||||||
|
"Docker/Armada. Failed to apply application "
|
||||||
|
"manifest %s: %s." % (manifest_file, exec_logs))
|
||||||
else:
|
else:
|
||||||
rc = False
|
rc = False
|
||||||
if exit_code == CONTAINER_ABNORMAL_EXIT_CODE:
|
if exit_code == CONTAINER_ABNORMAL_EXIT_CODE:
|
||||||
|
|
Loading…
Reference in New Issue