airship/armada
Code Issues Proposed changes

[feat] adding standard armada manifest

Browse Source 
- adding tmeplate armada manifest
- create Armada Manifest
- updated validation for new documents
- updated testing
- updated docs
This commit is contained in:
gardlt
2017-07-20 10:14:01 -05:00
committed by Alexis Rivera DeLa Torre
parent f8ff2c32df
commit afb7fe83ab
19 changed files with 1043 additions and 518 deletions
Download Patch File Download Diff File Expand all files Collapse all files

170
armada/handlers/armada.py
View File

@@ -21,9 +21,12 @@ from supermutes.dot import dotify
from chartbuilder import ChartBuilder
from tiller import Tiller
from manifest import Manifest
from ..utils.release import release_prefix
from ..utils import git
from ..utils import lint
from ..const import KEYWORD_ARMADA, KEYWORD_GROUPS, KEYWORD_CHARTS,\
KEYWORD_PREFIX
LOG = logging.getLogger(__name__)
@@ -33,13 +36,15 @@ DOMAIN = "armada"
logging.setup(CONF, DOMAIN)
class Armada(object):
'''
This is the main Armada class handling the Armada
workflows
'''
def __init__(self, config,
def __init__(self,
file,
disable_update_pre=False,
disable_update_post=False,
enable_chart_cleanup=False,
@@ -53,20 +58,26 @@ class Armada(object):
Initialize the Armada Engine and establish
a connection to Tiller
'''
self.disable_update_pre = disable_update_pre
self.disable_update_post = disable_update_post
self.enable_chart_cleanup = enable_chart_cleanup
self.dry_run = dry_run
self.wait = wait
self.timeout = timeout
self.config = yaml.load(config)
self.tiller = Tiller(tiller_host=tiller_host, tiller_port=tiller_port)
self.documents = yaml.safe_load_all(file)
self.config = self.get_armada_manifest()
self.tiller = Tiller()
self.debug = debug
# Set debug value
CONF.set_default('debug', self.debug)
logging.setup(CONF, DOMAIN)
def get_armada_manifest(self):
return Manifest(self.documents).get_manifest()
def find_release_chart(self, known_releases, name):
'''
Find a release given a list of known_releases and a release name
@@ -79,11 +90,14 @@ class Armada(object):
'''
Perform a series of checks and operations to ensure proper deployment
'''
# Ensure tiller is available and yaml is valid
if not self.tiller.tiller_status():
raise Exception("Tiller Services is not Available")
if not lint.valid_manifest(self.config):
raise Exception("Service: Tiller is not Available")
if not lint.validate_armada_documents(self.documents):
raise Exception("Invalid Armada Manifest")
if not lint.validate_armada_object(self.config):
raise Exception("Invalid Armada Object")
# Purge known releases that have failed and are in the current yaml
prefix = self.config.get('armada').get('release_prefix')
@@ -91,8 +105,9 @@ class Armada(object):
for release in failed_releases:
for group in self.config.get('armada').get('charts'):
for ch in group.get('chart_group'):
ch_release_name = release_prefix(prefix, ch.get('chart')
.get('name'))
ch_release_name = release_prefix(prefix,
ch.get('chart')
.get('name'))
if release[0] == ch_release_name:
LOG.info('Purging failed release %s '
'before deployment', release[0])
@@ -103,30 +118,35 @@ class Armada(object):
# We only support a git source type right now, which can also
# handle git:// local paths as well
repos = {}
for group in self.config.get('armada').get('charts'):
for ch in group.get('chart_group'):
location = ch.get('chart').get('source').get('location')
ct_type = ch.get('chart').get('source').get('type')
reference = ch.get('chart').get('source').get('reference')
subpath = ch.get('chart').get('source').get('subpath')
for group in self.config.get(KEYWORD_ARMADA).get(KEYWORD_GROUPS):
for ch in group.get(KEYWORD_CHARTS):
self.tag_cloned_repo(ch, repos)
if ct_type == 'local':
ch.get('chart')['source_dir'] = (location, subpath)
elif ct_type == 'git':
if location not in repos.keys():
try:
LOG.info('Cloning repo: %s', location)
repo_dir = git.git_clone(location, reference)
except Exception as e:
raise ValueError(e)
repos[location] = repo_dir
ch.get('chart')['source_dir'] = (repo_dir, subpath)
else:
ch.get('chart')['source_dir'] = (repos.get(location),
subpath)
else:
raise Exception("Unknown source type %s for chart %s",
ct_type, ch.get('chart').get('name'))
for dep in ch.get('chart').get('dependencies'):
self.tag_cloned_repo(dep, repos)
def tag_cloned_repo(self, ch, repos):
location = ch.get('chart').get('source').get('location')
ct_type = ch.get('chart').get('source').get('type')
reference = ch.get('chart').get('source').get('reference')
subpath = ch.get('chart').get('source').get('subpath')
if ct_type == 'local':
ch.get('chart')['source_dir'] = (location, subpath)
elif ct_type == 'git':
if location not in repos.keys():
try:
LOG.info('Cloning repo: %s', location)
repo_dir = git.git_clone(location, reference)
except Exception as e:
raise ValueError(e)
repos[location] = repo_dir
ch.get('chart')['source_dir'] = (repo_dir, subpath)
else:
ch.get('chart')['source_dir'] = (repos.get(location), subpath)
else:
raise Exception("Unknown source type %s for chart %s", ct_type,
ch.get('chart').get('name'))
def get_releases_by_status(self, status):
'''
@@ -154,16 +174,16 @@ class Armada(object):
# extract known charts on tiller right now
known_releases = self.tiller.list_charts()
prefix = self.config.get('armada').get('release_prefix')
prefix = self.config.get(KEYWORD_ARMADA).get(KEYWORD_PREFIX)
for release in known_releases:
LOG.debug("Release %s, Version %s found on tiller", release[0],
release[1])
for entry in self.config['armada']['charts']:
for entry in self.config[KEYWORD_ARMADA][KEYWORD_GROUPS]:
chart_wait = self.wait
desc = entry.get('description', 'A Chart Group')
chart_group = entry.get('chart_group', [])
chart_group = entry.get(KEYWORD_CHARTS, [])
if entry.get('sequenced', False):
chart_wait = True
@@ -175,9 +195,9 @@ class Armada(object):
values = gchart.get('chart').get('values', {})
pre_actions = {}
post_actions = {}
LOG.info('%s', chart.release_name)
LOG.info('%s', chart.release)
if chart.release_name is None:
if chart.release is None:
continue
# retrieve appropriate timeout value if 'wait' is specified
@@ -191,14 +211,14 @@ class Armada(object):
protoc_chart = chartbuilder.get_helm_chart()
# determine install or upgrade by examining known releases
LOG.debug("RELEASE: %s", chart.release_name)
LOG.debug("RELEASE: %s", chart.release)
deployed_releases = [x[0] for x in known_releases]
prefix_chart = release_prefix(prefix, chart.release_name)
prefix_chart = release_prefix(prefix, chart.release)
if prefix_chart in deployed_releases:
# indicate to the end user what path we are taking
LOG.info("Upgrading release %s", chart.release_name)
LOG.info("Upgrading release %s", chart.release)
# extract the installed chart and installed values from the
# latest release so we can compare to the intended state
LOG.info("Checking Pre/Post Actions")
@@ -209,12 +229,12 @@ class Armada(object):
upgrade = gchart.get('chart', {}).get('upgrade', False)
if upgrade:
if not self.disable_update_pre and upgrade.get('pre',
False):
if not self.disable_update_pre and upgrade.get(
'pre', False):
pre_actions = getattr(chart.upgrade, 'pre', {})
if not self.disable_update_post and upgrade.get('post',
False):
if not self.disable_update_post and upgrade.get(
'post', False):
post_actions = getattr(chart.upgrade, 'post', {})
# show delta for both the chart templates and the chart
@@ -231,29 +251,31 @@ class Armada(object):
continue
# do actual update
self.tiller.update_release(protoc_chart,
self.dry_run,
chart.release_name,
chart.namespace,
prefix, pre_actions,
post_actions,
disable_hooks=chart.
upgrade.no_hooks,
values=yaml.safe_dump(values),
wait=chart_wait,
timeout=chart_timeout)
self.tiller.update_release(
protoc_chart,
self.dry_run,
chart.release,
chart.namespace,
prefix,
pre_actions,
post_actions,
disable_hooks=chart.upgrade.no_hooks,
values=yaml.safe_dump(values),
wait=chart_wait,
timeout=chart_timeout)
# process install
else:
LOG.info("Installing release %s", chart.release_name)
self.tiller.install_release(protoc_chart,
self.dry_run,
chart.release_name,
chart.namespace,
prefix,
values=yaml.safe_dump(values),
wait=chart_wait,
timeout=chart_timeout)
LOG.info("Installing release %s", chart.release)
self.tiller.install_release(
protoc_chart,
self.dry_run,
chart.release,
chart.namespace,
prefix,
values=yaml.safe_dump(values),
wait=chart_wait,
timeout=chart_timeout)
LOG.debug("Cleaning up chart source in %s",
chartbuilder.source_directory)
@@ -262,7 +284,8 @@ class Armada(object):
self.post_flight_ops()
if self.enable_chart_cleanup:
self.tiller.chart_cleanup(prefix, self.config['armada']['charts'])
self.tiller.chart_cleanup(
prefix, self.config[KEYWORD_ARMADA][KEYWORD_GROUPS])
def post_flight_ops(self):
'''
@@ -274,8 +297,8 @@ class Armada(object):
if ch.get('chart').get('source').get('type') == 'git':
git.source_cleanup(ch.get('chart').get('source_dir')[0])
def show_diff(self, chart, installed_chart,
installed_values, target_chart, target_values):
def show_diff(self, chart, installed_chart, installed_values, target_chart,
target_values):
'''
Produce a unified diff of the installed chart vs our intention
@@ -283,20 +306,19 @@ class Armada(object):
unified diff output and avoid the use of print
'''
chart_diff = list(difflib.unified_diff(installed_chart
.SerializeToString()
.split('\n'),
target_chart.split('\n')))
chart_diff = list(
difflib.unified_diff(installed_chart.SerializeToString()
.split('\n'), target_chart.split('\n')))
if len(chart_diff) > 0:
LOG.info("Chart Unified Diff (%s)", chart.release_name)
LOG.info("Chart Unified Diff (%s)", chart.release)
for line in chart_diff:
LOG.debug(line)
values_diff = list(difflib.unified_diff(installed_values.split('\n'),
yaml
.safe_dump(target_values)
.split('\n')))
values_diff = list(
difflib.unified_diff(
installed_values.split('\n'),
yaml.safe_dump(target_values).split('\n')))
if len(values_diff) > 0:
LOG.info("Values Unified Diff (%s)", chart.release_name)
LOG.info("Values Unified Diff (%s)", chart.release)
for line in values_diff:
LOG.debug(line)

57
armada/handlers/chartbuilder.py
View File

@@ -31,6 +31,7 @@ DOMAIN = "armada"
logging.setup(CONF, DOMAIN)
class ChartBuilder(object):
'''
This class handles taking chart intentions as a paramter and
@@ -69,8 +70,7 @@ class ChartBuilder(object):
'''
Return the joined path of the source directory and subpath
'''
return os.path.join(self.chart.source_dir[0],
self.chart.source_dir[1])
return os.path.join(self.chart.source_dir[0], self.chart.source_dir[1])
def get_ignored_files(self):
'''
@@ -90,8 +90,8 @@ class ChartBuilder(object):
false otherwise
'''
for ignored_file in self.ignored_files:
if (ignored_file.startswith('*') and
filename.endswith(ignored_file.strip('*'))):
if (ignored_file.startswith('*')
and filename.endswith(ignored_file.strip('*'))):
return True
elif ignored_file == filename:
return True
@@ -102,15 +102,16 @@ class ChartBuilder(object):
Process metadata
'''
# extract Chart.yaml to construct metadata
chart_yaml = dotify(yaml.load(open(os.path.join(self.source_directory,
'Chart.yaml')).read()))
chart_yaml = dotify(
yaml.load(
open(os.path.join(self.source_directory, 'Chart.yaml'))
.read()))
# construct Metadata object
return Metadata(
description=chart_yaml.description,
name=chart_yaml.name,
version=chart_yaml.version
)
version=chart_yaml.version)
def get_files(self):
'''
@@ -127,8 +128,8 @@ class ChartBuilder(object):
# create config object representing unmarshaled values.yaml
if os.path.exists(os.path.join(self.source_directory, 'values.yaml')):
raw_values = open(os.path.join(self.source_directory,
'values.yaml')).read()
raw_values = open(
os.path.join(self.source_directory, 'values.yaml')).read()
else:
LOG.warn("No values.yaml in %s, using empty values",
self.source_directory)
@@ -144,24 +145,25 @@ class ChartBuilder(object):
# process all files in templates/ as a template to attach to the chart
# building a Template object
templates = []
if not os.path.exists(os.path.join(self.source_directory,
'templates')):
if not os.path.exists(
os.path.join(self.source_directory, 'templates')):
LOG.warn("Chart %s has no templates directory. "
"No templates will be deployed", self.chart.name)
for root, _, files in os.walk(os.path.join(self.source_directory,
'templates'), topdown=True):
"No templates will be deployed", self.chart.chart_name)
for root, _, files in os.walk(
os.path.join(self.source_directory, 'templates'),
topdown=True):
for tpl_file in files:
tname = os.path.relpath(os.path.join(root, tpl_file),
os.path.join(self.source_directory,
'templates'))
tname = os.path.relpath(
os.path.join(root, tpl_file),
os.path.join(self.source_directory, 'templates'))
if self.ignore_file(tname):
LOG.debug('Ignoring file %s', tname)
continue
templates.append(Template(name=tname,
data=open(os.path.join(root,
tpl_file),
'r').read()))
templates.append(
Template(
name=tname,
data=open(os.path.join(root, tpl_file), 'r').read()))
return templates
def get_helm_chart(self):
@@ -175,18 +177,17 @@ class ChartBuilder(object):
# [process_chart(x, is_dependency=True) for x in chart.dependencies]
dependencies = []
for chart in self.chart.dependencies:
LOG.info("Building dependency chart %s for release %s", chart.name,
self.chart.release_name)
dependencies.append(ChartBuilder(chart).get_helm_chart())
for dep in self.chart.dependencies:
LOG.info("Building dependency chart %s for release %s",
self.chart.chart_name, self.chart.release)
dependencies.append(ChartBuilder(dep.chart).get_helm_chart())
helm_chart = Chart(
metadata=self.get_metadata(),
templates=self.get_templates(),
dependencies=dependencies,
values=self.get_values(),
files=self.get_files(),
)
files=self.get_files(), )
self._helm_chart = helm_chart
return helm_chart

115
armada/handlers/manifest.py Normal file
View File

@@ -0,0 +1,115 @@
# Copyright 2017 The Armada Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ..const import DOCUMENT_CHART, DOCUMENT_GROUP, DOCUMENT_MANIFEST
class Manifest(object):
def __init__(self, documents):
self.config = None
self.documents = documents
self.charts = []
self.groups = []
self.manifest = None
self.get_documents()
def get_documents(self):
for document in self.documents:
if document.get('schema') == DOCUMENT_CHART:
self.charts.append(document)
if document.get('schema') == DOCUMENT_GROUP:
self.groups.append(document)
if document.get('schema') == DOCUMENT_MANIFEST:
self.manifest = document
def find_chart_document(self, name):
try:
for chart in self.charts:
if chart.get('metadata').get('name') == name:
return chart
except Exception:
raise Exception(
"Could not find {} in {}".format(name, DOCUMENT_CHART))
def find_chart_group_document(self, name):
try:
for group in self.groups:
if group.get('metadata').get('name') == name:
return group
except Exception:
raise Exception(
"Could not find {} in {}".format(name, DOCUMENT_GROUP))
def build_charts_deps(self):
for chart in self.charts:
self.build_chart_deps(chart)
def build_chart_groups(self):
for chart_group in self.groups:
self.build_chart_group(chart_group)
def build_chart_deps(self, chart):
try:
dep = None
for iter, dep in enumerate(chart.get('data').get('dependencies')):
if isinstance(dep, dict):
continue
chart_dep = self.find_chart_document(dep)
self.build_chart_deps(chart_dep)
chart['data']['dependencies'][iter] = {
'chart': chart_dep.get('data')
}
except Exception:
raise Exception(
"Could not find dependency chart {} in {}".format(
dep, DOCUMENT_CHART))
def build_chart_group(self, chart_group):
try:
chart = None
for iter, chart in enumerate(chart_group.get('data').get(
'chart_group', [])):
if isinstance(chart, dict):
continue
chart_dep = self.find_chart_document(chart)
chart_group['data']['chart_group'][iter] = {
'chart': chart_dep.get('data')
}
except Exception:
raise Exception(
"Could not find chart {} in {}".format(
chart, DOCUMENT_GROUP))
def build_armada_manifest(self):
try:
group = None
for iter, group in enumerate(self.manifest.get('data').get(
'chart_groups', [])):
if isinstance(group, dict):
continue
chart_grp = self.find_chart_group_document(group)
self.manifest['data']['chart_groups'][iter] = chart_grp.get(
'data')
except Exception:
raise Exception(
"Could not find chart group {} in {}".format(
group, DOCUMENT_MANIFEST))
def get_manifest(self):
self.build_charts_deps()
self.build_chart_groups()
self.build_armada_manifest()
return {
'armada': self.manifest.get('data')
}